No OneTemporary
Actions

Size

6 MB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/cddl/lib/libnvpair/Makefile b/cddl/lib/libnvpair/Makefile
	index fb80d7c22ea5..a93ffcbd00de 100644
	--- a/cddl/lib/libnvpair/Makefile
	+++ b/cddl/lib/libnvpair/Makefile
	@@ -1,32 +1,30 @@

	.PATH: ${SRCTOP}/sys/contrib/openzfs/module/nvpair
	.PATH: ${SRCTOP}/sys/contrib/openzfs/lib/libnvpair
	.PATH: ${SRCTOP}/sys/contrib/openzfs/include

	LIB= nvpair
	LIBADD= spl

	PACKAGE= zfs
	# user
	INCS= libnvpair.h
	SRCS= libnvpair.c \
	libnvpair_json.c \
	nvpair_alloc_system.c
	# kernel
	SRCS+= nvpair_alloc_fixed.c \
	nvpair.c \
	fnvpair.c

	WARNS?= 2
	CFLAGS+= -DIN_BASE -DHAVE_RPC_TYPES
	CFLAGS+= -I${SRCTOP}/sys/contrib/openzfs/include
	CFLAGS+= -I${SRCTOP}/sys/contrib/openzfs/lib/libspl/include/
	CFLAGS+= -I${SRCTOP}/sys/contrib/openzfs/lib/libspl/include/os/freebsd
	CFLAGS+= -I${SRCTOP}/sys
	CFLAGS+= -I${SRCTOP}/cddl/compat/opensolaris/include
	CFLAGS+= -include ${SRCTOP}/sys/contrib/openzfs/include/os/freebsd/spl/sys/ccompile.h
	CFLAGS+= -DHAVE_ISSETUGID -DHAVE_CONFIG_H -DHAVE_XDR_BYTESREC

	-
	-CFLAGS.nvpair.c+= -UHAVE_RPC_TYPES
	.include <bsd.lib.mk>
	diff --git a/sys/contrib/openzfs/.mailmap b/sys/contrib/openzfs/.mailmap
	index 46ef016b93f8..32bdb5209613 100644
	--- a/sys/contrib/openzfs/.mailmap
	+++ b/sys/contrib/openzfs/.mailmap
	@@ -1,189 +1,207 @@
	# This file maps the name+email seen in a commit back to a canonical
	# name+email. Git will replace the commit name/email with the canonical version
	# wherever it sees it.
	#
	# If there is a commit in the history with a "wrong" name or email, list it
	# here. If you regularly commit with an alternate name or email address and
	# would like to ensure that you are always listed consistently in the repo, add
	# mapping here.
	#
	# On the other hand, if you use multiple names or email addresses legitimately
	# (eg you use a company email address for your paid OpenZFS work, and a
	# personal address for your evening side projects), then don't map one to the
	# other here.
	#
	# The most common formats are:
	#
	# Canonical Name <canonical-email>
	# Canonical Name <canonical-email> <commit-email>
	# Canonical Name <canonical-email> Commit Name <commit-email>
	#
	# See https://git-scm.com/docs/gitmailmap for more info.

	# These maps are making names consistent where they have varied but the email
	# address has never changed. In most cases, the full name is in the
	# Signed-off-by of a commit with a matching author.
	Ahelenia Ziemiańska <nabijaczleweli@gmail.com>
	Ahelenia Ziemiańska <nabijaczleweli@nabijaczleweli.xyz>
	Alex John <alex@stty.io>
	Andreas Dilger <adilger@dilger.ca>
	Andrew Walker <awalker@ixsystems.com>
	Benedikt Neuffer <github@itfriend.de>
	Chengfei Zhu <chengfeix.zhu@intel.com>
	+ChenHao Lu <18302010006@fudan.edu.cn>
	Chris Lindee <chris.lindee+github@gmail.com>
	Colm Buckley <colm@tuatha.org>
	Crag Wang <crag0715@gmail.com>
	Damian Szuberski <szuberskidamian@gmail.com>
	Daniel Kolesa <daniel@octaforge.org>
	Debabrata Banerjee <dbavatar@gmail.com>
	Finix Yan <yanchongwen@hotmail.com>
	Gaurav Kumar <gauravk.18@gmail.com>
	Gionatan Danti <g.danti@assyoma.it>
	Glenn Washburn <development@efficientek.com>
	Gordan Bobic <gordan.bobic@gmail.com>
	Gregory Bartholomew <gregory.lee.bartholomew@gmail.com>
	hedong zhang <h_d_zhang@163.com>
	+Ilkka Sovanto <github@ilkka.kapsi.fi>
	InsanePrawn <Insane.Prawny@gmail.com>
	Jason Cohen <jwittlincohen@gmail.com>
	Jason Harmening <jason.harmening@gmail.com>
	Jeremy Faulkner <gldisater@gmail.com>
	Jinshan Xiong <jinshan.xiong@gmail.com>
	John Poduska <jpoduska@datto.com>
	Justin Scholz <git@justinscholz.de>
	Ka Ho Ng <khng300@gmail.com>
	Kash Pande <github@tripleback.net>
	Kay Pedersen <christianpe96@gmail.com>
	KernelOfTruth <kerneloftruth@gmail.com>
	Liu Hua <liu.hua130@zte.com.cn>
	Liu Qing <winglq@gmail.com>
	loli10K <ezomori.nozomu@gmail.com>
	+Mart Frauenlob <allkind@fastest.cc>
	Matthias Blankertz <matthias@blankertz.org>
	Michael Gmelin <grembo@FreeBSD.org>
	Olivier Mazouffre <olivier.mazouffre@ims-bordeaux.fr>
	Piotr Kubaj <pkubaj@anongoth.pl>
	Quentin Zdanis <zdanisq@gmail.com>
	Roberto Ricci <ricci@disroot.org>
	Rob Norris <robn@despairlabs.com>
	Rob Norris <rob.norris@klarasystems.com>
	Sam Lunt <samuel.j.lunt@gmail.com>
	Sanjeev Bagewadi <sanjeev.bagewadi@gmail.com>
	Stoiko Ivanov <github@nomore.at>
	Tamas TEVESZ <ice@extreme.hu>
	WHR <msl0000023508@gmail.com>
	Yanping Gao <yanping.gao@xtaotech.com>
	Youzhong Yang <youzhong@gmail.com>

	+# Signed-off-by: overriding Author:
	+Yuxin Wang <yuxinwang9999@gmail.com> <Bi11gates9999@gmail.com>
	+
	# Commits from strange places, long ago
	Brian Behlendorf <behlendorf1@llnl.gov> <behlendo@7e1ea52c-4ff2-0310-8f11-9dd32ca42a1c>
	Brian Behlendorf <behlendorf1@llnl.gov> <behlendo@fedora-17-amd64.(none)>
	Brian Behlendorf <behlendorf1@llnl.gov> <behlendo@myhost.(none)>
	Brian Behlendorf <behlendorf1@llnl.gov> <ubuntu@ip-172-31-16-145.us-west-1.compute.internal>
	Brian Behlendorf <behlendorf1@llnl.gov> <ubuntu@ip-172-31-20-6.us-west-1.compute.internal>
	Herb Wartens <wartens2@llnl.gov> <wartens2@7e1ea52c-4ff2-0310-8f11-9dd32ca42a1c>
	Ned Bass <bass6@llnl.gov> <bass6@zeno1.(none)>
	Tulsi Jain <tulsi.jain@delphix.com> <tulsi.jain@Tulsi-Jains-MacBook-Pro.local>

	# Mappings from Github no-reply addresses
	ajs124 <git@ajs124.de> <ajs124@users.noreply.github.com>
	Alek Pinchuk <apinchuk@axcient.com> <alek-p@users.noreply.github.com>
	Alexander Lobakin <alobakin@pm.me> <solbjorn@users.noreply.github.com>
	Alexey Smirnoff <fling@member.fsf.org> <fling-@users.noreply.github.com>
	Allen Holl <allen.m.holl@gmail.com> <65494904+allen-4@users.noreply.github.com>
	Ameer Hamza <ahamza@ixsystems.com> <106930537+ixhamza@users.noreply.github.com>
	Andrew J. Hesford <ajh@sideband.org> <48421688+ahesford@users.noreply.github.com>>
	Andrew Sun <me@andrewsun.com> <as-com@users.noreply.github.com>
	Aron Xu <happyaron.xu@gmail.com> <happyaron@users.noreply.github.com>
	Arun KV <arun.kv@datacore.com> <65647132+arun-kv@users.noreply.github.com>
	Ben Wolsieffer <benwolsieffer@gmail.com> <lopsided98@users.noreply.github.com>
	bernie1995 <bernie.pikes@gmail.com> <42413912+bernie1995@users.noreply.github.com>
	Boris Protopopov <boris.protopopov@actifio.com> <bprotopopov@users.noreply.github.com>
	Brad Forschinger <github@bnjf.id.au> <bnjf@users.noreply.github.com>
	Brandon Thetford <brandon@dodecatec.com> <dodexahedron@users.noreply.github.com>
	buzzingwires <buzzingwires@outlook.com> <131118055+buzzingwires@users.noreply.github.com>
	Cedric Maunoury <cedric.maunoury@gmail.com> <38213715+cedricmaunoury@users.noreply.github.com>
	Charles Suh <charles.suh@gmail.com> <charlessuh@users.noreply.github.com>
	+Chris Peredun <chris.peredun@ixsystems.com> <126915832+chrisperedun@users.noreply.github.com>
	Dacian Reece-Stremtan <dacianstremtan@gmail.com> <35844628+dacianstremtan@users.noreply.github.com>
	Damian Szuberski <szuberskidamian@gmail.com> <30863496+szubersk@users.noreply.github.com>
	Daniel Hiepler <d-git@coderdu.de> <32984777+heeplr@users.noreply.github.com>
	Daniel Kobras <d.kobras@science-computing.de> <sckobras@users.noreply.github.com>
	Daniel Reichelt <hacking@nachtgeist.net> <nachtgeist@users.noreply.github.com>
	David Quigley <david.quigley@intel.com> <dpquigl@users.noreply.github.com>
	+Dennis R. Friedrichsen <dennis.r.friedrichsen@gmail.com> <31087738+dennisfriedrichsen@users.noreply.github.com>
	+Dex Wood <slash2314@gmail.com> <slash2314@users.noreply.github.com>
	DHE <git@dehacked.net> <DeHackEd@users.noreply.github.com>
	Dmitri John Ledkov <dimitri.ledkov@canonical.com> <19779+xnox@users.noreply.github.com>
	Dries Michiels <driesm.michiels@gmail.com> <32487486+driesmp@users.noreply.github.com>
	Edmund Nadolski <edmund.nadolski@ixsystems.com> <137826107+ednadolski-ix@users.noreply.github.com>
	Érico Nogueira <erico.erc@gmail.com> <34201958+ericonr@users.noreply.github.com>
	Fedor Uporov <fuporov.vstack@gmail.com> <60701163+fuporovvStack@users.noreply.github.com>
	Felix Dörre <felix@dogcraft.de> <felixdoerre@users.noreply.github.com>
	Felix Neumärker <xdch47@posteo.de> <34678034+xdch47@users.noreply.github.com>
	Finix Yan <yancw@info2soft.com> <Finix1979@users.noreply.github.com>
	Gaurav Kumar <gauravk.18@gmail.com> <gaurkuma@users.noreply.github.com>
	George Gaydarov <git@gg7.io> <gg7@users.noreply.github.com>
	Georgy Yakovlev <gyakovlev@gentoo.org> <168902+gyakovlev@users.noreply.github.com>
	Gerardwx <gerardw@alum.mit.edu> <Gerardwx@users.noreply.github.com>
	Gian-Carlo DeFazio <defazio1@llnl.gov> <defaziogiancarlo@users.noreply.github.com>
	Giuseppe Di Natale <dinatale2@llnl.gov> <dinatale2@users.noreply.github.com>
	Hajo Möller <dasjoe@gmail.com> <dasjoe@users.noreply.github.com>
	Harry Mallon <hjmallon@gmail.com> <1816667+hjmallon@users.noreply.github.com>
	Hiếu Lê <leorize+oss@disroot.org> <alaviss@users.noreply.github.com>
	Jake Howard <git@theorangeone.net> <RealOrangeOne@users.noreply.github.com>
	James Cowgill <james.cowgill@mips.com> <jcowgill@users.noreply.github.com>
	+Jaron Kent-Dobias <jaron@kent-dobias.com> <kentdobias@users.noreply.github.com>
	Jason King <jason.king@joyent.com> <jasonbking@users.noreply.github.com>
	Jeff Dike <jdike@akamai.com> <52420226+jdike@users.noreply.github.com>
	Jitendra Patidar <jitendra.patidar@nutanix.com> <53164267+jsai20@users.noreply.github.com>
	João Carlos Mendes Luís <jonny@jonny.eng.br> <dioni21@users.noreply.github.com>
	John Eismeier <john.eismeier@gmail.com> <32205350+jeis2497052@users.noreply.github.com>
	John L. Hammond <john.hammond@intel.com> <35266395+jhammond-intel@users.noreply.github.com>
	John-Mark Gurney <jmg@funkthat.com> <jmgurney@users.noreply.github.com>
	John Ramsden <johnramsden@riseup.net> <johnramsden@users.noreply.github.com>
	Jonathon Fernyhough <jonathon@m2x.dev> <559369+jonathonf@users.noreply.github.com>
	+Jose Luis Duran <jlduran@gmail.com> <jlduran@users.noreply.github.com>
	Justin Hibbits <chmeeedalf@gmail.com> <chmeeedalf@users.noreply.github.com>
	+Kevin Greene <kevin.greene@delphix.com> <104801862+kxgreene@users.noreply.github.com>
	Kevin Jin <lostking2008@hotmail.com> <33590050+jxdking@users.noreply.github.com>
	Kevin P. Fleming <kevin@km6g.us> <kpfleming@users.noreply.github.com>
	Krzysztof Piecuch <piecuch@kpiecuch.pl> <3964215+pikrzysztof@users.noreply.github.com>
	Kyle Evans <kevans@FreeBSD.org> <kevans91@users.noreply.github.com>
	Laurențiu Nicola <lnicola@dend.ro> <lnicola@users.noreply.github.com>
	loli10K <ezomori.nozomu@gmail.com> <loli10K@users.noreply.github.com>
	Lorenz Hüdepohl <dev@stellardeath.org> <lhuedepohl@users.noreply.github.com>
	Luís Henriques <henrix@camandro.org> <73643340+lumigch@users.noreply.github.com>
	Marcin Skarbek <git@skarbek.name> <mskarbek@users.noreply.github.com>
	Matt Fiddaman <github@m.fiddaman.uk> <81489167+matt-fidd@users.noreply.github.com>
	+Maxim Filimonov <che@bein.link> <part1zano@users.noreply.github.com>
	Max Zettlmeißl <max@zettlmeissl.de> <6818198+maxz@users.noreply.github.com>
	Michael Niewöhner <foss@mniewoehner.de> <c0d3z3r0@users.noreply.github.com>
	Michael Zhivich <mzhivich@akamai.com> <33133421+mzhivich@users.noreply.github.com>
	+MigeljanImeri <ImeriMigel@gmail.com> <78048439+MigeljanImeri@users.noreply.github.com>
	Mo Zhou <cdluminate@gmail.com> <5723047+cdluminate@users.noreply.github.com>
	Nick Mattis <nickm970@gmail.com> <nmattis@users.noreply.github.com>
	omni <omni+vagant@hack.org> <79493359+omnivagant@users.noreply.github.com>
	Pablo Correa Gómez <ablocorrea@hotmail.com> <32678034+pablofsf@users.noreply.github.com>
	Paul Zuchowski <pzuchowski@datto.com> <31706010+PaulZ-98@users.noreply.github.com>
	Peter Ashford <ashford@accs.com> <pashford@users.noreply.github.com>
	Peter Dave Hello <hsu@peterdavehello.org> <PeterDaveHello@users.noreply.github.com>
	Peter Wirdemo <peter.wirdemo@gmail.com> <4224155+pewo@users.noreply.github.com>
	Petros Koutoupis <petros@petroskoutoupis.com> <pkoutoupis@users.noreply.github.com>
	Ping Huang <huangping@smartx.com> <101400146+hpingfs@users.noreply.github.com>
	Piotr P. Stefaniak <pstef@freebsd.org> <pstef@users.noreply.github.com>
	Richard Allen <belperite@gmail.com> <33836503+belperite@users.noreply.github.com>
	Rich Ercolani <rincebrain@gmail.com> <214141+rincebrain@users.noreply.github.com>
	+Rick Macklem <rmacklem@uoguelph.ca> <64620010+rmacklem@users.noreply.github.com>
	Rob Wing <rob.wing@klarasystems.com> <98866084+rob-wing@users.noreply.github.com>
	Roman Strashkin <roman.strashkin@nexenta.com> <Ramzec@users.noreply.github.com>
	Ryan Hirasaki <ryanhirasaki@gmail.com> <4690732+RyanHir@users.noreply.github.com>
	Samuel Wycliffe J <samwyc@hpe.com> <115969550+samwyc@users.noreply.github.com>
	Samuel Wycliffe <samuelwycliffe@gmail.com> <50765275+npc203@users.noreply.github.com>
	Savyasachee Jha <hi@savyasacheejha.com> <savyajha@users.noreply.github.com>
	Scott Colby <scott@scolby.com> <scolby33@users.noreply.github.com>
	Sean Eric Fagan <kithrup@mac.com> <kithrup@users.noreply.github.com>
	Spencer Kinny <spencerkinny1995@gmail.com> <30333052+Spencer-Kinny@users.noreply.github.com>
	Srikanth N S <srikanth.nagasubbaraoseetharaman@hpe.com> <75025422+nssrikanth@users.noreply.github.com>
	+Stefan Lendl <s.lendl@proxmox.com> <1321542+stfl@users.noreply.github.com>
	+Thomas Bertschinger <bertschinger@lanl.gov> <101425190+bertschinger@users.noreply.github.com>
	Thomas Geppert <geppi@digitx.de> <geppi@users.noreply.github.com>
	Tim Crawford <tcrawford@datto.com> <crawfxrd@users.noreply.github.com>
	Tom Matthews <tom@axiom-partners.com> <tomtastic@users.noreply.github.com>
	Tony Perkins <tperkins@datto.com> <62951051+tony-zfs@users.noreply.github.com>
	Torsten Wörtwein <twoertwein@gmail.com> <twoertwein@users.noreply.github.com>
	Tulsi Jain <tulsi.jain@delphix.com> <TulsiJain@users.noreply.github.com>
	Václav Skála <skala@vshosting.cz> <33496485+vaclavskala@users.noreply.github.com>
	+Vaibhav Bhanawat <vaibhav.bhanawat@delphix.com> <88050553+vaibhav-delphix@users.noreply.github.com>
	Violet Purcell <vimproved@inventati.org> <66446404+vimproved@users.noreply.github.com>
	Vipin Kumar Verma <vipin.verma@hpe.com> <75025470+vermavipinkumar@users.noreply.github.com>
	Wolfgang Bumiller <w.bumiller@proxmox.com> <Blub@users.noreply.github.com>
	xtouqh <xtouqh@hotmail.com> <72357159+xtouqh@users.noreply.github.com>
	Yuri Pankov <yuripv@FreeBSD.org> <113725409+yuripv@users.noreply.github.com>
	Yuri Pankov <yuripv@FreeBSD.org> <82001006+yuripv@users.noreply.github.com>
	diff --git a/sys/contrib/openzfs/AUTHORS b/sys/contrib/openzfs/AUTHORS
	index be1efb87b34c..d7d55f42d2e7 100644
	--- a/sys/contrib/openzfs/AUTHORS
	+++ b/sys/contrib/openzfs/AUTHORS
	@@ -1,624 +1,659 @@
	MAINTAINERS:

	Brian Behlendorf <behlendorf1@llnl.gov>
	Tony Hutter <hutter2@llnl.gov>

	PAST MAINTAINERS:

	Ned Bass <bass6@llnl.gov>

	CONTRIBUTORS:

	Aaron Fineman <abyxcos@gmail.com>
	Adam D. Moss <c@yotes.com>
	Adam Leventhal <ahl@delphix.com>
	Adam Stevko <adam.stevko@gmail.com>
	adisbladis <adis@blad.is>
	Adrian Chadd <adrian@freebsd.org>
	Ahelenia Ziemiańska <nabijaczleweli@nabijaczleweli.xyz>
	Ahmed G <ahmedg@delphix.com>
	Aidan Harris <me@aidanharr.is>
	AJ Jordan <alex@strugee.net>
	ajs124 <git@ajs124.de>
	Akash Ayare <aayare@delphix.com>
	Akash B <akash-b@hpe.com>
	Alan Somers <asomers@gmail.com>
	Alar Aun <spamtoaun@gmail.com>
	Albert Lee <trisk@nexenta.com>
	Alec Salazar <alec.j.salazar@gmail.com>
	Alejandro Colomar <Colomar.6.4.3@GMail.com>
	Alejandro R. Sedeño <asedeno@mit.edu>
	Alek Pinchuk <alek@nexenta.com>
	Aleksa Sarai <cyphar@cyphar.com>
	Alexander Eremin <a.eremin@nexenta.com>
	Alexander Lobakin <alobakin@pm.me>
	Alexander Motin <mav@freebsd.org>
	Alexander Pyhalov <apyhalov@gmail.com>
	Alexander Richardson <Alexander.Richardson@cl.cam.ac.uk>
	Alexander Stetsenko <ams@nexenta.com>
	Alex Braunegg <alex.braunegg@gmail.com>
	Alexey Shvetsov <alexxy@gentoo.org>
	Alexey Smirnoff <fling@member.fsf.org>
	Alex John <alex@stty.io>
	Alex McWhirter <alexmcwhirter@triadic.us>
	Alex Reece <alex@delphix.com>
	Alex Wilson <alex.wilson@joyent.com>
	Alex Zhuravlev <alexey.zhuravlev@intel.com>
	Allan Jude <allanjude@freebsd.org>
	Allen Holl <allen.m.holl@gmail.com>
	alteriks <alteriks@gmail.com>
	Alyssa Ross <hi@alyssa.is>
	Ameer Hamza <ahamza@ixsystems.com>
	Anatoly Borodin <anatoly.borodin@gmail.com>
	AndCycle <andcycle@andcycle.idv.tw>
	Andrea Gelmini <andrea.gelmini@gelma.net>
	Andrea Righi <andrea.righi@canonical.com>
	Andreas Buschmann <andreas.buschmann@tech.net.de>
	Andreas Dilger <adilger@intel.com>
	Andreas Vögele <andreas@andreasvoegele.com>
	Andrew Barnes <barnes333@gmail.com>
	Andrew Hamilton <ahamilto@tjhsst.edu>
	Andrew Innes <andrew.c12@gmail.com>
	Andrew J. Hesford <ajh@sideband.org>
	Andrew Reid <ColdCanuck@nailedtotheperch.com>
	Andrew Stormont <andrew.stormont@nexenta.com>
	Andrew Sun <me@andrewsun.com>
	Andrew Tselischev <andrewtselischev@gmail.com>
	Andrew Turner <andrew@fubar.geek.nz>
	Andrew Walker <awalker@ixsystems.com>
	Andrey Prokopenko <job@terem.fr>
	Andrey Vesnovaty <andrey.vesnovaty@gmail.com>
	Andriy Gapon <avg@freebsd.org>
	Andy Bakun <github@thwartedefforts.org>
	Andy Fiddaman <omnios@citrus-it.co.uk>
	Aniruddha Shankar <k@191a.net>
	Anton Gubarkov <anton.gubarkov@gmail.com>
	Antonio Russo <antonio.e.russo@gmail.com>
	Arkadiusz Bubała <arkadiusz.bubala@open-e.com>
	Armin Wehrfritz <dkxls23@gmail.com>
	Arne Jansen <arne@die-jansens.de>
	Aron Xu <happyaron.xu@gmail.com>
	Arshad Hussain <arshad.hussain@aeoncomputing.com>
	Arun KV <arun.kv@datacore.com>
	Arvind Sankar <nivedita@alum.mit.edu>
	Attila Fülöp <attila@fueloep.org>
	Avatat <kontakt@avatat.pl>
	Bart Coddens <bart.coddens@gmail.com>
	Basil Crow <basil.crow@delphix.com>
	Bassu <bassu@phi9.com>
	Ben Allen <bsallen@alcf.anl.gov>
	Ben Cordero <bencord0@condi.me>
	+ Benda Xu <orv@debian.org>
	Benedikt Neuffer <github@itfriend.de>
	Benjamin Albrecht <git@albrecht.io>
	Benjamin Gentil <benjgentil.pro@gmail.com>
	+ Benjamin Sherman <benjamin@holyarmy.org>
	Ben McGough <bmcgough@fredhutch.org>
	Ben Rubson <ben.rubson@gmail.com>
	Ben Wolsieffer <benwolsieffer@gmail.com>
	bernie1995 <bernie.pikes@gmail.com>
	Bill McGonigle <bill-github.com-public1@bfccomputing.com>
	Bill Pijewski <wdp@joyent.com>
	Boris Protopopov <boris.protopopov@nexenta.com>
	Brad Forschinger <github@bnjf.id.au>
	Brad Lewis <brad.lewis@delphix.com>
	Brandon Thetford <brandon@dodecatec.com>
	Brian Atkinson <bwa@g.clemson.edu>
	Brian Behlendorf <behlendorf1@llnl.gov>
	Brian J. Murrell <brian@sun.com>
	Brooks Davis <brooks@one-eyed-alien.net>
	BtbN <btbn@btbn.de>
	bunder2015 <omfgbunder@gmail.com>
	buzzingwires <buzzingwires@outlook.com>
	bzzz77 <bzzz.tomas@gmail.com>
	cable2999 <cable2999@users.noreply.github.com>
	Caleb James DeLisle <calebdelisle@lavabit.com>
	+ Cameron Harr <harr1@llnl.gov>
	Cao Xuewen <cao.xuewen@zte.com.cn>
	Carlo Landmeter <clandmeter@gmail.com>
	Carlos Alberto Lopez Perez <clopez@igalia.com>
	Cedric Maunoury <cedric.maunoury@gmail.com>
	Chaoyu Zhang <zhang.chaoyu@zte.com.cn>
	Charles Suh <charles.suh@gmail.com>
	Chen Can <chen.can2@zte.com.cn>
	Chengfei Zhu <chengfeix.zhu@intel.com>
	Chen Haiquan <oc@yunify.com>
	+ ChenHao Lu <18302010006@fudan.edu.cn>
	Chip Parker <aparker@enthought.com>
	Chris Burroughs <chris.burroughs@gmail.com>
	+ Chris Davidson <christopher.davidson@gmail.com>
	Chris Dunlap <cdunlap@llnl.gov>
	Chris Dunlop <chris@onthe.net.au>
	Chris Lindee <chris.lindee+github@gmail.com>
	Chris McDonough <chrism@plope.com>
	+ Chris Peredun <chris.peredun@ixsystems.com>
	Chris Siden <chris.siden@delphix.com>
	Chris Siebenmann <cks.github@cs.toronto.edu>
	Christer Ekholm <che@chrekh.se>
	Christian Kohlschütter <christian@kohlschutter.com>
	Christian Neukirchen <chneukirchen@gmail.com>
	Christian Schwarz <me@cschwarz.com>
	Christopher Voltz <cjunk@voltz.ws>
	Christ Schlacta <aarcane@aarcane.info>
	Chris Wedgwood <cw@f00f.org>
	Chris Williamson <chris.williamson@delphix.com>
	Chris Zubrzycki <github@mid-earth.net>
	Chuck Tuffli <ctuffli@gmail.com>
	Chunwei Chen <david.chen@nutanix.com>
	Clemens Fruhwirth <clemens@endorphin.org>
	Clemens Lang <cl@clang.name>
	Clint Armstrong <clint@clintarmstrong.net>
	Coleman Kane <ckane@colemankane.org>
	Colin Ian King <colin.king@canonical.com>
	+ Colin Percival <cperciva@tarsnap.com>
	Colm Buckley <colm@tuatha.org>
	Crag Wang <crag0715@gmail.com>
	Craig Loomis <cloomis@astro.princeton.edu>
	Craig Sanders <github@taz.net.au>
	Cyril Plisko <cyril.plisko@infinidat.com>
	Cy Schubert <cy@FreeBSD.org>
	Cédric Berger <cedric@precidata.com>
	Dacian Reece-Stremtan <dacianstremtan@gmail.com>
	Dag-Erling Smørgrav <des@FreeBSD.org>
	Damiano Albani <damiano.albani@gmail.com>
	Damian Szuberski <szuberskidamian@gmail.com>
	Damian Wojsław <damian@wojslaw.pl>
	+ Daniel Berlin <dberlin@dberlin.org>
	Daniel Hiepler <d-git@coderdu.de>
	Daniel Hoffman <dj.hoffman@delphix.com>
	Daniel Kobras <d.kobras@science-computing.de>
	Daniel Kolesa <daniel@octaforge.org>
	Daniel Reichelt <hacking@nachtgeist.net>
	Daniel Stevenson <bot@dstev.net>
	Daniel Verite <daniel@verite.pro>
	Daniil Lunev <d.lunev.mail@gmail.com>
	Dan Kimmel <dan.kimmel@delphix.com>
	Dan McDonald <danmcd@nexenta.com>
	Dan Swartzendruber <dswartz@druber.com>
	Dan Vatca <dan.vatca@gmail.com>
	Darik Horn <dajhorn@vanadac.com>
	Dave Eddy <dave@daveeddy.com>
	David Hedberg <david@qzx.se>
	David Lamparter <equinox@diac24.net>
	David Qian <david.qian@intel.com>
	David Quigley <david.quigley@intel.com>
	Debabrata Banerjee <dbanerje@akamai.com>
	D. Ebdrup <debdrup@freebsd.org>
	+ Dennis R. Friedrichsen <dennis.r.friedrichsen@gmail.com>
	Denys Rtveliashvili <denys@rtveliashvili.name>
	Derek Dai <daiderek@gmail.com>
	+ Dex Wood <slash2314@gmail.com>
	DHE <git@dehacked.net>
	Didier Roche <didrocks@ubuntu.com>
	Dimitri John Ledkov <xnox@ubuntu.com>
	Dimitry Andric <dimitry@andric.com>
	Dirkjan Bussink <d.bussink@gmail.com>
	Dmitry Khasanov <pik4ez@gmail.com>
	Dominic Pearson <dsp@technoanimal.net>
	Dominik Hassler <hadfl@omniosce.org>
	Dominik Honnef <dominikh@fork-bomb.org>
	Don Brady <don.brady@delphix.com>
	Doug Rabson <dfr@rabson.org>
	Dr. András Korn <korn-github.com@elan.rulez.org>
	Dries Michiels <driesm.michiels@gmail.com>
	Edmund Nadolski <edmund.nadolski@ixsystems.com>
	Eitan Adler <lists@eitanadler.com>
	Eli Rosenthal <eli.rosenthal@delphix.com>
	Eli Schwartz <eschwartz93@gmail.com>
	Eric Desrochers <eric.desrochers@canonical.com>
	Eric Dillmann <eric@jave.fr>
	Eric Schrock <Eric.Schrock@delphix.com>
	Ethan Coe-Renner <coerenner1@llnl.gov>
	Etienne Dechamps <etienne@edechamps.fr>
	Evan Allrich <eallrich@gmail.com>
	Evan Harris <eharris@puremagic.com>
	Evan Susarret <evansus@gmail.com>
	Fabian Grünbichler <f.gruenbichler@proxmox.com>
	Fabio Buso <dev.siroibaf@gmail.com>
	Fabio Scaccabarozzi <fsvm88@gmail.com>
	Fajar A. Nugraha <github@fajar.net>
	Fan Yong <fan.yong@intel.com>
	fbynite <fbynite@users.noreply.github.com>
	Fedor Uporov <fuporov.vstack@gmail.com>
	Felix Dörre <felix@dogcraft.de>
	Felix Neumärker <xdch47@posteo.de>
	Feng Sun <loyou85@gmail.com>
	Finix Yan <yancw@info2soft.com>
	Francesco Mazzoli <f@mazzo.li>
	Frederik Wessels <wessels147@gmail.com>
	Frédéric Vanniere <f.vanniere@planet-work.com>
	Gabriel A. Devenyi <gdevenyi@gmail.com>
	Garrett D'Amore <garrett@nexenta.com>
	Garrett Fields <ghfields@gmail.com>
	Garrison Jensen <garrison.jensen@gmail.com>
	Gary Mills <gary_mills@fastmail.fm>
	Gaurav Kumar <gauravk.18@gmail.com>
	GeLiXin <ge.lixin@zte.com.cn>
	George Amanakis <g_amanakis@yahoo.com>
	George Diamantopoulos <georgediam@gmail.com>
	George Gaydarov <git@gg7.io>
	George Melikov <mail@gmelikov.ru>
	George Wilson <gwilson@delphix.com>
	Georgy Yakovlev <ya@sysdump.net>
	Gerardwx <gerardw@alum.mit.edu>
	Gian-Carlo DeFazio <defazio1@llnl.gov>
	Gionatan Danti <g.danti@assyoma.it>
	Giuseppe Di Natale <guss80@gmail.com>
	Glenn Washburn <development@efficientek.com>
	+ gofaster <felix.gofaster@gmail.com>
	Gordan Bobic <gordan@redsleeve.org>
	Gordon Bergling <gbergling@googlemail.com>
	Gordon Ross <gwr@nexenta.com>
	+ Gordon Tetlow <gordon@freebsd.org>
	Graham Christensen <graham@grahamc.com>
	Graham Perrin <grahamperrin@gmail.com>
	Gregor Kopka <gregor@kopka.net>
	Gregory Bartholomew <gregory.lee.bartholomew@gmail.com>
	grembo <freebsd@grem.de>
	Grischa Zengel <github.zfsonlinux@zengel.info>
	grodik <pat@litke.dev>
	Gunnar Beutner <gunnar@beutner.name>
	Gvozden Neskovic <neskovic@gmail.com>
	Hajo Möller <dasjoe@gmail.com>
	Han Gao <rabenda.cn@gmail.com>
	Hans Rosenfeld <hans.rosenfeld@nexenta.com>
	Harald van Dijk <harald@gigawatt.nl>
	Harry Mallon <hjmallon@gmail.com>
	Harry Sintonen <github-piru@kyber.fi>
	HC <mmttdebbcc@yahoo.com>
	hedong zhang <h_d_zhang@163.com>
	Heitor Alves de Siqueira <halves@canonical.com>
	Henrik Riomar <henrik.riomar@gmail.com>
	Herb Wartens <wartens2@llnl.gov>
	Hiếu Lê <leorize+oss@disroot.org>
	Huang Liu <liu.huang@zte.com.cn>
	Håkan Johansson <f96hajo@chalmers.se>
	Igor K <igor@dilos.org>
	Igor Kozhukhov <ikozhukhov@gmail.com>
	Igor Lvovsky <ilvovsky@gmail.com>
	ilbsmart <wgqimut@gmail.com>
	+ Ilkka Sovanto <github@ilkka.kapsi.fi>
	illiliti <illiliti@protonmail.com>
	ilovezfs <ilovezfs@icloud.com>
	InsanePrawn <Insane.Prawny@gmail.com>
	Isaac Huang <he.huang@intel.com>
	Jacek Fefliński <feflik@gmail.com>
	Jacob Adams <tookmund@gmail.com>
	Jake Howard <git@theorangeone.net>
	James Cowgill <james.cowgill@mips.com>
	James H <james@kagisoft.co.uk>
	James Lee <jlee@thestaticvoid.com>
	James Pan <jiaming.pan@yahoo.com>
	James Wah <james@laird-wah.net>
	Jan Engelhardt <jengelh@inai.de>
	Jan Kryl <jan.kryl@nexenta.com>
	Jan Sanislo <oystr@cs.washington.edu>
	+ Jaron Kent-Dobias <jaron@kent-dobias.com>
	Jason Cohen <jwittlincohen@gmail.com>
	Jason Harmening <jason.harmening@gmail.com>
	Jason King <jason.brian.king@gmail.com>
	+ Jason Lee <jasonlee@lanl.gov>
	Jason Zaman <jasonzaman@gmail.com>
	Javen Wu <wu.javen@gmail.com>
	Jean-Baptiste Lallement <jean-baptiste@ubuntu.com>
	Jeff Dike <jdike@akamai.com>
	Jeremy Faulkner <gldisater@gmail.com>
	Jeremy Gill <jgill@parallax-innovations.com>
	Jeremy Jones <jeremy@delphix.com>
	Jeremy Visser <jeremy.visser@gmail.com>
	Jerry Jelinek <jerry.jelinek@joyent.com>
	Jessica Clarke <jrtc27@jrtc27.com>
	Jinshan Xiong <jinshan.xiong@intel.com>
	Jitendra Patidar <jitendra.patidar@nutanix.com>
	JK Dingwall <james@dingwall.me.uk>
	Joe Stein <joe.stein@delphix.com>
	John-Mark Gurney <jmg@funkthat.com>
	John Albietz <inthecloud247@gmail.com>
	John Eismeier <john.eismeier@gmail.com>
	John Gallagher <john.gallagher@delphix.com>
	John Layman <jlayman@sagecloud.com>
	John L. Hammond <john.hammond@intel.com>
	John M. Layman <jml@frijid.net>
	Johnny Stenback <github@jstenback.com>
	John Paul Adrian Glaubitz <glaubitz@physik.fu-berlin.de>
	John Poduska <jpoduska@datto.com>
	John Ramsden <johnramsden@riseup.net>
	John Wren Kennedy <john.kennedy@delphix.com>
	jokersus <lolivampireslave@gmail.com>
	Jonathon Fernyhough <jonathon@m2x.dev>
	Jorgen Lundman <lundman@lundman.net>
	Josef 'Jeff' Sipek <josef.sipek@nexenta.com>
	+ Jose Luis Duran <jlduran@gmail.com>
	Josh Soref <jsoref@users.noreply.github.com>
	Joshua M. Clulow <josh@sysmgr.org>
	José Luis Salvador Rufo <salvador.joseluis@gmail.com>
	João Carlos Mendes Luís <jonny@jonny.eng.br>
	Julian Brunner <julian.brunner@gmail.com>
	Julian Heuking <JulianH@beckhoff.com>
	jumbi77 <jumbi77@users.noreply.github.com>
	Justin Bedő <cu@cua0.org>
	Justin Gottula <justin@jgottula.com>
	Justin Hibbits <chmeeedalf@gmail.com>
	Justin Keogh <github.com@v6y.net>
	Justin Lecher <jlec@gentoo.org>
	Justin Scholz <git@justinscholz.de>
	Justin T. Gibbs <gibbs@FreeBSD.org>
	jyxent <jordanp@gmail.com>
	Jörg Thalheim <joerg@higgsboson.tk>
	ka7 <ka7@la-evento.com>
	Ka Ho Ng <khng@FreeBSD.org>
	Kamil Domański <kamil@domanski.co>
	Karsten Kretschmer <kkretschmer@gmail.com>
	Kash Pande <kash@tripleback.net>
	Kay Pedersen <christianpe96@gmail.com>
	Keith M Wesolowski <wesolows@foobazco.org>
	+ Kent Ross <k@mad.cash>
	KernelOfTruth <kerneloftruth@gmail.com>
	Kevin Bowling <kevin.bowling@kev009.com>
	+ Kevin Greene <kevin.greene@delphix.com>
	Kevin Jin <lostking2008@hotmail.com>
	Kevin P. Fleming <kevin@km6g.us>
	Kevin Tanguy <kevin.tanguy@ovh.net>
	KireinaHoro <i@jsteward.moe>
	Kjeld Schouten-Lebbing <kjeld@schouten-lebbing.nl>
	Kleber Tarcísio <klebertarcisio@yahoo.com.br>
	Kody A Kantor <kody.kantor@gmail.com>
	Kohsuke Kawaguchi <kk@kohsuke.org>
	Konstantin Khorenko <khorenko@virtuozzo.com>
	KORN Andras <korn@elan.rulez.org>
	Kristof Provost <github@sigsegv.be>
	Krzysztof Piecuch <piecuch@kpiecuch.pl>
	Kyle Blatter <kyleblatter@llnl.gov>
	Kyle Evans <kevans@FreeBSD.org>
	Kyle Fuller <inbox@kylefuller.co.uk>
	Laevos <Laevos@users.noreply.github.com>
	Lalufu <Lalufu@users.noreply.github.com>
	Lars Johannsen <laj@it.dk>
	Laura Hild <lsh@jlab.org>
	Laurențiu Nicola <lnicola@dend.ro>
	Lauri Tirkkonen <lauri@hacktheplanet.fi>
	liaoyuxiangqin <guo.yong33@zte.com.cn>
	Li Dongyang <dongyang.li@anu.edu.au>
	Liu Hua <liu.hua130@zte.com.cn>
	Liu Qing <winglq@gmail.com>
	Li Wei <W.Li@Sun.COM>
	Loli <ezomori.nozomu@gmail.com>
	lorddoskias <lorddoskias@gmail.com>
	Lorenz Brun <lorenz@dolansoft.org>
	Lorenz Hüdepohl <dev@stellardeath.org>
	louwrentius <louwrentius@gmail.com>
	Lukas Wunner <lukas@wunner.de>
	luozhengzheng <luo.zhengzheng@zte.com.cn>
	Luís Henriques <henrix@camandro.org>
	Madhav Suresh <madhav.suresh@delphix.com>
	manfromafar <jonsonb10@gmail.com>
	Manoj Joseph <manoj.joseph@delphix.com>
	Manuel Amador (Rudd-O) <rudd-o@rudd-o.com>
	Marcel Huber <marcelhuberfoo@gmail.com>
	Marcel Menzel <mail@mcl.gg>
	Marcel Schilling <marcel.schilling@uni-luebeck.de>
	Marcel Telka <marcel.telka@nexenta.com>
	Marcel Wysocki <maci.stgn@gmail.com>
	Marcin Skarbek <git@skarbek.name>
	Mariusz Zaborski <mariusz.zaborski@klarasystems.com>
	Mark Johnston <markj@FreeBSD.org>
	Mark Maybee <mark.maybee@delphix.com>
	Mark Roper <markroper@gmail.com>
	Mark Shellenbaum <Mark.Shellenbaum@Oracle.COM>
	marku89 <mar42@kola.li>
	Mark Wright <markwright@internode.on.net>
	+ Mart Frauenlob <allkind@fastest.cc>
	Martin Matuska <mm@FreeBSD.org>
	Martin Rüegg <martin.rueegg@metaworx.ch>
	Massimo Maggi <me@massimo-maggi.eu>
	Mateusz Guzik <mjguzik@gmail.com>
	Mateusz Piotrowski <0mp@FreeBSD.org>
	Mathieu Velten <matmaul@gmail.com>
	Matt Fiddaman <github@m.fiddaman.uk>
	Matthew Ahrens <matt@delphix.com>
	Matthew Thode <mthode@mthode.org>
	Matthias Blankertz <matthias@blankertz.org>
	Matt Johnston <matt@fugro-fsi.com.au>
	Matt Kemp <matt@mattikus.com>
	Matt Macy <mmacy@freebsd.org>
	Matus Kral <matuskral@me.com>
	Mauricio Faria de Oliveira <mfo@canonical.com>
	Max Grossman <max.grossman@delphix.com>
	+ Maxim Filimonov <che@bein.link>
	Maximilian Mehnert <maximilian.mehnert@gmx.de>
	Max Zettlmeißl <max@zettlmeissl.de>
	Md Islam <mdnahian@outlook.com>
	megari <megari@iki.fi>
	Michael D Labriola <michael.d.labriola@gmail.com>
	Michael Franzl <michael@franzl.name>
	Michael Gebetsroither <michael@mgeb.org>
	Michael Kjorling <michael@kjorling.se>
	Michael Martin <mgmartin.mgm@gmail.com>
	Michael Niewöhner <foss@mniewoehner.de>
	Michael Zhivich <mzhivich@akamai.com>
	Michal Vasilek <michal@vasilek.cz>
	+ MigeljanImeri <ImeriMigel@gmail.com>
	Mike Gerdts <mike.gerdts@joyent.com>
	Mike Harsch <mike@harschsystems.com>
	Mike Leddy <mike.leddy@gmail.com>
	Mike Swanson <mikeonthecomputer@gmail.com>
	Milan Jurik <milan.jurik@xylab.cz>
	Minsoo Choo <minsoochoo0122@proton.me>
	Mohamed Tawfik <m_tawfik@aucegypt.edu>
	Morgan Jones <mjones@rice.edu>
	Moritz Maxeiner <moritz@ucworks.org>
	Mo Zhou <cdluminate@gmail.com>
	naivekun <naivekun@outlook.com>
	nathancheek <myself@nathancheek.com>
	Nathaniel Clark <Nathaniel.Clark@misrule.us>
	Nathaniel Wesley Filardo <nwf@cs.jhu.edu>
	Nathan Lewis <linux.robotdude@gmail.com>
	Nav Ravindranath <nav@delphix.com>
	Neal Gompa (ニール・ゴンパ) <ngompa13@gmail.com>
	Ned Bass <bass6@llnl.gov>
	Neependra Khare <neependra@kqinfotech.com>
	Neil Stockbridge <neil@dist.ro>
	Nick Black <dank@qemfd.net>
	Nick Garvey <garvey.nick@gmail.com>
	Nick Mattis <nickm970@gmail.com>
	Nick Terrell <terrelln@fb.com>
	Niklas Haas <github-c6e1c8@haasn.xyz>
	Nikolay Borisov <n.borisov.lkml@gmail.com>
	nordaux <nordaux@gmail.com>
	ofthesun9 <olivier@ofthesun.net>
	Olaf Faaland <faaland1@llnl.gov>
	Oleg Drokin <green@linuxhacker.ru>
	Oleg Stepura <oleg@stepura.com>
	+ Olivier Certner <olce.freebsd@certner.fr>
	Olivier Mazouffre <olivier.mazouffre@ims-bordeaux.fr>
	omni <omni+vagant@hack.org>
	Orivej Desh <orivej@gmx.fr>
	Pablo Correa Gómez <ablocorrea@hotmail.com>
	Palash Gandhi <pbg4930@rit.edu>
	Patrick Mooney <pmooney@pfmooney.com>
	Patrik Greco <sikevux@sikevux.se>
	Paul B. Henson <henson@acm.org>
	Paul Dagnelie <pcd@delphix.com>
	Paul Zuchowski <pzuchowski@datto.com>
	Pavel Boldin <boldin.pavel@gmail.com>
	Pavel Snajdr <snajpa@snajpa.net>
	Pavel Zakharov <pavel.zakharov@delphix.com>
	Pawel Jakub Dawidek <pjd@FreeBSD.org>
	Pedro Giffuni <pfg@freebsd.org>
	Peng <peng.hse@xtaotech.com>
	Peter Ashford <ashford@accs.com>
	Peter Dave Hello <hsu@peterdavehello.org>
	Peter Levine <plevine457@gmail.com>
	Peter Wirdemo <peter.wirdemo@gmail.com>
	Petros Koutoupis <petros@petroskoutoupis.com>
	Philip Pokorny <ppokorny@penguincomputing.com>
	Philipp Riederer <pt@philipptoelke.de>
	Phil Kauffman <philip@kauffman.me>
	Ping Huang <huangping@smartx.com>
	Piotr Kubaj <pkubaj@anongoth.pl>
	Piotr P. Stefaniak <pstef@freebsd.org>
	Prakash Surya <prakash.surya@delphix.com>
	Prasad Joshi <prasadjoshi124@gmail.com>
	privb0x23 <privb0x23@users.noreply.github.com>
	P.SCH <p88@yahoo.com>
	+ Quartz <yyhran@163.com>
	Quentin Zdanis <zdanisq@gmail.com>
	Rafael Kitover <rkitover@gmail.com>
	RageLtMan <sempervictus@users.noreply.github.com>
	Ralf Ertzinger <ralf@skytale.net>
	Randall Mason <ClashTheBunny@gmail.com>
	Remy Blank <remy.blank@pobox.com>
	renelson <bnelson@nelsonbe.com>
	Reno Reckling <e-github@wthack.de>
	Ricardo M. Correia <ricardo.correia@oracle.com>
	Riccardo Schirone <rschirone91@gmail.com>
	Richard Allen <belperite@gmail.com>
	Richard Elling <Richard.Elling@RichardElling.com>
	+ Richard Kojedzinszky <richard@kojedz.in>
	Richard Laager <rlaager@wiktel.com>
	Richard Lowe <richlowe@richlowe.net>
	Richard Sharpe <rsharpe@samba.org>
	Richard Yao <ryao@gentoo.org>
	Rich Ercolani <rincebrain@gmail.com>
	+ Rick Macklem <rmacklem@uoguelph.ca>
	+ rilysh <nightquick@proton.me>
	+ Robert Evans <evansr@google.com>
	Robert Novak <sailnfool@gmail.com>
	Roberto Ricci <ricci@disroot.org>
	Rob Norris <robn@despairlabs.com>
	Rob Wing <rew@FreeBSD.org>
	Rohan Puri <rohan.puri15@gmail.com>
	Romain Dolbeau <romain.dolbeau@atos.net>
	Roman Strashkin <roman.strashkin@nexenta.com>
	Ross Williams <ross@ross-williams.net>
	Ruben Kerkhof <ruben@rubenkerkhof.com>
	Ryan Hirasaki <ryanhirasaki@gmail.com>
	Ryan Lahfa <masterancpp@gmail.com>
	Ryan Libby <rlibby@FreeBSD.org>
	Ryan Moeller <freqlabs@FreeBSD.org>
	+ Sam Atkinson <samatk@amazon.com>
	Sam Hathaway <github.com@munkynet.org>
	+ Sam James <sam@gentoo.org>
	Sam Lunt <samuel.j.lunt@gmail.com>
	Samuel VERSCHELDE <stormi-github@ylix.fr>
	Samuel Wycliffe <samuelwycliffe@gmail.com>
	Samuel Wycliffe J <samwyc@hpe.com>
	Sanjeev Bagewadi <sanjeev.bagewadi@gmail.com>
	Sara Hartse <sara.hartse@delphix.com>
	Saso Kiselkov <saso.kiselkov@nexenta.com>
	Satadru Pramanik <satadru@gmail.com>
	Savyasachee Jha <genghizkhan91@hawkradius.com>
	Scott Colby <scott@scolby.com>
	Scot W. Stevenson <scot.stevenson@gmail.com>
	Sean Eric Fagan <sef@ixsystems.com>
	Sebastian Gottschall <s.gottschall@dd-wrt.com>
	Sebastien Roy <seb@delphix.com>
	Sen Haerens <sen@senhaerens.be>
	Serapheim Dimitropoulos <serapheim@delphix.com>
	Seth Forshee <seth.forshee@canonical.com>
	Shaan Nobee <sniper111@gmail.com>
	Shampavman <sham.pavman@nexenta.com>
	Shaun Tancheff <shaun@aeonazure.com>
	+ Shawn Bayern <sbayern@law.fsu.edu>
	+ Shengqi Chen <harry-chen@outlook.com>
	Shen Yan <shenyanxxxy@qq.com>
	Simon Guest <simon.guest@tesujimath.org>
	Simon Klinkert <simon.klinkert@gmail.com>
	Sowrabha Gopal <sowrabha.gopal@delphix.com>
	Spencer Kinny <spencerkinny1995@gmail.com>
	Srikanth N S <srikanth.nagasubbaraoseetharaman@hpe.com>
	Stanislav Seletskiy <s.seletskiy@gmail.com>
	+ Stefan Lendl <s.lendl@proxmox.com>
	Steffen Müthing <steffen.muething@iwr.uni-heidelberg.de>
	Stephen Blinick <stephen.blinick@delphix.com>
	sterlingjensen <sterlingjensen@users.noreply.github.com>
	Steve Dougherty <sdougherty@barracuda.com>
	Steve Mokris <smokris@softpixel.com>
	Steven Burgess <sburgess@dattobackup.com>
	Steven Hartland <smh@freebsd.org>
	Steven Johnson <sjohnson@sakuraindustries.com>
	Steven Noonan <steven@uplinklabs.net>
	stf <s@ctrlc.hu>
	Stian Ellingsen <stian@plaimi.net>
	Stoiko Ivanov <github@nomore.at>
	Stéphane Lesimple <speed47_github@speed47.net>
	Suman Chakravartula <schakrava@gmail.com>
	Sydney Vanda <sydney.m.vanda@intel.com>
	Sören Tempel <soeren+git@soeren-tempel.net>
	Tamas TEVESZ <ice@extreme.hu>
	Teodor Spæren <teodor_spaeren@riseup.net>
	TerraTech <TerraTech@users.noreply.github.com>
	Thijs Cramer <thijs.cramer@gmail.com>
	+ Thomas Bertschinger <bertschinger@lanl.gov>
	Thomas Geppert <geppi@digitx.de>
	Thomas Lamprecht <guggentom@hotmail.de>
	Till Maas <opensource@till.name>
	Tim Chase <tim@chase2k.com>
	Tim Connors <tconnors@rather.puzzling.org>
	Tim Crawford <tcrawford@datto.com>
	Tim Haley <Tim.Haley@Sun.COM>
	timor <timor.dd@googlemail.com>
	Timothy Day <tday141@gmail.com>
	Tim Schumacher <timschumi@gmx.de>
	Tino Reichardt <milky-zfs@mcmilk.de>
	Tobin Harding <me@tobin.cc>
	Tom Caputi <tcaputi@datto.com>
	Tom Matthews <tom@axiom-partners.com>
	Tomohiro Kusumi <kusumi.tomohiro@gmail.com>
	Tom Prince <tom.prince@ualberta.net>
	Tony Hutter <hutter2@llnl.gov>
	Tony Nguyen <tony.nguyen@delphix.com>
	Tony Perkins <tperkins@datto.com>
	Toomas Soome <tsoome@me.com>
	Torsten Wörtwein <twoertwein@gmail.com>
	Toyam Cox <aviator45003@gmail.com>
	Trevor Bautista <trevrb@trevrb.net>
	Trey Dockendorf <treydock@gmail.com>
	Troels Nørgaard <tnn@tradeshift.com>
	Tulsi Jain <tulsi.jain@delphix.com>
	Turbo Fredriksson <turbo@bayour.com>
	Tyler J. Stachecki <stachecki.tyler@gmail.com>
	Umer Saleem <usaleem@ixsystems.com>
	+ Vaibhav Bhanawat <vaibhav.bhanawat@delphix.com>
	Valmiky Arquissandas <kayvlim@gmail.com>
	Val Packett <val@packett.cool>
	Vince van Oosten <techhazard@codeforyouand.me>
	Violet Purcell <vimproved@inventati.org>
	Vipin Kumar Verma <vipin.verma@hpe.com>
	Vitaut Bajaryn <vitaut.bayaryn@gmail.com>
	Volker Mauel <volkermauel@gmail.com>
	Václav Skála <skala@vshosting.cz>
	Walter Huf <hufman@gmail.com>
	Warner Losh <imp@bsdimp.com>
	Weigang Li <weigang.li@intel.com>
	WHR <msl0000023508@gmail.com>
	Will Andrews <will@freebsd.org>
	Will Rouesnel <w.rouesnel@gmail.com>
	Windel Bouwman <windel@windel.nl>
	Wojciech Małota-Wójcik <outofforest@users.noreply.github.com>
	Wolfgang Bumiller <w.bumiller@proxmox.com>
	Xin Li <delphij@FreeBSD.org>
	Xinliang Liu <xinliang.liu@linaro.org>
	xtouqh <xtouqh@hotmail.com>
	Yann Collet <cyan@fb.com>
	Yanping Gao <yanping.gao@xtaotech.com>
	Ying Zhu <casualfisher@gmail.com>
	Youzhong Yang <youzhong@gmail.com>
	yparitcher <y@paritcher.com>
	yuina822 <ayuichi@club.kyutech.ac.jp>
	YunQiang Su <syq@debian.org>
	Yuri Pankov <yuri.pankov@gmail.com>
	+ Yuxin Wang <yuxinwang9999@gmail.com>
	Yuxuan Shui <yshuiv7@gmail.com>
	Zachary Bedell <zac@thebedells.org>
	Zach Dykstra <dykstra.zachary@gmail.com>
	zgock <zgock@nuc.base.zgock-lab.net>
	Zhu Chuang <chuang@melty.land>
	Érico Nogueira <erico.erc@gmail.com>
	Đoàn Trần Công Danh <congdanhqx@gmail.com>
	韩朴宇 <w12101111@gmail.com>
	diff --git a/sys/contrib/openzfs/META b/sys/contrib/openzfs/META
	index d64414e32225..383fa37fd42a 100644
	--- a/sys/contrib/openzfs/META
	+++ b/sys/contrib/openzfs/META
	@@ -1,10 +1,10 @@
	Meta: 1
	Name: zfs
	Branch: 1.0
	-Version: 2.2.3
	+Version: 2.2.4
	Release: 1
	Release-Tags: relext
	License: CDDL
	Author: OpenZFS
	-Linux-Maximum: 6.7
	+Linux-Maximum: 6.8
	Linux-Minimum: 3.10
	diff --git a/sys/contrib/openzfs/cmd/arc_summary b/sys/contrib/openzfs/cmd/arc_summary
	index 9c69ec4f8ccc..100fb1987a8b 100755
	--- a/sys/contrib/openzfs/cmd/arc_summary
	+++ b/sys/contrib/openzfs/cmd/arc_summary
	@@ -1,1034 +1,1043 @@
	#!/usr/bin/env python3
	#
	# Copyright (c) 2008 Ben Rockwood <benr@cuddletech.com>,
	# Copyright (c) 2010 Martin Matuska <mm@FreeBSD.org>,
	# Copyright (c) 2010-2011 Jason J. Hellenthal <jhell@DataIX.net>,
	# Copyright (c) 2017 Scot W. Stevenson <scot.stevenson@gmail.com>
	# All rights reserved.
	#
	# Redistribution and use in source and binary forms, with or without
	# modification, are permitted provided that the following conditions
	# are met:
	#
	# 1. Redistributions of source code must retain the above copyright
	# notice, this list of conditions and the following disclaimer.
	# 2. Redistributions in binary form must reproduce the above copyright
	# notice, this list of conditions and the following disclaimer in the
	# documentation and/or other materials provided with the distribution.
	#
	# THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	# SUCH DAMAGE.
	"""Print statistics on the ZFS ARC Cache and other information

	Provides basic information on the ARC, its efficiency, the L2ARC (if present),
	the Data Management Unit (DMU), Virtual Devices (VDEVs), and tunables. See
	the in-source documentation and code at
	https://github.com/openzfs/zfs/blob/master/module/zfs/arc.c for details.
	The original introduction to arc_summary can be found at
	http://cuddletech.com/?p=454
	"""

	import argparse
	import os
	import subprocess
	import sys
	import time
	import errno

	# We can't use env -S portably, and we need python3 -u to handle pipes in
	# the shell abruptly closing the way we want to, so...
	import io
	if isinstance(sys.__stderr__.buffer, io.BufferedWriter):
	os.execv(sys.executable, [sys.executable, "-u"] + sys.argv)

	DESCRIPTION = 'Print ARC and other statistics for OpenZFS'
	INDENT = ' '*8
	LINE_LENGTH = 72
	DATE_FORMAT = '%a %b %d %H:%M:%S %Y'
	TITLE = 'ZFS Subsystem Report'

	SECTIONS = 'arc archits dmu l2arc spl tunables vdev zil'.split()
	SECTION_HELP = 'print info from one section ('+' '.join(SECTIONS)+')'

	# Tunables and SPL are handled separately because they come from
	# different sources
	SECTION_PATHS = {'arc': 'arcstats',
	'dmu': 'dmu_tx',
	'l2arc': 'arcstats', # L2ARC stuff lives in arcstats
	'zfetch': 'zfetchstats',
	'zil': 'zil'}

	parser = argparse.ArgumentParser(description=DESCRIPTION)
	parser.add_argument('-a', '--alternate', action='store_true', default=False,
	help='use alternate formatting for tunables and SPL',
	dest='alt')
	parser.add_argument('-d', '--description', action='store_true', default=False,
	help='print descriptions with tunables and SPL',
	dest='desc')
	parser.add_argument('-g', '--graph', action='store_true', default=False,
	help='print graph on ARC use and exit', dest='graph')
	parser.add_argument('-p', '--page', type=int, dest='page',
	help='print page by number (DEPRECATED, use "-s")')
	parser.add_argument('-r', '--raw', action='store_true', default=False,
	help='dump all available data with minimal formatting',
	dest='raw')
	parser.add_argument('-s', '--section', dest='section', help=SECTION_HELP)
	ARGS = parser.parse_args()


	if sys.platform.startswith('freebsd'):
	# Requires py36-sysctl on FreeBSD
	import sysctl

	def is_value(ctl):
	return ctl.type != sysctl.CTLTYPE_NODE

	def namefmt(ctl, base='vfs.zfs.'):
	# base is removed from the name
	cut = len(base)
	return ctl.name[cut:]

	def load_kstats(section):
	base = 'kstat.zfs.misc.{section}.'.format(section=section)
	fmt = lambda kstat: '{name} : {value}'.format(name=namefmt(kstat, base),
	value=kstat.value)
	kstats = sysctl.filter(base)
	return [fmt(kstat) for kstat in kstats if is_value(kstat)]

	def get_params(base):
	ctls = sysctl.filter(base)
	return {namefmt(ctl): str(ctl.value) for ctl in ctls if is_value(ctl)}

	def get_tunable_params():
	return get_params('vfs.zfs')

	def get_vdev_params():
	return get_params('vfs.zfs.vdev')

	def get_version_impl(request):
	# FreeBSD reports versions for zpl and spa instead of zfs and spl.
	name = {'zfs': 'zpl',
	'spl': 'spa'}[request]
	mib = 'vfs.zfs.version.{}'.format(name)
	version = sysctl.filter(mib)[0].value
	return '{} version {}'.format(name, version)

	def get_descriptions(_request):
	ctls = sysctl.filter('vfs.zfs')
	return {namefmt(ctl): ctl.description for ctl in ctls if is_value(ctl)}


	elif sys.platform.startswith('linux'):
	KSTAT_PATH = '/proc/spl/kstat/zfs'
	SPL_PATH = '/sys/module/spl/parameters'
	TUNABLES_PATH = '/sys/module/zfs/parameters'

	def load_kstats(section):
	path = os.path.join(KSTAT_PATH, section)
	with open(path) as f:
	return list(f)[2:] # Get rid of header

	def get_params(basepath):
	"""Collect information on the Solaris Porting Layer (SPL) or the
	tunables, depending on the PATH given. Does not check if PATH is
	legal.
	"""
	result = {}
	for name in os.listdir(basepath):
	path = os.path.join(basepath, name)
	with open(path) as f:
	value = f.read()
	result[name] = value.strip()
	return result

	def get_spl_params():
	return get_params(SPL_PATH)

	def get_tunable_params():
	return get_params(TUNABLES_PATH)

	def get_vdev_params():
	return get_params(TUNABLES_PATH)

	def get_version_impl(request):
	# The original arc_summary called /sbin/modinfo/{spl,zfs} to get
	# the version information. We switch to /sys/module/{spl,zfs}/version
	# to make sure we get what is really loaded in the kernel
	try:
	with open("/sys/module/{}/version".format(request)) as f:
	return f.read().strip()
	except:
	return "(unknown)"

	def get_descriptions(request):
	"""Get the descriptions of the Solaris Porting Layer (SPL) or the
	tunables, return with minimal formatting.
	"""

	if request not in ('spl', 'zfs'):
	print('ERROR: description of "{0}" requested)'.format(request))
	sys.exit(1)

	descs = {}
	target_prefix = 'parm:'

	# We would prefer to do this with /sys/modules -- see the discussion at
	# get_version() -- but there isn't a way to get the descriptions from
	# there, so we fall back on modinfo
	command = ["/sbin/modinfo", request, "-0"]

	info = ''

	try:

	info = subprocess.run(command, stdout=subprocess.PIPE,
	check=True, universal_newlines=True)
	raw_output = info.stdout.split('\0')

	except subprocess.CalledProcessError:
	print("Error: Descriptions not available",
	"(can't access kernel module)")
	sys.exit(1)

	for line in raw_output:

	if not line.startswith(target_prefix):
	continue

	line = line[len(target_prefix):].strip()
	name, raw_desc = line.split(':', 1)
	desc = raw_desc.rsplit('(', 1)[0]

	if desc == '':
	desc = '(No description found)'

	descs[name.strip()] = desc.strip()

	return descs

	def handle_unraisableException(exc_type, exc_value=None, exc_traceback=None,
	err_msg=None, object=None):
	handle_Exception(exc_type, object, exc_traceback)

	def handle_Exception(ex_cls, ex, tb):
	if ex_cls is KeyboardInterrupt:
	sys.exit()

	if ex_cls is BrokenPipeError:
	# It turns out that while sys.exit() triggers an exception
	# not handled message on Python 3.8+, os._exit() does not.
	os._exit(0)

	if ex_cls is OSError:
	if ex.errno == errno.ENOTCONN:
	sys.exit()

	raise ex

	if hasattr(sys,'unraisablehook'): # Python 3.8+
	sys.unraisablehook = handle_unraisableException
	sys.excepthook = handle_Exception


	def cleanup_line(single_line):
	"""Format a raw line of data from /proc and isolate the name value
	part, returning a tuple with each. Currently, this gets rid of the
	middle '4'. For example "arc_no_grow 4 0" returns the tuple
	("arc_no_grow", "0").
	"""
	name, _, value = single_line.split()

	return name, value


	def draw_graph(kstats_dict):
	"""Draw a primitive graph representing the basic information on the
	ARC -- its size and the proportion used by MFU and MRU -- and quit.
	We use max size of the ARC to calculate how full it is. This is a
	very rough representation.
	"""

	arc_stats = isolate_section('arcstats', kstats_dict)

	GRAPH_INDENT = ' '*4
	GRAPH_WIDTH = 60
	arc_size = f_bytes(arc_stats['size'])
	arc_perc = f_perc(arc_stats['size'], arc_stats['c_max'])
	mfu_size = f_bytes(arc_stats['mfu_size'])
	mru_size = f_bytes(arc_stats['mru_size'])
	meta_size = f_bytes(arc_stats['arc_meta_used'])
	dnode_limit = f_bytes(arc_stats['arc_dnode_limit'])
	dnode_size = f_bytes(arc_stats['dnode_size'])

	info_form = ('ARC: {0} ({1}) MFU: {2} MRU: {3} META: {4} '
	'DNODE {5} ({6})')
	info_line = info_form.format(arc_size, arc_perc, mfu_size, mru_size,
	meta_size, dnode_size, dnode_limit)
	info_spc = ' '*int((GRAPH_WIDTH-len(info_line))/2)
	info_line = GRAPH_INDENT+info_spc+info_line

	graph_line = GRAPH_INDENT+'+'+('-'*(GRAPH_WIDTH-2))+'+'

	mfu_perc = float(int(arc_stats['mfu_size'])/int(arc_stats['c_max']))
	mru_perc = float(int(arc_stats['mru_size'])/int(arc_stats['c_max']))
	arc_perc = float(int(arc_stats['size'])/int(arc_stats['c_max']))
	total_ticks = float(arc_perc)*GRAPH_WIDTH
	mfu_ticks = mfu_perc*GRAPH_WIDTH
	mru_ticks = mru_perc*GRAPH_WIDTH
	other_ticks = total_ticks-(mfu_ticks+mru_ticks)

	core_form = 'F'int(mfu_ticks)+'R'int(mru_ticks)+'O'*int(other_ticks)
	core_spc = ' '*(GRAPH_WIDTH-(2+len(core_form)))
	core_line = GRAPH_INDENT+'\|'+core_form+core_spc+'\|'

	for line in ('', info_line, graph_line, core_line, graph_line, ''):
	print(line)


	def f_bytes(byte_string):
	"""Return human-readable representation of a byte value in
	powers of 2 (eg "KiB" for "kibibytes", etc) to two decimal
	points. Values smaller than one KiB are returned without
	decimal points. Note "bytes" is a reserved keyword.
	"""

	prefixes = ([2**80, "YiB"], # yobibytes (yotta)
	[2**70, "ZiB"], # zebibytes (zetta)
	[2**60, "EiB"], # exbibytes (exa)
	[2**50, "PiB"], # pebibytes (peta)
	[2**40, "TiB"], # tebibytes (tera)
	[2**30, "GiB"], # gibibytes (giga)
	[2**20, "MiB"], # mebibytes (mega)
	[2**10, "KiB"]) # kibibytes (kilo)

	bites = int(byte_string)

	if bites >= 2**10:
	for limit, unit in prefixes:

	if bites >= limit:
	value = bites / limit
	break

	result = '{0:.1f} {1}'.format(value, unit)
	else:
	result = '{0} Bytes'.format(bites)

	return result


	def f_hits(hits_string):
	"""Create a human-readable representation of the number of hits.
	The single-letter symbols used are SI to avoid the confusion caused
	by the different "short scale" and "long scale" representations in
	English, which use the same words for different values. See
	https://en.wikipedia.org/wiki/Names_of_large_numbers and:
	https://physics.nist.gov/cuu/Units/prefixes.html
	"""

	numbers = ([10**24, 'Y'], # yotta (septillion)
	[10**21, 'Z'], # zetta (sextillion)
	[10**18, 'E'], # exa (quintrillion)
	[10**15, 'P'], # peta (quadrillion)
	[10**12, 'T'], # tera (trillion)
	[10**9, 'G'], # giga (billion)
	[10**6, 'M'], # mega (million)
	[10**3, 'k']) # kilo (thousand)

	hits = int(hits_string)

	if hits >= 1000:
	for limit, symbol in numbers:

	if hits >= limit:
	value = hits/limit
	break

	result = "%0.1f%s" % (value, symbol)
	else:
	result = "%d" % hits

	return result


	def f_perc(value1, value2):
	"""Calculate percentage and return in human-readable form. If
	rounding produces the result '0.0' though the first number is
	not zero, include a 'less-than' symbol to avoid confusion.
	Division by zero is handled by returning 'n/a'; no error
	is called.
	"""

	v1 = float(value1)
	v2 = float(value2)

	try:
	perc = 100 * v1/v2
	except ZeroDivisionError:
	result = 'n/a'
	else:
	result = '{0:0.1f} %'.format(perc)

	if result == '0.0 %' and v1 > 0:
	result = '< 0.1 %'

	return result


	def format_raw_line(name, value):
	"""For the --raw option for the tunable and SPL outputs, decide on the
	correct formatting based on the --alternate flag.
	"""

	if ARGS.alt:
	result = '{0}{1}={2}'.format(INDENT, name, value)
	else:
	# Right-align the value within the line length if it fits,
	# otherwise just separate it from the name by a single space.
	fit = LINE_LENGTH - len(INDENT) - len(name)
	overflow = len(value) + 1
	w = max(fit, overflow)
	result = '{0}{1}{2:>{w}}'.format(INDENT, name, value, w=w)

	return result


	def get_kstats():
	"""Collect information on the ZFS subsystem. The step does not perform any
	further processing, giving us the option to only work on what is actually
	needed. The name "kstat" is a holdover from the Solaris utility of the same
	name.
	"""

	result = {}

	for section in SECTION_PATHS.values():
	if section not in result:
	result[section] = load_kstats(section)

	return result


	def get_version(request):
	"""Get the version number of ZFS or SPL on this machine for header.
	Returns an error string, but does not raise an error, if we can't
	get the ZFS/SPL version.
	"""

	if request not in ('spl', 'zfs'):
	error_msg = '(ERROR: "{0}" requested)'.format(request)
	return error_msg

	return get_version_impl(request)


	def print_header():
	"""Print the initial heading with date and time as well as info on the
	kernel and ZFS versions. This is not called for the graph.
	"""

	# datetime is now recommended over time but we keep the exact formatting
	# from the older version of arc_summary in case there are scripts
	# that expect it in this way
	daydate = time.strftime(DATE_FORMAT)
	spc_date = LINE_LENGTH-len(daydate)
	sys_version = os.uname()

	sys_msg = sys_version.sysname+' '+sys_version.release
	zfs = get_version('zfs')
	spc_zfs = LINE_LENGTH-len(zfs)

	machine_msg = 'Machine: '+sys_version.nodename+' ('+sys_version.machine+')'
	spl = get_version('spl')
	spc_spl = LINE_LENGTH-len(spl)

	print('\n'+('-'*LINE_LENGTH))
	print('{0:<{spc}}{1}'.format(TITLE, daydate, spc=spc_date))
	print('{0:<{spc}}{1}'.format(sys_msg, zfs, spc=spc_zfs))
	print('{0:<{spc}}{1}\n'.format(machine_msg, spl, spc=spc_spl))


	def print_raw(kstats_dict):
	"""Print all available data from the system in a minimally sorted format.
	This can be used as a source to be piped through 'grep'.
	"""

	sections = sorted(kstats_dict.keys())

	for section in sections:

	print('\n{0}:'.format(section.upper()))
	lines = sorted(kstats_dict[section])

	for line in lines:
	name, value = cleanup_line(line)
	print(format_raw_line(name, value))

	# Tunables and SPL must be handled separately because they come from a
	# different source and have descriptions the user might request
	print()
	section_spl()
	section_tunables()


	def isolate_section(section_name, kstats_dict):
	"""From the complete information on all sections, retrieve only those
	for one section.
	"""

	try:
	section_data = kstats_dict[section_name]
	except KeyError:
	print('ERROR: Data on {0} not available'.format(section_data))
	sys.exit(1)

	section_dict = dict(cleanup_line(l) for l in section_data)

	return section_dict


	# Formatted output helper functions


	def prt_1(text, value):
	"""Print text and one value, no indent"""
	spc = ' '*(LINE_LENGTH-(len(text)+len(value)))
	print('{0}{spc}{1}'.format(text, value, spc=spc))


	def prt_i1(text, value):
	"""Print text and one value, with indent"""
	spc = ' '*(LINE_LENGTH-(len(INDENT)+len(text)+len(value)))
	print(INDENT+'{0}{spc}{1}'.format(text, value, spc=spc))


	def prt_2(text, value1, value2):
	"""Print text and two values, no indent"""
	values = '{0:>9} {1:>9}'.format(value1, value2)
	spc = ' '*(LINE_LENGTH-(len(text)+len(values)+2))
	print('{0}{spc} {1}'.format(text, values, spc=spc))


	def prt_i2(text, value1, value2):
	"""Print text and two values, with indent"""
	values = '{0:>9} {1:>9}'.format(value1, value2)
	spc = ' '*(LINE_LENGTH-(len(INDENT)+len(text)+len(values)+2))
	print(INDENT+'{0}{spc} {1}'.format(text, values, spc=spc))


	# The section output concentrates on important parameters instead of
	# being exhaustive (that is what the --raw parameter is for)


	def section_arc(kstats_dict):
	"""Give basic information on the ARC, MRU and MFU. This is the first
	and most used section.
	"""

	arc_stats = isolate_section('arcstats', kstats_dict)

	throttle = arc_stats['memory_throttle_count']

	if throttle == '0':
	health = 'HEALTHY'
	else:
	health = 'THROTTLED'

	prt_1('ARC status:', health)
	prt_i1('Memory throttle count:', throttle)
	print()

	arc_size = arc_stats['size']
	arc_target_size = arc_stats['c']
	arc_max = arc_stats['c_max']
	arc_min = arc_stats['c_min']
	meta = arc_stats['meta']
	pd = arc_stats['pd']
	pm = arc_stats['pm']
	anon_data = arc_stats['anon_data']
	anon_metadata = arc_stats['anon_metadata']
	mfu_data = arc_stats['mfu_data']
	mfu_metadata = arc_stats['mfu_metadata']
	mru_data = arc_stats['mru_data']
	mru_metadata = arc_stats['mru_metadata']
	mfug_data = arc_stats['mfu_ghost_data']
	mfug_metadata = arc_stats['mfu_ghost_metadata']
	mrug_data = arc_stats['mru_ghost_data']
	mrug_metadata = arc_stats['mru_ghost_metadata']
	unc_data = arc_stats['uncached_data']
	unc_metadata = arc_stats['uncached_metadata']
	bonus_size = arc_stats['bonus_size']
	dnode_limit = arc_stats['arc_dnode_limit']
	dnode_size = arc_stats['dnode_size']
	dbuf_size = arc_stats['dbuf_size']
	hdr_size = arc_stats['hdr_size']
	l2_hdr_size = arc_stats['l2_hdr_size']
	abd_chunk_waste_size = arc_stats['abd_chunk_waste_size']
	target_size_ratio = '{0}:1'.format(int(arc_max) // int(arc_min))

	prt_2('ARC size (current):',
	f_perc(arc_size, arc_max), f_bytes(arc_size))
	prt_i2('Target size (adaptive):',
	f_perc(arc_target_size, arc_max), f_bytes(arc_target_size))
	prt_i2('Min size (hard limit):',
	f_perc(arc_min, arc_max), f_bytes(arc_min))
	prt_i2('Max size (high water):',
	target_size_ratio, f_bytes(arc_max))
	caches_size = int(anon_data)+int(anon_metadata)+\
	int(mfu_data)+int(mfu_metadata)+int(mru_data)+int(mru_metadata)+\
	int(unc_data)+int(unc_metadata)
	prt_i2('Anonymous data size:',
	f_perc(anon_data, caches_size), f_bytes(anon_data))
	prt_i2('Anonymous metadata size:',
	f_perc(anon_metadata, caches_size), f_bytes(anon_metadata))
	s = 4294967296
	v = (s-int(pd))*(s-int(meta))/s
	prt_i2('MFU data target:', f_perc(v, s),
	f_bytes(v / 65536 * caches_size / 65536))
	prt_i2('MFU data size:',
	f_perc(mfu_data, caches_size), f_bytes(mfu_data))
	prt_i1('MFU ghost data size:', f_bytes(mfug_data))
	v = (s-int(pm))*int(meta)/s
	prt_i2('MFU metadata target:', f_perc(v, s),
	f_bytes(v / 65536 * caches_size / 65536))
	prt_i2('MFU metadata size:',
	f_perc(mfu_metadata, caches_size), f_bytes(mfu_metadata))
	prt_i1('MFU ghost metadata size:', f_bytes(mfug_metadata))
	v = int(pd)*(s-int(meta))/s
	prt_i2('MRU data target:', f_perc(v, s),
	f_bytes(v / 65536 * caches_size / 65536))
	prt_i2('MRU data size:',
	f_perc(mru_data, caches_size), f_bytes(mru_data))
	prt_i1('MRU ghost data size:', f_bytes(mrug_data))
	v = int(pm)*int(meta)/s
	prt_i2('MRU metadata target:', f_perc(v, s),
	f_bytes(v / 65536 * caches_size / 65536))
	prt_i2('MRU metadata size:',
	f_perc(mru_metadata, caches_size), f_bytes(mru_metadata))
	prt_i1('MRU ghost metadata size:', f_bytes(mrug_metadata))
	prt_i2('Uncached data size:',
	f_perc(unc_data, caches_size), f_bytes(unc_data))
	prt_i2('Uncached metadata size:',
	f_perc(unc_metadata, caches_size), f_bytes(unc_metadata))
	prt_i2('Bonus size:',
	f_perc(bonus_size, arc_size), f_bytes(bonus_size))
	prt_i2('Dnode cache target:',
	f_perc(dnode_limit, arc_max), f_bytes(dnode_limit))
	prt_i2('Dnode cache size:',
	f_perc(dnode_size, dnode_limit), f_bytes(dnode_size))
	prt_i2('Dbuf size:',
	f_perc(dbuf_size, arc_size), f_bytes(dbuf_size))
	prt_i2('Header size:',
	f_perc(hdr_size, arc_size), f_bytes(hdr_size))
	prt_i2('L2 header size:',
	f_perc(l2_hdr_size, arc_size), f_bytes(l2_hdr_size))
	prt_i2('ABD chunk waste size:',
	f_perc(abd_chunk_waste_size, arc_size), f_bytes(abd_chunk_waste_size))
	print()

	print('ARC hash breakdown:')
	prt_i1('Elements max:', f_hits(arc_stats['hash_elements_max']))
	prt_i2('Elements current:',
	f_perc(arc_stats['hash_elements'], arc_stats['hash_elements_max']),
	f_hits(arc_stats['hash_elements']))
	prt_i1('Collisions:', f_hits(arc_stats['hash_collisions']))

	prt_i1('Chain max:', f_hits(arc_stats['hash_chain_max']))
	prt_i1('Chains:', f_hits(arc_stats['hash_chains']))
	print()

	print('ARC misc:')
	prt_i1('Deleted:', f_hits(arc_stats['deleted']))
	prt_i1('Mutex misses:', f_hits(arc_stats['mutex_miss']))
	prt_i1('Eviction skips:', f_hits(arc_stats['evict_skip']))
	prt_i1('Eviction skips due to L2 writes:',
	f_hits(arc_stats['evict_l2_skip']))
	prt_i1('L2 cached evictions:', f_bytes(arc_stats['evict_l2_cached']))
	prt_i1('L2 eligible evictions:', f_bytes(arc_stats['evict_l2_eligible']))
	prt_i2('L2 eligible MFU evictions:',
	f_perc(arc_stats['evict_l2_eligible_mfu'],
	arc_stats['evict_l2_eligible']),
	f_bytes(arc_stats['evict_l2_eligible_mfu']))
	prt_i2('L2 eligible MRU evictions:',
	f_perc(arc_stats['evict_l2_eligible_mru'],
	arc_stats['evict_l2_eligible']),
	f_bytes(arc_stats['evict_l2_eligible_mru']))
	prt_i1('L2 ineligible evictions:',
	f_bytes(arc_stats['evict_l2_ineligible']))
	print()


	def section_archits(kstats_dict):
	"""Print information on how the caches are accessed ("arc hits").
	"""

	arc_stats = isolate_section('arcstats', kstats_dict)
	all_accesses = int(arc_stats['hits'])+int(arc_stats['iohits'])+\
	int(arc_stats['misses'])

	prt_1('ARC total accesses:', f_hits(all_accesses))
	ta_todo = (('Total hits:', arc_stats['hits']),
	('Total I/O hits:', arc_stats['iohits']),
	('Total misses:', arc_stats['misses']))
	for title, value in ta_todo:
	prt_i2(title, f_perc(value, all_accesses), f_hits(value))
	print()

	dd_total = int(arc_stats['demand_data_hits']) +\
	int(arc_stats['demand_data_iohits']) +\
	int(arc_stats['demand_data_misses'])
	prt_2('ARC demand data accesses:', f_perc(dd_total, all_accesses),
	f_hits(dd_total))
	dd_todo = (('Demand data hits:', arc_stats['demand_data_hits']),
	('Demand data I/O hits:', arc_stats['demand_data_iohits']),
	('Demand data misses:', arc_stats['demand_data_misses']))
	for title, value in dd_todo:
	prt_i2(title, f_perc(value, dd_total), f_hits(value))
	print()

	dm_total = int(arc_stats['demand_metadata_hits']) +\
	int(arc_stats['demand_metadata_iohits']) +\
	int(arc_stats['demand_metadata_misses'])
	prt_2('ARC demand metadata accesses:', f_perc(dm_total, all_accesses),
	f_hits(dm_total))
	dm_todo = (('Demand metadata hits:', arc_stats['demand_metadata_hits']),
	('Demand metadata I/O hits:',
	arc_stats['demand_metadata_iohits']),
	('Demand metadata misses:', arc_stats['demand_metadata_misses']))
	for title, value in dm_todo:
	prt_i2(title, f_perc(value, dm_total), f_hits(value))
	print()

	pd_total = int(arc_stats['prefetch_data_hits']) +\
	int(arc_stats['prefetch_data_iohits']) +\
	int(arc_stats['prefetch_data_misses'])
	prt_2('ARC prefetch data accesses:', f_perc(pd_total, all_accesses),
	f_hits(pd_total))
	pd_todo = (('Prefetch data hits:', arc_stats['prefetch_data_hits']),
	('Prefetch data I/O hits:', arc_stats['prefetch_data_iohits']),
	('Prefetch data misses:', arc_stats['prefetch_data_misses']))
	for title, value in pd_todo:
	prt_i2(title, f_perc(value, pd_total), f_hits(value))
	print()

	pm_total = int(arc_stats['prefetch_metadata_hits']) +\
	int(arc_stats['prefetch_metadata_iohits']) +\
	int(arc_stats['prefetch_metadata_misses'])
	prt_2('ARC prefetch metadata accesses:', f_perc(pm_total, all_accesses),
	f_hits(pm_total))
	pm_todo = (('Prefetch metadata hits:',
	arc_stats['prefetch_metadata_hits']),
	('Prefetch metadata I/O hits:',
	arc_stats['prefetch_metadata_iohits']),
	('Prefetch metadata misses:',
	arc_stats['prefetch_metadata_misses']))
	for title, value in pm_todo:
	prt_i2(title, f_perc(value, pm_total), f_hits(value))
	print()

	all_prefetches = int(arc_stats['predictive_prefetch'])+\
	int(arc_stats['prescient_prefetch'])
	prt_2('ARC predictive prefetches:',
	f_perc(arc_stats['predictive_prefetch'], all_prefetches),
	f_hits(arc_stats['predictive_prefetch']))
	prt_i2('Demand hits after predictive:',
	f_perc(arc_stats['demand_hit_predictive_prefetch'],
	arc_stats['predictive_prefetch']),
	f_hits(arc_stats['demand_hit_predictive_prefetch']))
	prt_i2('Demand I/O hits after predictive:',
	f_perc(arc_stats['demand_iohit_predictive_prefetch'],
	arc_stats['predictive_prefetch']),
	f_hits(arc_stats['demand_iohit_predictive_prefetch']))
	never = int(arc_stats['predictive_prefetch']) -\
	int(arc_stats['demand_hit_predictive_prefetch']) -\
	int(arc_stats['demand_iohit_predictive_prefetch'])
	prt_i2('Never demanded after predictive:',
	f_perc(never, arc_stats['predictive_prefetch']),
	f_hits(never))
	print()

	prt_2('ARC prescient prefetches:',
	f_perc(arc_stats['prescient_prefetch'], all_prefetches),
	f_hits(arc_stats['prescient_prefetch']))
	prt_i2('Demand hits after prescient:',
	f_perc(arc_stats['demand_hit_prescient_prefetch'],
	arc_stats['prescient_prefetch']),
	f_hits(arc_stats['demand_hit_prescient_prefetch']))
	prt_i2('Demand I/O hits after prescient:',
	f_perc(arc_stats['demand_iohit_prescient_prefetch'],
	arc_stats['prescient_prefetch']),
	f_hits(arc_stats['demand_iohit_prescient_prefetch']))
	never = int(arc_stats['prescient_prefetch'])-\
	int(arc_stats['demand_hit_prescient_prefetch'])-\
	int(arc_stats['demand_iohit_prescient_prefetch'])
	prt_i2('Never demanded after prescient:',
	f_perc(never, arc_stats['prescient_prefetch']),
	f_hits(never))
	print()

	print('ARC states hits of all accesses:')
	cl_todo = (('Most frequently used (MFU):', arc_stats['mfu_hits']),
	('Most recently used (MRU):', arc_stats['mru_hits']),
	('Most frequently used (MFU) ghost:',
	arc_stats['mfu_ghost_hits']),
	('Most recently used (MRU) ghost:',
	arc_stats['mru_ghost_hits']),
	('Uncached:', arc_stats['uncached_hits']))
	for title, value in cl_todo:
	prt_i2(title, f_perc(value, all_accesses), f_hits(value))
	print()


	def section_dmu(kstats_dict):
	"""Collect information on the DMU"""

	zfetch_stats = isolate_section('zfetchstats', kstats_dict)

	- zfetch_access_total = int(zfetch_stats['hits'])+int(zfetch_stats['misses'])
	+ zfetch_access_total = int(zfetch_stats['hits']) +\
	+ int(zfetch_stats['future']) + int(zfetch_stats['stride']) +\
	+ int(zfetch_stats['past']) + int(zfetch_stats['misses'])

	prt_1('DMU predictive prefetcher calls:', f_hits(zfetch_access_total))
	prt_i2('Stream hits:',
	f_perc(zfetch_stats['hits'], zfetch_access_total),
	f_hits(zfetch_stats['hits']))
	+ future = int(zfetch_stats['future']) + int(zfetch_stats['stride'])
	+ prt_i2('Hits ahead of stream:', f_perc(future, zfetch_access_total),
	+ f_hits(future))
	+ prt_i2('Hits behind stream:',
	+ f_perc(zfetch_stats['past'], zfetch_access_total),
	+ f_hits(zfetch_stats['past']))
	prt_i2('Stream misses:',
	f_perc(zfetch_stats['misses'], zfetch_access_total),
	f_hits(zfetch_stats['misses']))
	prt_i2('Streams limit reached:',
	f_perc(zfetch_stats['max_streams'], zfetch_stats['misses']),
	f_hits(zfetch_stats['max_streams']))
	+ prt_i1('Stream strides:', f_hits(zfetch_stats['stride']))
	prt_i1('Prefetches issued', f_hits(zfetch_stats['io_issued']))
	print()


	def section_l2arc(kstats_dict):
	"""Collect information on L2ARC device if present. If not, tell user
	that we're skipping the section.
	"""

	# The L2ARC statistics live in the same section as the normal ARC stuff
	arc_stats = isolate_section('arcstats', kstats_dict)

	if arc_stats['l2_size'] == '0':
	print('L2ARC not detected, skipping section\n')
	return

	l2_errors = int(arc_stats['l2_writes_error']) +\
	int(arc_stats['l2_cksum_bad']) +\
	int(arc_stats['l2_io_error'])

	l2_access_total = int(arc_stats['l2_hits'])+int(arc_stats['l2_misses'])
	health = 'HEALTHY'

	if l2_errors > 0:
	health = 'DEGRADED'

	prt_1('L2ARC status:', health)

	l2_todo = (('Low memory aborts:', 'l2_abort_lowmem'),
	('Free on write:', 'l2_free_on_write'),
	('R/W clashes:', 'l2_rw_clash'),
	('Bad checksums:', 'l2_cksum_bad'),
	('Read errors:', 'l2_io_error'),
	('Write errors:', 'l2_writes_error'))

	for title, value in l2_todo:
	prt_i1(title, f_hits(arc_stats[value]))

	print()
	prt_1('L2ARC size (adaptive):', f_bytes(arc_stats['l2_size']))
	prt_i2('Compressed:', f_perc(arc_stats['l2_asize'], arc_stats['l2_size']),
	f_bytes(arc_stats['l2_asize']))
	prt_i2('Header size:',
	f_perc(arc_stats['l2_hdr_size'], arc_stats['l2_size']),
	f_bytes(arc_stats['l2_hdr_size']))
	prt_i2('MFU allocated size:',
	f_perc(arc_stats['l2_mfu_asize'], arc_stats['l2_asize']),
	f_bytes(arc_stats['l2_mfu_asize']))
	prt_i2('MRU allocated size:',
	f_perc(arc_stats['l2_mru_asize'], arc_stats['l2_asize']),
	f_bytes(arc_stats['l2_mru_asize']))
	prt_i2('Prefetch allocated size:',
	f_perc(arc_stats['l2_prefetch_asize'], arc_stats['l2_asize']),
	f_bytes(arc_stats['l2_prefetch_asize']))
	prt_i2('Data (buffer content) allocated size:',
	f_perc(arc_stats['l2_bufc_data_asize'], arc_stats['l2_asize']),
	f_bytes(arc_stats['l2_bufc_data_asize']))
	prt_i2('Metadata (buffer content) allocated size:',
	f_perc(arc_stats['l2_bufc_metadata_asize'], arc_stats['l2_asize']),
	f_bytes(arc_stats['l2_bufc_metadata_asize']))

	print()
	prt_1('L2ARC breakdown:', f_hits(l2_access_total))
	prt_i2('Hit ratio:',
	f_perc(arc_stats['l2_hits'], l2_access_total),
	f_hits(arc_stats['l2_hits']))
	prt_i2('Miss ratio:',
	f_perc(arc_stats['l2_misses'], l2_access_total),
	f_hits(arc_stats['l2_misses']))

	print()
	print('L2ARC I/O:')
	prt_i2('Reads:',
	f_bytes(arc_stats['l2_read_bytes']),
	f_hits(arc_stats['l2_hits']))
	prt_i2('Writes:',
	f_bytes(arc_stats['l2_write_bytes']),
	f_hits(arc_stats['l2_writes_sent']))

	print()
	print('L2ARC evicts:')
	prt_i1('L1 cached:', f_hits(arc_stats['l2_evict_l1cached']))
	prt_i1('While reading:', f_hits(arc_stats['l2_evict_reading']))
	print()


	def section_spl(*_):
	"""Print the SPL parameters, if requested with alternative format
	and/or descriptions. This does not use kstats.
	"""

	if sys.platform.startswith('freebsd'):
	# No SPL support in FreeBSD
	return

	spls = get_spl_params()
	keylist = sorted(spls.keys())
	print('Solaris Porting Layer (SPL):')

	if ARGS.desc:
	descriptions = get_descriptions('spl')

	for key in keylist:
	value = spls[key]

	if ARGS.desc:
	try:
	print(INDENT+'#', descriptions[key])
	except KeyError:
	print(INDENT+'# (No description found)') # paranoid

	print(format_raw_line(key, value))

	print()


	def section_tunables(*_):
	"""Print the tunables, if requested with alternative format and/or
	descriptions. This does not use kstasts.
	"""

	tunables = get_tunable_params()
	keylist = sorted(tunables.keys())
	print('Tunables:')

	if ARGS.desc:
	descriptions = get_descriptions('zfs')

	for key in keylist:
	value = tunables[key]

	if ARGS.desc:
	try:
	print(INDENT+'#', descriptions[key])
	except KeyError:
	print(INDENT+'# (No description found)') # paranoid

	print(format_raw_line(key, value))

	print()


	def section_zil(kstats_dict):
	"""Collect information on the ZFS Intent Log. Some of the information
	taken from https://github.com/openzfs/zfs/blob/master/include/sys/zil.h
	"""

	zil_stats = isolate_section('zil', kstats_dict)

	prt_1('ZIL committed transactions:',
	f_hits(zil_stats['zil_itx_count']))
	prt_i1('Commit requests:', f_hits(zil_stats['zil_commit_count']))
	prt_i1('Flushes to stable storage:',
	f_hits(zil_stats['zil_commit_writer_count']))
	prt_i2('Transactions to SLOG storage pool:',
	f_bytes(zil_stats['zil_itx_metaslab_slog_bytes']),
	f_hits(zil_stats['zil_itx_metaslab_slog_count']))
	prt_i2('Transactions to non-SLOG storage pool:',
	f_bytes(zil_stats['zil_itx_metaslab_normal_bytes']),
	f_hits(zil_stats['zil_itx_metaslab_normal_count']))
	print()


	section_calls = {'arc': section_arc,
	'archits': section_archits,
	'dmu': section_dmu,
	'l2arc': section_l2arc,
	'spl': section_spl,
	'tunables': section_tunables,
	'zil': section_zil}


	def main():
	"""Run program. The options to draw a graph and to print all data raw are
	treated separately because they come with their own call.
	"""

	kstats = get_kstats()

	if ARGS.graph:
	draw_graph(kstats)
	sys.exit(0)

	print_header()

	if ARGS.raw:
	print_raw(kstats)

	elif ARGS.section:

	try:
	section_calls[ARGS.section](kstats)
	except KeyError:
	print('Error: Section "{0}" unknown'.format(ARGS.section))
	sys.exit(1)

	elif ARGS.page:
	print('WARNING: Pages are deprecated, please use "--section"\n')

	pages_to_calls = {1: 'arc',
	2: 'archits',
	3: 'l2arc',
	4: 'dmu',
	5: 'vdev',
	6: 'tunables'}

	try:
	call = pages_to_calls[ARGS.page]
	except KeyError:
	print('Error: Page "{0}" not supported'.format(ARGS.page))
	sys.exit(1)
	else:
	section_calls[call](kstats)

	else:
	# If no parameters were given, we print all sections. We might want to
	# change the sequence by hand
	calls = sorted(section_calls.keys())

	for section in calls:
	section_calls[section](kstats)

	sys.exit(0)


	if __name__ == '__main__':
	main()
	diff --git a/sys/contrib/openzfs/cmd/arcstat.in b/sys/contrib/openzfs/cmd/arcstat.in
	index 8df1c62f7e86..c4f10a1d6d3b 100755
	--- a/sys/contrib/openzfs/cmd/arcstat.in
	+++ b/sys/contrib/openzfs/cmd/arcstat.in
	@@ -1,637 +1,680 @@
	#!/usr/bin/env @PYTHON_SHEBANG@
	#
	# Print out ZFS ARC Statistics exported via kstat(1)
	# For a definition of fields, or usage, use arcstat -v
	#
	# This script was originally a fork of the original arcstat.pl (0.1)
	# by Neelakanth Nadgir, originally published on his Sun blog on
	# 09/18/2007
	# http://blogs.sun.com/realneel/entry/zfs_arc_statistics
	#
	# A new version aimed to improve upon the original by adding features
	# and fixing bugs as needed. This version was maintained by Mike
	# Harsch and was hosted in a public open source repository:
	# http://github.com/mharsch/arcstat
	#
	# but has since moved to the illumos-gate repository.
	#
	# This Python port was written by John Hixson for FreeNAS, introduced
	# in commit e2c29f:
	# https://github.com/freenas/freenas
	#
	# and has been improved by many people since.
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License, Version 1.0 only
	# (the "License"). You may not use this file except in compliance
	# with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END
	#
	#
	# Fields have a fixed width. Every interval, we fill the "v"
	# hash with its corresponding value (v[field]=value) using calculate().
	# @hdr is the array of fields that needs to be printed, so we
	# just iterate over this array and print the values using our pretty printer.
	#
	# This script must remain compatible with Python 3.6+.
	#

	import sys
	import time
	import getopt
	import re
	import copy

	from signal import signal, SIGINT, SIGWINCH, SIG_DFL


	cols = {
	# HDR: [Size, Scale, Description]
	"time": [8, -1, "Time"],
	"hits": [4, 1000, "ARC hits per second"],
	"iohs": [4, 1000, "ARC I/O hits per second"],
	"miss": [4, 1000, "ARC misses per second"],
	"read": [4, 1000, "Total ARC accesses per second"],
	"hit%": [4, 100, "ARC hit percentage"],
	"ioh%": [4, 100, "ARC I/O hit percentage"],
	"miss%": [5, 100, "ARC miss percentage"],
	"dhit": [4, 1000, "Demand hits per second"],
	"dioh": [4, 1000, "Demand I/O hits per second"],
	"dmis": [4, 1000, "Demand misses per second"],
	"dh%": [3, 100, "Demand hit percentage"],
	"di%": [3, 100, "Demand I/O hit percentage"],
	"dm%": [3, 100, "Demand miss percentage"],
	"ddhit": [5, 1000, "Demand data hits per second"],
	"ddioh": [5, 1000, "Demand data I/O hits per second"],
	"ddmis": [5, 1000, "Demand data misses per second"],
	"ddh%": [4, 100, "Demand data hit percentage"],
	"ddi%": [4, 100, "Demand data I/O hit percentage"],
	"ddm%": [4, 100, "Demand data miss percentage"],
	"dmhit": [5, 1000, "Demand metadata hits per second"],
	"dmioh": [5, 1000, "Demand metadata I/O hits per second"],
	"dmmis": [5, 1000, "Demand metadata misses per second"],
	"dmh%": [4, 100, "Demand metadata hit percentage"],
	"dmi%": [4, 100, "Demand metadata I/O hit percentage"],
	"dmm%": [4, 100, "Demand metadata miss percentage"],
	"phit": [4, 1000, "Prefetch hits per second"],
	"pioh": [4, 1000, "Prefetch I/O hits per second"],
	"pmis": [4, 1000, "Prefetch misses per second"],
	"ph%": [3, 100, "Prefetch hits percentage"],
	"pi%": [3, 100, "Prefetch I/O hits percentage"],
	"pm%": [3, 100, "Prefetch miss percentage"],
	"pdhit": [5, 1000, "Prefetch data hits per second"],
	"pdioh": [5, 1000, "Prefetch data I/O hits per second"],
	"pdmis": [5, 1000, "Prefetch data misses per second"],
	"pdh%": [4, 100, "Prefetch data hits percentage"],
	"pdi%": [4, 100, "Prefetch data I/O hits percentage"],
	"pdm%": [4, 100, "Prefetch data miss percentage"],
	"pmhit": [5, 1000, "Prefetch metadata hits per second"],
	"pmioh": [5, 1000, "Prefetch metadata I/O hits per second"],
	"pmmis": [5, 1000, "Prefetch metadata misses per second"],
	"pmh%": [4, 100, "Prefetch metadata hits percentage"],
	"pmi%": [4, 100, "Prefetch metadata I/O hits percentage"],
	"pmm%": [4, 100, "Prefetch metadata miss percentage"],
	"mhit": [4, 1000, "Metadata hits per second"],
	"mioh": [4, 1000, "Metadata I/O hits per second"],
	"mmis": [4, 1000, "Metadata misses per second"],
	"mread": [5, 1000, "Metadata accesses per second"],
	"mh%": [3, 100, "Metadata hit percentage"],
	"mi%": [3, 100, "Metadata I/O hit percentage"],
	"mm%": [3, 100, "Metadata miss percentage"],
	"arcsz": [5, 1024, "ARC size"],
	"size": [5, 1024, "ARC size"],
	"c": [5, 1024, "ARC target size"],
	"mfu": [4, 1000, "MFU list hits per second"],
	"mru": [4, 1000, "MRU list hits per second"],
	"mfug": [4, 1000, "MFU ghost list hits per second"],
	"mrug": [4, 1000, "MRU ghost list hits per second"],
	"unc": [4, 1000, "Uncached list hits per second"],
	"eskip": [5, 1000, "evict_skip per second"],
	"el2skip": [7, 1000, "evict skip, due to l2 writes, per second"],
	"el2cach": [7, 1024, "Size of L2 cached evictions per second"],
	"el2el": [5, 1024, "Size of L2 eligible evictions per second"],
	"el2mfu": [6, 1024, "Size of L2 eligible MFU evictions per second"],
	"el2mru": [6, 1024, "Size of L2 eligible MRU evictions per second"],
	"el2inel": [7, 1024, "Size of L2 ineligible evictions per second"],
	"mtxmis": [6, 1000, "mutex_miss per second"],
	"dread": [5, 1000, "Demand accesses per second"],
	"ddread": [6, 1000, "Demand data accesses per second"],
	"dmread": [6, 1000, "Demand metadata accesses per second"],
	"pread": [5, 1000, "Prefetch accesses per second"],
	"pdread": [6, 1000, "Prefetch data accesses per second"],
	"pmread": [6, 1000, "Prefetch metadata accesses per second"],
	"l2hits": [6, 1000, "L2ARC hits per second"],
	"l2miss": [6, 1000, "L2ARC misses per second"],
	"l2read": [6, 1000, "Total L2ARC accesses per second"],
	"l2hit%": [6, 100, "L2ARC access hit percentage"],
	"l2miss%": [7, 100, "L2ARC access miss percentage"],
	"l2pref": [6, 1024, "L2ARC prefetch allocated size"],
	"l2mfu": [5, 1024, "L2ARC MFU allocated size"],
	"l2mru": [5, 1024, "L2ARC MRU allocated size"],
	"l2data": [6, 1024, "L2ARC data allocated size"],
	"l2meta": [6, 1024, "L2ARC metadata allocated size"],
	"l2pref%": [7, 100, "L2ARC prefetch percentage"],
	"l2mfu%": [6, 100, "L2ARC MFU percentage"],
	"l2mru%": [6, 100, "L2ARC MRU percentage"],
	"l2data%": [7, 100, "L2ARC data percentage"],
	"l2meta%": [7, 100, "L2ARC metadata percentage"],
	"l2asize": [7, 1024, "Actual (compressed) size of the L2ARC"],
	"l2size": [6, 1024, "Size of the L2ARC"],
	"l2bytes": [7, 1024, "Bytes read per second from the L2ARC"],
	"grow": [4, 1000, "ARC grow disabled"],
	"need": [5, 1024, "ARC reclaim need"],
	"free": [5, 1024, "ARC free memory"],
	"avail": [5, 1024, "ARC available memory"],
	"waste": [5, 1024, "Wasted memory due to round up to pagesize"],
	+ "ztotal": [6, 1000, "zfetch total prefetcher calls per second"],
	+ "zhits": [5, 1000, "zfetch stream hits per second"],
	+ "zahead": [6, 1000, "zfetch hits ahead of streams per second"],
	+ "zpast": [5, 1000, "zfetch hits behind streams per second"],
	+ "zmisses": [7, 1000, "zfetch stream misses per second"],
	+ "zmax": [4, 1000, "zfetch limit reached per second"],
	+ "zfuture": [7, 1000, "zfetch stream future per second"],
	+ "zstride": [7, 1000, "zfetch stream strides per second"],
	+ "zissued": [7, 1000, "zfetch prefetches issued per second"],
	+ "zactive": [7, 1000, "zfetch prefetches active per second"],
	}

	v = {}
	hdr = ["time", "read", "ddread", "ddh%", "dmread", "dmh%", "pread", "ph%",
	"size", "c", "avail"]
	xhdr = ["time", "mfu", "mru", "mfug", "mrug", "unc", "eskip", "mtxmis",
	"dread", "pread", "read"]
	+zhdr = ["time", "ztotal", "zhits", "zahead", "zpast", "zmisses", "zmax",
	+ "zfuture", "zstride", "zissued", "zactive"]
	sint = 1 # Default interval is 1 second
	count = 1 # Default count is 1
	hdr_intr = 20 # Print header every 20 lines of output
	opfile = None
	sep = " " # Default separator is 2 spaces
	l2exist = False
	cmd = ("Usage: arcstat [-havxp] [-f fields] [-o file] [-s string] [interval "
	"[count]]\n")
	cur = {}
	d = {}
	out = None
	kstat = None
	pretty_print = True


	if sys.platform.startswith('freebsd'):
	# Requires py-sysctl on FreeBSD
	import sysctl

	def kstat_update():
	global kstat

	k = [ctl for ctl in sysctl.filter('kstat.zfs.misc.arcstats')
	if ctl.type != sysctl.CTLTYPE_NODE]
	+ k += [ctl for ctl in sysctl.filter('kstat.zfs.misc.zfetchstats')
	+ if ctl.type != sysctl.CTLTYPE_NODE]

	if not k:
	sys.exit(1)

	kstat = {}

	for s in k:
	if not s:
	continue

	name, value = s.name, s.value
	- # Trims 'kstat.zfs.misc.arcstats' from the name
	- kstat[name[24:]] = int(value)
	+
	+ if "arcstats" in name:
	+ # Trims 'kstat.zfs.misc.arcstats' from the name
	+ kstat[name[24:]] = int(value)
	+ else:
	+ kstat["zfetch_" + name[27:]] = int(value)

	elif sys.platform.startswith('linux'):
	def kstat_update():
	global kstat

	- k = [line.strip() for line in open('/proc/spl/kstat/zfs/arcstats')]
	+ k1 = [line.strip() for line in open('/proc/spl/kstat/zfs/arcstats')]

	- if not k:
	+ k2 = ["zfetch_" + line.strip() for line in
	+ open('/proc/spl/kstat/zfs/zfetchstats')]
	+
	+ if k1 is None or k2 is None:
	sys.exit(1)

	- del k[0:2]
	+ del k1[0:2]
	+ del k2[0:2]
	+ k = k1 + k2
	kstat = {}

	for s in k:
	if not s:
	continue

	name, unused, value = s.split()
	kstat[name] = int(value)


	def detailed_usage():
	sys.stderr.write("%s\n" % cmd)
	sys.stderr.write("Field definitions are as follows:\n")
	for key in cols:
	sys.stderr.write("%11s : %s\n" % (key, cols[key][2]))
	sys.stderr.write("\n")

	sys.exit(0)


	def usage():
	sys.stderr.write("%s\n" % cmd)
	sys.stderr.write("\t -h : Print this help message\n")
	sys.stderr.write("\t -a : Print all possible stats\n")
	sys.stderr.write("\t -v : List all possible field headers and definitions"
	"\n")
	sys.stderr.write("\t -x : Print extended stats\n")
	+ sys.stderr.write("\t -z : Print zfetch stats\n")
	sys.stderr.write("\t -f : Specify specific fields to print (see -v)\n")
	sys.stderr.write("\t -o : Redirect output to the specified file\n")
	sys.stderr.write("\t -s : Override default field separator with custom "
	"character or string\n")
	sys.stderr.write("\t -p : Disable auto-scaling of numerical fields\n")
	sys.stderr.write("\nExamples:\n")
	sys.stderr.write("\tarcstat -o /tmp/a.log 2 10\n")
	sys.stderr.write("\tarcstat -s \",\" -o /tmp/a.log 2 10\n")
	sys.stderr.write("\tarcstat -v\n")
	sys.stderr.write("\tarcstat -f time,hit%,dh%,ph%,mh% 1\n")
	sys.stderr.write("\n")

	sys.exit(1)


	def snap_stats():
	global cur
	global kstat

	prev = copy.deepcopy(cur)
	kstat_update()

	cur = kstat
	for key in cur:
	if re.match(key, "class"):
	continue
	if key in prev:
	d[key] = cur[key] - prev[key]
	else:
	d[key] = cur[key]


	def prettynum(sz, scale, num=0):
	suffix = [' ', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']
	index = 0
	save = 0

	# Special case for date field
	if scale == -1:
	return "%s" % num

	# Rounding error, return 0
	elif 0 < num < 1:
	num = 0

	while abs(num) > scale and index < 5:
	save = num
	num = num / scale
	index += 1

	if index == 0:
	return "%*d" % (sz, num)

	if abs(save / scale) < 10:
	return "%*.1f%s" % (sz - 1, num, suffix[index])
	else:
	return "%*d%s" % (sz - 1, num, suffix[index])


	def print_values():
	global hdr
	global sep
	global v
	global pretty_print

	if pretty_print:
	fmt = lambda col: prettynum(cols[col][0], cols[col][1], v[col])
	else:
	fmt = lambda col: str(v[col])

	sys.stdout.write(sep.join(fmt(col) for col in hdr))
	sys.stdout.write("\n")
	sys.stdout.flush()


	def print_header():
	global hdr
	global sep
	global pretty_print

	if pretty_print:
	fmt = lambda col: "%*s" % (cols[col][0], col)
	else:
	fmt = lambda col: col

	sys.stdout.write(sep.join(fmt(col) for col in hdr))
	sys.stdout.write("\n")


	def get_terminal_lines():
	try:
	import fcntl
	import termios
	import struct
	data = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, '1234')
	sz = struct.unpack('hh', data)
	return sz[0]
	except Exception:
	pass


	def update_hdr_intr():
	global hdr_intr

	lines = get_terminal_lines()
	if lines and lines > 3:
	hdr_intr = lines - 3


	def resize_handler(signum, frame):
	update_hdr_intr()


	def init():
	global sint
	global count
	global hdr
	global xhdr
	+ global zhdr
	global opfile
	global sep
	global out
	global l2exist
	global pretty_print

	desired_cols = None
	aflag = False
	xflag = False
	hflag = False
	vflag = False
	+ zflag = False
	i = 1

	try:
	opts, args = getopt.getopt(
	sys.argv[1:],
	- "axo:hvs:f:p",
	+ "axzo:hvs:f:p",
	[
	"all",
	"extended",
	+ "zfetch",
	"outfile",
	"help",
	"verbose",
	"separator",
	"columns",
	"parsable"
	]
	)
	except getopt.error as msg:
	sys.stderr.write("Error: %s\n" % str(msg))
	usage()
	opts = None

	for opt, arg in opts:
	if opt in ('-a', '--all'):
	aflag = True
	if opt in ('-x', '--extended'):
	xflag = True
	if opt in ('-o', '--outfile'):
	opfile = arg
	i += 1
	if opt in ('-h', '--help'):
	hflag = True
	if opt in ('-v', '--verbose'):
	vflag = True
	if opt in ('-s', '--separator'):
	sep = arg
	i += 1
	if opt in ('-f', '--columns'):
	desired_cols = arg
	i += 1
	if opt in ('-p', '--parsable'):
	pretty_print = False
	+ if opt in ('-z', '--zfetch'):
	+ zflag = True
	i += 1

	argv = sys.argv[i:]
	sint = int(argv[0]) if argv else sint
	count = int(argv[1]) if len(argv) > 1 else (0 if len(argv) > 0 else 1)

	- if hflag or (xflag and desired_cols):
	+ if hflag or (xflag and zflag) or ((zflag or xflag) and desired_cols):
	usage()

	if vflag:
	detailed_usage()

	if xflag:
	hdr = xhdr

	+ if zflag:
	+ hdr = zhdr
	+
	update_hdr_intr()

	# check if L2ARC exists
	snap_stats()
	l2_size = cur.get("l2_size")
	if l2_size:
	l2exist = True

	if desired_cols:
	hdr = desired_cols.split(",")

	invalid = []
	incompat = []
	for ele in hdr:
	if ele not in cols:
	invalid.append(ele)
	elif not l2exist and ele.startswith("l2"):
	sys.stdout.write("No L2ARC Here\n%s\n" % ele)
	incompat.append(ele)

	if len(invalid) > 0:
	sys.stderr.write("Invalid column definition! -- %s\n" % invalid)
	usage()

	if len(incompat) > 0:
	sys.stderr.write("Incompatible field specified! -- %s\n" %
	incompat)
	usage()

	if aflag:
	if l2exist:
	hdr = cols.keys()
	else:
	hdr = [col for col in cols.keys() if not col.startswith("l2")]

	if opfile:
	try:
	out = open(opfile, "w")
	sys.stdout = out

	except IOError:
	sys.stderr.write("Cannot open %s for writing\n" % opfile)
	sys.exit(1)


	def calculate():
	global d
	global v
	global l2exist

	v = dict()
	v["time"] = time.strftime("%H:%M:%S", time.localtime())
	v["hits"] = d["hits"] // sint
	v["iohs"] = d["iohits"] // sint
	v["miss"] = d["misses"] // sint
	v["read"] = v["hits"] + v["iohs"] + v["miss"]
	v["hit%"] = 100 * v["hits"] // v["read"] if v["read"] > 0 else 0
	v["ioh%"] = 100 * v["iohs"] // v["read"] if v["read"] > 0 else 0
	v["miss%"] = 100 - v["hit%"] - v["ioh%"] if v["read"] > 0 else 0

	v["dhit"] = (d["demand_data_hits"] + d["demand_metadata_hits"]) // sint
	v["dioh"] = (d["demand_data_iohits"] + d["demand_metadata_iohits"]) // sint
	v["dmis"] = (d["demand_data_misses"] + d["demand_metadata_misses"]) // sint

	v["dread"] = v["dhit"] + v["dioh"] + v["dmis"]
	v["dh%"] = 100 * v["dhit"] // v["dread"] if v["dread"] > 0 else 0
	v["di%"] = 100 * v["dioh"] // v["dread"] if v["dread"] > 0 else 0
	v["dm%"] = 100 - v["dh%"] - v["di%"] if v["dread"] > 0 else 0

	v["ddhit"] = d["demand_data_hits"] // sint
	v["ddioh"] = d["demand_data_iohits"] // sint
	v["ddmis"] = d["demand_data_misses"] // sint

	v["ddread"] = v["ddhit"] + v["ddioh"] + v["ddmis"]
	v["ddh%"] = 100 * v["ddhit"] // v["ddread"] if v["ddread"] > 0 else 0
	v["ddi%"] = 100 * v["ddioh"] // v["ddread"] if v["ddread"] > 0 else 0
	v["ddm%"] = 100 - v["ddh%"] - v["ddi%"] if v["ddread"] > 0 else 0

	v["dmhit"] = d["demand_metadata_hits"] // sint
	v["dmioh"] = d["demand_metadata_iohits"] // sint
	v["dmmis"] = d["demand_metadata_misses"] // sint

	v["dmread"] = v["dmhit"] + v["dmioh"] + v["dmmis"]
	v["dmh%"] = 100 * v["dmhit"] // v["dmread"] if v["dmread"] > 0 else 0
	v["dmi%"] = 100 * v["dmioh"] // v["dmread"] if v["dmread"] > 0 else 0
	v["dmm%"] = 100 - v["dmh%"] - v["dmi%"] if v["dmread"] > 0 else 0

	v["phit"] = (d["prefetch_data_hits"] + d["prefetch_metadata_hits"]) // sint
	v["pioh"] = (d["prefetch_data_iohits"] +
	d["prefetch_metadata_iohits"]) // sint
	v["pmis"] = (d["prefetch_data_misses"] +
	d["prefetch_metadata_misses"]) // sint

	v["pread"] = v["phit"] + v["pioh"] + v["pmis"]
	v["ph%"] = 100 * v["phit"] // v["pread"] if v["pread"] > 0 else 0
	v["pi%"] = 100 * v["pioh"] // v["pread"] if v["pread"] > 0 else 0
	v["pm%"] = 100 - v["ph%"] - v["pi%"] if v["pread"] > 0 else 0

	v["pdhit"] = d["prefetch_data_hits"] // sint
	v["pdioh"] = d["prefetch_data_iohits"] // sint
	v["pdmis"] = d["prefetch_data_misses"] // sint

	v["pdread"] = v["pdhit"] + v["pdioh"] + v["pdmis"]
	v["pdh%"] = 100 * v["pdhit"] // v["pdread"] if v["pdread"] > 0 else 0
	v["pdi%"] = 100 * v["pdioh"] // v["pdread"] if v["pdread"] > 0 else 0
	v["pdm%"] = 100 - v["pdh%"] - v["pdi%"] if v["pdread"] > 0 else 0

	v["pmhit"] = d["prefetch_metadata_hits"] // sint
	v["pmioh"] = d["prefetch_metadata_iohits"] // sint
	v["pmmis"] = d["prefetch_metadata_misses"] // sint

	v["pmread"] = v["pmhit"] + v["pmioh"] + v["pmmis"]
	v["pmh%"] = 100 * v["pmhit"] // v["pmread"] if v["pmread"] > 0 else 0
	v["pmi%"] = 100 * v["pmioh"] // v["pmread"] if v["pmread"] > 0 else 0
	v["pmm%"] = 100 - v["pmh%"] - v["pmi%"] if v["pmread"] > 0 else 0

	v["mhit"] = (d["prefetch_metadata_hits"] +
	d["demand_metadata_hits"]) // sint
	v["mioh"] = (d["prefetch_metadata_iohits"] +
	d["demand_metadata_iohits"]) // sint
	v["mmis"] = (d["prefetch_metadata_misses"] +
	d["demand_metadata_misses"]) // sint

	v["mread"] = v["mhit"] + v["mioh"] + v["mmis"]
	v["mh%"] = 100 * v["mhit"] // v["mread"] if v["mread"] > 0 else 0
	v["mi%"] = 100 * v["mioh"] // v["mread"] if v["mread"] > 0 else 0
	v["mm%"] = 100 - v["mh%"] - v["mi%"] if v["mread"] > 0 else 0

	v["arcsz"] = cur["size"]
	v["size"] = cur["size"]
	v["c"] = cur["c"]
	v["mfu"] = d["mfu_hits"] // sint
	v["mru"] = d["mru_hits"] // sint
	v["mrug"] = d["mru_ghost_hits"] // sint
	v["mfug"] = d["mfu_ghost_hits"] // sint
	v["unc"] = d["uncached_hits"] // sint
	v["eskip"] = d["evict_skip"] // sint
	v["el2skip"] = d["evict_l2_skip"] // sint
	v["el2cach"] = d["evict_l2_cached"] // sint
	v["el2el"] = d["evict_l2_eligible"] // sint
	v["el2mfu"] = d["evict_l2_eligible_mfu"] // sint
	v["el2mru"] = d["evict_l2_eligible_mru"] // sint
	v["el2inel"] = d["evict_l2_ineligible"] // sint
	v["mtxmis"] = d["mutex_miss"] // sint
	+ v["ztotal"] = (d["zfetch_hits"] + d["zfetch_future"] + d["zfetch_stride"] +
	+ d["zfetch_past"] + d["zfetch_misses"]) // sint
	+ v["zhits"] = d["zfetch_hits"] // sint
	+ v["zahead"] = (d["zfetch_future"] + d["zfetch_stride"]) // sint
	+ v["zpast"] = d["zfetch_past"] // sint
	+ v["zmisses"] = d["zfetch_misses"] // sint
	+ v["zmax"] = d["zfetch_max_streams"] // sint
	+ v["zfuture"] = d["zfetch_future"] // sint
	+ v["zstride"] = d["zfetch_stride"] // sint
	+ v["zissued"] = d["zfetch_io_issued"] // sint
	+ v["zactive"] = d["zfetch_io_active"] // sint

	if l2exist:
	v["l2hits"] = d["l2_hits"] // sint
	v["l2miss"] = d["l2_misses"] // sint
	v["l2read"] = v["l2hits"] + v["l2miss"]
	v["l2hit%"] = 100 * v["l2hits"] // v["l2read"] if v["l2read"] > 0 else 0

	v["l2miss%"] = 100 - v["l2hit%"] if v["l2read"] > 0 else 0
	v["l2asize"] = cur["l2_asize"]
	v["l2size"] = cur["l2_size"]
	v["l2bytes"] = d["l2_read_bytes"] // sint

	v["l2pref"] = cur["l2_prefetch_asize"]
	v["l2mfu"] = cur["l2_mfu_asize"]
	v["l2mru"] = cur["l2_mru_asize"]
	v["l2data"] = cur["l2_bufc_data_asize"]
	v["l2meta"] = cur["l2_bufc_metadata_asize"]
	v["l2pref%"] = 100 * v["l2pref"] // v["l2asize"]
	v["l2mfu%"] = 100 * v["l2mfu"] // v["l2asize"]
	v["l2mru%"] = 100 * v["l2mru"] // v["l2asize"]
	v["l2data%"] = 100 * v["l2data"] // v["l2asize"]
	v["l2meta%"] = 100 * v["l2meta"] // v["l2asize"]

	v["grow"] = 0 if cur["arc_no_grow"] else 1
	v["need"] = cur["arc_need_free"]
	v["free"] = cur["memory_free_bytes"]
	v["avail"] = cur["memory_available_bytes"]
	v["waste"] = cur["abd_chunk_waste_size"]


	def main():
	global sint
	global count
	global hdr_intr

	i = 0
	count_flag = 0

	init()
	if count > 0:
	count_flag = 1

	signal(SIGINT, SIG_DFL)
	signal(SIGWINCH, resize_handler)
	while True:
	if i == 0:
	print_header()

	snap_stats()
	calculate()
	print_values()

	if count_flag == 1:
	if count <= 1:
	break
	count -= 1

	i = 0 if i >= hdr_intr else i + 1
	time.sleep(sint)

	if out:
	out.close()


	if __name__ == '__main__':
	main()
	diff --git a/sys/contrib/openzfs/cmd/zed/agents/fmd_api.c b/sys/contrib/openzfs/cmd/zed/agents/fmd_api.c
	index 4a6cfbf8c05c..fe43e2ab971e 100644
	--- a/sys/contrib/openzfs/cmd/zed/agents/fmd_api.c
	+++ b/sys/contrib/openzfs/cmd/zed/agents/fmd_api.c
	@@ -1,781 +1,776 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
	*
	* Copyright (c) 2016, Intel Corporation.
	+ * Copyright (c) 2023, Klara Inc.
	*/

	/*
	* This file implements the minimal FMD module API required to support the
	* fault logic modules in ZED. This support includes module registration,
	* memory allocation, module property accessors, basic case management,
	* one-shot timers and SERD engines.
	*
	* In the ZED runtime, the modules are called from a single thread so no
	* locking is required in this emulated FMD environment.
	*/

	#include <sys/types.h>
	#include <sys/fm/protocol.h>
	#include <uuid/uuid.h>
	#include <signal.h>
	#include <string.h>
	#include <time.h>

	#include "fmd_api.h"
	#include "fmd_serd.h"

	#include "zfs_agents.h"
	#include "../zed_log.h"

	typedef struct fmd_modstat {
	fmd_stat_t ms_accepted; /* total events accepted by module */
	fmd_stat_t ms_caseopen; /* cases currently open */
	fmd_stat_t ms_casesolved; /* total cases solved by module */
	fmd_stat_t ms_caseclosed; /* total cases closed by module */
	} fmd_modstat_t;

	typedef struct fmd_module {
	const char mod_name; / basename of module (ro) */
	const fmd_hdl_info_t mod_info; / module info registered with handle */
	void mod_spec; / fmd_hdl_get/setspecific data value */
	fmd_stat_t mod_ustat; / module specific custom stats */
	uint_t mod_ustat_cnt; /* count of ustat stats */
	fmd_modstat_t mod_stats; /* fmd built-in per-module statistics */
	fmd_serd_hash_t mod_serds; /* hash of serd engs owned by module */
	char mod_vers; / a copy of module version string */
	} fmd_module_t;

	/*
	* ZED has two FMD hardwired module instances
	*/
	fmd_module_t zfs_retire_module;
	fmd_module_t zfs_diagnosis_module;

	/*
	* Enable a reasonable set of defaults for libumem debugging on DEBUG builds.
	*/

	#ifdef DEBUG
	const char *
	_umem_debug_init(void)
	{
	return ("default,verbose"); /* $UMEM_DEBUG setting */
	}

	const char *
	_umem_logging_init(void)
	{
	return ("fail,contents"); /* $UMEM_LOGGING setting */
	}
	#endif

	/*
	* Register a module with fmd and finish module initialization.
	* Returns an integer indicating whether it succeeded (zero) or
	* failed (non-zero).
	*/
	int
	fmd_hdl_register(fmd_hdl_t hdl, int version, const fmd_hdl_info_t mip)
	{
	(void) version;
	fmd_module_t mp = (fmd_module_t )hdl;

	mp->mod_info = mip;
	mp->mod_name = mip->fmdi_desc + 4; /* drop 'ZFS ' prefix */
	mp->mod_spec = NULL;

	/* bare minimum module stats */
	(void) strcpy(mp->mod_stats.ms_accepted.fmds_name, "fmd.accepted");
	(void) strcpy(mp->mod_stats.ms_caseopen.fmds_name, "fmd.caseopen");
	(void) strcpy(mp->mod_stats.ms_casesolved.fmds_name, "fmd.casesolved");
	(void) strcpy(mp->mod_stats.ms_caseclosed.fmds_name, "fmd.caseclosed");

	fmd_serd_hash_create(&mp->mod_serds);

	fmd_hdl_debug(hdl, "register module");

	return (0);
	}

	void
	fmd_hdl_unregister(fmd_hdl_t *hdl)
	{
	fmd_module_t mp = (fmd_module_t )hdl;
	fmd_modstat_t *msp = &mp->mod_stats;
	const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;

	/* dump generic module stats */
	fmd_hdl_debug(hdl, "%s: %llu", msp->ms_accepted.fmds_name,
	msp->ms_accepted.fmds_value.ui64);
	if (ops->fmdo_close != NULL) {
	fmd_hdl_debug(hdl, "%s: %llu", msp->ms_caseopen.fmds_name,
	msp->ms_caseopen.fmds_value.ui64);
	fmd_hdl_debug(hdl, "%s: %llu", msp->ms_casesolved.fmds_name,
	msp->ms_casesolved.fmds_value.ui64);
	fmd_hdl_debug(hdl, "%s: %llu", msp->ms_caseclosed.fmds_name,
	msp->ms_caseclosed.fmds_value.ui64);
	}

	/* dump module specific stats */
	if (mp->mod_ustat != NULL) {
	int i;

	for (i = 0; i < mp->mod_ustat_cnt; i++) {
	fmd_hdl_debug(hdl, "%s: %llu",
	mp->mod_ustat[i].fmds_name,
	mp->mod_ustat[i].fmds_value.ui64);
	}
	}

	fmd_serd_hash_destroy(&mp->mod_serds);

	fmd_hdl_debug(hdl, "unregister module");
	}

	/*
	* fmd_hdl_setspecific() is used to associate a data pointer with
	* the specified handle for the duration of the module's lifetime.
	* This pointer can be retrieved using fmd_hdl_getspecific().
	*/
	void
	fmd_hdl_setspecific(fmd_hdl_t hdl, void spec)
	{
	fmd_module_t mp = (fmd_module_t )hdl;

	mp->mod_spec = spec;
	}

	/*
	* Return the module-specific data pointer previously associated
	* with the handle using fmd_hdl_setspecific().
	*/
	void *
	fmd_hdl_getspecific(fmd_hdl_t *hdl)
	{
	fmd_module_t mp = (fmd_module_t )hdl;

	return (mp->mod_spec);
	}

	void *
	fmd_hdl_alloc(fmd_hdl_t *hdl, size_t size, int flags)
	{
	(void) hdl;
	return (umem_alloc(size, flags));
	}

	void *
	fmd_hdl_zalloc(fmd_hdl_t *hdl, size_t size, int flags)
	{
	(void) hdl;
	return (umem_zalloc(size, flags));
	}

	void
	fmd_hdl_free(fmd_hdl_t hdl, void data, size_t size)
	{
	(void) hdl;
	umem_free(data, size);
	}

	/*
	* Record a module debug message using the specified format.
	*/
	void
	fmd_hdl_debug(fmd_hdl_t hdl, const char format, ...)
	{
	char message[256];
	va_list vargs;
	fmd_module_t mp = (fmd_module_t )hdl;

	va_start(vargs, format);
	(void) vsnprintf(message, sizeof (message), format, vargs);
	va_end(vargs);

	/* prefix message with module name */
	zed_log_msg(LOG_INFO, "%s: %s", mp->mod_name, message);
	}

	/* Property Retrieval */

	int32_t
	fmd_prop_get_int32(fmd_hdl_t hdl, const char name)
	{
	(void) hdl;

	/*
	* These can be looked up in mp->modinfo->fmdi_props
	* For now we just hard code for phase 2. In the
	* future, there can be a ZED based override.
	*/
	if (strcmp(name, "spare_on_remove") == 0)
	return (1);

	- if (strcmp(name, "io_N") == 0 \|\| strcmp(name, "checksum_N") == 0)
	- return (10); /* N = 10 events */
	-
	- return (0);
	-}
	-
	-int64_t
	-fmd_prop_get_int64(fmd_hdl_t hdl, const char name)
	-{
	- (void) hdl;
	-
	- /*
	- * These can be looked up in mp->modinfo->fmdi_props
	- * For now we just hard code for phase 2. In the
	- * future, there can be a ZED based override.
	- */
	- if (strcmp(name, "remove_timeout") == 0)
	- return (15ULL * 1000ULL * 1000ULL * 1000ULL); /* 15 sec */
	-
	- if (strcmp(name, "io_T") == 0 \|\| strcmp(name, "checksum_T") == 0)
	- return (1000ULL * 1000ULL * 1000ULL * 600ULL); /* 10 min */
	-
	return (0);
	}

	/* FMD Statistics */

	fmd_stat_t *
	fmd_stat_create(fmd_hdl_t hdl, uint_t flags, uint_t nstats, fmd_stat_t statv)
	{
	fmd_module_t mp = (fmd_module_t )hdl;

	if (flags == FMD_STAT_NOALLOC) {
	mp->mod_ustat = statv;
	mp->mod_ustat_cnt = nstats;
	}

	return (statv);
	}

	/* Case Management */

	fmd_case_t *
	fmd_case_open(fmd_hdl_t hdl, void data)
	{
	fmd_module_t mp = (fmd_module_t )hdl;
	uuid_t uuid;

	fmd_case_t *cp;

	cp = fmd_hdl_zalloc(hdl, sizeof (fmd_case_t), FMD_SLEEP);
	cp->ci_mod = hdl;
	cp->ci_state = FMD_CASE_UNSOLVED;
	cp->ci_flags = FMD_CF_DIRTY;
	cp->ci_data = data;
	cp->ci_bufptr = NULL;
	cp->ci_bufsiz = 0;

	uuid_generate(uuid);
	uuid_unparse(uuid, cp->ci_uuid);

	fmd_hdl_debug(hdl, "case opened (%s)", cp->ci_uuid);
	mp->mod_stats.ms_caseopen.fmds_value.ui64++;

	return (cp);
	}

	void
	fmd_case_solve(fmd_hdl_t hdl, fmd_case_t cp)
	{
	fmd_module_t mp = (fmd_module_t )hdl;

	/*
	* For ZED, the event was already sent from fmd_case_add_suspect()
	*/

	if (cp->ci_state >= FMD_CASE_SOLVED)
	fmd_hdl_debug(hdl, "case is already solved or closed");

	cp->ci_state = FMD_CASE_SOLVED;

	fmd_hdl_debug(hdl, "case solved (%s)", cp->ci_uuid);
	mp->mod_stats.ms_casesolved.fmds_value.ui64++;
	}

	void
	fmd_case_close(fmd_hdl_t hdl, fmd_case_t cp)
	{
	fmd_module_t mp = (fmd_module_t )hdl;
	const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;

	fmd_hdl_debug(hdl, "case closed (%s)", cp->ci_uuid);

	if (ops->fmdo_close != NULL)
	ops->fmdo_close(hdl, cp);

	mp->mod_stats.ms_caseopen.fmds_value.ui64--;
	mp->mod_stats.ms_caseclosed.fmds_value.ui64++;

	if (cp->ci_bufptr != NULL && cp->ci_bufsiz > 0)
	fmd_hdl_free(hdl, cp->ci_bufptr, cp->ci_bufsiz);

	fmd_hdl_free(hdl, cp, sizeof (fmd_case_t));
	}

	void
	fmd_case_uuresolved(fmd_hdl_t hdl, const char uuid)
	{
	fmd_hdl_debug(hdl, "case resolved by uuid (%s)", uuid);
	}

	boolean_t
	fmd_case_solved(fmd_hdl_t hdl, fmd_case_t cp)
	{
	(void) hdl;
	return (cp->ci_state >= FMD_CASE_SOLVED);
	}

	void
	fmd_case_add_ereport(fmd_hdl_t hdl, fmd_case_t cp, fmd_event_t *ep)
	{
	(void) hdl, (void) cp, (void) ep;
	}

	static void
	zed_log_fault(nvlist_t nvl, const char uuid, const char *code)
	{
	nvlist_t *rsrc;
	const char *strval;
	uint64_t guid;
	uint8_t byte;

	zed_log_msg(LOG_INFO, "\nzed_fault_event:");

	if (uuid != NULL)
	zed_log_msg(LOG_INFO, "\t%s: %s", FM_SUSPECT_UUID, uuid);
	if (nvlist_lookup_string(nvl, FM_CLASS, &strval) == 0)
	zed_log_msg(LOG_INFO, "\t%s: %s", FM_CLASS, strval);
	if (code != NULL)
	zed_log_msg(LOG_INFO, "\t%s: %s", FM_SUSPECT_DIAG_CODE, code);
	if (nvlist_lookup_uint8(nvl, FM_FAULT_CERTAINTY, &byte) == 0)
	zed_log_msg(LOG_INFO, "\t%s: %hhu", FM_FAULT_CERTAINTY, byte);
	if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) == 0) {
	if (nvlist_lookup_string(rsrc, FM_FMRI_SCHEME, &strval) == 0)
	zed_log_msg(LOG_INFO, "\t%s: %s", FM_FMRI_SCHEME,
	strval);
	if (nvlist_lookup_uint64(rsrc, FM_FMRI_ZFS_POOL, &guid) == 0)
	zed_log_msg(LOG_INFO, "\t%s: %llu", FM_FMRI_ZFS_POOL,
	guid);
	if (nvlist_lookup_uint64(rsrc, FM_FMRI_ZFS_VDEV, &guid) == 0)
	zed_log_msg(LOG_INFO, "\t%s: %llu \n", FM_FMRI_ZFS_VDEV,
	guid);
	}
	}

	static const char *
	fmd_fault_mkcode(nvlist_t *fault)
	{
	const char *class;
	const char *code = "-";

	/*
	* Note: message codes come from: openzfs/usr/src/cmd/fm/dicts/ZFS.po
	*/
	if (nvlist_lookup_string(fault, FM_CLASS, &class) == 0) {
	if (strcmp(class, "fault.fs.zfs.vdev.io") == 0)
	code = "ZFS-8000-FD";
	else if (strcmp(class, "fault.fs.zfs.vdev.checksum") == 0)
	code = "ZFS-8000-GH";
	else if (strcmp(class, "fault.fs.zfs.io_failure_wait") == 0)
	code = "ZFS-8000-HC";
	else if (strcmp(class, "fault.fs.zfs.io_failure_continue") == 0)
	code = "ZFS-8000-JQ";
	else if (strcmp(class, "fault.fs.zfs.log_replay") == 0)
	code = "ZFS-8000-K4";
	else if (strcmp(class, "fault.fs.zfs.pool") == 0)
	code = "ZFS-8000-CS";
	else if (strcmp(class, "fault.fs.zfs.device") == 0)
	code = "ZFS-8000-D3";

	}
	return (code);
	}

	void
	fmd_case_add_suspect(fmd_hdl_t hdl, fmd_case_t cp, nvlist_t *fault)
	{
	nvlist_t *nvl;
	const char *code = fmd_fault_mkcode(fault);
	int64_t tod[2];
	int err = 0;

	/*
	* payload derived from fmd_protocol_list()
	*/

	(void) gettimeofday(&cp->ci_tv, NULL);
	tod[0] = cp->ci_tv.tv_sec;
	tod[1] = cp->ci_tv.tv_usec;

	nvl = fmd_nvl_alloc(hdl, FMD_SLEEP);

	err \|= nvlist_add_uint8(nvl, FM_VERSION, FM_SUSPECT_VERSION);
	err \|= nvlist_add_string(nvl, FM_CLASS, FM_LIST_SUSPECT_CLASS);
	err \|= nvlist_add_string(nvl, FM_SUSPECT_UUID, cp->ci_uuid);
	err \|= nvlist_add_string(nvl, FM_SUSPECT_DIAG_CODE, code);
	err \|= nvlist_add_int64_array(nvl, FM_SUSPECT_DIAG_TIME, tod, 2);
	err \|= nvlist_add_uint32(nvl, FM_SUSPECT_FAULT_SZ, 1);
	err \|= nvlist_add_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
	(const nvlist_t **)&fault, 1);

	if (err)
	zed_log_die("failed to populate nvlist");

	zed_log_fault(fault, cp->ci_uuid, code);
	zfs_agent_post_event(FM_LIST_SUSPECT_CLASS, NULL, nvl);

	nvlist_free(nvl);
	nvlist_free(fault);
	}

	void
	fmd_case_setspecific(fmd_hdl_t hdl, fmd_case_t cp, void *data)
	{
	(void) hdl;
	cp->ci_data = data;
	}

	void *
	fmd_case_getspecific(fmd_hdl_t hdl, fmd_case_t cp)
	{
	(void) hdl;
	return (cp->ci_data);
	}

	void
	fmd_buf_create(fmd_hdl_t hdl, fmd_case_t cp, const char *name, size_t size)
	{
	assert(strcmp(name, "data") == 0), (void) name;
	assert(cp->ci_bufptr == NULL);
	assert(size < (1024 * 1024));

	cp->ci_bufptr = fmd_hdl_alloc(hdl, size, FMD_SLEEP);
	cp->ci_bufsiz = size;
	}

	void
	fmd_buf_read(fmd_hdl_t hdl, fmd_case_t cp,
	const char name, void buf, size_t size)
	{
	(void) hdl;
	assert(strcmp(name, "data") == 0), (void) name;
	assert(cp->ci_bufptr != NULL);
	assert(size <= cp->ci_bufsiz);

	memcpy(buf, cp->ci_bufptr, size);
	}

	void
	fmd_buf_write(fmd_hdl_t hdl, fmd_case_t cp,
	const char name, const void buf, size_t size)
	{
	(void) hdl;
	assert(strcmp(name, "data") == 0), (void) name;
	assert(cp->ci_bufptr != NULL);
	assert(cp->ci_bufsiz >= size);

	memcpy(cp->ci_bufptr, buf, size);
	}

	/* SERD Engines */

	void
	fmd_serd_create(fmd_hdl_t hdl, const char name, uint_t n, hrtime_t t)
	{
	fmd_module_t mp = (fmd_module_t )hdl;

	if (fmd_serd_eng_lookup(&mp->mod_serds, name) != NULL) {
	zed_log_msg(LOG_ERR, "failed to create SERD engine '%s': "
	" name already exists", name);
	return;
	}

	(void) fmd_serd_eng_insert(&mp->mod_serds, name, n, t);
	}

	void
	fmd_serd_destroy(fmd_hdl_t hdl, const char name)
	{
	fmd_module_t mp = (fmd_module_t )hdl;

	fmd_serd_eng_delete(&mp->mod_serds, name);

	fmd_hdl_debug(hdl, "serd_destroy %s", name);
	}

	int
	fmd_serd_exists(fmd_hdl_t hdl, const char name)
	{
	fmd_module_t mp = (fmd_module_t )hdl;

	return (fmd_serd_eng_lookup(&mp->mod_serds, name) != NULL);
	}

	-void
	-fmd_serd_reset(fmd_hdl_t hdl, const char name)
	+int
	+fmd_serd_active(fmd_hdl_t hdl, const char name)
	{
	fmd_module_t mp = (fmd_module_t )hdl;
	fmd_serd_eng_t *sgp;

	if ((sgp = fmd_serd_eng_lookup(&mp->mod_serds, name)) == NULL) {
	zed_log_msg(LOG_ERR, "serd engine '%s' does not exist", name);
	- return;
	+ return (0);
	}
	+ return (fmd_serd_eng_fired(sgp) \|\| !fmd_serd_eng_empty(sgp));
	+}

	- fmd_serd_eng_reset(sgp);
	+void
	+fmd_serd_reset(fmd_hdl_t hdl, const char name)
	+{
	+ fmd_module_t mp = (fmd_module_t )hdl;
	+ fmd_serd_eng_t *sgp;

	- fmd_hdl_debug(hdl, "serd_reset %s", name);
	+ if ((sgp = fmd_serd_eng_lookup(&mp->mod_serds, name)) == NULL) {
	+ zed_log_msg(LOG_ERR, "serd engine '%s' does not exist", name);
	+ } else {
	+ fmd_serd_eng_reset(sgp);
	+ fmd_hdl_debug(hdl, "serd_reset %s", name);
	+ }
	}

	int
	fmd_serd_record(fmd_hdl_t hdl, const char name, fmd_event_t *ep)
	{
	fmd_module_t mp = (fmd_module_t )hdl;
	fmd_serd_eng_t *sgp;
	- int err;

	if ((sgp = fmd_serd_eng_lookup(&mp->mod_serds, name)) == NULL) {
	zed_log_msg(LOG_ERR, "failed to add record to SERD engine '%s'",
	name);
	return (0);
	}
	- err = fmd_serd_eng_record(sgp, ep->ev_hrt);
	+ return (fmd_serd_eng_record(sgp, ep->ev_hrt));
	+}
	+
	+void
	+fmd_serd_gc(fmd_hdl_t *hdl)
	+{
	+ fmd_module_t mp = (fmd_module_t )hdl;

	- return (err);
	+ fmd_serd_hash_apply(&mp->mod_serds, fmd_serd_eng_gc, NULL);
	}

	/* FMD Timers */

	static void
	_timer_notify(union sigval sv)
	{
	fmd_timer_t *ftp = sv.sival_ptr;
	fmd_hdl_t *hdl = ftp->ft_hdl;
	fmd_module_t mp = (fmd_module_t )hdl;
	const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;
	struct itimerspec its;

	- fmd_hdl_debug(hdl, "timer fired (%p)", ftp->ft_tid);
	+ fmd_hdl_debug(hdl, "%s timer fired (%p)", mp->mod_name, ftp->ft_tid);

	/* disarm the timer */
	memset(&its, 0, sizeof (struct itimerspec));
	timer_settime(ftp->ft_tid, 0, &its, NULL);

	/* Note that the fmdo_timeout can remove this timer */
	if (ops->fmdo_timeout != NULL)
	ops->fmdo_timeout(hdl, ftp, ftp->ft_arg);
	}

	/*
	* Install a new timer which will fire at least delta nanoseconds after the
	* current time. After the timeout has expired, the module's fmdo_timeout
	* entry point is called.
	*/
	fmd_timer_t *
	fmd_timer_install(fmd_hdl_t hdl, void arg, fmd_event_t *ep, hrtime_t delta)
	{
	(void) ep;
	struct sigevent sev;
	struct itimerspec its;
	fmd_timer_t *ftp;

	ftp = fmd_hdl_alloc(hdl, sizeof (fmd_timer_t), FMD_SLEEP);
	ftp->ft_arg = arg;
	ftp->ft_hdl = hdl;

	its.it_value.tv_sec = delta / 1000000000;
	its.it_value.tv_nsec = delta % 1000000000;
	its.it_interval.tv_sec = its.it_value.tv_sec;
	its.it_interval.tv_nsec = its.it_value.tv_nsec;

	sev.sigev_notify = SIGEV_THREAD;
	sev.sigev_notify_function = _timer_notify;
	sev.sigev_notify_attributes = NULL;
	sev.sigev_value.sival_ptr = ftp;
	sev.sigev_signo = 0;

	timer_create(CLOCK_REALTIME, &sev, &ftp->ft_tid);
	timer_settime(ftp->ft_tid, 0, &its, NULL);

	fmd_hdl_debug(hdl, "installing timer for %d secs (%p)",
	(int)its.it_value.tv_sec, ftp->ft_tid);

	return (ftp);
	}

	void
	fmd_timer_remove(fmd_hdl_t hdl, fmd_timer_t ftp)
	{
	fmd_hdl_debug(hdl, "removing timer (%p)", ftp->ft_tid);

	timer_delete(ftp->ft_tid);

	fmd_hdl_free(hdl, ftp, sizeof (fmd_timer_t));
	}

	/* Name-Value Pair Lists */

	nvlist_t *
	fmd_nvl_create_fault(fmd_hdl_t hdl, const char class, uint8_t certainty,
	nvlist_t asru, nvlist_t fru, nvlist_t *resource)
	{
	(void) hdl;
	nvlist_t *nvl;
	int err = 0;

	if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0)
	zed_log_die("failed to xalloc fault nvlist");

	err \|= nvlist_add_uint8(nvl, FM_VERSION, FM_FAULT_VERSION);
	err \|= nvlist_add_string(nvl, FM_CLASS, class);
	err \|= nvlist_add_uint8(nvl, FM_FAULT_CERTAINTY, certainty);

	if (asru != NULL)
	err \|= nvlist_add_nvlist(nvl, FM_FAULT_ASRU, asru);
	if (fru != NULL)
	err \|= nvlist_add_nvlist(nvl, FM_FAULT_FRU, fru);
	if (resource != NULL)
	err \|= nvlist_add_nvlist(nvl, FM_FAULT_RESOURCE, resource);

	if (err)
	zed_log_die("failed to populate nvlist: %s\n", strerror(err));

	return (nvl);
	}

	/*
	* sourced from fmd_string.c
	*/
	static int
	fmd_strmatch(const char s, const char p)
	{
	char c;

	if (p == NULL)
	return (0);

	if (s == NULL)
	s = ""; /* treat NULL string as the empty string */

	do {
	if ((c = *p++) == '\0')
	return (*s == '\0');

	if (c == '*') {
	while (p == '')
	p++; /* consecutive 's can be collapsed /

	if (*p == '\0')
	return (1);

	while (*s != '\0') {
	if (fmd_strmatch(s++, p) != 0)
	return (1);
	}

	return (0);
	}
	} while (c == *s++);

	return (0);
	}

	int
	fmd_nvl_class_match(fmd_hdl_t hdl, nvlist_t nvl, const char *pattern)
	{
	(void) hdl;
	const char *class;

	return (nvl != NULL &&
	nvlist_lookup_string(nvl, FM_CLASS, &class) == 0 &&
	fmd_strmatch(class, pattern));
	}

	nvlist_t *
	fmd_nvl_alloc(fmd_hdl_t *hdl, int flags)
	{
	(void) hdl, (void) flags;
	nvlist_t *nvl = NULL;

	if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0)
	return (NULL);

	return (nvl);
	}


	/*
	* ZED Agent specific APIs
	*/

	fmd_hdl_t *
	fmd_module_hdl(const char *name)
	{
	if (strcmp(name, "zfs-retire") == 0)
	return ((fmd_hdl_t *)&zfs_retire_module);
	if (strcmp(name, "zfs-diagnosis") == 0)
	return ((fmd_hdl_t *)&zfs_diagnosis_module);

	return (NULL);
	}

	boolean_t
	fmd_module_initialized(fmd_hdl_t *hdl)
	{
	fmd_module_t mp = (fmd_module_t )hdl;

	return (mp->mod_info != NULL);
	}

	/*
	* fmd_module_recv is called for each event that is received by
	* the fault manager that has a class that matches one of the
	* module's subscriptions.
	*/
	void
	fmd_module_recv(fmd_hdl_t hdl, nvlist_t nvl, const char *class)
	{
	fmd_module_t mp = (fmd_module_t )hdl;
	const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;
	fmd_event_t faux_event = {0};
	int64_t *tv;
	uint_t n;

	/*
	* Will need to normalized this if we persistently store the case data
	*/
	if (nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tv, &n) == 0)
	faux_event.ev_hrt = tv[0] * NANOSEC + tv[1];
	else
	faux_event.ev_hrt = 0;

	ops->fmdo_recv(hdl, &faux_event, nvl, class);

	mp->mod_stats.ms_accepted.fmds_value.ui64++;

	/* TBD - should we initiate fm_module_gc() periodically? */
	}
	diff --git a/sys/contrib/openzfs/cmd/zed/agents/fmd_api.h b/sys/contrib/openzfs/cmd/zed/agents/fmd_api.h
	index b940d0d395ec..8471feecf33f 100644
	--- a/sys/contrib/openzfs/cmd/zed/agents/fmd_api.h
	+++ b/sys/contrib/openzfs/cmd/zed/agents/fmd_api.h
	@@ -1,241 +1,242 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
	*
	* Copyright (c) 2016, Intel Corporation.
	*/

	#ifndef _FMD_API_H
	#define _FMD_API_H

	#include <sys/types.h>
	#include <sys/time.h>
	#include <time.h>
	#include <libnvpair.h>
	#include <stdarg.h>
	#include <umem.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*
	* Fault Management Daemon Client Interfaces
	*/

	#define FMD_API_VERSION 5

	typedef struct fmd_hdl fmd_hdl_t;

	typedef struct fmd_timer {
	timer_t ft_tid;
	void *ft_arg;
	fmd_hdl_t *ft_hdl;
	} fmd_timer_t;

	#define id_t fmd_timer_t *


	typedef struct fmd_event {
	hrtime_t ev_hrt; /* event time used by SERD engines */
	} fmd_event_t;

	typedef struct fmd_case {
	char ci_uuid[48]; /* uuid string for this case */
	fmd_hdl_t ci_mod; / module that owns this case */
	void ci_data; / data from fmd_case_setspecific() */
	ushort_t ci_state; /* case state (see below) */
	ushort_t ci_flags; /* case flags (see below) */
	struct timeval ci_tv; /* time of original diagnosis */
	void ci_bufptr; / case data serialization buffer */
	size_t ci_bufsiz;
	} fmd_case_t;


	#define FMD_CASE_UNSOLVED 0 /* case is not yet solved (waiting) */
	#define FMD_CASE_SOLVED 1 /* case is solved (suspects added) */
	#define FMD_CASE_CLOSE_WAIT 2 /* case is executing fmdo_close() */
	#define FMD_CASE_CLOSED 3 /* case is closed (reconfig done) */
	#define FMD_CASE_REPAIRED 4 /* case is repaired */
	#define FMD_CASE_RESOLVED 5 /* case is resolved (can be freed) */

	#define FMD_CF_DIRTY 0x01 /* case is in need of checkpoint */
	#define FMD_CF_SOLVED 0x02 /* case has been solved */
	#define FMD_CF_ISOLATED 0x04 /* case has been isolated */
	#define FMD_CF_REPAIRED 0x08 /* case has been repaired */
	#define FMD_CF_RESOLVED 0x10 /* case has been resolved */


	#define FMD_TYPE_BOOL 0 /* int */
	#define FMD_TYPE_INT32 1 /* int32_t */
	#define FMD_TYPE_UINT32 2 /* uint32_t */
	#define FMD_TYPE_INT64 3 /* int64_t */
	#define FMD_TYPE_UINT64 4 /* uint64_t */
	#define FMD_TYPE_TIME 5 /* uint64_t */
	#define FMD_TYPE_SIZE 6 /* uint64_t */

	typedef struct fmd_prop {
	const char fmdp_name; / property name */
	uint_t fmdp_type; /* property type (see above) */
	const char fmdp_defv; / default value */
	} fmd_prop_t;

	typedef struct fmd_stat {
	char fmds_name[32]; /* statistic name */
	uint_t fmds_type; /* statistic type (see above) */
	char fmds_desc[64]; /* statistic description */
	union {
	int bool; /* FMD_TYPE_BOOL */
	int32_t i32; /* FMD_TYPE_INT32 */
	uint32_t ui32; /* FMD_TYPE_UINT32 */
	int64_t i64; /* FMD_TYPE_INT64 */
	uint64_t ui64; /* FMD_TYPE_UINT64 */
	} fmds_value;
	} fmd_stat_t;

	typedef struct fmd_hdl_ops {
	void (fmdo_recv)(fmd_hdl_t , fmd_event_t , nvlist_t , const char *);
	void (fmdo_timeout)(fmd_hdl_t , id_t, void *);
	void (fmdo_close)(fmd_hdl_t , fmd_case_t *);
	void (fmdo_stats)(fmd_hdl_t );
	void (fmdo_gc)(fmd_hdl_t );
	} fmd_hdl_ops_t;

	#define FMD_SEND_SUCCESS 0 /* fmdo_send queued event */
	#define FMD_SEND_FAILED 1 /* fmdo_send unrecoverable error */
	#define FMD_SEND_RETRY 2 /* fmdo_send requests retry */

	typedef struct fmd_hdl_info {
	const char fmdi_desc; / fmd client description string */
	const char fmdi_vers; / fmd client version string */
	const fmd_hdl_ops_t fmdi_ops; / ops vector for client */
	const fmd_prop_t fmdi_props; / array of configuration props */
	} fmd_hdl_info_t;

	extern int fmd_hdl_register(fmd_hdl_t , int, const fmd_hdl_info_t );
	extern void fmd_hdl_unregister(fmd_hdl_t *);

	extern void fmd_hdl_setspecific(fmd_hdl_t , void );
	extern void fmd_hdl_getspecific(fmd_hdl_t );

	#define FMD_SLEEP UMEM_NOFAIL

	extern void fmd_hdl_alloc(fmd_hdl_t , size_t, int);
	extern void fmd_hdl_zalloc(fmd_hdl_t , size_t, int);
	extern void fmd_hdl_free(fmd_hdl_t , void , size_t);

	extern char fmd_hdl_strdup(fmd_hdl_t , const char *, int);
	extern void fmd_hdl_strfree(fmd_hdl_t , char );

	extern void fmd_hdl_vdebug(fmd_hdl_t , const char , va_list);
	extern void fmd_hdl_debug(fmd_hdl_t , const char , ...);

	extern int32_t fmd_prop_get_int32(fmd_hdl_t , const char );
	-extern int64_t fmd_prop_get_int64(fmd_hdl_t , const char );

	#define FMD_STAT_NOALLOC 0x0 /* fmd should use caller's memory */
	#define FMD_STAT_ALLOC 0x1 /* fmd should allocate stats memory */

	extern fmd_stat_t fmd_stat_create(fmd_hdl_t , uint_t, uint_t, fmd_stat_t *);
	extern void fmd_stat_destroy(fmd_hdl_t , uint_t, fmd_stat_t );
	extern void fmd_stat_setstr(fmd_hdl_t , fmd_stat_t , const char *);

	extern fmd_case_t fmd_case_open(fmd_hdl_t , void *);
	extern void fmd_case_reset(fmd_hdl_t , fmd_case_t );
	extern void fmd_case_solve(fmd_hdl_t , fmd_case_t );
	extern void fmd_case_close(fmd_hdl_t , fmd_case_t );

	extern const char fmd_case_uuid(fmd_hdl_t , fmd_case_t *);
	extern fmd_case_t fmd_case_uulookup(fmd_hdl_t , const char *);
	extern void fmd_case_uuclose(fmd_hdl_t , const char );
	extern int fmd_case_uuclosed(fmd_hdl_t , const char );
	extern int fmd_case_uuisresolved(fmd_hdl_t , const char );
	extern void fmd_case_uuresolved(fmd_hdl_t , const char );

	extern boolean_t fmd_case_solved(fmd_hdl_t , fmd_case_t );

	extern void fmd_case_add_ereport(fmd_hdl_t , fmd_case_t , fmd_event_t *);
	extern void fmd_case_add_serd(fmd_hdl_t , fmd_case_t , const char *);
	extern void fmd_case_add_suspect(fmd_hdl_t , fmd_case_t , nvlist_t *);

	extern void fmd_case_setspecific(fmd_hdl_t , fmd_case_t , void *);
	extern void fmd_case_getspecific(fmd_hdl_t , fmd_case_t *);

	extern fmd_case_t fmd_case_next(fmd_hdl_t , fmd_case_t *);
	extern fmd_case_t fmd_case_prev(fmd_hdl_t , fmd_case_t *);

	extern void fmd_buf_create(fmd_hdl_t , fmd_case_t , const char *, size_t);
	extern void fmd_buf_destroy(fmd_hdl_t , fmd_case_t , const char *);
	extern void fmd_buf_read(fmd_hdl_t , fmd_case_t ,
	const char , void , size_t);
	extern void fmd_buf_write(fmd_hdl_t , fmd_case_t ,
	const char , const void , size_t);
	extern size_t fmd_buf_size(fmd_hdl_t , fmd_case_t , const char *);

	extern void fmd_serd_create(fmd_hdl_t , const char , uint_t, hrtime_t);
	extern void fmd_serd_destroy(fmd_hdl_t , const char );
	extern int fmd_serd_exists(fmd_hdl_t , const char );
	+extern int fmd_serd_active(fmd_hdl_t , const char );
	extern void fmd_serd_reset(fmd_hdl_t , const char );
	extern int fmd_serd_record(fmd_hdl_t , const char , fmd_event_t *);
	extern int fmd_serd_fired(fmd_hdl_t , const char );
	extern int fmd_serd_empty(fmd_hdl_t , const char );
	+extern void fmd_serd_gc(fmd_hdl_t *);

	extern id_t fmd_timer_install(fmd_hdl_t , void , fmd_event_t *, hrtime_t);
	extern void fmd_timer_remove(fmd_hdl_t *, id_t);

	extern nvlist_t fmd_nvl_create_fault(fmd_hdl_t ,
	const char , uint8_t, nvlist_t , nvlist_t , nvlist_t );

	extern int fmd_nvl_class_match(fmd_hdl_t , nvlist_t , const char *);

	#define FMD_HAS_FAULT_FRU 0
	#define FMD_HAS_FAULT_ASRU 1
	#define FMD_HAS_FAULT_RESOURCE 2

	extern void fmd_repair_fru(fmd_hdl_t , const char );
	extern int fmd_repair_asru(fmd_hdl_t , const char );

	extern nvlist_t fmd_nvl_alloc(fmd_hdl_t , int);
	extern nvlist_t fmd_nvl_dup(fmd_hdl_t , nvlist_t *, int);

	/*
	* ZED Specific Interfaces
	*/

	extern fmd_hdl_t fmd_module_hdl(const char );
	extern boolean_t fmd_module_initialized(fmd_hdl_t *);
	extern void fmd_module_recv(fmd_hdl_t , nvlist_t , const char *);

	/* ZFS FMA Retire Agent */
	extern void _zfs_retire_init(fmd_hdl_t *);
	extern void _zfs_retire_fini(fmd_hdl_t *);

	/* ZFS FMA Diagnosis Engine */
	extern void _zfs_diagnosis_init(fmd_hdl_t *);
	extern void _zfs_diagnosis_fini(fmd_hdl_t *);

	#ifdef __cplusplus
	}
	#endif

	#endif /* _FMD_API_H */
	diff --git a/sys/contrib/openzfs/cmd/zed/agents/fmd_serd.c b/sys/contrib/openzfs/cmd/zed/agents/fmd_serd.c
	index 0bb2c535f094..f942e62b3f48 100644
	--- a/sys/contrib/openzfs/cmd/zed/agents/fmd_serd.c
	+++ b/sys/contrib/openzfs/cmd/zed/agents/fmd_serd.c
	@@ -1,335 +1,336 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License, Version 1.0 only
	* (the "License"). You may not use this file except in compliance
	* with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright 2004 Sun Microsystems, Inc. All rights reserved.
	* Use is subject to license terms.
	*
	* Copyright (c) 2016, Intel Corporation.
	*/

	#include <assert.h>
	#include <stddef.h>
	#include <stdlib.h>
	#include <string.h>
	#include <sys/list.h>
	#include <sys/time.h>

	#include "fmd_api.h"
	#include "fmd_serd.h"
	#include "../zed_log.h"


	#define FMD_STR_BUCKETS 211


	#ifdef SERD_ENG_DEBUG
	#define serd_log_msg(fmt, ...) \
	zed_log_msg(LOG_INFO, fmt, __VA_ARGS__)
	#else
	#define serd_log_msg(fmt, ...)
	#endif


	/*
	* SERD Engine Backend
	*/

	/*
	* Compute the delta between events in nanoseconds. To account for very old
	* events which are replayed, we must handle the case where time is negative.
	* We convert the hrtime_t's to unsigned 64-bit integers and then handle the
	* case where 'old' is greater than 'new' (i.e. high-res time has wrapped).
	*/
	static hrtime_t
	fmd_event_delta(hrtime_t t1, hrtime_t t2)
	{
	uint64_t old = t1;
	uint64_t new = t2;

	return (new >= old ? new - old : (UINT64_MAX - old) + new + 1);
	}

	static fmd_serd_eng_t *
	fmd_serd_eng_alloc(const char *name, uint64_t n, hrtime_t t)
	{
	fmd_serd_eng_t *sgp;

	sgp = malloc(sizeof (fmd_serd_eng_t));
	if (sgp == NULL) {
	perror("malloc");
	exit(EXIT_FAILURE);
	}
	memset(sgp, 0, sizeof (fmd_serd_eng_t));

	sgp->sg_name = strdup(name);
	if (sgp->sg_name == NULL) {
	perror("strdup");
	exit(EXIT_FAILURE);
	}

	sgp->sg_flags = FMD_SERD_DIRTY;
	sgp->sg_n = n;
	sgp->sg_t = t;

	list_create(&sgp->sg_list, sizeof (fmd_serd_elem_t),
	offsetof(fmd_serd_elem_t, se_list));

	return (sgp);
	}

	static void
	fmd_serd_eng_free(fmd_serd_eng_t *sgp)
	{
	fmd_serd_eng_reset(sgp);
	free(sgp->sg_name);
	list_destroy(&sgp->sg_list);
	free(sgp);
	}

	/*
	* sourced from fmd_string.c
	*/
	static ulong_t
	fmd_strhash(const char *key)
	{
	ulong_t g, h = 0;
	const char *p;

	for (p = key; *p != '\0'; p++) {
	h = (h << 4) + *p;

	if ((g = (h & 0xf0000000)) != 0) {
	h ^= (g >> 24);
	h ^= g;
	}
	}

	return (h);
	}

	void
	fmd_serd_hash_create(fmd_serd_hash_t *shp)
	{
	shp->sh_hashlen = FMD_STR_BUCKETS;
	shp->sh_hash = calloc(shp->sh_hashlen, sizeof (void *));
	shp->sh_count = 0;

	if (shp->sh_hash == NULL) {
	perror("calloc");
	exit(EXIT_FAILURE);
	}

	}

	void
	fmd_serd_hash_destroy(fmd_serd_hash_t *shp)
	{
	fmd_serd_eng_t sgp, ngp;
	uint_t i;

	for (i = 0; i < shp->sh_hashlen; i++) {
	for (sgp = shp->sh_hash[i]; sgp != NULL; sgp = ngp) {
	ngp = sgp->sg_next;
	fmd_serd_eng_free(sgp);
	}
	}

	free(shp->sh_hash);
	memset(shp, 0, sizeof (fmd_serd_hash_t));
	}

	void
	fmd_serd_hash_apply(fmd_serd_hash_t shp, fmd_serd_eng_f func, void *arg)
	{
	fmd_serd_eng_t *sgp;
	uint_t i;

	for (i = 0; i < shp->sh_hashlen; i++) {
	for (sgp = shp->sh_hash[i]; sgp != NULL; sgp = sgp->sg_next)
	func(sgp, arg);
	}
	}

	fmd_serd_eng_t *
	fmd_serd_eng_insert(fmd_serd_hash_t shp, const char name,
	uint_t n, hrtime_t t)
	{
	uint_t h = fmd_strhash(name) % shp->sh_hashlen;
	fmd_serd_eng_t *sgp = fmd_serd_eng_alloc(name, n, t);

	serd_log_msg(" SERD Engine: inserting %s N %d T %llu",
	name, (int)n, (long long unsigned)t);

	sgp->sg_next = shp->sh_hash[h];
	shp->sh_hash[h] = sgp;
	shp->sh_count++;

	return (sgp);
	}

	fmd_serd_eng_t *
	fmd_serd_eng_lookup(fmd_serd_hash_t shp, const char name)
	{
	uint_t h = fmd_strhash(name) % shp->sh_hashlen;
	fmd_serd_eng_t *sgp;

	for (sgp = shp->sh_hash[h]; sgp != NULL; sgp = sgp->sg_next) {
	if (strcmp(name, sgp->sg_name) == 0)
	return (sgp);
	}

	return (NULL);
	}

	void
	fmd_serd_eng_delete(fmd_serd_hash_t shp, const char name)
	{
	uint_t h = fmd_strhash(name) % shp->sh_hashlen;
	fmd_serd_eng_t sgp, *pp = &shp->sh_hash[h];

	serd_log_msg(" SERD Engine: deleting %s", name);

	for (sgp = *pp; sgp != NULL; sgp = sgp->sg_next) {
	if (strcmp(sgp->sg_name, name) != 0)
	pp = &sgp->sg_next;
	else
	break;
	}

	if (sgp != NULL) {
	*pp = sgp->sg_next;
	fmd_serd_eng_free(sgp);
	assert(shp->sh_count != 0);
	shp->sh_count--;
	}
	}

	static void
	fmd_serd_eng_discard(fmd_serd_eng_t sgp, fmd_serd_elem_t sep)
	{
	list_remove(&sgp->sg_list, sep);
	sgp->sg_count--;

	serd_log_msg(" SERD Engine: discarding %s, %d remaining",
	sgp->sg_name, (int)sgp->sg_count);

	free(sep);
	}

	int
	fmd_serd_eng_record(fmd_serd_eng_t *sgp, hrtime_t hrt)
	{
	fmd_serd_elem_t sep, oep;

	/*
	* If the fired flag is already set, return false and discard the
	* event. This means that the caller will only see the engine "fire"
	* once until fmd_serd_eng_reset() is called. The fmd_serd_eng_fired()
	* function can also be used in combination with fmd_serd_eng_record().
	*/
	if (sgp->sg_flags & FMD_SERD_FIRED) {
	serd_log_msg(" SERD Engine: record %s already fired!",
	sgp->sg_name);
	return (B_FALSE);
	}

	while (sgp->sg_count >= sgp->sg_n)
	fmd_serd_eng_discard(sgp, list_tail(&sgp->sg_list));

	sep = malloc(sizeof (fmd_serd_elem_t));
	if (sep == NULL) {
	perror("malloc");
	exit(EXIT_FAILURE);
	}
	sep->se_hrt = hrt;

	list_insert_head(&sgp->sg_list, sep);
	sgp->sg_count++;

	serd_log_msg(" SERD Engine: recording %s of %d (%llu)",
	sgp->sg_name, (int)sgp->sg_count, (long long unsigned)hrt);

	/*
	* Pick up the oldest element pointer for comparison to 'sep'. We must
	* do this after adding 'sep' because 'oep' and 'sep' can be the same.
	*/
	oep = list_tail(&sgp->sg_list);

	if (sgp->sg_count >= sgp->sg_n &&
	fmd_event_delta(oep->se_hrt, sep->se_hrt) <= sgp->sg_t) {
	sgp->sg_flags \|= FMD_SERD_FIRED \| FMD_SERD_DIRTY;
	serd_log_msg(" SERD Engine: fired %s", sgp->sg_name);
	return (B_TRUE);
	}

	sgp->sg_flags \|= FMD_SERD_DIRTY;
	return (B_FALSE);
	}

	int
	fmd_serd_eng_fired(fmd_serd_eng_t *sgp)
	{
	return (sgp->sg_flags & FMD_SERD_FIRED);
	}

	int
	fmd_serd_eng_empty(fmd_serd_eng_t *sgp)
	{
	return (sgp->sg_count == 0);
	}

	void
	fmd_serd_eng_reset(fmd_serd_eng_t *sgp)
	{
	serd_log_msg(" SERD Engine: resetting %s", sgp->sg_name);

	while (sgp->sg_count != 0)
	fmd_serd_eng_discard(sgp, list_head(&sgp->sg_list));

	sgp->sg_flags &= ~FMD_SERD_FIRED;
	sgp->sg_flags \|= FMD_SERD_DIRTY;
	}

	void
	-fmd_serd_eng_gc(fmd_serd_eng_t *sgp)
	+fmd_serd_eng_gc(fmd_serd_eng_t sgp, void arg)
	{
	+ (void) arg;
	fmd_serd_elem_t sep, nep;
	hrtime_t hrt;

	if (sgp->sg_count == 0 \|\| (sgp->sg_flags & FMD_SERD_FIRED))
	return; /* no garbage collection needed if empty or fired */

	sep = list_head(&sgp->sg_list);
	if (sep == NULL)
	return;

	hrt = sep->se_hrt - sgp->sg_t;

	for (sep = list_head(&sgp->sg_list); sep != NULL; sep = nep) {
	if (sep->se_hrt >= hrt)
	break; /* sep and subsequent events are all within T */

	nep = list_next(&sgp->sg_list, sep);
	fmd_serd_eng_discard(sgp, sep);
	sgp->sg_flags \|= FMD_SERD_DIRTY;
	}
	}
	diff --git a/sys/contrib/openzfs/cmd/zed/agents/fmd_serd.h b/sys/contrib/openzfs/cmd/zed/agents/fmd_serd.h
	index 25b6888e61f2..80ff9a3b25b8 100644
	--- a/sys/contrib/openzfs/cmd/zed/agents/fmd_serd.h
	+++ b/sys/contrib/openzfs/cmd/zed/agents/fmd_serd.h
	@@ -1,86 +1,86 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License, Version 1.0 only
	* (the "License"). You may not use this file except in compliance
	* with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright 2004 Sun Microsystems, Inc. All rights reserved.
	* Use is subject to license terms.
	*
	* Copyright (c) 2016, Intel Corporation.
	*/

	#ifndef _FMD_SERD_H
	#define _FMD_SERD_H

	#ifdef __cplusplus
	extern "C" {
	#endif

	#include <sys/list.h>
	#include <sys/time.h>

	typedef struct fmd_serd_elem {
	list_node_t se_list; /* linked list forward/back pointers */
	hrtime_t se_hrt; /* upper bound on event hrtime */
	} fmd_serd_elem_t;

	typedef struct fmd_serd_eng {
	char sg_name; / string name for this engine */
	struct fmd_serd_eng sg_next; / next engine on hash chain */
	list_t sg_list; /* list of fmd_serd_elem_t's */
	uint_t sg_count; /* count of events in sg_list */
	uint_t sg_flags; /* engine flags (see below) */
	uint_t sg_n; /* engine N parameter (event count) */
	hrtime_t sg_t; /* engine T parameter (nanoseconds) */
	} fmd_serd_eng_t;

	#define FMD_SERD_FIRED 0x1 /* error rate has exceeded threshold */
	#define FMD_SERD_DIRTY 0x2 /* engine needs to be checkpointed */

	typedef void fmd_serd_eng_f(fmd_serd_eng_t , void );

	typedef struct fmd_serd_hash {
	fmd_serd_eng_t *sh_hash; / hash bucket array for buffers */
	uint_t sh_hashlen; /* length of hash bucket array */
	uint_t sh_count; /* count of engines in hash */
	} fmd_serd_hash_t;

	extern void fmd_serd_hash_create(fmd_serd_hash_t *);
	extern void fmd_serd_hash_destroy(fmd_serd_hash_t *);
	extern void fmd_serd_hash_apply(fmd_serd_hash_t , fmd_serd_eng_f , void *);

	extern fmd_serd_eng_t fmd_serd_eng_insert(fmd_serd_hash_t ,
	const char *, uint32_t, hrtime_t);

	extern fmd_serd_eng_t fmd_serd_eng_lookup(fmd_serd_hash_t , const char *);
	extern void fmd_serd_eng_delete(fmd_serd_hash_t , const char );

	extern int fmd_serd_eng_record(fmd_serd_eng_t *, hrtime_t);
	extern int fmd_serd_eng_fired(fmd_serd_eng_t *);
	extern int fmd_serd_eng_empty(fmd_serd_eng_t *);

	extern void fmd_serd_eng_reset(fmd_serd_eng_t *);
	-extern void fmd_serd_eng_gc(fmd_serd_eng_t *);
	+extern void fmd_serd_eng_gc(fmd_serd_eng_t , void );

	#ifdef __cplusplus
	}
	#endif

	#endif /* _FMD_SERD_H */
	diff --git a/sys/contrib/openzfs/cmd/zed/agents/zfs_diagnosis.c b/sys/contrib/openzfs/cmd/zed/agents/zfs_diagnosis.c
	index f6ba334a3ba3..e0ad00800add 100644
	--- a/sys/contrib/openzfs/cmd/zed/agents/zfs_diagnosis.c
	+++ b/sys/contrib/openzfs/cmd/zed/agents/zfs_diagnosis.c
	@@ -1,1028 +1,1119 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright 2015 Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2016, Intel Corporation.
	+ * Copyright (c) 2023, Klara Inc.
	*/

	#include <stddef.h>
	#include <string.h>
	#include <libuutil.h>
	#include <libzfs.h>
	#include <sys/types.h>
	#include <sys/time.h>
	#include <sys/fs/zfs.h>
	#include <sys/fm/protocol.h>
	#include <sys/fm/fs/zfs.h>
	#include <sys/zio.h>

	#include "zfs_agents.h"
	#include "fmd_api.h"

	/*
	* Default values for the serd engine when processing checksum or io errors. The
	* semantics are N <events> in T <seconds>.
	*/
	#define DEFAULT_CHECKSUM_N 10 /* events */
	#define DEFAULT_CHECKSUM_T 600 /* seconds */
	#define DEFAULT_IO_N 10 /* events */
	#define DEFAULT_IO_T 600 /* seconds */
	+#define DEFAULT_SLOW_IO_N 10 /* events */
	+#define DEFAULT_SLOW_IO_T 30 /* seconds */
	+
	+#define CASE_GC_TIMEOUT_SECS 43200 /* 12 hours */

	/*
	- * Our serd engines are named 'zfs_<pool_guid>_<vdev_guid>_{checksum,io}'. This
	- * #define reserves enough space for two 64-bit hex values plus the length of
	- * the longest string.
	+ * Our serd engines are named in the following format:
	+ * 'zfs_<pool_guid>_<vdev_guid>_{checksum,io,slow_io}'
	+ * This #define reserves enough space for two 64-bit hex values plus the
	+ * length of the longest string.
	*/
	#define MAX_SERDLEN (16 * 2 + sizeof ("zfs___checksum"))

	/*
	* On-disk case structure. This must maintain backwards compatibility with
	* previous versions of the DE. By default, any members appended to the end
	* will be filled with zeros if they don't exist in a previous version.
	*/
	typedef struct zfs_case_data {
	uint64_t zc_version;
	uint64_t zc_ena;
	uint64_t zc_pool_guid;
	uint64_t zc_vdev_guid;
	int zc_pool_state;
	char zc_serd_checksum[MAX_SERDLEN];
	char zc_serd_io[MAX_SERDLEN];
	+ char zc_serd_slow_io[MAX_SERDLEN];
	int zc_has_remove_timer;
	} zfs_case_data_t;

	/*
	* Time-of-day
	*/
	typedef struct er_timeval {
	uint64_t ertv_sec;
	uint64_t ertv_nsec;
	} er_timeval_t;

	/*
	* In-core case structure.
	*/
	typedef struct zfs_case {
	boolean_t zc_present;
	uint32_t zc_version;
	zfs_case_data_t zc_data;
	fmd_case_t *zc_case;
	uu_list_node_t zc_node;
	id_t zc_remove_timer;
	char *zc_fru;
	er_timeval_t zc_when;
	} zfs_case_t;

	#define CASE_DATA "data"
	#define CASE_FRU "fru"
	#define CASE_DATA_VERSION_INITIAL 1
	#define CASE_DATA_VERSION_SERD 2

	typedef struct zfs_de_stats {
	fmd_stat_t old_drops;
	fmd_stat_t dev_drops;
	fmd_stat_t vdev_drops;
	fmd_stat_t import_drops;
	fmd_stat_t resource_drops;
	} zfs_de_stats_t;

	zfs_de_stats_t zfs_stats = {
	{ "old_drops", FMD_TYPE_UINT64, "ereports dropped (from before load)" },
	{ "dev_drops", FMD_TYPE_UINT64, "ereports dropped (dev during open)"},
	{ "vdev_drops", FMD_TYPE_UINT64, "ereports dropped (weird vdev types)"},
	{ "import_drops", FMD_TYPE_UINT64, "ereports dropped (during import)" },
	{ "resource_drops", FMD_TYPE_UINT64, "resource related ereports" }
	};

	-static hrtime_t zfs_remove_timeout;
	+/* wait 15 seconds after a removal */
	+static hrtime_t zfs_remove_timeout = SEC2NSEC(15);

	uu_list_pool_t *zfs_case_pool;
	uu_list_t *zfs_cases;

	#define ZFS_MAKE_RSRC(type) \
	FM_RSRC_CLASS "." ZFS_ERROR_CLASS "." type
	#define ZFS_MAKE_EREPORT(type) \
	FM_EREPORT_CLASS "." ZFS_ERROR_CLASS "." type

	+static void zfs_purge_cases(fmd_hdl_t *hdl);
	+
	/*
	* Write out the persistent representation of an active case.
	*/
	static void
	zfs_case_serialize(zfs_case_t *zcp)
	{
	zcp->zc_data.zc_version = CASE_DATA_VERSION_SERD;
	}

	/*
	* Read back the persistent representation of an active case.
	*/
	static zfs_case_t *
	zfs_case_unserialize(fmd_hdl_t hdl, fmd_case_t cp)
	{
	zfs_case_t *zcp;

	zcp = fmd_hdl_zalloc(hdl, sizeof (zfs_case_t), FMD_SLEEP);
	zcp->zc_case = cp;

	fmd_buf_read(hdl, cp, CASE_DATA, &zcp->zc_data,
	sizeof (zcp->zc_data));

	if (zcp->zc_data.zc_version > CASE_DATA_VERSION_SERD) {
	fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
	return (NULL);
	}

	/*
	* fmd_buf_read() will have already zeroed out the remainder of the
	* buffer, so we don't have to do anything special if the version
	* doesn't include the SERD engine name.
	*/

	if (zcp->zc_data.zc_has_remove_timer)
	zcp->zc_remove_timer = fmd_timer_install(hdl, zcp,
	NULL, zfs_remove_timeout);

	uu_list_node_init(zcp, &zcp->zc_node, zfs_case_pool);
	(void) uu_list_insert_before(zfs_cases, NULL, zcp);

	fmd_case_setspecific(hdl, cp, zcp);

	return (zcp);
	}

	+/*
	+ * count other unique slow-io cases in a pool
	+ */
	+static uint_t
	+zfs_other_slow_cases(fmd_hdl_t hdl, const zfs_case_data_t zfs_case)
	+{
	+ zfs_case_t *zcp;
	+ uint_t cases = 0;
	+ static hrtime_t next_check = 0;
	+
	+ /*
	+ * Note that plumbing in some external GC would require adding locking,
	+ * since most of this module code is not thread safe and assumes there
	+ * is only one thread running against the module. So we perform GC here
	+ * inline periodically so that future delay induced faults will be
	+ * possible once the issue causing multiple vdev delays is resolved.
	+ */
	+ if (gethrestime_sec() > next_check) {
	+ /* Periodically purge old SERD entries and stale cases */
	+ fmd_serd_gc(hdl);
	+ zfs_purge_cases(hdl);
	+ next_check = gethrestime_sec() + CASE_GC_TIMEOUT_SECS;
	+ }
	+
	+ for (zcp = uu_list_first(zfs_cases); zcp != NULL;
	+ zcp = uu_list_next(zfs_cases, zcp)) {
	+ if (zcp->zc_data.zc_pool_guid == zfs_case->zc_pool_guid &&
	+ zcp->zc_data.zc_vdev_guid != zfs_case->zc_vdev_guid &&
	+ zcp->zc_data.zc_serd_slow_io[0] != '\0' &&
	+ fmd_serd_active(hdl, zcp->zc_data.zc_serd_slow_io)) {
	+ cases++;
	+ }
	+ }
	+ return (cases);
	+}
	+
	/*
	* Iterate over any active cases. If any cases are associated with a pool or
	* vdev which is no longer present on the system, close the associated case.
	*/
	static void
	zfs_mark_vdev(uint64_t pool_guid, nvlist_t vd, er_timeval_t loaded)
	{
	uint64_t vdev_guid = 0;
	uint_t c, children;
	nvlist_t **child;
	zfs_case_t *zcp;

	(void) nvlist_lookup_uint64(vd, ZPOOL_CONFIG_GUID, &vdev_guid);

	/*
	* Mark any cases associated with this (pool, vdev) pair.
	*/
	for (zcp = uu_list_first(zfs_cases); zcp != NULL;
	zcp = uu_list_next(zfs_cases, zcp)) {
	if (zcp->zc_data.zc_pool_guid == pool_guid &&
	zcp->zc_data.zc_vdev_guid == vdev_guid) {
	zcp->zc_present = B_TRUE;
	zcp->zc_when = *loaded;
	}
	}

	/*
	* Iterate over all children.
	*/
	if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_CHILDREN, &child,
	&children) == 0) {
	for (c = 0; c < children; c++)
	zfs_mark_vdev(pool_guid, child[c], loaded);
	}

	if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_L2CACHE, &child,
	&children) == 0) {
	for (c = 0; c < children; c++)
	zfs_mark_vdev(pool_guid, child[c], loaded);
	}

	if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_SPARES, &child,
	&children) == 0) {
	for (c = 0; c < children; c++)
	zfs_mark_vdev(pool_guid, child[c], loaded);
	}
	}

	static int
	zfs_mark_pool(zpool_handle_t zhp, void unused)
	{
	(void) unused;
	zfs_case_t *zcp;
	uint64_t pool_guid;
	uint64_t *tod;
	er_timeval_t loaded = { 0 };
	nvlist_t config, vd;
	uint_t nelem = 0;
	int ret;

	pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
	/*
	* Mark any cases associated with just this pool.
	*/
	for (zcp = uu_list_first(zfs_cases); zcp != NULL;
	zcp = uu_list_next(zfs_cases, zcp)) {
	if (zcp->zc_data.zc_pool_guid == pool_guid &&
	zcp->zc_data.zc_vdev_guid == 0)
	zcp->zc_present = B_TRUE;
	}

	if ((config = zpool_get_config(zhp, NULL)) == NULL) {
	zpool_close(zhp);
	return (-1);
	}

	(void) nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
	&tod, &nelem);
	if (nelem == 2) {
	loaded.ertv_sec = tod[0];
	loaded.ertv_nsec = tod[1];
	for (zcp = uu_list_first(zfs_cases); zcp != NULL;
	zcp = uu_list_next(zfs_cases, zcp)) {
	if (zcp->zc_data.zc_pool_guid == pool_guid &&
	zcp->zc_data.zc_vdev_guid == 0) {
	zcp->zc_when = loaded;
	}
	}
	}

	ret = nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &vd);
	if (ret) {
	zpool_close(zhp);
	return (-1);
	}

	zfs_mark_vdev(pool_guid, vd, &loaded);

	zpool_close(zhp);

	return (0);
	}

	struct load_time_arg {
	uint64_t lt_guid;
	er_timeval_t *lt_time;
	boolean_t lt_found;
	};

	static int
	zpool_find_load_time(zpool_handle_t zhp, void arg)
	{
	struct load_time_arg *lta = arg;
	uint64_t pool_guid;
	uint64_t *tod;
	nvlist_t *config;
	uint_t nelem;

	if (lta->lt_found) {
	zpool_close(zhp);
	return (0);
	}

	pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
	if (pool_guid != lta->lt_guid) {
	zpool_close(zhp);
	return (0);
	}

	if ((config = zpool_get_config(zhp, NULL)) == NULL) {
	zpool_close(zhp);
	return (-1);
	}

	if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
	&tod, &nelem) == 0 && nelem == 2) {
	lta->lt_found = B_TRUE;
	lta->lt_time->ertv_sec = tod[0];
	lta->lt_time->ertv_nsec = tod[1];
	}

	zpool_close(zhp);

	return (0);
	}

	static void
	zfs_purge_cases(fmd_hdl_t *hdl)
	{
	zfs_case_t *zcp;
	uu_list_walk_t *walk;
	libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);

	/*
	* There is no way to open a pool by GUID, or lookup a vdev by GUID. No
	* matter what we do, we're going to have to stomach an O(vdevs * cases)
	* algorithm. In reality, both quantities are likely so small that
	* neither will matter. Given that iterating over pools is more
	* expensive than iterating over the in-memory case list, we opt for a
	* 'present' flag in each case that starts off cleared. We then iterate
	* over all pools, marking those that are still present, and removing
	* those that aren't found.
	*
	* Note that we could also construct an FMRI and rely on
	* fmd_nvl_fmri_present(), but this would end up doing the same search.
	*/

	/*
	* Mark the cases as not present.
	*/
	for (zcp = uu_list_first(zfs_cases); zcp != NULL;
	zcp = uu_list_next(zfs_cases, zcp))
	zcp->zc_present = B_FALSE;

	/*
	* Iterate over all pools and mark the pools and vdevs found. If this
	* fails (most probably because we're out of memory), then don't close
	* any of the cases and we cannot be sure they are accurate.
	*/
	if (zpool_iter(zhdl, zfs_mark_pool, NULL) != 0)
	return;

	/*
	* Remove those cases which were not found.
	*/
	walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
	while ((zcp = uu_list_walk_next(walk)) != NULL) {
	if (!zcp->zc_present)
	fmd_case_close(hdl, zcp->zc_case);
	}
	uu_list_walk_end(walk);
	}

	/*
	* Construct the name of a serd engine given the pool/vdev GUID and type (io or
	* checksum).
	*/
	static void
	zfs_serd_name(char *buf, uint64_t pool_guid, uint64_t vdev_guid,
	const char *type)
	{
	(void) snprintf(buf, MAX_SERDLEN, "zfs_%llx_%llx_%s",
	(long long unsigned int)pool_guid,
	(long long unsigned int)vdev_guid, type);
	}

	+static void
	+zfs_case_retire(fmd_hdl_t hdl, zfs_case_t zcp)
	+{
	+ fmd_hdl_debug(hdl, "retiring case");
	+
	+ fmd_case_close(hdl, zcp->zc_case);
	+}
	+
	/*
	* Solve a given ZFS case. This first checks to make sure the diagnosis is
	* still valid, as well as cleaning up any pending timer associated with the
	* case.
	*/
	static void
	zfs_case_solve(fmd_hdl_t hdl, zfs_case_t zcp, const char *faultname)
	{
	nvlist_t detector, fault;
	boolean_t serialize;
	nvlist_t *fru = NULL;
	fmd_hdl_debug(hdl, "solving fault '%s'", faultname);

	/*
	* Construct the detector from the case data. The detector is in the
	* ZFS scheme, and is either the pool or the vdev, depending on whether
	* this is a vdev or pool fault.
	*/
	detector = fmd_nvl_alloc(hdl, FMD_SLEEP);

	(void) nvlist_add_uint8(detector, FM_VERSION, ZFS_SCHEME_VERSION0);
	(void) nvlist_add_string(detector, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS);
	(void) nvlist_add_uint64(detector, FM_FMRI_ZFS_POOL,
	zcp->zc_data.zc_pool_guid);
	if (zcp->zc_data.zc_vdev_guid != 0) {
	(void) nvlist_add_uint64(detector, FM_FMRI_ZFS_VDEV,
	zcp->zc_data.zc_vdev_guid);
	}

	fault = fmd_nvl_create_fault(hdl, faultname, 100, detector,
	fru, detector);
	fmd_case_add_suspect(hdl, zcp->zc_case, fault);

	nvlist_free(fru);

	fmd_case_solve(hdl, zcp->zc_case);

	serialize = B_FALSE;
	if (zcp->zc_data.zc_has_remove_timer) {
	fmd_timer_remove(hdl, zcp->zc_remove_timer);
	zcp->zc_data.zc_has_remove_timer = 0;
	serialize = B_TRUE;
	}
	if (serialize)
	zfs_case_serialize(zcp);

	nvlist_free(detector);
	}

	static boolean_t
	timeval_earlier(er_timeval_t a, er_timeval_t b)
	{
	return (a->ertv_sec < b->ertv_sec \|\|
	(a->ertv_sec == b->ertv_sec && a->ertv_nsec < b->ertv_nsec));
	}

	static void
	zfs_ereport_when(fmd_hdl_t hdl, nvlist_t nvl, er_timeval_t *when)
	{
	(void) hdl;
	int64_t *tod;
	uint_t nelem;

	if (nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tod,
	&nelem) == 0 && nelem == 2) {
	when->ertv_sec = tod[0];
	when->ertv_nsec = tod[1];
	} else {
	when->ertv_sec = when->ertv_nsec = UINT64_MAX;
	}
	}

	/*
	* Main fmd entry point.
	*/
	static void
	zfs_fm_recv(fmd_hdl_t hdl, fmd_event_t ep, nvlist_t nvl, const char class)
	{
	zfs_case_t zcp, dcp;
	int32_t pool_state;
	uint64_t ena, pool_guid, vdev_guid;
	uint64_t checksum_n, checksum_t;
	uint64_t io_n, io_t;
	er_timeval_t pool_load;
	er_timeval_t er_when;
	nvlist_t *detector;
	boolean_t pool_found = B_FALSE;
	boolean_t isresource;
	const char *type;

	/*
	* We subscribe to notifications for vdev or pool removal. In these
	* cases, there may be cases that no longer apply. Purge any cases
	* that no longer apply.
	*/
	if (fmd_nvl_class_match(hdl, nvl, "sysevent.fs.zfs.*")) {
	fmd_hdl_debug(hdl, "purging orphaned cases from %s",
	strrchr(class, '.') + 1);
	zfs_purge_cases(hdl);
	zfs_stats.resource_drops.fmds_value.ui64++;
	return;
	}

	isresource = fmd_nvl_class_match(hdl, nvl, "resource.fs.zfs.*");

	if (isresource) {
	/*
	* For resources, we don't have a normal payload.
	*/
	if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
	&vdev_guid) != 0)
	pool_state = SPA_LOAD_OPEN;
	else
	pool_state = SPA_LOAD_NONE;
	detector = NULL;
	} else {
	(void) nvlist_lookup_nvlist(nvl,
	FM_EREPORT_DETECTOR, &detector);
	(void) nvlist_lookup_int32(nvl,
	FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, &pool_state);
	}

	/*
	* We also ignore all ereports generated during an import of a pool,
	* since the only possible fault (.pool) would result in import failure,
	* and hence no persistent fault. Some day we may want to do something
	* with these ereports, so we continue generating them internally.
	*/
	if (pool_state == SPA_LOAD_IMPORT) {
	zfs_stats.import_drops.fmds_value.ui64++;
	fmd_hdl_debug(hdl, "ignoring '%s' during import", class);
	return;
	}

	/*
	* Device I/O errors are ignored during pool open.
	*/
	if (pool_state == SPA_LOAD_OPEN &&
	(fmd_nvl_class_match(hdl, nvl,
	ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) \|\|
	fmd_nvl_class_match(hdl, nvl,
	ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) \|\|
	fmd_nvl_class_match(hdl, nvl,
	ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE)))) {
	fmd_hdl_debug(hdl, "ignoring '%s' during pool open", class);
	zfs_stats.dev_drops.fmds_value.ui64++;
	return;
	}

	/*
	* We ignore ereports for anything except disks and files.
	*/
	if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
	&type) == 0) {
	if (strcmp(type, VDEV_TYPE_DISK) != 0 &&
	strcmp(type, VDEV_TYPE_FILE) != 0) {
	zfs_stats.vdev_drops.fmds_value.ui64++;
	return;
	}
	}

	/*
	* Determine if this ereport corresponds to an open case.
	* Each vdev or pool can have a single case.
	*/
	(void) nvlist_lookup_uint64(nvl,
	FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, &pool_guid);
	if (nvlist_lookup_uint64(nvl,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
	vdev_guid = 0;
	if (nvlist_lookup_uint64(nvl, FM_EREPORT_ENA, &ena) != 0)
	ena = 0;

	zfs_ereport_when(hdl, nvl, &er_when);

	for (zcp = uu_list_first(zfs_cases); zcp != NULL;
	zcp = uu_list_next(zfs_cases, zcp)) {
	if (zcp->zc_data.zc_pool_guid == pool_guid) {
	pool_found = B_TRUE;
	pool_load = zcp->zc_when;
	}
	if (zcp->zc_data.zc_vdev_guid == vdev_guid)
	break;
	}

	/*
	* Avoid falsely accusing a pool of being faulty. Do so by
	* not replaying ereports that were generated prior to the
	* current import. If the failure that generated them was
	* transient because the device was actually removed but we
	* didn't receive the normal asynchronous notification, we
	* don't want to mark it as faulted and potentially panic. If
	* there is still a problem we'd expect not to be able to
	* import the pool, or that new ereports will be generated
	* once the pool is used.
	*/
	if (pool_found && timeval_earlier(&er_when, &pool_load)) {
	fmd_hdl_debug(hdl, "ignoring pool %llx, "
	"ereport time %lld.%lld, pool load time = %lld.%lld",
	pool_guid, er_when.ertv_sec, er_when.ertv_nsec,
	pool_load.ertv_sec, pool_load.ertv_nsec);
	zfs_stats.old_drops.fmds_value.ui64++;
	return;
	}

	if (!pool_found) {
	/*
	* Haven't yet seen this pool, but same situation
	* may apply.
	*/
	libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
	struct load_time_arg la;

	la.lt_guid = pool_guid;
	la.lt_time = &pool_load;
	la.lt_found = B_FALSE;

	if (zhdl != NULL &&
	zpool_iter(zhdl, zpool_find_load_time, &la) == 0 &&
	la.lt_found == B_TRUE) {
	pool_found = B_TRUE;

	if (timeval_earlier(&er_when, &pool_load)) {
	fmd_hdl_debug(hdl, "ignoring pool %llx, "
	"ereport time %lld.%lld, "
	"pool load time = %lld.%lld",
	pool_guid, er_when.ertv_sec,
	er_when.ertv_nsec, pool_load.ertv_sec,
	pool_load.ertv_nsec);
	zfs_stats.old_drops.fmds_value.ui64++;
	return;
	}
	}
	}

	if (zcp == NULL) {
	fmd_case_t *cs;
	zfs_case_data_t data = { 0 };

	/*
	* If this is one of our 'fake' resource ereports, and there is
	* no case open, simply discard it.
	*/
	if (isresource) {
	zfs_stats.resource_drops.fmds_value.ui64++;
	fmd_hdl_debug(hdl, "discarding '%s for vdev %llu",
	class, vdev_guid);
	return;
	}

	/*
	* Skip tracking some ereports
	*/
	if (strcmp(class,
	ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DATA)) == 0 \|\|
	strcmp(class,
	- ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CONFIG_CACHE_WRITE)) == 0 \|\|
	- strcmp(class,
	- ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY)) == 0) {
	+ ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CONFIG_CACHE_WRITE)) == 0) {
	zfs_stats.resource_drops.fmds_value.ui64++;
	return;
	}

	/*
	* Open a new case.
	*/
	cs = fmd_case_open(hdl, NULL);

	fmd_hdl_debug(hdl, "opening case for vdev %llu due to '%s'",
	vdev_guid, class);

	/*
	* Initialize the case buffer. To commonize code, we actually
	* create the buffer with existing data, and then call
	* zfs_case_unserialize() to instantiate the in-core structure.
	*/
	fmd_buf_create(hdl, cs, CASE_DATA, sizeof (zfs_case_data_t));

	data.zc_version = CASE_DATA_VERSION_SERD;
	data.zc_ena = ena;
	data.zc_pool_guid = pool_guid;
	data.zc_vdev_guid = vdev_guid;
	data.zc_pool_state = (int)pool_state;

	fmd_buf_write(hdl, cs, CASE_DATA, &data, sizeof (data));

	zcp = zfs_case_unserialize(hdl, cs);
	assert(zcp != NULL);
	if (pool_found)
	zcp->zc_when = pool_load;
	}

	if (isresource) {
	fmd_hdl_debug(hdl, "resource event '%s'", class);

	if (fmd_nvl_class_match(hdl, nvl,
	ZFS_MAKE_RSRC(FM_RESOURCE_AUTOREPLACE))) {
	/*
	* The 'resource.fs.zfs.autoreplace' event indicates
	* that the pool was loaded with the 'autoreplace'
	* property set. In this case, any pending device
	* failures should be ignored, as the asynchronous
	* autoreplace handling will take care of them.
	*/
	fmd_case_close(hdl, zcp->zc_case);
	} else if (fmd_nvl_class_match(hdl, nvl,
	ZFS_MAKE_RSRC(FM_RESOURCE_REMOVED))) {
	/*
	* The 'resource.fs.zfs.removed' event indicates that
	* device removal was detected, and the device was
	* closed asynchronously. If this is the case, we
	* assume that any recent I/O errors were due to the
	* device removal, not any fault of the device itself.
	* We reset the SERD engine, and cancel any pending
	* timers.
	*/
	if (zcp->zc_data.zc_has_remove_timer) {
	fmd_timer_remove(hdl, zcp->zc_remove_timer);
	zcp->zc_data.zc_has_remove_timer = 0;
	zfs_case_serialize(zcp);
	}
	if (zcp->zc_data.zc_serd_io[0] != '\0')
	fmd_serd_reset(hdl, zcp->zc_data.zc_serd_io);
	if (zcp->zc_data.zc_serd_checksum[0] != '\0')
	fmd_serd_reset(hdl,
	zcp->zc_data.zc_serd_checksum);
	+ if (zcp->zc_data.zc_serd_slow_io[0] != '\0')
	+ fmd_serd_reset(hdl,
	+ zcp->zc_data.zc_serd_slow_io);
	} else if (fmd_nvl_class_match(hdl, nvl,
	ZFS_MAKE_RSRC(FM_RESOURCE_STATECHANGE))) {
	uint64_t state = 0;

	if (zcp != NULL &&
	nvlist_lookup_uint64(nvl,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE, &state) == 0 &&
	state == VDEV_STATE_HEALTHY) {
	fmd_hdl_debug(hdl, "closing case after a "
	"device statechange to healthy");
	fmd_case_close(hdl, zcp->zc_case);
	}
	}
	zfs_stats.resource_drops.fmds_value.ui64++;
	return;
	}

	/*
	* Associate the ereport with this case.
	*/
	fmd_case_add_ereport(hdl, zcp->zc_case, ep);

	/*
	* Don't do anything else if this case is already solved.
	*/
	if (fmd_case_solved(hdl, zcp->zc_case))
	return;

	- fmd_hdl_debug(hdl, "error event '%s'", class);
	+ if (vdev_guid)
	+ fmd_hdl_debug(hdl, "error event '%s', vdev %llu", class,
	+ vdev_guid);
	+ else
	+ fmd_hdl_debug(hdl, "error event '%s'", class);

	/*
	* Determine if we should solve the case and generate a fault. We solve
	* a case if:
	*
	* a. A pool failed to open (ereport.fs.zfs.pool)
	* b. A device failed to open (ereport.fs.zfs.pool) while a pool
	* was up and running.
	*
	* We may see a series of ereports associated with a pool open, all
	* chained together by the same ENA. If the pool open succeeds, then
	* we'll see no further ereports. To detect when a pool open has
	* succeeded, we associate a timer with the event. When it expires, we
	* close the case.
	*/
	if (fmd_nvl_class_match(hdl, nvl,
	ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_POOL))) {
	/*
	* Pool level fault. Before solving the case, go through and
	* close any open device cases that may be pending.
	*/
	for (dcp = uu_list_first(zfs_cases); dcp != NULL;
	dcp = uu_list_next(zfs_cases, dcp)) {
	if (dcp->zc_data.zc_pool_guid ==
	zcp->zc_data.zc_pool_guid &&
	dcp->zc_data.zc_vdev_guid != 0)
	fmd_case_close(hdl, dcp->zc_case);
	}

	zfs_case_solve(hdl, zcp, "fault.fs.zfs.pool");
	} else if (fmd_nvl_class_match(hdl, nvl,
	ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_LOG_REPLAY))) {
	/*
	* Pool level fault for reading the intent logs.
	*/
	zfs_case_solve(hdl, zcp, "fault.fs.zfs.log_replay");
	} else if (fmd_nvl_class_match(hdl, nvl, "ereport.fs.zfs.vdev.*")) {
	/*
	* Device fault.
	*/
	zfs_case_solve(hdl, zcp, "fault.fs.zfs.device");
	} else if (fmd_nvl_class_match(hdl, nvl,
	ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) \|\|
	fmd_nvl_class_match(hdl, nvl,
	ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) \|\|
	fmd_nvl_class_match(hdl, nvl,
	ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) \|\|
	fmd_nvl_class_match(hdl, nvl,
	+ ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY)) \|\|
	+ fmd_nvl_class_match(hdl, nvl,
	ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
	const char *failmode = NULL;
	boolean_t checkremove = B_FALSE;
	uint32_t pri = 0;
	int32_t flags = 0;

	/*
	* If this is a checksum or I/O error, then toss it into the
	* appropriate SERD engine and check to see if it has fired.
	* Ideally, we want to do something more sophisticated,
	* (persistent errors for a single data block, etc). For now,
	* a single SERD engine is sufficient.
	*/
	if (fmd_nvl_class_match(hdl, nvl,
	ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO))) {
	if (zcp->zc_data.zc_serd_io[0] == '\0') {
	if (nvlist_lookup_uint64(nvl,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_N,
	&io_n) != 0) {
	io_n = DEFAULT_IO_N;
	}
	if (nvlist_lookup_uint64(nvl,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_T,
	&io_t) != 0) {
	io_t = DEFAULT_IO_T;
	}
	zfs_serd_name(zcp->zc_data.zc_serd_io,
	pool_guid, vdev_guid, "io");
	fmd_serd_create(hdl, zcp->zc_data.zc_serd_io,
	io_n,
	SEC2NSEC(io_t));
	zfs_case_serialize(zcp);
	}
	if (fmd_serd_record(hdl, zcp->zc_data.zc_serd_io, ep))
	checkremove = B_TRUE;
	+ } else if (fmd_nvl_class_match(hdl, nvl,
	+ ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY))) {
	+ uint64_t slow_io_n, slow_io_t;
	+
	+ /*
	+ * Create a slow io SERD engine when the VDEV has the
	+ * 'vdev_slow_io_n' and 'vdev_slow_io_n' properties.
	+ */
	+ if (zcp->zc_data.zc_serd_slow_io[0] == '\0' &&
	+ nvlist_lookup_uint64(nvl,
	+ FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_N,
	+ &slow_io_n) == 0 &&
	+ nvlist_lookup_uint64(nvl,
	+ FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_T,
	+ &slow_io_t) == 0) {
	+ zfs_serd_name(zcp->zc_data.zc_serd_slow_io,
	+ pool_guid, vdev_guid, "slow_io");
	+ fmd_serd_create(hdl,
	+ zcp->zc_data.zc_serd_slow_io,
	+ slow_io_n,
	+ SEC2NSEC(slow_io_t));
	+ zfs_case_serialize(zcp);
	+ }
	+ /* Pass event to SERD engine and see if this triggers */
	+ if (zcp->zc_data.zc_serd_slow_io[0] != '\0' &&
	+ fmd_serd_record(hdl, zcp->zc_data.zc_serd_slow_io,
	+ ep)) {
	+ /*
	+ * Ignore a slow io diagnosis when other
	+ * VDEVs in the pool show signs of being slow.
	+ */
	+ if (zfs_other_slow_cases(hdl, &zcp->zc_data)) {
	+ zfs_case_retire(hdl, zcp);
	+ fmd_hdl_debug(hdl, "pool %llu has "
	+ "multiple slow io cases -- skip "
	+ "degrading vdev %llu",
	+ (u_longlong_t)
	+ zcp->zc_data.zc_pool_guid,
	+ (u_longlong_t)
	+ zcp->zc_data.zc_vdev_guid);
	+ } else {
	+ zfs_case_solve(hdl, zcp,
	+ "fault.fs.zfs.vdev.slow_io");
	+ }
	+ }
	} else if (fmd_nvl_class_match(hdl, nvl,
	ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM))) {
	/*
	* We ignore ereports for checksum errors generated by
	* scrub/resilver I/O to avoid potentially further
	* degrading the pool while it's being repaired.
	*/
	if (((nvlist_lookup_uint32(nvl,
	FM_EREPORT_PAYLOAD_ZFS_ZIO_PRIORITY, &pri) == 0) &&
	(pri == ZIO_PRIORITY_SCRUB \|\|
	pri == ZIO_PRIORITY_REBUILD)) \|\|
	((nvlist_lookup_int32(nvl,
	FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS, &flags) == 0) &&
	(flags & (ZIO_FLAG_SCRUB \| ZIO_FLAG_RESILVER)))) {
	fmd_hdl_debug(hdl, "ignoring '%s' for "
	"scrub/resilver I/O", class);
	return;
	}

	if (zcp->zc_data.zc_serd_checksum[0] == '\0') {
	if (nvlist_lookup_uint64(nvl,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_N,
	&checksum_n) != 0) {
	checksum_n = DEFAULT_CHECKSUM_N;
	}
	if (nvlist_lookup_uint64(nvl,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_T,
	&checksum_t) != 0) {
	checksum_t = DEFAULT_CHECKSUM_T;
	}

	zfs_serd_name(zcp->zc_data.zc_serd_checksum,
	pool_guid, vdev_guid, "checksum");
	fmd_serd_create(hdl,
	zcp->zc_data.zc_serd_checksum,
	checksum_n,
	SEC2NSEC(checksum_t));
	zfs_case_serialize(zcp);
	}
	if (fmd_serd_record(hdl,
	zcp->zc_data.zc_serd_checksum, ep)) {
	zfs_case_solve(hdl, zcp,
	"fault.fs.zfs.vdev.checksum");
	}
	} else if (fmd_nvl_class_match(hdl, nvl,
	ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) &&
	(nvlist_lookup_string(nvl,
	FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE, &failmode) == 0) &&
	failmode != NULL) {
	if (strncmp(failmode, FM_EREPORT_FAILMODE_CONTINUE,
	strlen(FM_EREPORT_FAILMODE_CONTINUE)) == 0) {
	zfs_case_solve(hdl, zcp,
	"fault.fs.zfs.io_failure_continue");
	} else if (strncmp(failmode, FM_EREPORT_FAILMODE_WAIT,
	strlen(FM_EREPORT_FAILMODE_WAIT)) == 0) {
	zfs_case_solve(hdl, zcp,
	"fault.fs.zfs.io_failure_wait");
	}
	} else if (fmd_nvl_class_match(hdl, nvl,
	ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
	#ifndef __linux__
	/* This causes an unexpected fault diagnosis on linux */
	checkremove = B_TRUE;
	#endif
	}

	/*
	* Because I/O errors may be due to device removal, we postpone
	* any diagnosis until we're sure that we aren't about to
	* receive a 'resource.fs.zfs.removed' event.
	*/
	if (checkremove) {
	if (zcp->zc_data.zc_has_remove_timer)
	fmd_timer_remove(hdl, zcp->zc_remove_timer);
	zcp->zc_remove_timer = fmd_timer_install(hdl, zcp, NULL,
	zfs_remove_timeout);
	if (!zcp->zc_data.zc_has_remove_timer) {
	zcp->zc_data.zc_has_remove_timer = 1;
	zfs_case_serialize(zcp);
	}
	}
	}
	}

	/*
	* The timeout is fired when we diagnosed an I/O error, and it was not due to
	* device removal (which would cause the timeout to be cancelled).
	*/
	static void
	zfs_fm_timeout(fmd_hdl_t hdl, id_t id, void data)
	{
	zfs_case_t *zcp = data;

	if (id == zcp->zc_remove_timer)
	zfs_case_solve(hdl, zcp, "fault.fs.zfs.vdev.io");
	}

	/*
	* The specified case has been closed and any case-specific
	* data structures should be deallocated.
	*/
	static void
	zfs_fm_close(fmd_hdl_t hdl, fmd_case_t cs)
	{
	zfs_case_t *zcp = fmd_case_getspecific(hdl, cs);

	if (zcp->zc_data.zc_serd_checksum[0] != '\0')
	fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_checksum);
	if (zcp->zc_data.zc_serd_io[0] != '\0')
	fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_io);
	+ if (zcp->zc_data.zc_serd_slow_io[0] != '\0')
	+ fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_slow_io);
	if (zcp->zc_data.zc_has_remove_timer)
	fmd_timer_remove(hdl, zcp->zc_remove_timer);

	uu_list_remove(zfs_cases, zcp);
	uu_list_node_fini(zcp, &zcp->zc_node, zfs_case_pool);
	fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
	}

	-/*
	- * We use the fmd gc entry point to look for old cases that no longer apply.
	- * This allows us to keep our set of case data small in a long running system.
	- */
	-static void
	-zfs_fm_gc(fmd_hdl_t *hdl)
	-{
	- zfs_purge_cases(hdl);
	-}
	-
	static const fmd_hdl_ops_t fmd_ops = {
	zfs_fm_recv, /* fmdo_recv */
	zfs_fm_timeout, /* fmdo_timeout */
	zfs_fm_close, /* fmdo_close */
	NULL, /* fmdo_stats */
	- zfs_fm_gc, /* fmdo_gc */
	+ NULL, /* fmdo_gc */
	};

	static const fmd_prop_t fmd_props[] = {
	- { "checksum_N", FMD_TYPE_UINT32, "10" },
	- { "checksum_T", FMD_TYPE_TIME, "10min" },
	- { "io_N", FMD_TYPE_UINT32, "10" },
	- { "io_T", FMD_TYPE_TIME, "10min" },
	- { "remove_timeout", FMD_TYPE_TIME, "15sec" },
	{ NULL, 0, NULL }
	};

	static const fmd_hdl_info_t fmd_info = {
	"ZFS Diagnosis Engine", "1.0", &fmd_ops, fmd_props
	};

	void
	_zfs_diagnosis_init(fmd_hdl_t *hdl)
	{
	libzfs_handle_t *zhdl;

	if ((zhdl = libzfs_init()) == NULL)
	return;

	if ((zfs_case_pool = uu_list_pool_create("zfs_case_pool",
	sizeof (zfs_case_t), offsetof(zfs_case_t, zc_node),
	NULL, UU_LIST_POOL_DEBUG)) == NULL) {
	libzfs_fini(zhdl);
	return;
	}

	if ((zfs_cases = uu_list_create(zfs_case_pool, NULL,
	UU_LIST_DEBUG)) == NULL) {
	uu_list_pool_destroy(zfs_case_pool);
	libzfs_fini(zhdl);
	return;
	}

	if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
	uu_list_destroy(zfs_cases);
	uu_list_pool_destroy(zfs_case_pool);
	libzfs_fini(zhdl);
	return;
	}

	fmd_hdl_setspecific(hdl, zhdl);

	(void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (zfs_stats) /
	sizeof (fmd_stat_t), (fmd_stat_t *)&zfs_stats);
	-
	- zfs_remove_timeout = fmd_prop_get_int64(hdl, "remove_timeout");
	}

	void
	_zfs_diagnosis_fini(fmd_hdl_t *hdl)
	{
	zfs_case_t *zcp;
	uu_list_walk_t *walk;
	libzfs_handle_t *zhdl;

	/*
	* Remove all active cases.
	*/
	walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
	while ((zcp = uu_list_walk_next(walk)) != NULL) {
	fmd_hdl_debug(hdl, "removing case ena %llu",
	(long long unsigned)zcp->zc_data.zc_ena);
	uu_list_remove(zfs_cases, zcp);
	uu_list_node_fini(zcp, &zcp->zc_node, zfs_case_pool);
	fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
	}
	uu_list_walk_end(walk);

	uu_list_destroy(zfs_cases);
	uu_list_pool_destroy(zfs_case_pool);

	zhdl = fmd_hdl_getspecific(hdl);
	libzfs_fini(zhdl);
	}
	diff --git a/sys/contrib/openzfs/cmd/zed/agents/zfs_retire.c b/sys/contrib/openzfs/cmd/zed/agents/zfs_retire.c
	index a0e377a4a0c8..1ef5c631a438 100644
	--- a/sys/contrib/openzfs/cmd/zed/agents/zfs_retire.c
	+++ b/sys/contrib/openzfs/cmd/zed/agents/zfs_retire.c
	@@ -1,668 +1,671 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
	*
	* Copyright (c) 2016, Intel Corporation.
	* Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>
	*/

	/*
	* The ZFS retire agent is responsible for managing hot spares across all pools.
	* When we see a device fault or a device removal, we try to open the associated
	* pool and look for any hot spares. We iterate over any available hot spares
	* and attempt a 'zpool replace' for each one.
	*
	* For vdevs diagnosed as faulty, the agent is also responsible for proactively
	* marking the vdev FAULTY (for I/O errors) or DEGRADED (for checksum errors).
	*/

	#include <sys/fs/zfs.h>
	#include <sys/fm/protocol.h>
	#include <sys/fm/fs/zfs.h>
	#include <libzutil.h>
	#include <libzfs.h>
	#include <string.h>
	#include <libgen.h>

	#include "zfs_agents.h"
	#include "fmd_api.h"


	typedef struct zfs_retire_repaired {
	struct zfs_retire_repaired *zrr_next;
	uint64_t zrr_pool;
	uint64_t zrr_vdev;
	} zfs_retire_repaired_t;

	typedef struct zfs_retire_data {
	libzfs_handle_t *zrd_hdl;
	zfs_retire_repaired_t *zrd_repaired;
	} zfs_retire_data_t;

	static void
	zfs_retire_clear_data(fmd_hdl_t hdl, zfs_retire_data_t zdp)
	{
	zfs_retire_repaired_t *zrp;

	while ((zrp = zdp->zrd_repaired) != NULL) {
	zdp->zrd_repaired = zrp->zrr_next;
	fmd_hdl_free(hdl, zrp, sizeof (zfs_retire_repaired_t));
	}
	}

	/*
	* Find a pool with a matching GUID.
	*/
	typedef struct find_cbdata {
	uint64_t cb_guid;
	zpool_handle_t *cb_zhp;
	nvlist_t *cb_vdev;
	uint64_t cb_vdev_guid;
	uint64_t cb_num_spares;
	} find_cbdata_t;

	static int
	find_pool(zpool_handle_t zhp, void data)
	{
	find_cbdata_t *cbp = data;

	if (cbp->cb_guid ==
	zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL)) {
	cbp->cb_zhp = zhp;
	return (1);
	}

	zpool_close(zhp);
	return (0);
	}

	/*
	* Find a vdev within a tree with a matching GUID.
	*/
	static nvlist_t *
	find_vdev(libzfs_handle_t zhdl, nvlist_t nv, uint64_t search_guid)
	{
	uint64_t guid;
	nvlist_t **child;
	uint_t c, children;
	nvlist_t *ret;

	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) == 0 &&
	guid == search_guid) {
	fmd_hdl_debug(fmd_module_hdl("zfs-retire"),
	"matched vdev %llu", guid);
	return (nv);
	}

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
	&child, &children) != 0)
	return (NULL);

	for (c = 0; c < children; c++) {
	if ((ret = find_vdev(zhdl, child[c], search_guid)) != NULL)
	return (ret);
	}

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
	&child, &children) != 0)
	return (NULL);

	for (c = 0; c < children; c++) {
	if ((ret = find_vdev(zhdl, child[c], search_guid)) != NULL)
	return (ret);
	}

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
	&child, &children) != 0)
	return (NULL);

	for (c = 0; c < children; c++) {
	if ((ret = find_vdev(zhdl, child[c], search_guid)) != NULL)
	return (ret);
	}

	return (NULL);
	}

	static int
	remove_spares(zpool_handle_t zhp, void data)
	{
	nvlist_t config, nvroot;
	nvlist_t **spares;
	uint_t nspares;
	char *devname;
	find_cbdata_t *cbp = data;
	uint64_t spareguid = 0;
	vdev_stat_t *vs;
	unsigned int c;

	config = zpool_get_config(zhp, NULL);
	if (nvlist_lookup_nvlist(config,
	ZPOOL_CONFIG_VDEV_TREE, &nvroot) != 0) {
	zpool_close(zhp);
	return (0);
	}

	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
	&spares, &nspares) != 0) {
	zpool_close(zhp);
	return (0);
	}

	for (int i = 0; i < nspares; i++) {
	if (nvlist_lookup_uint64(spares[i], ZPOOL_CONFIG_GUID,
	&spareguid) == 0 && spareguid == cbp->cb_vdev_guid) {
	devname = zpool_vdev_name(NULL, zhp, spares[i],
	B_FALSE);
	nvlist_lookup_uint64_array(spares[i],
	ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &c);
	if (vs->vs_state != VDEV_STATE_REMOVED &&
	zpool_vdev_remove_wanted(zhp, devname) == 0)
	cbp->cb_num_spares++;
	break;
	}
	}

	zpool_close(zhp);
	return (0);
	}

	/*
	* Given a vdev guid, find and remove all spares associated with it.
	*/
	static int
	find_and_remove_spares(libzfs_handle_t *zhdl, uint64_t vdev_guid)
	{
	find_cbdata_t cb;

	cb.cb_num_spares = 0;
	cb.cb_vdev_guid = vdev_guid;
	zpool_iter(zhdl, remove_spares, &cb);

	return (cb.cb_num_spares);
	}

	/*
	* Given a (pool, vdev) GUID pair, find the matching pool and vdev.
	*/
	static zpool_handle_t *
	find_by_guid(libzfs_handle_t *zhdl, uint64_t pool_guid, uint64_t vdev_guid,
	nvlist_t **vdevp)
	{
	find_cbdata_t cb;
	zpool_handle_t *zhp;
	nvlist_t config, nvroot;

	/*
	* Find the corresponding pool and make sure the vdev still exists.
	*/
	cb.cb_guid = pool_guid;
	if (zpool_iter(zhdl, find_pool, &cb) != 1)
	return (NULL);

	zhp = cb.cb_zhp;
	config = zpool_get_config(zhp, NULL);
	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
	&nvroot) != 0) {
	zpool_close(zhp);
	return (NULL);
	}

	if (vdev_guid != 0) {
	if ((*vdevp = find_vdev(zhdl, nvroot, vdev_guid)) == NULL) {
	zpool_close(zhp);
	return (NULL);
	}
	}

	return (zhp);
	}

	/*
	* Given a vdev, attempt to replace it with every known spare until one
	* succeeds or we run out of devices to try.
	* Return whether we were successful or not in replacing the device.
	*/
	static boolean_t
	replace_with_spare(fmd_hdl_t hdl, zpool_handle_t zhp, nvlist_t *vdev)
	{
	nvlist_t config, nvroot, *replacement;
	nvlist_t **spares;
	uint_t s, nspares;
	char *dev_name;
	zprop_source_t source;
	int ashift;

	config = zpool_get_config(zhp, NULL);
	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
	&nvroot) != 0)
	return (B_FALSE);

	/*
	* Find out if there are any hot spares available in the pool.
	*/
	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
	&spares, &nspares) != 0)
	return (B_FALSE);

	/*
	* lookup "ashift" pool property, we may need it for the replacement
	*/
	ashift = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &source);

	replacement = fmd_nvl_alloc(hdl, FMD_SLEEP);

	(void) nvlist_add_string(replacement, ZPOOL_CONFIG_TYPE,
	VDEV_TYPE_ROOT);

	dev_name = zpool_vdev_name(NULL, zhp, vdev, B_FALSE);

	/*
	* Try to replace each spare, ending when we successfully
	* replace it.
	*/
	for (s = 0; s < nspares; s++) {
	boolean_t rebuild = B_FALSE;
	const char spare_name, type;

	if (nvlist_lookup_string(spares[s], ZPOOL_CONFIG_PATH,
	&spare_name) != 0)
	continue;

	/* prefer sequential resilvering for distributed spares */
	if ((nvlist_lookup_string(spares[s], ZPOOL_CONFIG_TYPE,
	&type) == 0) && strcmp(type, VDEV_TYPE_DRAID_SPARE) == 0)
	rebuild = B_TRUE;

	/* if set, add the "ashift" pool property to the spare nvlist */
	if (source != ZPROP_SRC_DEFAULT)
	(void) nvlist_add_uint64(spares[s],
	ZPOOL_CONFIG_ASHIFT, ashift);

	(void) nvlist_add_nvlist_array(replacement,
	ZPOOL_CONFIG_CHILDREN, (const nvlist_t **)&spares[s], 1);

	fmd_hdl_debug(hdl, "zpool_vdev_replace '%s' with spare '%s'",
	dev_name, zfs_basename(spare_name));

	if (zpool_vdev_attach(zhp, dev_name, spare_name,
	replacement, B_TRUE, rebuild) == 0) {
	free(dev_name);
	nvlist_free(replacement);
	return (B_TRUE);
	}
	}

	free(dev_name);
	nvlist_free(replacement);

	return (B_FALSE);
	}

	/*
	* Repair this vdev if we had diagnosed a 'fault.fs.zfs.device' and
	* ASRU is now usable. ZFS has found the device to be present and
	* functioning.
	*/
	static void
	zfs_vdev_repair(fmd_hdl_t hdl, nvlist_t nvl)
	{
	zfs_retire_data_t *zdp = fmd_hdl_getspecific(hdl);
	zfs_retire_repaired_t *zrp;
	uint64_t pool_guid, vdev_guid;
	if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_POOL_GUID,
	&pool_guid) != 0 \|\| nvlist_lookup_uint64(nvl,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
	return;

	/*
	* Before checking the state of the ASRU, go through and see if we've
	* already made an attempt to repair this ASRU. This list is cleared
	* whenever we receive any kind of list event, and is designed to
	* prevent us from generating a feedback loop when we attempt repairs
	* against a faulted pool. The problem is that checking the unusable
	* state of the ASRU can involve opening the pool, which can post
	* statechange events but otherwise leave the pool in the faulted
	* state. This list allows us to detect when a statechange event is
	* due to our own request.
	*/
	for (zrp = zdp->zrd_repaired; zrp != NULL; zrp = zrp->zrr_next) {
	if (zrp->zrr_pool == pool_guid &&
	zrp->zrr_vdev == vdev_guid)
	return;
	}

	zrp = fmd_hdl_alloc(hdl, sizeof (zfs_retire_repaired_t), FMD_SLEEP);
	zrp->zrr_next = zdp->zrd_repaired;
	zrp->zrr_pool = pool_guid;
	zrp->zrr_vdev = vdev_guid;
	zdp->zrd_repaired = zrp;

	fmd_hdl_debug(hdl, "marking repaired vdev %llu on pool %llu",
	vdev_guid, pool_guid);
	}

	static void
	zfs_retire_recv(fmd_hdl_t hdl, fmd_event_t ep, nvlist_t *nvl,
	const char *class)
	{
	(void) ep;
	uint64_t pool_guid, vdev_guid;
	zpool_handle_t *zhp;
	nvlist_t resource, fault;
	nvlist_t **faults;
	uint_t f, nfaults;
	zfs_retire_data_t *zdp = fmd_hdl_getspecific(hdl);
	libzfs_handle_t *zhdl = zdp->zrd_hdl;
	boolean_t fault_device, degrade_device;
	boolean_t is_repair;
	boolean_t l2arc = B_FALSE;
	boolean_t spare = B_FALSE;
	const char *scheme;
	nvlist_t *vdev = NULL;
	const char *uuid;
	int repair_done = 0;
	boolean_t retire;
	boolean_t is_disk;
	vdev_aux_t aux;
	uint64_t state = 0;
	vdev_stat_t *vs;
	unsigned int c;

	fmd_hdl_debug(hdl, "zfs_retire_recv: '%s'", class);

	(void) nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE,
	&state);

	/*
	* If this is a resource notifying us of device removal then simply
	* check for an available spare and continue unless the device is a
	* l2arc vdev, in which case we just offline it.
	*/
	if (strcmp(class, "resource.fs.zfs.removed") == 0 \|\|
	(strcmp(class, "resource.fs.zfs.statechange") == 0 &&
	(state == VDEV_STATE_REMOVED \|\| state == VDEV_STATE_FAULTED))) {
	const char *devtype;
	char *devname;

	if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
	&devtype) == 0) {
	if (strcmp(devtype, VDEV_TYPE_SPARE) == 0)
	spare = B_TRUE;
	else if (strcmp(devtype, VDEV_TYPE_L2CACHE) == 0)
	l2arc = B_TRUE;
	}

	if (nvlist_lookup_uint64(nvl,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
	return;

	if (vdev_guid == 0) {
	fmd_hdl_debug(hdl, "Got a zero GUID");
	return;
	}

	if (spare) {
	int nspares = find_and_remove_spares(zhdl, vdev_guid);
	fmd_hdl_debug(hdl, "%d spares removed", nspares);
	return;
	}

	if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_POOL_GUID,
	&pool_guid) != 0)
	return;

	if ((zhp = find_by_guid(zhdl, pool_guid, vdev_guid,
	&vdev)) == NULL)
	return;

	devname = zpool_vdev_name(NULL, zhp, vdev, B_FALSE);

	nvlist_lookup_uint64_array(vdev, ZPOOL_CONFIG_VDEV_STATS,
	(uint64_t **)&vs, &c);

	/*
	* If state removed is requested for already removed vdev,
	* its a loopback event from spa_async_remove(). Just
	* ignore it.
	*/
	if (vs->vs_state == VDEV_STATE_REMOVED &&
	state == VDEV_STATE_REMOVED)
	return;

	/* Remove the vdev since device is unplugged */
	int remove_status = 0;
	if (l2arc \|\| (strcmp(class, "resource.fs.zfs.removed") == 0)) {
	remove_status = zpool_vdev_remove_wanted(zhp, devname);
	fmd_hdl_debug(hdl, "zpool_vdev_remove_wanted '%s'"
	", err:%d", devname, libzfs_errno(zhdl));
	}

	/* Replace the vdev with a spare if its not a l2arc */
	if (!l2arc && !remove_status &&
	(!fmd_prop_get_int32(hdl, "spare_on_remove") \|\|
	replace_with_spare(hdl, zhp, vdev) == B_FALSE)) {
	/* Could not handle with spare */
	fmd_hdl_debug(hdl, "no spare for '%s'", devname);
	}

	free(devname);
	zpool_close(zhp);
	return;
	}

	if (strcmp(class, FM_LIST_RESOLVED_CLASS) == 0)
	return;

	/*
	* Note: on Linux statechange events are more than just
	* healthy ones so we need to confirm the actual state value.
	*/
	if (strcmp(class, "resource.fs.zfs.statechange") == 0 &&
	state == VDEV_STATE_HEALTHY) {
	zfs_vdev_repair(hdl, nvl);
	return;
	}
	if (strcmp(class, "sysevent.fs.zfs.vdev_remove") == 0) {
	zfs_vdev_repair(hdl, nvl);
	return;
	}

	zfs_retire_clear_data(hdl, zdp);

	if (strcmp(class, FM_LIST_REPAIRED_CLASS) == 0)
	is_repair = B_TRUE;
	else
	is_repair = B_FALSE;

	/*
	* We subscribe to zfs faults as well as all repair events.
	*/
	if (nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
	&faults, &nfaults) != 0)
	return;

	for (f = 0; f < nfaults; f++) {
	fault = faults[f];

	fault_device = B_FALSE;
	degrade_device = B_FALSE;
	is_disk = B_FALSE;

	if (nvlist_lookup_boolean_value(fault, FM_SUSPECT_RETIRE,
	&retire) == 0 && retire == 0)
	continue;

	/*
	* While we subscribe to fault.fs.zfs.*, we only take action
	* for faults targeting a specific vdev (open failure or SERD
	* failure). We also subscribe to fault.io.* events, so that
	* faulty disks will be faulted in the ZFS configuration.
	*/
	if (fmd_nvl_class_match(hdl, fault, "fault.fs.zfs.vdev.io")) {
	fault_device = B_TRUE;
	} else if (fmd_nvl_class_match(hdl, fault,
	"fault.fs.zfs.vdev.checksum")) {
	degrade_device = B_TRUE;
	+ } else if (fmd_nvl_class_match(hdl, fault,
	+ "fault.fs.zfs.vdev.slow_io")) {
	+ degrade_device = B_TRUE;
	} else if (fmd_nvl_class_match(hdl, fault,
	"fault.fs.zfs.device")) {
	fault_device = B_FALSE;
	} else if (fmd_nvl_class_match(hdl, fault, "fault.io.*")) {
	is_disk = B_TRUE;
	fault_device = B_TRUE;
	} else {
	continue;
	}

	if (is_disk) {
	continue;
	} else {
	/*
	* This is a ZFS fault. Lookup the resource, and
	* attempt to find the matching vdev.
	*/
	if (nvlist_lookup_nvlist(fault, FM_FAULT_RESOURCE,
	&resource) != 0 \|\|
	nvlist_lookup_string(resource, FM_FMRI_SCHEME,
	&scheme) != 0)
	continue;

	if (strcmp(scheme, FM_FMRI_SCHEME_ZFS) != 0)
	continue;

	if (nvlist_lookup_uint64(resource, FM_FMRI_ZFS_POOL,
	&pool_guid) != 0)
	continue;

	if (nvlist_lookup_uint64(resource, FM_FMRI_ZFS_VDEV,
	&vdev_guid) != 0) {
	if (is_repair)
	vdev_guid = 0;
	else
	continue;
	}

	if ((zhp = find_by_guid(zhdl, pool_guid, vdev_guid,
	&vdev)) == NULL)
	continue;

	aux = VDEV_AUX_ERR_EXCEEDED;
	}

	if (vdev_guid == 0) {
	/*
	* For pool-level repair events, clear the entire pool.
	*/
	fmd_hdl_debug(hdl, "zpool_clear of pool '%s'",
	zpool_get_name(zhp));
	(void) zpool_clear(zhp, NULL, NULL);
	zpool_close(zhp);
	continue;
	}

	/*
	* If this is a repair event, then mark the vdev as repaired and
	* continue.
	*/
	if (is_repair) {
	repair_done = 1;
	fmd_hdl_debug(hdl, "zpool_clear of pool '%s' vdev %llu",
	zpool_get_name(zhp), vdev_guid);
	(void) zpool_vdev_clear(zhp, vdev_guid);
	zpool_close(zhp);
	continue;
	}

	/*
	* Actively fault the device if needed.
	*/
	if (fault_device)
	(void) zpool_vdev_fault(zhp, vdev_guid, aux);
	if (degrade_device)
	(void) zpool_vdev_degrade(zhp, vdev_guid, aux);

	if (fault_device \|\| degrade_device)
	fmd_hdl_debug(hdl, "zpool_vdev_%s: vdev %llu on '%s'",
	fault_device ? "fault" : "degrade", vdev_guid,
	zpool_get_name(zhp));

	/*
	* Attempt to substitute a hot spare.
	*/
	(void) replace_with_spare(hdl, zhp, vdev);

	zpool_close(zhp);
	}

	if (strcmp(class, FM_LIST_REPAIRED_CLASS) == 0 && repair_done &&
	nvlist_lookup_string(nvl, FM_SUSPECT_UUID, &uuid) == 0)
	fmd_case_uuresolved(hdl, uuid);
	}

	static const fmd_hdl_ops_t fmd_ops = {
	zfs_retire_recv, /* fmdo_recv */
	NULL, /* fmdo_timeout */
	NULL, /* fmdo_close */
	NULL, /* fmdo_stats */
	NULL, /* fmdo_gc */
	};

	static const fmd_prop_t fmd_props[] = {
	{ "spare_on_remove", FMD_TYPE_BOOL, "true" },
	{ NULL, 0, NULL }
	};

	static const fmd_hdl_info_t fmd_info = {
	"ZFS Retire Agent", "1.0", &fmd_ops, fmd_props
	};

	void
	_zfs_retire_init(fmd_hdl_t *hdl)
	{
	zfs_retire_data_t *zdp;
	libzfs_handle_t *zhdl;

	if ((zhdl = libzfs_init()) == NULL)
	return;

	if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
	libzfs_fini(zhdl);
	return;
	}

	zdp = fmd_hdl_zalloc(hdl, sizeof (zfs_retire_data_t), FMD_SLEEP);
	zdp->zrd_hdl = zhdl;

	fmd_hdl_setspecific(hdl, zdp);
	}

	void
	_zfs_retire_fini(fmd_hdl_t *hdl)
	{
	zfs_retire_data_t *zdp = fmd_hdl_getspecific(hdl);

	if (zdp != NULL) {
	zfs_retire_clear_data(hdl, zdp);
	libzfs_fini(zdp->zrd_hdl);
	fmd_hdl_free(hdl, zdp, sizeof (zfs_retire_data_t));
	}
	}
	diff --git a/sys/contrib/openzfs/cmd/zfs/zfs_main.c b/sys/contrib/openzfs/cmd/zfs/zfs_main.c
	index f7e07b8199d3..a5dbf3f5a2b1 100644
	--- a/sys/contrib/openzfs/cmd/zfs/zfs_main.c
	+++ b/sys/contrib/openzfs/cmd/zfs/zfs_main.c
	@@ -1,8997 +1,9044 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
	* Copyright 2012 Milan Jurik. All rights reserved.
	* Copyright (c) 2012, Joyent, Inc. All rights reserved.
	* Copyright (c) 2013 Steven Hartland. All rights reserved.
	* Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
	* Copyright 2016 Nexenta Systems, Inc.
	* Copyright (c) 2019 Datto Inc.
	* Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>
	* Copyright 2019 Joyent, Inc.
	* Copyright (c) 2019, 2020 by Christian Schwarz. All rights reserved.
	*/

	#include <assert.h>
	#include <ctype.h>
	#include <sys/debug.h>
	#include <errno.h>
	#include <getopt.h>
	#include <libgen.h>
	#include <libintl.h>
	#include <libuutil.h>
	#include <libnvpair.h>
	#include <locale.h>
	#include <stddef.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <unistd.h>
	#include <fcntl.h>
	#include <zone.h>
	#include <grp.h>
	#include <pwd.h>
	#include <umem.h>
	#include <pthread.h>
	#include <signal.h>
	#include <sys/list.h>
	#include <sys/mkdev.h>
	#include <sys/mntent.h>
	#include <sys/mnttab.h>
	#include <sys/mount.h>
	#include <sys/stat.h>
	#include <sys/fs/zfs.h>
	#include <sys/systeminfo.h>
	#include <sys/types.h>
	#include <time.h>
	#include <sys/zfs_project.h>

	#include <libzfs.h>
	#include <libzfs_core.h>
	#include <zfs_prop.h>
	#include <zfs_deleg.h>
	#include <libzutil.h>
	#ifdef HAVE_IDMAP
	#include <aclutils.h>
	#include <directory.h>
	#endif /* HAVE_IDMAP */

	#include "zfs_iter.h"
	#include "zfs_util.h"
	#include "zfs_comutil.h"
	#include "zfs_projectutil.h"

	libzfs_handle_t *g_zfs;

	static char history_str[HIS_MAX_RECORD_LEN];
	static boolean_t log_history = B_TRUE;

	static int zfs_do_clone(int argc, char **argv);
	static int zfs_do_create(int argc, char **argv);
	static int zfs_do_destroy(int argc, char **argv);
	static int zfs_do_get(int argc, char **argv);
	static int zfs_do_inherit(int argc, char **argv);
	static int zfs_do_list(int argc, char **argv);
	static int zfs_do_mount(int argc, char **argv);
	static int zfs_do_rename(int argc, char **argv);
	static int zfs_do_rollback(int argc, char **argv);
	static int zfs_do_set(int argc, char **argv);
	static int zfs_do_upgrade(int argc, char **argv);
	static int zfs_do_snapshot(int argc, char **argv);
	static int zfs_do_unmount(int argc, char **argv);
	static int zfs_do_share(int argc, char **argv);
	static int zfs_do_unshare(int argc, char **argv);
	static int zfs_do_send(int argc, char **argv);
	static int zfs_do_receive(int argc, char **argv);
	static int zfs_do_promote(int argc, char **argv);
	static int zfs_do_userspace(int argc, char **argv);
	static int zfs_do_allow(int argc, char **argv);
	static int zfs_do_unallow(int argc, char **argv);
	static int zfs_do_hold(int argc, char **argv);
	static int zfs_do_holds(int argc, char **argv);
	static int zfs_do_release(int argc, char **argv);
	static int zfs_do_diff(int argc, char **argv);
	static int zfs_do_bookmark(int argc, char **argv);
	static int zfs_do_channel_program(int argc, char **argv);
	static int zfs_do_load_key(int argc, char **argv);
	static int zfs_do_unload_key(int argc, char **argv);
	static int zfs_do_change_key(int argc, char **argv);
	static int zfs_do_project(int argc, char **argv);
	static int zfs_do_version(int argc, char **argv);
	static int zfs_do_redact(int argc, char **argv);
	static int zfs_do_wait(int argc, char **argv);

	#ifdef __FreeBSD__
	static int zfs_do_jail(int argc, char **argv);
	static int zfs_do_unjail(int argc, char **argv);
	#endif

	#ifdef __linux__
	static int zfs_do_zone(int argc, char **argv);
	static int zfs_do_unzone(int argc, char **argv);
	#endif

	static int zfs_do_help(int argc, char **argv);

	/*
	* Enable a reasonable set of defaults for libumem debugging on DEBUG builds.
	*/

	#ifdef DEBUG
	const char *
	_umem_debug_init(void)
	{
	return ("default,verbose"); /* $UMEM_DEBUG setting */
	}

	const char *
	_umem_logging_init(void)
	{
	return ("fail,contents"); /* $UMEM_LOGGING setting */
	}
	#endif

	typedef enum {
	HELP_CLONE,
	HELP_CREATE,
	HELP_DESTROY,
	HELP_GET,
	HELP_INHERIT,
	HELP_UPGRADE,
	HELP_LIST,
	HELP_MOUNT,
	HELP_PROMOTE,
	HELP_RECEIVE,
	HELP_RENAME,
	HELP_ROLLBACK,
	HELP_SEND,
	HELP_SET,
	HELP_SHARE,
	HELP_SNAPSHOT,
	HELP_UNMOUNT,
	HELP_UNSHARE,
	HELP_ALLOW,
	HELP_UNALLOW,
	HELP_USERSPACE,
	HELP_GROUPSPACE,
	HELP_PROJECTSPACE,
	HELP_PROJECT,
	HELP_HOLD,
	HELP_HOLDS,
	HELP_RELEASE,
	HELP_DIFF,
	HELP_BOOKMARK,
	HELP_CHANNEL_PROGRAM,
	HELP_LOAD_KEY,
	HELP_UNLOAD_KEY,
	HELP_CHANGE_KEY,
	HELP_VERSION,
	HELP_REDACT,
	HELP_JAIL,
	HELP_UNJAIL,
	HELP_WAIT,
	HELP_ZONE,
	HELP_UNZONE,
	} zfs_help_t;

	typedef struct zfs_command {
	const char *name;
	int (func)(int argc, char *argv);
	zfs_help_t usage;
	} zfs_command_t;

	/*
	* Master command table. Each ZFS command has a name, associated function, and
	* usage message. The usage messages need to be internationalized, so we have
	* to have a function to return the usage message based on a command index.
	*
	* These commands are organized according to how they are displayed in the usage
	* message. An empty command (one with a NULL name) indicates an empty line in
	* the generic usage message.
	*/
	static zfs_command_t command_table[] = {
	{ "version", zfs_do_version, HELP_VERSION },
	{ NULL },
	{ "create", zfs_do_create, HELP_CREATE },
	{ "destroy", zfs_do_destroy, HELP_DESTROY },
	{ NULL },
	{ "snapshot", zfs_do_snapshot, HELP_SNAPSHOT },
	{ "rollback", zfs_do_rollback, HELP_ROLLBACK },
	{ "clone", zfs_do_clone, HELP_CLONE },
	{ "promote", zfs_do_promote, HELP_PROMOTE },
	{ "rename", zfs_do_rename, HELP_RENAME },
	{ "bookmark", zfs_do_bookmark, HELP_BOOKMARK },
	{ "program", zfs_do_channel_program, HELP_CHANNEL_PROGRAM },
	{ NULL },
	{ "list", zfs_do_list, HELP_LIST },
	{ NULL },
	{ "set", zfs_do_set, HELP_SET },
	{ "get", zfs_do_get, HELP_GET },
	{ "inherit", zfs_do_inherit, HELP_INHERIT },
	{ "upgrade", zfs_do_upgrade, HELP_UPGRADE },
	{ NULL },
	{ "userspace", zfs_do_userspace, HELP_USERSPACE },
	{ "groupspace", zfs_do_userspace, HELP_GROUPSPACE },
	{ "projectspace", zfs_do_userspace, HELP_PROJECTSPACE },
	{ NULL },
	{ "project", zfs_do_project, HELP_PROJECT },
	{ NULL },
	{ "mount", zfs_do_mount, HELP_MOUNT },
	{ "unmount", zfs_do_unmount, HELP_UNMOUNT },
	{ "share", zfs_do_share, HELP_SHARE },
	{ "unshare", zfs_do_unshare, HELP_UNSHARE },
	{ NULL },
	{ "send", zfs_do_send, HELP_SEND },
	{ "receive", zfs_do_receive, HELP_RECEIVE },
	{ NULL },
	{ "allow", zfs_do_allow, HELP_ALLOW },
	{ NULL },
	{ "unallow", zfs_do_unallow, HELP_UNALLOW },
	{ NULL },
	{ "hold", zfs_do_hold, HELP_HOLD },
	{ "holds", zfs_do_holds, HELP_HOLDS },
	{ "release", zfs_do_release, HELP_RELEASE },
	{ "diff", zfs_do_diff, HELP_DIFF },
	{ "load-key", zfs_do_load_key, HELP_LOAD_KEY },
	{ "unload-key", zfs_do_unload_key, HELP_UNLOAD_KEY },
	{ "change-key", zfs_do_change_key, HELP_CHANGE_KEY },
	{ "redact", zfs_do_redact, HELP_REDACT },
	{ "wait", zfs_do_wait, HELP_WAIT },

	#ifdef __FreeBSD__
	{ "jail", zfs_do_jail, HELP_JAIL },
	{ "unjail", zfs_do_unjail, HELP_UNJAIL },
	#endif

	#ifdef __linux__
	{ "zone", zfs_do_zone, HELP_ZONE },
	{ "unzone", zfs_do_unzone, HELP_UNZONE },
	#endif
	};

	#define NCOMMAND (sizeof (command_table) / sizeof (command_table[0]))

	zfs_command_t *current_command;

	static const char *
	get_usage(zfs_help_t idx)
	{
	switch (idx) {
	case HELP_CLONE:
	return (gettext("\tclone [-p] [-o property=value] ... "
	"<snapshot> <filesystem\|volume>\n"));
	case HELP_CREATE:
	return (gettext("\tcreate [-Pnpuv] [-o property=value] ... "
	"<filesystem>\n"
	"\tcreate [-Pnpsv] [-b blocksize] [-o property=value] ... "
	"-V <size> <volume>\n"));
	case HELP_DESTROY:
	return (gettext("\tdestroy [-fnpRrv] <filesystem\|volume>\n"
	"\tdestroy [-dnpRrv] "
	"<filesystem\|volume>@<snap>[%<snap>][,...]\n"
	"\tdestroy <filesystem\|volume>#<bookmark>\n"));
	case HELP_GET:
	return (gettext("\tget [-rHp] [-d max] "
	"[-o \"all\" \| field[,...]]\n"
	"\t [-t type[,...]] [-s source[,...]]\n"
	"\t <\"all\" \| property[,...]> "
	"[filesystem\|volume\|snapshot\|bookmark] ...\n"));
	case HELP_INHERIT:
	return (gettext("\tinherit [-rS] <property> "
	"<filesystem\|volume\|snapshot> ...\n"));
	case HELP_UPGRADE:
	return (gettext("\tupgrade [-v]\n"
	"\tupgrade [-r] [-V version] <-a \| filesystem ...>\n"));
	case HELP_LIST:
	return (gettext("\tlist [-Hp] [-r\|-d max] [-o property[,...]] "
	"[-s property]...\n\t [-S property]... [-t type[,...]] "
	"[filesystem\|volume\|snapshot] ...\n"));
	case HELP_MOUNT:
	return (gettext("\tmount\n"
	- "\tmount [-flvO] [-o opts] <-a \| filesystem>\n"));
	+ "\tmount [-flvO] [-o opts] <-a\|-R filesystem\|"
	+ "filesystem>\n"));
	case HELP_PROMOTE:
	return (gettext("\tpromote <clone-filesystem>\n"));
	case HELP_RECEIVE:
	return (gettext("\treceive [-vMnsFhu] "
	"[-o <property>=<value>] ... [-x <property>] ...\n"
	"\t <filesystem\|volume\|snapshot>\n"
	"\treceive [-vMnsFhu] [-o <property>=<value>] ... "
	"[-x <property>] ... \n"
	"\t [-d \| -e] <filesystem>\n"
	"\treceive -A <filesystem\|volume>\n"));
	case HELP_RENAME:
	return (gettext("\trename [-f] <filesystem\|volume\|snapshot> "
	"<filesystem\|volume\|snapshot>\n"
	"\trename -p [-f] <filesystem\|volume> <filesystem\|volume>\n"
	"\trename -u [-f] <filesystem> <filesystem>\n"
	"\trename -r <snapshot> <snapshot>\n"));
	case HELP_ROLLBACK:
	return (gettext("\trollback [-rRf] <snapshot>\n"));
	case HELP_SEND:
	return (gettext("\tsend [-DLPbcehnpsVvw] "
	"[-i\|-I snapshot]\n"
	"\t [-R [-X dataset[,dataset]...]] <snapshot>\n"
	"\tsend [-DnVvPLecw] [-i snapshot\|bookmark] "
	"<filesystem\|volume\|snapshot>\n"
	"\tsend [-DnPpVvLec] [-i bookmark\|snapshot] "
	"--redact <bookmark> <snapshot>\n"
	"\tsend [-nVvPe] -t <receive_resume_token>\n"
	"\tsend [-PnVv] --saved filesystem\n"));
	case HELP_SET:
	return (gettext("\tset [-u] <property=value> ... "
	"<filesystem\|volume\|snapshot> ...\n"));
	case HELP_SHARE:
	return (gettext("\tshare [-l] <-a [nfs\|smb] \| filesystem>\n"));
	case HELP_SNAPSHOT:
	return (gettext("\tsnapshot [-r] [-o property=value] ... "
	"<filesystem\|volume>@<snap> ...\n"));
	case HELP_UNMOUNT:
	return (gettext("\tunmount [-fu] "
	"<-a \| filesystem\|mountpoint>\n"));
	case HELP_UNSHARE:
	return (gettext("\tunshare "
	"<-a [nfs\|smb] \| filesystem\|mountpoint>\n"));
	case HELP_ALLOW:
	return (gettext("\tallow <filesystem\|volume>\n"
	"\tallow [-ldug] "
	"<\"everyone\"\|user\|group>[,...] <perm\|@setname>[,...]\n"
	"\t <filesystem\|volume>\n"
	"\tallow [-ld] -e <perm\|@setname>[,...] "
	"<filesystem\|volume>\n"
	"\tallow -c <perm\|@setname>[,...] <filesystem\|volume>\n"
	"\tallow -s @setname <perm\|@setname>[,...] "
	"<filesystem\|volume>\n"));
	case HELP_UNALLOW:
	return (gettext("\tunallow [-rldug] "
	"<\"everyone\"\|user\|group>[,...]\n"
	"\t [<perm\|@setname>[,...]] <filesystem\|volume>\n"
	"\tunallow [-rld] -e [<perm\|@setname>[,...]] "
	"<filesystem\|volume>\n"
	"\tunallow [-r] -c [<perm\|@setname>[,...]] "
	"<filesystem\|volume>\n"
	"\tunallow [-r] -s @setname [<perm\|@setname>[,...]] "
	"<filesystem\|volume>\n"));
	case HELP_USERSPACE:
	return (gettext("\tuserspace [-Hinp] [-o field[,...]] "
	"[-s field] ...\n"
	"\t [-S field] ... [-t type[,...]] "
	"<filesystem\|snapshot\|path>\n"));
	case HELP_GROUPSPACE:
	return (gettext("\tgroupspace [-Hinp] [-o field[,...]] "
	"[-s field] ...\n"
	"\t [-S field] ... [-t type[,...]] "
	"<filesystem\|snapshot\|path>\n"));
	case HELP_PROJECTSPACE:
	return (gettext("\tprojectspace [-Hp] [-o field[,...]] "
	"[-s field] ... \n"
	"\t [-S field] ... <filesystem\|snapshot\|path>\n"));
	case HELP_PROJECT:
	return (gettext("\tproject [-d\|-r] <directory\|file ...>\n"
	"\tproject -c [-0] [-d\|-r] [-p id] <directory\|file ...>\n"
	"\tproject -C [-k] [-r] <directory ...>\n"
	"\tproject [-p id] [-r] [-s] <directory ...>\n"));
	case HELP_HOLD:
	return (gettext("\thold [-r] <tag> <snapshot> ...\n"));
	case HELP_HOLDS:
	return (gettext("\tholds [-rHp] <snapshot> ...\n"));
	case HELP_RELEASE:
	return (gettext("\trelease [-r] <tag> <snapshot> ...\n"));
	case HELP_DIFF:
	return (gettext("\tdiff [-FHth] <snapshot> "
	"[snapshot\|filesystem]\n"));
	case HELP_BOOKMARK:
	return (gettext("\tbookmark <snapshot\|bookmark> "
	"<newbookmark>\n"));
	case HELP_CHANNEL_PROGRAM:
	return (gettext("\tprogram [-jn] [-t <instruction limit>] "
	"[-m <memory limit (b)>]\n"
	"\t <pool> <program file> [lua args...]\n"));
	case HELP_LOAD_KEY:
	return (gettext("\tload-key [-rn] [-L <keylocation>] "
	"<-a \| filesystem\|volume>\n"));
	case HELP_UNLOAD_KEY:
	return (gettext("\tunload-key [-r] "
	"<-a \| filesystem\|volume>\n"));
	case HELP_CHANGE_KEY:
	return (gettext("\tchange-key [-l] [-o keyformat=<value>]\n"
	"\t [-o keylocation=<value>] [-o pbkdf2iters=<value>]\n"
	"\t <filesystem\|volume>\n"
	"\tchange-key -i [-l] <filesystem\|volume>\n"));
	case HELP_VERSION:
	return (gettext("\tversion\n"));
	case HELP_REDACT:
	return (gettext("\tredact <snapshot> <bookmark> "
	"<redaction_snapshot> ...\n"));
	case HELP_JAIL:
	return (gettext("\tjail <jailid\|jailname> <filesystem>\n"));
	case HELP_UNJAIL:
	return (gettext("\tunjail <jailid\|jailname> <filesystem>\n"));
	case HELP_WAIT:
	return (gettext("\twait [-t <activity>] <filesystem>\n"));
	case HELP_ZONE:
	return (gettext("\tzone <nsfile> <filesystem>\n"));
	case HELP_UNZONE:
	return (gettext("\tunzone <nsfile> <filesystem>\n"));
	default:
	__builtin_unreachable();
	}
	}

	void
	nomem(void)
	{
	(void) fprintf(stderr, gettext("internal error: out of memory\n"));
	exit(1);
	}

	/*
	* Utility function to guarantee malloc() success.
	*/

	void *
	safe_malloc(size_t size)
	{
	void *data;

	if ((data = calloc(1, size)) == NULL)
	nomem();

	return (data);
	}

	static void *
	safe_realloc(void *data, size_t size)
	{
	void *newp;
	if ((newp = realloc(data, size)) == NULL) {
	free(data);
	nomem();
	}

	return (newp);
	}

	static char *
	safe_strdup(const char *str)
	{
	char *dupstr = strdup(str);

	if (dupstr == NULL)
	nomem();

	return (dupstr);
	}

	/*
	* Callback routine that will print out information for each of
	* the properties.
	*/
	static int
	usage_prop_cb(int prop, void *cb)
	{
	FILE *fp = cb;

	(void) fprintf(fp, "\t%-15s ", zfs_prop_to_name(prop));

	if (zfs_prop_readonly(prop))
	(void) fprintf(fp, " NO ");
	else
	(void) fprintf(fp, "YES ");

	if (zfs_prop_inheritable(prop))
	(void) fprintf(fp, " YES ");
	else
	(void) fprintf(fp, " NO ");

	(void) fprintf(fp, "%s\n", zfs_prop_values(prop) ?: "-");

	return (ZPROP_CONT);
	}

	/*
	* Display usage message. If we're inside a command, display only the usage for
	* that command. Otherwise, iterate over the entire command table and display
	* a complete usage message.
	*/
	static __attribute__((noreturn)) void
	usage(boolean_t requested)
	{
	int i;
	boolean_t show_properties = B_FALSE;
	FILE *fp = requested ? stdout : stderr;

	if (current_command == NULL) {

	(void) fprintf(fp, gettext("usage: zfs command args ...\n"));
	(void) fprintf(fp,
	gettext("where 'command' is one of the following:\n\n"));

	for (i = 0; i < NCOMMAND; i++) {
	if (command_table[i].name == NULL)
	(void) fprintf(fp, "\n");
	else
	(void) fprintf(fp, "%s",
	get_usage(command_table[i].usage));
	}

	(void) fprintf(fp, gettext("\nEach dataset is of the form: "
	"pool/[dataset/]*dataset[@name]\n"));
	} else {
	(void) fprintf(fp, gettext("usage:\n"));
	(void) fprintf(fp, "%s", get_usage(current_command->usage));
	}

	if (current_command != NULL &&
	(strcmp(current_command->name, "set") == 0 \|\|
	strcmp(current_command->name, "get") == 0 \|\|
	strcmp(current_command->name, "inherit") == 0 \|\|
	strcmp(current_command->name, "list") == 0))
	show_properties = B_TRUE;

	if (show_properties) {
	(void) fprintf(fp, "%s",
	gettext("\nThe following properties are supported:\n"));

	(void) fprintf(fp, "\n\t%-14s %s %s %s\n\n",
	"PROPERTY", "EDIT", "INHERIT", "VALUES");

	/* Iterate over all properties */
	(void) zprop_iter(usage_prop_cb, fp, B_FALSE, B_TRUE,
	ZFS_TYPE_DATASET);

	(void) fprintf(fp, "\t%-15s ", "userused@...");
	(void) fprintf(fp, " NO NO <size>\n");
	(void) fprintf(fp, "\t%-15s ", "groupused@...");
	(void) fprintf(fp, " NO NO <size>\n");
	(void) fprintf(fp, "\t%-15s ", "projectused@...");
	(void) fprintf(fp, " NO NO <size>\n");
	(void) fprintf(fp, "\t%-15s ", "userobjused@...");
	(void) fprintf(fp, " NO NO <size>\n");
	(void) fprintf(fp, "\t%-15s ", "groupobjused@...");
	(void) fprintf(fp, " NO NO <size>\n");
	(void) fprintf(fp, "\t%-15s ", "projectobjused@...");
	(void) fprintf(fp, " NO NO <size>\n");
	(void) fprintf(fp, "\t%-15s ", "userquota@...");
	(void) fprintf(fp, "YES NO <size> \| none\n");
	(void) fprintf(fp, "\t%-15s ", "groupquota@...");
	(void) fprintf(fp, "YES NO <size> \| none\n");
	(void) fprintf(fp, "\t%-15s ", "projectquota@...");
	(void) fprintf(fp, "YES NO <size> \| none\n");
	(void) fprintf(fp, "\t%-15s ", "userobjquota@...");
	(void) fprintf(fp, "YES NO <size> \| none\n");
	(void) fprintf(fp, "\t%-15s ", "groupobjquota@...");
	(void) fprintf(fp, "YES NO <size> \| none\n");
	(void) fprintf(fp, "\t%-15s ", "projectobjquota@...");
	(void) fprintf(fp, "YES NO <size> \| none\n");
	(void) fprintf(fp, "\t%-15s ", "written@<snap>");
	(void) fprintf(fp, " NO NO <size>\n");
	(void) fprintf(fp, "\t%-15s ", "written#<bookmark>");
	(void) fprintf(fp, " NO NO <size>\n");

	(void) fprintf(fp, gettext("\nSizes are specified in bytes "
	"with standard units such as K, M, G, etc.\n"));
	(void) fprintf(fp, "%s", gettext("\nUser-defined properties "
	"can be specified by using a name containing a colon "
	"(:).\n"));
	(void) fprintf(fp, gettext("\nThe {user\|group\|project}"
	"[obj]{used\|quota}@ properties must be appended with\n"
	"a user\|group\|project specifier of one of these forms:\n"
	" POSIX name (eg: \"matt\")\n"
	" POSIX id (eg: \"126829\")\n"
	" SMB name@domain (eg: \"matt@sun\")\n"
	" SMB SID (eg: \"S-1-234-567-89\")\n"));
	} else {
	(void) fprintf(fp,
	gettext("\nFor the property list, run: %s\n"),
	"zfs set\|get");
	(void) fprintf(fp,
	gettext("\nFor the delegated permission list, run: %s\n"),
	"zfs allow\|unallow");
	(void) fprintf(fp,
	gettext("\nFor further help on a command or topic, "
	"run: %s\n"), "zfs help [<topic>]");
	}

	/*
	* See comments at end of main().
	*/
	if (getenv("ZFS_ABORT") != NULL) {
	(void) printf("dumping core by request\n");
	abort();
	}

	exit(requested ? 0 : 2);
	}

	/*
	* Take a property=value argument string and add it to the given nvlist.
	* Modifies the argument inplace.
	*/
	static boolean_t
	parseprop(nvlist_t props, char propname)
	{
	char *propval;

	if ((propval = strchr(propname, '=')) == NULL) {
	(void) fprintf(stderr, gettext("missing "
	"'=' for property=value argument\n"));
	return (B_FALSE);
	}
	*propval = '\0';
	propval++;
	if (nvlist_exists(props, propname)) {
	(void) fprintf(stderr, gettext("property '%s' "
	"specified multiple times\n"), propname);
	return (B_FALSE);
	}
	if (nvlist_add_string(props, propname, propval) != 0)
	nomem();
	return (B_TRUE);
	}

	/*
	* Take a property name argument and add it to the given nvlist.
	* Modifies the argument inplace.
	*/
	static boolean_t
	parsepropname(nvlist_t props, char propname)
	{
	if (strchr(propname, '=') != NULL) {
	(void) fprintf(stderr, gettext("invalid character "
	"'=' in property argument\n"));
	return (B_FALSE);
	}
	if (nvlist_exists(props, propname)) {
	(void) fprintf(stderr, gettext("property '%s' "
	"specified multiple times\n"), propname);
	return (B_FALSE);
	}
	if (nvlist_add_boolean(props, propname) != 0)
	nomem();
	return (B_TRUE);
	}

	static int
	parse_depth(char opt, int flags)
	{
	char *tmp;
	int depth;

	depth = (int)strtol(opt, &tmp, 0);
	if (*tmp) {
	(void) fprintf(stderr,
	gettext("%s is not an integer\n"), optarg);
	usage(B_FALSE);
	}
	if (depth < 0) {
	(void) fprintf(stderr,
	gettext("Depth can not be negative.\n"));
	usage(B_FALSE);
	}
	*flags \|= (ZFS_ITER_DEPTH_LIMIT\|ZFS_ITER_RECURSE);
	return (depth);
	}

	#define PROGRESS_DELAY 2 /* seconds */

	static const char *pt_reverse =
	"\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b";
	static time_t pt_begin;
	static char *pt_header = NULL;
	static boolean_t pt_shown;

	static void
	start_progress_timer(void)
	{
	pt_begin = time(NULL) + PROGRESS_DELAY;
	pt_shown = B_FALSE;
	}

	static void
	set_progress_header(const char *header)
	{
	assert(pt_header == NULL);
	pt_header = safe_strdup(header);
	if (pt_shown) {
	(void) printf("%s: ", header);
	(void) fflush(stdout);
	}
	}

	static void
	update_progress(const char *update)
	{
	if (!pt_shown && time(NULL) > pt_begin) {
	int len = strlen(update);

	(void) printf("%s: %s%.s", pt_header, update, len, len,
	pt_reverse);
	(void) fflush(stdout);
	pt_shown = B_TRUE;
	} else if (pt_shown) {
	int len = strlen(update);

	(void) printf("%s%.s", update, len, len, pt_reverse);
	(void) fflush(stdout);
	}
	}

	static void
	finish_progress(const char *done)
	{
	if (pt_shown) {
	(void) puts(done);
	(void) fflush(stdout);
	}
	free(pt_header);
	pt_header = NULL;
	}

	static int
	zfs_mount_and_share(libzfs_handle_t hdl, const char dataset, zfs_type_t type)
	{
	zfs_handle_t *zhp = NULL;
	int ret = 0;

	zhp = zfs_open(hdl, dataset, type);
	if (zhp == NULL)
	return (1);

	/*
	* Volumes may neither be mounted or shared. Potentially in the
	* future filesystems detected on these volumes could be mounted.
	*/
	if (zfs_get_type(zhp) == ZFS_TYPE_VOLUME) {
	zfs_close(zhp);
	return (0);
	}

	/*
	* Mount and/or share the new filesystem as appropriate. We provide a
	* verbose error message to let the user know that their filesystem was
	* in fact created, even if we failed to mount or share it.
	*
	* If the user doesn't want the dataset automatically mounted, then
	* skip the mount/share step
	*/
	if (zfs_prop_valid_for_type(ZFS_PROP_CANMOUNT, type, B_FALSE) &&
	zfs_prop_get_int(zhp, ZFS_PROP_CANMOUNT) == ZFS_CANMOUNT_ON) {
	if (zfs_mount_delegation_check()) {
	(void) fprintf(stderr, gettext("filesystem "
	"successfully created, but it may only be "
	"mounted by root\n"));
	ret = 1;
	} else if (zfs_mount(zhp, NULL, 0) != 0) {
	(void) fprintf(stderr, gettext("filesystem "
	"successfully created, but not mounted\n"));
	ret = 1;
	} else if (zfs_share(zhp, NULL) != 0) {
	(void) fprintf(stderr, gettext("filesystem "
	"successfully created, but not shared\n"));
	ret = 1;
	}
	zfs_commit_shares(NULL);
	}

	zfs_close(zhp);

	return (ret);
	}

	/*
	* zfs clone [-p] [-o prop=value] ... <snap> <fs \| vol>
	*
	* Given an existing dataset, create a writable copy whose initial contents
	* are the same as the source. The newly created dataset maintains a
	* dependency on the original; the original cannot be destroyed so long as
	* the clone exists.
	*
	* The '-p' flag creates all the non-existing ancestors of the target first.
	*/
	static int
	zfs_do_clone(int argc, char **argv)
	{
	zfs_handle_t *zhp = NULL;
	boolean_t parents = B_FALSE;
	nvlist_t *props;
	int ret = 0;
	int c;

	if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0)
	nomem();

	/* check options */
	while ((c = getopt(argc, argv, "o:p")) != -1) {
	switch (c) {
	case 'o':
	if (!parseprop(props, optarg)) {
	nvlist_free(props);
	return (1);
	}
	break;
	case 'p':
	parents = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	goto usage;
	}
	}

	argc -= optind;
	argv += optind;

	/* check number of arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing source dataset "
	"argument\n"));
	goto usage;
	}
	if (argc < 2) {
	(void) fprintf(stderr, gettext("missing target dataset "
	"argument\n"));
	goto usage;
	}
	if (argc > 2) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	goto usage;
	}

	/* open the source dataset */
	if ((zhp = zfs_open(g_zfs, argv[0], ZFS_TYPE_SNAPSHOT)) == NULL) {
	nvlist_free(props);
	return (1);
	}

	if (parents && zfs_name_valid(argv[1], ZFS_TYPE_FILESYSTEM \|
	ZFS_TYPE_VOLUME)) {
	/*
	* Now create the ancestors of the target dataset. If the
	* target already exists and '-p' option was used we should not
	* complain.
	*/
	if (zfs_dataset_exists(g_zfs, argv[1], ZFS_TYPE_FILESYSTEM \|
	ZFS_TYPE_VOLUME)) {
	zfs_close(zhp);
	nvlist_free(props);
	return (0);
	}
	if (zfs_create_ancestors(g_zfs, argv[1]) != 0) {
	zfs_close(zhp);
	nvlist_free(props);
	return (1);
	}
	}

	/* pass to libzfs */
	ret = zfs_clone(zhp, argv[1], props);

	/* create the mountpoint if necessary */
	if (ret == 0) {
	if (log_history) {
	(void) zpool_log_history(g_zfs, history_str);
	log_history = B_FALSE;
	}

	ret = zfs_mount_and_share(g_zfs, argv[1], ZFS_TYPE_DATASET);
	}

	zfs_close(zhp);
	nvlist_free(props);

	return (!!ret);

	usage:
	ASSERT3P(zhp, ==, NULL);
	nvlist_free(props);
	usage(B_FALSE);
	return (-1);
	}

	/*
	* Return a default volblocksize for the pool which always uses more than
	* half of the data sectors. This primarily applies to dRAID which always
	* writes full stripe widths.
	*/
	static uint64_t
	default_volblocksize(zpool_handle_t zhp, nvlist_t props)
	{
	uint64_t volblocksize, asize = SPA_MINBLOCKSIZE;
	nvlist_t tree, *vdevs;
	uint_t nvdevs;

	nvlist_t *config = zpool_get_config(zhp, NULL);

	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &tree) != 0 \|\|
	nvlist_lookup_nvlist_array(tree, ZPOOL_CONFIG_CHILDREN,
	&vdevs, &nvdevs) != 0) {
	return (ZVOL_DEFAULT_BLOCKSIZE);
	}

	for (int i = 0; i < nvdevs; i++) {
	nvlist_t *nv = vdevs[i];
	uint64_t ashift, ndata, nparity;

	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_ASHIFT, &ashift) != 0)
	continue;

	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_DRAID_NDATA,
	&ndata) == 0) {
	/* dRAID minimum allocation width */
	asize = MAX(asize, ndata * (1ULL << ashift));
	} else if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NPARITY,
	&nparity) == 0) {
	/* raidz minimum allocation width */
	if (nparity == 1)
	asize = MAX(asize, 2 * (1ULL << ashift));
	else
	asize = MAX(asize, 4 * (1ULL << ashift));
	} else {
	/* mirror or (non-redundant) leaf vdev */
	asize = MAX(asize, 1ULL << ashift);
	}
	}

	/*
	* Calculate the target volblocksize such that more than half
	* of the asize is used. The following table is for 4k sectors.
	*
	* n asize blksz used \| n asize blksz used
	* -------------------------+---------------------------------
	* 1 4,096 8,192 100% \| 9 36,864 32,768 88%
	* 2 8,192 8,192 100% \| 10 40,960 32,768 80%
	* 3 12,288 8,192 66% \| 11 45,056 32,768 72%
	* 4 16,384 16,384 100% \| 12 49,152 32,768 66%
	* 5 20,480 16,384 80% \| 13 53,248 32,768 61%
	* 6 24,576 16,384 66% \| 14 57,344 32,768 57%
	* 7 28,672 16,384 57% \| 15 61,440 32,768 53%
	* 8 32,768 32,768 100% \| 16 65,536 65,636 100%
	*
	* This is primarily a concern for dRAID which always allocates
	* a full stripe width. For dRAID the default stripe width is
	* n=8 in which case the volblocksize is set to 32k. Ignoring
	* compression there are no unused sectors. This same reasoning
	* applies to raidz[2,3] so target 4 sectors to minimize waste.
	*/
	uint64_t tgt_volblocksize = ZVOL_DEFAULT_BLOCKSIZE;
	while (tgt_volblocksize * 2 <= asize)
	tgt_volblocksize *= 2;

	const char *prop = zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE);
	if (nvlist_lookup_uint64(props, prop, &volblocksize) == 0) {

	/* Issue a warning when a non-optimal size is requested. */
	if (volblocksize < ZVOL_DEFAULT_BLOCKSIZE) {
	(void) fprintf(stderr, gettext("Warning: "
	"volblocksize (%llu) is less than the default "
	"minimum block size (%llu).\nTo reduce wasted "
	"space a volblocksize of %llu is recommended.\n"),
	(u_longlong_t)volblocksize,
	(u_longlong_t)ZVOL_DEFAULT_BLOCKSIZE,
	(u_longlong_t)tgt_volblocksize);
	} else if (volblocksize < tgt_volblocksize) {
	(void) fprintf(stderr, gettext("Warning: "
	"volblocksize (%llu) is much less than the "
	"minimum allocation\nunit (%llu), which wastes "
	"at least %llu%% of space. To reduce wasted "
	"space,\nuse a larger volblocksize (%llu is "
	"recommended), fewer dRAID data disks\n"
	"per group, or smaller sector size (ashift).\n"),
	(u_longlong_t)volblocksize, (u_longlong_t)asize,
	(u_longlong_t)((100 * (asize - volblocksize)) /
	asize), (u_longlong_t)tgt_volblocksize);
	}
	} else {
	volblocksize = tgt_volblocksize;
	fnvlist_add_uint64(props, prop, volblocksize);
	}

	return (volblocksize);
	}

	/*
	* zfs create [-Pnpv] [-o prop=value] ... fs
	* zfs create [-Pnpsv] [-b blocksize] [-o prop=value] ... -V vol size
	*
	* Create a new dataset. This command can be used to create filesystems
	* and volumes. Snapshot creation is handled by 'zfs snapshot'.
	* For volumes, the user must specify a size to be used.
	*
	* The '-s' flag applies only to volumes, and indicates that we should not try
	* to set the reservation for this volume. By default we set a reservation
	* equal to the size for any volume. For pools with SPA_VERSION >=
	* SPA_VERSION_REFRESERVATION, we set a refreservation instead.
	*
	* The '-p' flag creates all the non-existing ancestors of the target first.
	*
	* The '-n' flag is no-op (dry run) mode. This will perform a user-space sanity
	* check of arguments and properties, but does not check for permissions,
	* available space, etc.
	*
	* The '-u' flag prevents the newly created file system from being mounted.
	*
	* The '-v' flag is for verbose output.
	*
	* The '-P' flag is used for parseable output. It implies '-v'.
	*/
	static int
	zfs_do_create(int argc, char **argv)
	{
	zfs_type_t type = ZFS_TYPE_FILESYSTEM;
	zpool_handle_t *zpool_handle = NULL;
	nvlist_t *real_props = NULL;
	uint64_t volsize = 0;
	int c;
	boolean_t noreserve = B_FALSE;
	boolean_t bflag = B_FALSE;
	boolean_t parents = B_FALSE;
	boolean_t dryrun = B_FALSE;
	boolean_t nomount = B_FALSE;
	boolean_t verbose = B_FALSE;
	boolean_t parseable = B_FALSE;
	int ret = 1;
	nvlist_t *props;
	uint64_t intval;
	const char *strval;

	if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0)
	nomem();

	/* check options */
	while ((c = getopt(argc, argv, ":PV:b:nso:puv")) != -1) {
	switch (c) {
	case 'V':
	type = ZFS_TYPE_VOLUME;
	if (zfs_nicestrtonum(g_zfs, optarg, &intval) != 0) {
	(void) fprintf(stderr, gettext("bad volume "
	"size '%s': %s\n"), optarg,
	libzfs_error_description(g_zfs));
	goto error;
	}

	if (nvlist_add_uint64(props,
	zfs_prop_to_name(ZFS_PROP_VOLSIZE), intval) != 0)
	nomem();
	volsize = intval;
	break;
	case 'P':
	verbose = B_TRUE;
	parseable = B_TRUE;
	break;
	case 'p':
	parents = B_TRUE;
	break;
	case 'b':
	bflag = B_TRUE;
	if (zfs_nicestrtonum(g_zfs, optarg, &intval) != 0) {
	(void) fprintf(stderr, gettext("bad volume "
	"block size '%s': %s\n"), optarg,
	libzfs_error_description(g_zfs));
	goto error;
	}

	if (nvlist_add_uint64(props,
	zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE),
	intval) != 0)
	nomem();
	break;
	case 'n':
	dryrun = B_TRUE;
	break;
	case 'o':
	if (!parseprop(props, optarg))
	goto error;
	break;
	case 's':
	noreserve = B_TRUE;
	break;
	case 'u':
	nomount = B_TRUE;
	break;
	case 'v':
	verbose = B_TRUE;
	break;
	case ':':
	(void) fprintf(stderr, gettext("missing size "
	"argument\n"));
	goto badusage;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	goto badusage;
	}
	}

	if ((bflag \|\| noreserve) && type != ZFS_TYPE_VOLUME) {
	(void) fprintf(stderr, gettext("'-s' and '-b' can only be "
	"used when creating a volume\n"));
	goto badusage;
	}
	if (nomount && type != ZFS_TYPE_FILESYSTEM) {
	(void) fprintf(stderr, gettext("'-u' can only be "
	"used when creating a filesystem\n"));
	goto badusage;
	}

	argc -= optind;
	argv += optind;

	/* check number of arguments */
	if (argc == 0) {
	(void) fprintf(stderr, gettext("missing %s argument\n"),
	zfs_type_to_name(type));
	goto badusage;
	}
	if (argc > 1) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	goto badusage;
	}

	if (dryrun \|\| type == ZFS_TYPE_VOLUME) {
	char msg[ZFS_MAX_DATASET_NAME_LEN * 2];
	char *p;

	if ((p = strchr(argv[0], '/')) != NULL)
	*p = '\0';
	zpool_handle = zpool_open(g_zfs, argv[0]);
	if (p != NULL)
	*p = '/';
	if (zpool_handle == NULL)
	goto error;

	(void) snprintf(msg, sizeof (msg),
	dryrun ? gettext("cannot verify '%s'") :
	gettext("cannot create '%s'"), argv[0]);
	if (props && (real_props = zfs_valid_proplist(g_zfs, type,
	props, 0, NULL, zpool_handle, B_TRUE, msg)) == NULL) {
	zpool_close(zpool_handle);
	goto error;
	}
	}

	if (type == ZFS_TYPE_VOLUME) {
	const char *prop = zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE);
	uint64_t volblocksize = default_volblocksize(zpool_handle,
	real_props);

	if (volblocksize != ZVOL_DEFAULT_BLOCKSIZE &&
	nvlist_lookup_string(props, prop, &strval) != 0) {
	char *tmp;
	if (asprintf(&tmp, "%llu",
	(u_longlong_t)volblocksize) == -1)
	nomem();
	nvlist_add_string(props, prop, tmp);
	free(tmp);
	}

	/*
	* If volsize is not a multiple of volblocksize, round it
	* up to the nearest multiple of the volblocksize.
	*/
	if (volsize % volblocksize) {
	volsize = P2ROUNDUP_TYPED(volsize, volblocksize,
	uint64_t);

	if (nvlist_add_uint64(props,
	zfs_prop_to_name(ZFS_PROP_VOLSIZE), volsize) != 0) {
	nvlist_free(props);
	nomem();
	}
	}
	}

	if (type == ZFS_TYPE_VOLUME && !noreserve) {
	uint64_t spa_version;
	zfs_prop_t resv_prop;

	spa_version = zpool_get_prop_int(zpool_handle,
	ZPOOL_PROP_VERSION, NULL);
	if (spa_version >= SPA_VERSION_REFRESERVATION)
	resv_prop = ZFS_PROP_REFRESERVATION;
	else
	resv_prop = ZFS_PROP_RESERVATION;

	volsize = zvol_volsize_to_reservation(zpool_handle, volsize,
	real_props);

	if (nvlist_lookup_string(props, zfs_prop_to_name(resv_prop),
	&strval) != 0) {
	if (nvlist_add_uint64(props,
	zfs_prop_to_name(resv_prop), volsize) != 0) {
	nvlist_free(props);
	nomem();
	}
	}
	}
	if (zpool_handle != NULL) {
	zpool_close(zpool_handle);
	nvlist_free(real_props);
	}

	if (parents && zfs_name_valid(argv[0], type)) {
	/*
	* Now create the ancestors of target dataset. If the target
	* already exists and '-p' option was used we should not
	* complain.
	*/
	if (zfs_dataset_exists(g_zfs, argv[0], type)) {
	ret = 0;
	goto error;
	}
	if (verbose) {
	(void) printf(parseable ? "create_ancestors\t%s\n" :
	dryrun ? "would create ancestors of %s\n" :
	"create ancestors of %s\n", argv[0]);
	}
	if (!dryrun) {
	if (zfs_create_ancestors(g_zfs, argv[0]) != 0) {
	goto error;
	}
	}
	}

	if (verbose) {
	nvpair_t *nvp = NULL;
	(void) printf(parseable ? "create\t%s\n" :
	dryrun ? "would create %s\n" : "create %s\n", argv[0]);
	while ((nvp = nvlist_next_nvpair(props, nvp)) != NULL) {
	uint64_t uval;
	const char *sval;

	switch (nvpair_type(nvp)) {
	case DATA_TYPE_UINT64:
	VERIFY0(nvpair_value_uint64(nvp, &uval));
	(void) printf(parseable ?
	"property\t%s\t%llu\n" : "\t%s=%llu\n",
	nvpair_name(nvp), (u_longlong_t)uval);
	break;
	case DATA_TYPE_STRING:
	VERIFY0(nvpair_value_string(nvp, &sval));
	(void) printf(parseable ?
	"property\t%s\t%s\n" : "\t%s=%s\n",
	nvpair_name(nvp), sval);
	break;
	default:
	(void) fprintf(stderr, "property '%s' "
	"has illegal type %d\n",
	nvpair_name(nvp), nvpair_type(nvp));
	abort();
	}
	}
	}
	if (dryrun) {
	ret = 0;
	goto error;
	}

	/* pass to libzfs */
	if (zfs_create(g_zfs, argv[0], type, props) != 0)
	goto error;

	if (log_history) {
	(void) zpool_log_history(g_zfs, history_str);
	log_history = B_FALSE;
	}

	if (nomount) {
	ret = 0;
	goto error;
	}

	ret = zfs_mount_and_share(g_zfs, argv[0], ZFS_TYPE_DATASET);
	error:
	nvlist_free(props);
	return (ret);
	badusage:
	nvlist_free(props);
	usage(B_FALSE);
	return (2);
	}

	/*
	* zfs destroy [-rRf] <fs, vol>
	* zfs destroy [-rRd] <snap>
	*
	* -r Recursively destroy all children
	* -R Recursively destroy all dependents, including clones
	* -f Force unmounting of any dependents
	* -d If we can't destroy now, mark for deferred destruction
	*
	* Destroys the given dataset. By default, it will unmount any filesystems,
	* and refuse to destroy a dataset that has any dependents. A dependent can
	* either be a child, or a clone of a child.
	*/
	typedef struct destroy_cbdata {
	boolean_t cb_first;
	boolean_t cb_force;
	boolean_t cb_recurse;
	boolean_t cb_error;
	boolean_t cb_doclones;
	zfs_handle_t *cb_target;
	boolean_t cb_defer_destroy;
	boolean_t cb_verbose;
	boolean_t cb_parsable;
	boolean_t cb_dryrun;
	nvlist_t *cb_nvl;
	nvlist_t *cb_batchedsnaps;

	/* first snap in contiguous run */
	char *cb_firstsnap;
	/* previous snap in contiguous run */
	char *cb_prevsnap;
	int64_t cb_snapused;
	char *cb_snapspec;
	char *cb_bookmark;
	uint64_t cb_snap_count;
	} destroy_cbdata_t;

	/*
	* Check for any dependents based on the '-r' or '-R' flags.
	*/
	static int
	destroy_check_dependent(zfs_handle_t zhp, void data)
	{
	destroy_cbdata_t *cbp = data;
	const char *tname = zfs_get_name(cbp->cb_target);
	const char *name = zfs_get_name(zhp);

	if (strncmp(tname, name, strlen(tname)) == 0 &&
	(name[strlen(tname)] == '/' \|\| name[strlen(tname)] == '@')) {
	/*
	* This is a direct descendant, not a clone somewhere else in
	* the hierarchy.
	*/
	if (cbp->cb_recurse)
	goto out;

	if (cbp->cb_first) {
	(void) fprintf(stderr, gettext("cannot destroy '%s': "
	"%s has children\n"),
	zfs_get_name(cbp->cb_target),
	zfs_type_to_name(zfs_get_type(cbp->cb_target)));
	(void) fprintf(stderr, gettext("use '-r' to destroy "
	"the following datasets:\n"));
	cbp->cb_first = B_FALSE;
	cbp->cb_error = B_TRUE;
	}

	(void) fprintf(stderr, "%s\n", zfs_get_name(zhp));
	} else {
	/*
	* This is a clone. We only want to report this if the '-r'
	* wasn't specified, or the target is a snapshot.
	*/
	if (!cbp->cb_recurse &&
	zfs_get_type(cbp->cb_target) != ZFS_TYPE_SNAPSHOT)
	goto out;

	if (cbp->cb_first) {
	(void) fprintf(stderr, gettext("cannot destroy '%s': "
	"%s has dependent clones\n"),
	zfs_get_name(cbp->cb_target),
	zfs_type_to_name(zfs_get_type(cbp->cb_target)));
	(void) fprintf(stderr, gettext("use '-R' to destroy "
	"the following datasets:\n"));
	cbp->cb_first = B_FALSE;
	cbp->cb_error = B_TRUE;
	cbp->cb_dryrun = B_TRUE;
	}

	(void) fprintf(stderr, "%s\n", zfs_get_name(zhp));
	}

	out:
	zfs_close(zhp);
	return (0);
	}

	static int
	destroy_batched(destroy_cbdata_t *cb)
	{
	int error = zfs_destroy_snaps_nvl(g_zfs,
	cb->cb_batchedsnaps, B_FALSE);
	fnvlist_free(cb->cb_batchedsnaps);
	cb->cb_batchedsnaps = fnvlist_alloc();
	return (error);
	}

	static int
	destroy_callback(zfs_handle_t zhp, void data)
	{
	destroy_cbdata_t *cb = data;
	const char *name = zfs_get_name(zhp);
	int error;

	if (cb->cb_verbose) {
	if (cb->cb_parsable) {
	(void) printf("destroy\t%s\n", name);
	} else if (cb->cb_dryrun) {
	(void) printf(gettext("would destroy %s\n"),
	name);
	} else {
	(void) printf(gettext("will destroy %s\n"),
	name);
	}
	}

	/*
	* Ignore pools (which we've already flagged as an error before getting
	* here).
	*/
	if (strchr(zfs_get_name(zhp), '/') == NULL &&
	zfs_get_type(zhp) == ZFS_TYPE_FILESYSTEM) {
	zfs_close(zhp);
	return (0);
	}
	if (cb->cb_dryrun) {
	zfs_close(zhp);
	return (0);
	}

	/*
	* We batch up all contiguous snapshots (even of different
	* filesystems) and destroy them with one ioctl. We can't
	* simply do all snap deletions and then all fs deletions,
	* because we must delete a clone before its origin.
	*/
	if (zfs_get_type(zhp) == ZFS_TYPE_SNAPSHOT) {
	cb->cb_snap_count++;
	fnvlist_add_boolean(cb->cb_batchedsnaps, name);
	if (cb->cb_snap_count % 10 == 0 && cb->cb_defer_destroy) {
	error = destroy_batched(cb);
	if (error != 0) {
	zfs_close(zhp);
	return (-1);
	}
	}
	} else {
	error = destroy_batched(cb);
	if (error != 0 \|\|
	zfs_unmount(zhp, NULL, cb->cb_force ? MS_FORCE : 0) != 0 \|\|
	zfs_destroy(zhp, cb->cb_defer_destroy) != 0) {
	zfs_close(zhp);
	/*
	* When performing a recursive destroy we ignore errors
	* so that the recursive destroy could continue
	* destroying past problem datasets
	*/
	if (cb->cb_recurse) {
	cb->cb_error = B_TRUE;
	return (0);
	}
	return (-1);
	}
	}

	zfs_close(zhp);
	return (0);
	}

	static int
	destroy_print_cb(zfs_handle_t zhp, void arg)
	{
	destroy_cbdata_t *cb = arg;
	const char *name = zfs_get_name(zhp);
	int err = 0;

	if (nvlist_exists(cb->cb_nvl, name)) {
	if (cb->cb_firstsnap == NULL)
	cb->cb_firstsnap = strdup(name);
	if (cb->cb_prevsnap != NULL)
	free(cb->cb_prevsnap);
	/* this snap continues the current range */
	cb->cb_prevsnap = strdup(name);
	if (cb->cb_firstsnap == NULL \|\| cb->cb_prevsnap == NULL)
	nomem();
	if (cb->cb_verbose) {
	if (cb->cb_parsable) {
	(void) printf("destroy\t%s\n", name);
	} else if (cb->cb_dryrun) {
	(void) printf(gettext("would destroy %s\n"),
	name);
	} else {
	(void) printf(gettext("will destroy %s\n"),
	name);
	}
	}
	} else if (cb->cb_firstsnap != NULL) {
	/* end of this range */
	uint64_t used = 0;
	err = lzc_snaprange_space(cb->cb_firstsnap,
	cb->cb_prevsnap, &used);
	cb->cb_snapused += used;
	free(cb->cb_firstsnap);
	cb->cb_firstsnap = NULL;
	free(cb->cb_prevsnap);
	cb->cb_prevsnap = NULL;
	}
	zfs_close(zhp);
	return (err);
	}

	static int
	destroy_print_snapshots(zfs_handle_t fs_zhp, destroy_cbdata_t cb)
	{
	int err;
	assert(cb->cb_firstsnap == NULL);
	assert(cb->cb_prevsnap == NULL);
	err = zfs_iter_snapshots_sorted_v2(fs_zhp, 0, destroy_print_cb, cb, 0,
	0);
	if (cb->cb_firstsnap != NULL) {
	uint64_t used = 0;
	if (err == 0) {
	err = lzc_snaprange_space(cb->cb_firstsnap,
	cb->cb_prevsnap, &used);
	}
	cb->cb_snapused += used;
	free(cb->cb_firstsnap);
	cb->cb_firstsnap = NULL;
	free(cb->cb_prevsnap);
	cb->cb_prevsnap = NULL;
	}
	return (err);
	}

	static int
	snapshot_to_nvl_cb(zfs_handle_t zhp, void arg)
	{
	destroy_cbdata_t *cb = arg;
	int err = 0;

	/* Check for clones. */
	if (!cb->cb_doclones && !cb->cb_defer_destroy) {
	cb->cb_target = zhp;
	cb->cb_first = B_TRUE;
	err = zfs_iter_dependents_v2(zhp, 0, B_TRUE,
	destroy_check_dependent, cb);
	}

	if (err == 0) {
	if (nvlist_add_boolean(cb->cb_nvl, zfs_get_name(zhp)))
	nomem();
	}
	zfs_close(zhp);
	return (err);
	}

	static int
	gather_snapshots(zfs_handle_t zhp, void arg)
	{
	destroy_cbdata_t *cb = arg;
	int err = 0;

	err = zfs_iter_snapspec_v2(zhp, 0, cb->cb_snapspec,
	snapshot_to_nvl_cb, cb);
	if (err == ENOENT)
	err = 0;
	if (err != 0)
	goto out;

	if (cb->cb_verbose) {
	err = destroy_print_snapshots(zhp, cb);
	if (err != 0)
	goto out;
	}

	if (cb->cb_recurse)
	err = zfs_iter_filesystems_v2(zhp, 0, gather_snapshots, cb);

	out:
	zfs_close(zhp);
	return (err);
	}

	static int
	destroy_clones(destroy_cbdata_t *cb)
	{
	nvpair_t *pair;
	for (pair = nvlist_next_nvpair(cb->cb_nvl, NULL);
	pair != NULL;
	pair = nvlist_next_nvpair(cb->cb_nvl, pair)) {
	zfs_handle_t *zhp = zfs_open(g_zfs, nvpair_name(pair),
	ZFS_TYPE_SNAPSHOT);
	if (zhp != NULL) {
	boolean_t defer = cb->cb_defer_destroy;
	int err;

	/*
	* We can't defer destroy non-snapshots, so set it to
	* false while destroying the clones.
	*/
	cb->cb_defer_destroy = B_FALSE;
	err = zfs_iter_dependents_v2(zhp, 0, B_FALSE,
	destroy_callback, cb);
	cb->cb_defer_destroy = defer;
	zfs_close(zhp);
	if (err != 0)
	return (err);
	}
	}
	return (0);
	}

	static int
	zfs_do_destroy(int argc, char **argv)
	{
	destroy_cbdata_t cb = { 0 };
	int rv = 0;
	int err = 0;
	int c;
	zfs_handle_t *zhp = NULL;
	char at, pound;
	zfs_type_t type = ZFS_TYPE_DATASET;

	/* check options */
	while ((c = getopt(argc, argv, "vpndfrR")) != -1) {
	switch (c) {
	case 'v':
	cb.cb_verbose = B_TRUE;
	break;
	case 'p':
	cb.cb_verbose = B_TRUE;
	cb.cb_parsable = B_TRUE;
	break;
	case 'n':
	cb.cb_dryrun = B_TRUE;
	break;
	case 'd':
	cb.cb_defer_destroy = B_TRUE;
	type = ZFS_TYPE_SNAPSHOT;
	break;
	case 'f':
	cb.cb_force = B_TRUE;
	break;
	case 'r':
	cb.cb_recurse = B_TRUE;
	break;
	case 'R':
	cb.cb_recurse = B_TRUE;
	cb.cb_doclones = B_TRUE;
	break;
	case '?':
	default:
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* check number of arguments */
	if (argc == 0) {
	(void) fprintf(stderr, gettext("missing dataset argument\n"));
	usage(B_FALSE);
	}
	if (argc > 1) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	at = strchr(argv[0], '@');
	pound = strchr(argv[0], '#');
	if (at != NULL) {

	/* Build the list of snaps to destroy in cb_nvl. */
	cb.cb_nvl = fnvlist_alloc();

	*at = '\0';
	zhp = zfs_open(g_zfs, argv[0],
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME);
	if (zhp == NULL) {
	nvlist_free(cb.cb_nvl);
	return (1);
	}

	cb.cb_snapspec = at + 1;
	if (gather_snapshots(zfs_handle_dup(zhp), &cb) != 0 \|\|
	cb.cb_error) {
	rv = 1;
	goto out;
	}

	if (nvlist_empty(cb.cb_nvl)) {
	(void) fprintf(stderr, gettext("could not find any "
	"snapshots to destroy; check snapshot names.\n"));
	rv = 1;
	goto out;
	}

	if (cb.cb_verbose) {
	char buf[16];
	zfs_nicebytes(cb.cb_snapused, buf, sizeof (buf));
	if (cb.cb_parsable) {
	(void) printf("reclaim\t%llu\n",
	(u_longlong_t)cb.cb_snapused);
	} else if (cb.cb_dryrun) {
	(void) printf(gettext("would reclaim %s\n"),
	buf);
	} else {
	(void) printf(gettext("will reclaim %s\n"),
	buf);
	}
	}

	if (!cb.cb_dryrun) {
	if (cb.cb_doclones) {
	cb.cb_batchedsnaps = fnvlist_alloc();
	err = destroy_clones(&cb);
	if (err == 0) {
	err = zfs_destroy_snaps_nvl(g_zfs,
	cb.cb_batchedsnaps, B_FALSE);
	}
	if (err != 0) {
	rv = 1;
	goto out;
	}
	}
	if (err == 0) {
	err = zfs_destroy_snaps_nvl(g_zfs, cb.cb_nvl,
	cb.cb_defer_destroy);
	}
	}

	if (err != 0)
	rv = 1;
	} else if (pound != NULL) {
	int err;
	nvlist_t *nvl;

	if (cb.cb_dryrun) {
	(void) fprintf(stderr,
	"dryrun is not supported with bookmark\n");
	return (-1);
	}

	if (cb.cb_defer_destroy) {
	(void) fprintf(stderr,
	"defer destroy is not supported with bookmark\n");
	return (-1);
	}

	if (cb.cb_recurse) {
	(void) fprintf(stderr,
	"recursive is not supported with bookmark\n");
	return (-1);
	}

	/*
	* Unfortunately, zfs_bookmark() doesn't honor the
	* casesensitivity setting. However, we can't simply
	* remove this check, because lzc_destroy_bookmarks()
	* ignores non-existent bookmarks, so this is necessary
	* to get a proper error message.
	*/
	if (!zfs_bookmark_exists(argv[0])) {
	(void) fprintf(stderr, gettext("bookmark '%s' "
	"does not exist.\n"), argv[0]);
	return (1);
	}

	nvl = fnvlist_alloc();
	fnvlist_add_boolean(nvl, argv[0]);

	err = lzc_destroy_bookmarks(nvl, NULL);
	if (err != 0) {
	(void) zfs_standard_error(g_zfs, err,
	"cannot destroy bookmark");
	}

	nvlist_free(nvl);

	return (err);
	} else {
	/* Open the given dataset */
	if ((zhp = zfs_open(g_zfs, argv[0], type)) == NULL)
	return (1);

	cb.cb_target = zhp;

	/*
	* Perform an explicit check for pools before going any further.
	*/
	if (!cb.cb_recurse && strchr(zfs_get_name(zhp), '/') == NULL &&
	zfs_get_type(zhp) == ZFS_TYPE_FILESYSTEM) {
	(void) fprintf(stderr, gettext("cannot destroy '%s': "
	"operation does not apply to pools\n"),
	zfs_get_name(zhp));
	(void) fprintf(stderr, gettext("use 'zfs destroy -r "
	"%s' to destroy all datasets in the pool\n"),
	zfs_get_name(zhp));
	(void) fprintf(stderr, gettext("use 'zpool destroy %s' "
	"to destroy the pool itself\n"), zfs_get_name(zhp));
	rv = 1;
	goto out;
	}

	/*
	* Check for any dependents and/or clones.
	*/
	cb.cb_first = B_TRUE;
	if (!cb.cb_doclones && zfs_iter_dependents_v2(zhp, 0, B_TRUE,
	destroy_check_dependent, &cb) != 0) {
	rv = 1;
	goto out;
	}

	if (cb.cb_error) {
	rv = 1;
	goto out;
	}
	cb.cb_batchedsnaps = fnvlist_alloc();
	if (zfs_iter_dependents_v2(zhp, 0, B_FALSE, destroy_callback,
	&cb) != 0) {
	rv = 1;
	goto out;
	}

	/*
	* Do the real thing. The callback will close the
	* handle regardless of whether it succeeds or not.
	*/
	err = destroy_callback(zhp, &cb);
	zhp = NULL;
	if (err == 0) {
	err = zfs_destroy_snaps_nvl(g_zfs,
	cb.cb_batchedsnaps, cb.cb_defer_destroy);
	}
	if (err != 0 \|\| cb.cb_error == B_TRUE)
	rv = 1;
	}

	out:
	fnvlist_free(cb.cb_batchedsnaps);
	fnvlist_free(cb.cb_nvl);
	if (zhp != NULL)
	zfs_close(zhp);
	return (rv);
	}

	static boolean_t
	is_recvd_column(zprop_get_cbdata_t *cbp)
	{
	int i;
	zfs_get_column_t col;

	for (i = 0; i < ZFS_GET_NCOLS &&
	(col = cbp->cb_columns[i]) != GET_COL_NONE; i++)
	if (col == GET_COL_RECVD)
	return (B_TRUE);
	return (B_FALSE);
	}

	/*
	* zfs get [-rHp] [-o all \| field[,field]...] [-s source[,source]...]
	* < all \| property[,property]... > < fs \| snap \| vol > ...
	*
	* -r recurse over any child datasets
	* -H scripted mode. Headers are stripped, and fields are separated
	* by tabs instead of spaces.
	* -o Set of fields to display. One of "name,property,value,
	* received,source". Default is "name,property,value,source".
	* "all" is an alias for all five.
	* -s Set of sources to allow. One of
	* "local,default,inherited,received,temporary,none". Default is
	* all six.
	* -p Display values in parsable (literal) format.
	*
	* Prints properties for the given datasets. The user can control which
	* columns to display as well as which property types to allow.
	*/

	/*
	* Invoked to display the properties for a single dataset.
	*/
	static int
	get_callback(zfs_handle_t zhp, void data)
	{
	char buf[ZFS_MAXPROPLEN];
	char rbuf[ZFS_MAXPROPLEN];
	zprop_source_t sourcetype;
	char source[ZFS_MAX_DATASET_NAME_LEN];
	zprop_get_cbdata_t *cbp = data;
	nvlist_t *user_props = zfs_get_user_props(zhp);
	zprop_list_t *pl = cbp->cb_proplist;
	nvlist_t *propval;
	const char *strval;
	const char *sourceval;
	boolean_t received = is_recvd_column(cbp);

	for (; pl != NULL; pl = pl->pl_next) {
	char *recvdval = NULL;
	/*
	* Skip the special fake placeholder. This will also skip over
	* the name property when 'all' is specified.
	*/
	if (pl->pl_prop == ZFS_PROP_NAME &&
	pl == cbp->cb_proplist)
	continue;

	if (pl->pl_prop != ZPROP_USERPROP) {
	if (zfs_prop_get(zhp, pl->pl_prop, buf,
	sizeof (buf), &sourcetype, source,
	sizeof (source),
	cbp->cb_literal) != 0) {
	if (pl->pl_all)
	continue;
	if (!zfs_prop_valid_for_type(pl->pl_prop,
	ZFS_TYPE_DATASET, B_FALSE)) {
	(void) fprintf(stderr,
	gettext("No such property '%s'\n"),
	zfs_prop_to_name(pl->pl_prop));
	continue;
	}
	sourcetype = ZPROP_SRC_NONE;
	(void) strlcpy(buf, "-", sizeof (buf));
	}

	if (received && (zfs_prop_get_recvd(zhp,
	zfs_prop_to_name(pl->pl_prop), rbuf, sizeof (rbuf),
	cbp->cb_literal) == 0))
	recvdval = rbuf;

	zprop_print_one_property(zfs_get_name(zhp), cbp,
	zfs_prop_to_name(pl->pl_prop),
	buf, sourcetype, source, recvdval);
	} else if (zfs_prop_userquota(pl->pl_user_prop)) {
	sourcetype = ZPROP_SRC_LOCAL;

	if (zfs_prop_get_userquota(zhp, pl->pl_user_prop,
	buf, sizeof (buf), cbp->cb_literal) != 0) {
	sourcetype = ZPROP_SRC_NONE;
	(void) strlcpy(buf, "-", sizeof (buf));
	}

	zprop_print_one_property(zfs_get_name(zhp), cbp,
	pl->pl_user_prop, buf, sourcetype, source, NULL);
	} else if (zfs_prop_written(pl->pl_user_prop)) {
	sourcetype = ZPROP_SRC_LOCAL;

	if (zfs_prop_get_written(zhp, pl->pl_user_prop,
	buf, sizeof (buf), cbp->cb_literal) != 0) {
	sourcetype = ZPROP_SRC_NONE;
	(void) strlcpy(buf, "-", sizeof (buf));
	}

	zprop_print_one_property(zfs_get_name(zhp), cbp,
	pl->pl_user_prop, buf, sourcetype, source, NULL);
	} else {
	if (nvlist_lookup_nvlist(user_props,
	pl->pl_user_prop, &propval) != 0) {
	if (pl->pl_all)
	continue;
	sourcetype = ZPROP_SRC_NONE;
	strval = "-";
	} else {
	strval = fnvlist_lookup_string(propval,
	ZPROP_VALUE);
	sourceval = fnvlist_lookup_string(propval,
	ZPROP_SOURCE);

	if (strcmp(sourceval,
	zfs_get_name(zhp)) == 0) {
	sourcetype = ZPROP_SRC_LOCAL;
	} else if (strcmp(sourceval,
	ZPROP_SOURCE_VAL_RECVD) == 0) {
	sourcetype = ZPROP_SRC_RECEIVED;
	} else {
	sourcetype = ZPROP_SRC_INHERITED;
	(void) strlcpy(source,
	sourceval, sizeof (source));
	}
	}

	if (received && (zfs_prop_get_recvd(zhp,
	pl->pl_user_prop, rbuf, sizeof (rbuf),
	cbp->cb_literal) == 0))
	recvdval = rbuf;

	zprop_print_one_property(zfs_get_name(zhp), cbp,
	pl->pl_user_prop, strval, sourcetype,
	source, recvdval);
	}
	}

	return (0);
	}

	static int
	zfs_do_get(int argc, char **argv)
	{
	zprop_get_cbdata_t cb = { 0 };
	int i, c, flags = ZFS_ITER_ARGS_CAN_BE_PATHS;
	int types = ZFS_TYPE_DATASET \| ZFS_TYPE_BOOKMARK;
	char *fields;
	int ret = 0;
	int limit = 0;
	zprop_list_t fake_name = { 0 };

	/*
	* Set up default columns and sources.
	*/
	cb.cb_sources = ZPROP_SRC_ALL;
	cb.cb_columns[0] = GET_COL_NAME;
	cb.cb_columns[1] = GET_COL_PROPERTY;
	cb.cb_columns[2] = GET_COL_VALUE;
	cb.cb_columns[3] = GET_COL_SOURCE;
	cb.cb_type = ZFS_TYPE_DATASET;

	/* check options */
	while ((c = getopt(argc, argv, ":d:o:s:rt:Hp")) != -1) {
	switch (c) {
	case 'p':
	cb.cb_literal = B_TRUE;
	break;
	case 'd':
	limit = parse_depth(optarg, &flags);
	break;
	case 'r':
	flags \|= ZFS_ITER_RECURSE;
	break;
	case 'H':
	cb.cb_scripted = B_TRUE;
	break;
	case ':':
	(void) fprintf(stderr, gettext("missing argument for "
	"'%c' option\n"), optopt);
	usage(B_FALSE);
	break;
	case 'o':
	/*
	* Process the set of columns to display. We zero out
	* the structure to give us a blank slate.
	*/
	memset(&cb.cb_columns, 0, sizeof (cb.cb_columns));

	i = 0;
	for (char *tok; (tok = strsep(&optarg, ",")); ) {
	static const char *const col_subopts[] =
	{ "name", "property", "value",
	"received", "source", "all" };
	static const zfs_get_column_t col_subopt_col[] =
	{ GET_COL_NAME, GET_COL_PROPERTY, GET_COL_VALUE,
	GET_COL_RECVD, GET_COL_SOURCE };
	static const int col_subopt_flags[] =
	{ 0, 0, 0, ZFS_ITER_RECVD_PROPS, 0 };

	if (i == ZFS_GET_NCOLS) {
	(void) fprintf(stderr, gettext("too "
	"many fields given to -o "
	"option\n"));
	usage(B_FALSE);
	}

	for (c = 0; c < ARRAY_SIZE(col_subopts); ++c)
	if (strcmp(tok, col_subopts[c]) == 0)
	goto found;

	(void) fprintf(stderr,
	gettext("invalid column name '%s'\n"), tok);
	usage(B_FALSE);

	found:
	if (c >= 5) {
	if (i > 0) {
	(void) fprintf(stderr,
	gettext("\"all\" conflicts "
	"with specific fields "
	"given to -o option\n"));
	usage(B_FALSE);
	}

	memcpy(cb.cb_columns, col_subopt_col,
	sizeof (col_subopt_col));
	flags \|= ZFS_ITER_RECVD_PROPS;
	i = ZFS_GET_NCOLS;
	} else {
	cb.cb_columns[i++] = col_subopt_col[c];
	flags \|= col_subopt_flags[c];
	}
	}
	break;

	case 's':
	cb.cb_sources = 0;

	for (char *tok; (tok = strsep(&optarg, ",")); ) {
	static const char *const source_opt[] = {
	"local", "default",
	"inherited", "received",
	"temporary", "none" };
	static const int source_flg[] = {
	ZPROP_SRC_LOCAL, ZPROP_SRC_DEFAULT,
	ZPROP_SRC_INHERITED, ZPROP_SRC_RECEIVED,
	ZPROP_SRC_TEMPORARY, ZPROP_SRC_NONE };

	for (i = 0; i < ARRAY_SIZE(source_opt); ++i)
	if (strcmp(tok, source_opt[i]) == 0) {
	cb.cb_sources \|= source_flg[i];
	goto found2;
	}

	(void) fprintf(stderr,
	gettext("invalid source '%s'\n"), tok);
	usage(B_FALSE);
	found2:;
	}
	break;

	case 't':
	types = 0;
	flags &= ~ZFS_ITER_PROP_LISTSNAPS;

	for (char *tok; (tok = strsep(&optarg, ",")); ) {
	static const char *const type_opts[] = {
	"filesystem", "volume",
	"snapshot", "snap",
	"bookmark",
	"all" };
	static const int type_types[] = {
	ZFS_TYPE_FILESYSTEM, ZFS_TYPE_VOLUME,
	ZFS_TYPE_SNAPSHOT, ZFS_TYPE_SNAPSHOT,
	ZFS_TYPE_BOOKMARK,
	ZFS_TYPE_DATASET \| ZFS_TYPE_BOOKMARK };

	for (i = 0; i < ARRAY_SIZE(type_opts); ++i)
	if (strcmp(tok, type_opts[i]) == 0) {
	types \|= type_types[i];
	goto found3;
	}

	(void) fprintf(stderr,
	gettext("invalid type '%s'\n"), tok);
	usage(B_FALSE);
	found3:;
	}
	break;

	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing property "
	"argument\n"));
	usage(B_FALSE);
	}

	fields = argv[0];

	/*
	* Handle users who want to get all snapshots or bookmarks
	* of a dataset (ex. 'zfs get -t snapshot refer <dataset>').
	*/
	if ((types == ZFS_TYPE_SNAPSHOT \|\| types == ZFS_TYPE_BOOKMARK) &&
	argc > 1 && (flags & ZFS_ITER_RECURSE) == 0 && limit == 0) {
	flags \|= (ZFS_ITER_DEPTH_LIMIT \| ZFS_ITER_RECURSE);
	limit = 1;
	}

	if (zprop_get_list(g_zfs, fields, &cb.cb_proplist, ZFS_TYPE_DATASET)
	!= 0)
	usage(B_FALSE);

	argc--;
	argv++;

	/*
	* As part of zfs_expand_proplist(), we keep track of the maximum column
	* width for each property. For the 'NAME' (and 'SOURCE') columns, we
	* need to know the maximum name length. However, the user likely did
	* not specify 'name' as one of the properties to fetch, so we need to
	* make sure we always include at least this property for
	* print_get_headers() to work properly.
	*/
	if (cb.cb_proplist != NULL) {
	fake_name.pl_prop = ZFS_PROP_NAME;
	fake_name.pl_width = strlen(gettext("NAME"));
	fake_name.pl_next = cb.cb_proplist;
	cb.cb_proplist = &fake_name;
	}

	cb.cb_first = B_TRUE;

	/* run for each object */
	ret = zfs_for_each(argc, argv, flags, types, NULL,
	&cb.cb_proplist, limit, get_callback, &cb);

	if (cb.cb_proplist == &fake_name)
	zprop_free_list(fake_name.pl_next);
	else
	zprop_free_list(cb.cb_proplist);

	return (ret);
	}

	/*
	* inherit [-rS] <property> <fs\|vol> ...
	*
	* -r Recurse over all children
	* -S Revert to received value, if any
	*
	* For each dataset specified on the command line, inherit the given property
	* from its parent. Inheriting a property at the pool level will cause it to
	* use the default value. The '-r' flag will recurse over all children, and is
	* useful for setting a property on a hierarchy-wide basis, regardless of any
	* local modifications for each dataset.
	*/

	typedef struct inherit_cbdata {
	const char *cb_propname;
	boolean_t cb_received;
	} inherit_cbdata_t;

	static int
	inherit_recurse_cb(zfs_handle_t zhp, void data)
	{
	inherit_cbdata_t *cb = data;
	zfs_prop_t prop = zfs_name_to_prop(cb->cb_propname);

	/*
	* If we're doing it recursively, then ignore properties that
	* are not valid for this type of dataset.
	*/
	if (prop != ZPROP_INVAL &&
	!zfs_prop_valid_for_type(prop, zfs_get_type(zhp), B_FALSE))
	return (0);

	return (zfs_prop_inherit(zhp, cb->cb_propname, cb->cb_received) != 0);
	}

	static int
	inherit_cb(zfs_handle_t zhp, void data)
	{
	inherit_cbdata_t *cb = data;

	return (zfs_prop_inherit(zhp, cb->cb_propname, cb->cb_received) != 0);
	}

	static int
	zfs_do_inherit(int argc, char **argv)
	{
	int c;
	zfs_prop_t prop;
	inherit_cbdata_t cb = { 0 };
	char *propname;
	int ret = 0;
	int flags = 0;
	boolean_t received = B_FALSE;

	/* check options */
	while ((c = getopt(argc, argv, "rS")) != -1) {
	switch (c) {
	case 'r':
	flags \|= ZFS_ITER_RECURSE;
	break;
	case 'S':
	received = B_TRUE;
	break;
	case '?':
	default:
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* check number of arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing property argument\n"));
	usage(B_FALSE);
	}
	if (argc < 2) {
	(void) fprintf(stderr, gettext("missing dataset argument\n"));
	usage(B_FALSE);
	}

	propname = argv[0];
	argc--;
	argv++;

	if ((prop = zfs_name_to_prop(propname)) != ZPROP_USERPROP) {
	if (zfs_prop_readonly(prop)) {
	(void) fprintf(stderr, gettext(
	"%s property is read-only\n"),
	propname);
	return (1);
	}
	if (!zfs_prop_inheritable(prop) && !received) {
	(void) fprintf(stderr, gettext("'%s' property cannot "
	"be inherited\n"), propname);
	if (prop == ZFS_PROP_QUOTA \|\|
	prop == ZFS_PROP_RESERVATION \|\|
	prop == ZFS_PROP_REFQUOTA \|\|
	prop == ZFS_PROP_REFRESERVATION) {
	(void) fprintf(stderr, gettext("use 'zfs set "
	"%s=none' to clear\n"), propname);
	(void) fprintf(stderr, gettext("use 'zfs "
	"inherit -S %s' to revert to received "
	"value\n"), propname);
	}
	return (1);
	}
	if (received && (prop == ZFS_PROP_VOLSIZE \|\|
	prop == ZFS_PROP_VERSION)) {
	(void) fprintf(stderr, gettext("'%s' property cannot "
	"be reverted to a received value\n"), propname);
	return (1);
	}
	} else if (!zfs_prop_user(propname)) {
	(void) fprintf(stderr, gettext("invalid property '%s'\n"),
	propname);
	usage(B_FALSE);
	}

	cb.cb_propname = propname;
	cb.cb_received = received;

	if (flags & ZFS_ITER_RECURSE) {
	ret = zfs_for_each(argc, argv, flags, ZFS_TYPE_DATASET,
	NULL, NULL, 0, inherit_recurse_cb, &cb);
	} else {
	ret = zfs_for_each(argc, argv, flags, ZFS_TYPE_DATASET,
	NULL, NULL, 0, inherit_cb, &cb);
	}

	return (ret);
	}

	typedef struct upgrade_cbdata {
	uint64_t cb_numupgraded;
	uint64_t cb_numsamegraded;
	uint64_t cb_numfailed;
	uint64_t cb_version;
	boolean_t cb_newer;
	boolean_t cb_foundone;
	char cb_lastfs[ZFS_MAX_DATASET_NAME_LEN];
	} upgrade_cbdata_t;

	static int
	same_pool(zfs_handle_t zhp, const char name)
	{
	int len1 = strcspn(name, "/@");
	const char *zhname = zfs_get_name(zhp);
	int len2 = strcspn(zhname, "/@");

	if (len1 != len2)
	return (B_FALSE);
	return (strncmp(name, zhname, len1) == 0);
	}

	static int
	upgrade_list_callback(zfs_handle_t zhp, void data)
	{
	upgrade_cbdata_t *cb = data;
	int version = zfs_prop_get_int(zhp, ZFS_PROP_VERSION);

	/* list if it's old/new */
	if ((!cb->cb_newer && version < ZPL_VERSION) \|\|
	(cb->cb_newer && version > ZPL_VERSION)) {
	char *str;
	if (cb->cb_newer) {
	str = gettext("The following filesystems are "
	"formatted using a newer software version and\n"
	"cannot be accessed on the current system.\n\n");
	} else {
	str = gettext("The following filesystems are "
	"out of date, and can be upgraded. After being\n"
	"upgraded, these filesystems (and any 'zfs send' "
	"streams generated from\n"
	"subsequent snapshots) will no longer be "
	"accessible by older software versions.\n\n");
	}

	if (!cb->cb_foundone) {
	(void) puts(str);
	(void) printf(gettext("VER FILESYSTEM\n"));
	(void) printf(gettext("--- ------------\n"));
	cb->cb_foundone = B_TRUE;
	}

	(void) printf("%2u %s\n", version, zfs_get_name(zhp));
	}

	return (0);
	}

	static int
	upgrade_set_callback(zfs_handle_t zhp, void data)
	{
	upgrade_cbdata_t *cb = data;
	int version = zfs_prop_get_int(zhp, ZFS_PROP_VERSION);
	int needed_spa_version;
	int spa_version;

	if (zfs_spa_version(zhp, &spa_version) < 0)
	return (-1);

	needed_spa_version = zfs_spa_version_map(cb->cb_version);

	if (needed_spa_version < 0)
	return (-1);

	if (spa_version < needed_spa_version) {
	/* can't upgrade */
	(void) printf(gettext("%s: can not be "
	"upgraded; the pool version needs to first "
	"be upgraded\nto version %d\n\n"),
	zfs_get_name(zhp), needed_spa_version);
	cb->cb_numfailed++;
	return (0);
	}

	/* upgrade */
	if (version < cb->cb_version) {
	char verstr[24];
	(void) snprintf(verstr, sizeof (verstr),
	"%llu", (u_longlong_t)cb->cb_version);
	if (cb->cb_lastfs[0] && !same_pool(zhp, cb->cb_lastfs)) {
	/*
	* If they did "zfs upgrade -a", then we could
	* be doing ioctls to different pools. We need
	* to log this history once to each pool, and bypass
	* the normal history logging that happens in main().
	*/
	(void) zpool_log_history(g_zfs, history_str);
	log_history = B_FALSE;
	}
	if (zfs_prop_set(zhp, "version", verstr) == 0)
	cb->cb_numupgraded++;
	else
	cb->cb_numfailed++;
	(void) strlcpy(cb->cb_lastfs, zfs_get_name(zhp),
	sizeof (cb->cb_lastfs));
	} else if (version > cb->cb_version) {
	/* can't downgrade */
	(void) printf(gettext("%s: can not be downgraded; "
	"it is already at version %u\n"),
	zfs_get_name(zhp), version);
	cb->cb_numfailed++;
	} else {
	cb->cb_numsamegraded++;
	}
	return (0);
	}

	/*
	* zfs upgrade
	* zfs upgrade -v
	* zfs upgrade [-r] [-V <version>] <-a \| filesystem>
	*/
	static int
	zfs_do_upgrade(int argc, char **argv)
	{
	boolean_t all = B_FALSE;
	boolean_t showversions = B_FALSE;
	int ret = 0;
	upgrade_cbdata_t cb = { 0 };
	int c;
	int flags = ZFS_ITER_ARGS_CAN_BE_PATHS;

	/* check options */
	while ((c = getopt(argc, argv, "rvV:a")) != -1) {
	switch (c) {
	case 'r':
	flags \|= ZFS_ITER_RECURSE;
	break;
	case 'v':
	showversions = B_TRUE;
	break;
	case 'V':
	if (zfs_prop_string_to_index(ZFS_PROP_VERSION,
	optarg, &cb.cb_version) != 0) {
	(void) fprintf(stderr,
	gettext("invalid version %s\n"), optarg);
	usage(B_FALSE);
	}
	break;
	case 'a':
	all = B_TRUE;
	break;
	case '?':
	default:
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	if ((!all && !argc) && ((flags & ZFS_ITER_RECURSE) \| cb.cb_version))
	usage(B_FALSE);
	if (showversions && (flags & ZFS_ITER_RECURSE \|\| all \|\|
	cb.cb_version \|\| argc))
	usage(B_FALSE);
	if ((all \|\| argc) && (showversions))
	usage(B_FALSE);
	if (all && argc)
	usage(B_FALSE);

	if (showversions) {
	/* Show info on available versions. */
	(void) printf(gettext("The following filesystem versions are "
	"supported:\n\n"));
	(void) printf(gettext("VER DESCRIPTION\n"));
	(void) printf("--- -----------------------------------------"
	"---------------\n");
	(void) printf(gettext(" 1 Initial ZFS filesystem version\n"));
	(void) printf(gettext(" 2 Enhanced directory entries\n"));
	(void) printf(gettext(" 3 Case insensitive and filesystem "
	"user identifier (FUID)\n"));
	(void) printf(gettext(" 4 userquota, groupquota "
	"properties\n"));
	(void) printf(gettext(" 5 System attributes\n"));
	(void) printf(gettext("\nFor more information on a particular "
	"version, including supported releases,\n"));
	(void) printf("see the ZFS Administration Guide.\n\n");
	ret = 0;
	} else if (argc \|\| all) {
	/* Upgrade filesystems */
	if (cb.cb_version == 0)
	cb.cb_version = ZPL_VERSION;
	ret = zfs_for_each(argc, argv, flags, ZFS_TYPE_FILESYSTEM,
	NULL, NULL, 0, upgrade_set_callback, &cb);
	(void) printf(gettext("%llu filesystems upgraded\n"),
	(u_longlong_t)cb.cb_numupgraded);
	if (cb.cb_numsamegraded) {
	(void) printf(gettext("%llu filesystems already at "
	"this version\n"),
	(u_longlong_t)cb.cb_numsamegraded);
	}
	if (cb.cb_numfailed != 0)
	ret = 1;
	} else {
	/* List old-version filesystems */
	boolean_t found;
	(void) printf(gettext("This system is currently running "
	"ZFS filesystem version %llu.\n\n"), ZPL_VERSION);

	flags \|= ZFS_ITER_RECURSE;
	ret = zfs_for_each(0, NULL, flags, ZFS_TYPE_FILESYSTEM,
	NULL, NULL, 0, upgrade_list_callback, &cb);

	found = cb.cb_foundone;
	cb.cb_foundone = B_FALSE;
	cb.cb_newer = B_TRUE;

	ret \|= zfs_for_each(0, NULL, flags, ZFS_TYPE_FILESYSTEM,
	NULL, NULL, 0, upgrade_list_callback, &cb);

	if (!cb.cb_foundone && !found) {
	(void) printf(gettext("All filesystems are "
	"formatted with the current version.\n"));
	}
	}

	return (ret);
	}

	/*
	* zfs userspace [-Hinp] [-o field[,...]] [-s field [-s field]...]
	* [-S field [-S field]...] [-t type[,...]]
	* filesystem \| snapshot \| path
	* zfs groupspace [-Hinp] [-o field[,...]] [-s field [-s field]...]
	* [-S field [-S field]...] [-t type[,...]]
	* filesystem \| snapshot \| path
	* zfs projectspace [-Hp] [-o field[,...]] [-s field [-s field]...]
	* [-S field [-S field]...] filesystem \| snapshot \| path
	*
	* -H Scripted mode; elide headers and separate columns by tabs.
	* -i Translate SID to POSIX ID.
	* -n Print numeric ID instead of user/group name.
	* -o Control which fields to display.
	* -p Use exact (parsable) numeric output.
	* -s Specify sort columns, descending order.
	* -S Specify sort columns, ascending order.
	* -t Control which object types to display.
	*
	* Displays space consumed by, and quotas on, each user in the specified
	* filesystem or snapshot.
	*/

	/* us_field_types, us_field_hdr and us_field_names should be kept in sync */
	enum us_field_types {
	USFIELD_TYPE,
	USFIELD_NAME,
	USFIELD_USED,
	USFIELD_QUOTA,
	USFIELD_OBJUSED,
	USFIELD_OBJQUOTA
	};
	static const char *const us_field_hdr[] = { "TYPE", "NAME", "USED", "QUOTA",
	"OBJUSED", "OBJQUOTA" };
	static const char *const us_field_names[] = { "type", "name", "used", "quota",
	"objused", "objquota" };
	#define USFIELD_LAST (sizeof (us_field_names) / sizeof (char *))

	#define USTYPE_PSX_GRP (1 << 0)
	#define USTYPE_PSX_USR (1 << 1)
	#define USTYPE_SMB_GRP (1 << 2)
	#define USTYPE_SMB_USR (1 << 3)
	#define USTYPE_PROJ (1 << 4)
	#define USTYPE_ALL \
	(USTYPE_PSX_GRP \| USTYPE_PSX_USR \| USTYPE_SMB_GRP \| USTYPE_SMB_USR \| \
	USTYPE_PROJ)

	static int us_type_bits[] = {
	USTYPE_PSX_GRP,
	USTYPE_PSX_USR,
	USTYPE_SMB_GRP,
	USTYPE_SMB_USR,
	USTYPE_ALL
	};
	static const char *const us_type_names[] = { "posixgroup", "posixuser",
	"smbgroup", "smbuser", "all" };

	typedef struct us_node {
	nvlist_t *usn_nvl;
	uu_avl_node_t usn_avlnode;
	uu_list_node_t usn_listnode;
	} us_node_t;

	typedef struct us_cbdata {
	nvlist_t **cb_nvlp;
	uu_avl_pool_t *cb_avl_pool;
	uu_avl_t *cb_avl;
	boolean_t cb_numname;
	boolean_t cb_nicenum;
	boolean_t cb_sid2posix;
	zfs_userquota_prop_t cb_prop;
	zfs_sort_column_t *cb_sortcol;
	size_t cb_width[USFIELD_LAST];
	} us_cbdata_t;

	static boolean_t us_populated = B_FALSE;

	typedef struct {
	zfs_sort_column_t *si_sortcol;
	boolean_t si_numname;
	} us_sort_info_t;

	static int
	us_field_index(const char *field)
	{
	for (int i = 0; i < USFIELD_LAST; i++) {
	if (strcmp(field, us_field_names[i]) == 0)
	return (i);
	}

	return (-1);
	}

	static int
	us_compare(const void larg, const void rarg, void *unused)
	{
	const us_node_t *l = larg;
	const us_node_t *r = rarg;
	us_sort_info_t si = (us_sort_info_t )unused;
	zfs_sort_column_t *sortcol = si->si_sortcol;
	boolean_t numname = si->si_numname;
	nvlist_t *lnvl = l->usn_nvl;
	nvlist_t *rnvl = r->usn_nvl;
	int rc = 0;
	boolean_t lvb, rvb;

	for (; sortcol != NULL; sortcol = sortcol->sc_next) {
	const char *lvstr = "";
	const char *rvstr = "";
	uint32_t lv32 = 0;
	uint32_t rv32 = 0;
	uint64_t lv64 = 0;
	uint64_t rv64 = 0;
	zfs_prop_t prop = sortcol->sc_prop;
	const char *propname = NULL;
	boolean_t reverse = sortcol->sc_reverse;

	switch (prop) {
	case ZFS_PROP_TYPE:
	propname = "type";
	(void) nvlist_lookup_uint32(lnvl, propname, &lv32);
	(void) nvlist_lookup_uint32(rnvl, propname, &rv32);
	if (rv32 != lv32)
	rc = (rv32 < lv32) ? 1 : -1;
	break;
	case ZFS_PROP_NAME:
	propname = "name";
	if (numname) {
	compare_nums:
	(void) nvlist_lookup_uint64(lnvl, propname,
	&lv64);
	(void) nvlist_lookup_uint64(rnvl, propname,
	&rv64);
	if (rv64 != lv64)
	rc = (rv64 < lv64) ? 1 : -1;
	} else {
	if ((nvlist_lookup_string(lnvl, propname,
	&lvstr) == ENOENT) \|\|
	(nvlist_lookup_string(rnvl, propname,
	&rvstr) == ENOENT)) {
	goto compare_nums;
	}
	rc = strcmp(lvstr, rvstr);
	}
	break;
	case ZFS_PROP_USED:
	case ZFS_PROP_QUOTA:
	if (!us_populated)
	break;
	if (prop == ZFS_PROP_USED)
	propname = "used";
	else
	propname = "quota";
	(void) nvlist_lookup_uint64(lnvl, propname, &lv64);
	(void) nvlist_lookup_uint64(rnvl, propname, &rv64);
	if (rv64 != lv64)
	rc = (rv64 < lv64) ? 1 : -1;
	break;

	default:
	break;
	}

	if (rc != 0) {
	if (rc < 0)
	return (reverse ? 1 : -1);
	else
	return (reverse ? -1 : 1);
	}
	}

	/*
	* If entries still seem to be the same, check if they are of the same
	* type (smbentity is added only if we are doing SID to POSIX ID
	* translation where we can have duplicate type/name combinations).
	*/
	if (nvlist_lookup_boolean_value(lnvl, "smbentity", &lvb) == 0 &&
	nvlist_lookup_boolean_value(rnvl, "smbentity", &rvb) == 0 &&
	lvb != rvb)
	return (lvb < rvb ? -1 : 1);

	return (0);
	}

	static boolean_t
	zfs_prop_is_user(unsigned p)
	{
	return (p == ZFS_PROP_USERUSED \|\| p == ZFS_PROP_USERQUOTA \|\|
	p == ZFS_PROP_USEROBJUSED \|\| p == ZFS_PROP_USEROBJQUOTA);
	}

	static boolean_t
	zfs_prop_is_group(unsigned p)
	{
	return (p == ZFS_PROP_GROUPUSED \|\| p == ZFS_PROP_GROUPQUOTA \|\|
	p == ZFS_PROP_GROUPOBJUSED \|\| p == ZFS_PROP_GROUPOBJQUOTA);
	}

	static boolean_t
	zfs_prop_is_project(unsigned p)
	{
	return (p == ZFS_PROP_PROJECTUSED \|\| p == ZFS_PROP_PROJECTQUOTA \|\|
	p == ZFS_PROP_PROJECTOBJUSED \|\| p == ZFS_PROP_PROJECTOBJQUOTA);
	}

	static inline const char *
	us_type2str(unsigned field_type)
	{
	switch (field_type) {
	case USTYPE_PSX_USR:
	return ("POSIX User");
	case USTYPE_PSX_GRP:
	return ("POSIX Group");
	case USTYPE_SMB_USR:
	return ("SMB User");
	case USTYPE_SMB_GRP:
	return ("SMB Group");
	case USTYPE_PROJ:
	return ("Project");
	default:
	return ("Undefined");
	}
	}

	static int
	userspace_cb(void arg, const char domain, uid_t rid, uint64_t space)
	{
	us_cbdata_t cb = (us_cbdata_t )arg;
	zfs_userquota_prop_t prop = cb->cb_prop;
	char *name = NULL;
	const char *propname;
	char sizebuf[32];
	us_node_t *node;
	uu_avl_pool_t *avl_pool = cb->cb_avl_pool;
	uu_avl_t *avl = cb->cb_avl;
	uu_avl_index_t idx;
	nvlist_t *props;
	us_node_t *n;
	zfs_sort_column_t *sortcol = cb->cb_sortcol;
	unsigned type = 0;
	const char *typestr;
	size_t namelen;
	size_t typelen;
	size_t sizelen;
	int typeidx, nameidx, sizeidx;
	us_sort_info_t sortinfo = { sortcol, cb->cb_numname };
	boolean_t smbentity = B_FALSE;

	if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0)
	nomem();
	node = safe_malloc(sizeof (us_node_t));
	uu_avl_node_init(node, &node->usn_avlnode, avl_pool);
	node->usn_nvl = props;

	if (domain != NULL && domain[0] != '\0') {
	#ifdef HAVE_IDMAP
	/* SMB */
	char sid[MAXNAMELEN + 32];
	uid_t id;
	uint64_t classes;
	int err;
	directory_error_t e;

	smbentity = B_TRUE;

	(void) snprintf(sid, sizeof (sid), "%s-%u", domain, rid);

	if (prop == ZFS_PROP_GROUPUSED \|\| prop == ZFS_PROP_GROUPQUOTA) {
	type = USTYPE_SMB_GRP;
	err = sid_to_id(sid, B_FALSE, &id);
	} else {
	type = USTYPE_SMB_USR;
	err = sid_to_id(sid, B_TRUE, &id);
	}

	if (err == 0) {
	rid = id;
	if (!cb->cb_sid2posix) {
	e = directory_name_from_sid(NULL, sid, &name,
	&classes);
	if (e != NULL)
	directory_error_free(e);
	if (name == NULL)
	name = sid;
	}
	}
	#else
	nvlist_free(props);
	free(node);

	return (-1);
	#endif /* HAVE_IDMAP */
	}

	if (cb->cb_sid2posix \|\| domain == NULL \|\| domain[0] == '\0') {
	/* POSIX or -i */
	if (zfs_prop_is_group(prop)) {
	type = USTYPE_PSX_GRP;
	if (!cb->cb_numname) {
	struct group *g;

	if ((g = getgrgid(rid)) != NULL)
	name = g->gr_name;
	}
	} else if (zfs_prop_is_user(prop)) {
	type = USTYPE_PSX_USR;
	if (!cb->cb_numname) {
	struct passwd *p;

	if ((p = getpwuid(rid)) != NULL)
	name = p->pw_name;
	}
	} else {
	type = USTYPE_PROJ;
	}
	}

	/*
	* Make sure that the type/name combination is unique when doing
	* SID to POSIX ID translation (hence changing the type from SMB to
	* POSIX).
	*/
	if (cb->cb_sid2posix &&
	nvlist_add_boolean_value(props, "smbentity", smbentity) != 0)
	nomem();

	/* Calculate/update width of TYPE field */
	typestr = us_type2str(type);
	typelen = strlen(gettext(typestr));
	typeidx = us_field_index("type");
	if (typelen > cb->cb_width[typeidx])
	cb->cb_width[typeidx] = typelen;
	if (nvlist_add_uint32(props, "type", type) != 0)
	nomem();

	/* Calculate/update width of NAME field */
	if ((cb->cb_numname && cb->cb_sid2posix) \|\| name == NULL) {
	if (nvlist_add_uint64(props, "name", rid) != 0)
	nomem();
	namelen = snprintf(NULL, 0, "%u", rid);
	} else {
	if (nvlist_add_string(props, "name", name) != 0)
	nomem();
	namelen = strlen(name);
	}
	nameidx = us_field_index("name");
	if (nameidx >= 0 && namelen > cb->cb_width[nameidx])
	cb->cb_width[nameidx] = namelen;

	/*
	* Check if this type/name combination is in the list and update it;
	* otherwise add new node to the list.
	*/
	if ((n = uu_avl_find(avl, node, &sortinfo, &idx)) == NULL) {
	uu_avl_insert(avl, node, idx);
	} else {
	nvlist_free(props);
	free(node);
	node = n;
	props = node->usn_nvl;
	}

	/* Calculate/update width of USED/QUOTA fields */
	if (cb->cb_nicenum) {
	if (prop == ZFS_PROP_USERUSED \|\| prop == ZFS_PROP_GROUPUSED \|\|
	prop == ZFS_PROP_USERQUOTA \|\| prop == ZFS_PROP_GROUPQUOTA \|\|
	prop == ZFS_PROP_PROJECTUSED \|\|
	prop == ZFS_PROP_PROJECTQUOTA) {
	zfs_nicebytes(space, sizebuf, sizeof (sizebuf));
	} else {
	zfs_nicenum(space, sizebuf, sizeof (sizebuf));
	}
	} else {
	(void) snprintf(sizebuf, sizeof (sizebuf), "%llu",
	(u_longlong_t)space);
	}
	sizelen = strlen(sizebuf);
	if (prop == ZFS_PROP_USERUSED \|\| prop == ZFS_PROP_GROUPUSED \|\|
	prop == ZFS_PROP_PROJECTUSED) {
	propname = "used";
	if (!nvlist_exists(props, "quota"))
	(void) nvlist_add_uint64(props, "quota", 0);
	} else if (prop == ZFS_PROP_USERQUOTA \|\| prop == ZFS_PROP_GROUPQUOTA \|\|
	prop == ZFS_PROP_PROJECTQUOTA) {
	propname = "quota";
	if (!nvlist_exists(props, "used"))
	(void) nvlist_add_uint64(props, "used", 0);
	} else if (prop == ZFS_PROP_USEROBJUSED \|\|
	prop == ZFS_PROP_GROUPOBJUSED \|\| prop == ZFS_PROP_PROJECTOBJUSED) {
	propname = "objused";
	if (!nvlist_exists(props, "objquota"))
	(void) nvlist_add_uint64(props, "objquota", 0);
	} else if (prop == ZFS_PROP_USEROBJQUOTA \|\|
	prop == ZFS_PROP_GROUPOBJQUOTA \|\|
	prop == ZFS_PROP_PROJECTOBJQUOTA) {
	propname = "objquota";
	if (!nvlist_exists(props, "objused"))
	(void) nvlist_add_uint64(props, "objused", 0);
	} else {
	return (-1);
	}
	sizeidx = us_field_index(propname);
	if (sizeidx >= 0 && sizelen > cb->cb_width[sizeidx])
	cb->cb_width[sizeidx] = sizelen;

	if (nvlist_add_uint64(props, propname, space) != 0)
	nomem();

	return (0);
	}

	static void
	print_us_node(boolean_t scripted, boolean_t parsable, int *fields, int types,
	size_t width, us_node_t node)
	{
	nvlist_t *nvl = node->usn_nvl;
	char valstr[MAXNAMELEN];
	boolean_t first = B_TRUE;
	int cfield = 0;
	int field;
	uint32_t ustype;

	/* Check type */
	(void) nvlist_lookup_uint32(nvl, "type", &ustype);
	if (!(ustype & types))
	return;

	while ((field = fields[cfield]) != USFIELD_LAST) {
	nvpair_t *nvp = NULL;
	data_type_t type;
	uint32_t val32 = -1;
	uint64_t val64 = -1;
	const char *strval = "-";

	while ((nvp = nvlist_next_nvpair(nvl, nvp)) != NULL)
	if (strcmp(nvpair_name(nvp),
	us_field_names[field]) == 0)
	break;

	type = nvp == NULL ? DATA_TYPE_UNKNOWN : nvpair_type(nvp);
	switch (type) {
	case DATA_TYPE_UINT32:
	val32 = fnvpair_value_uint32(nvp);
	break;
	case DATA_TYPE_UINT64:
	val64 = fnvpair_value_uint64(nvp);
	break;
	case DATA_TYPE_STRING:
	strval = fnvpair_value_string(nvp);
	break;
	case DATA_TYPE_UNKNOWN:
	break;
	default:
	(void) fprintf(stderr, "invalid data type\n");
	}

	switch (field) {
	case USFIELD_TYPE:
	if (type == DATA_TYPE_UINT32)
	strval = us_type2str(val32);
	break;
	case USFIELD_NAME:
	if (type == DATA_TYPE_UINT64) {
	(void) sprintf(valstr, "%llu",
	(u_longlong_t)val64);
	strval = valstr;
	}
	break;
	case USFIELD_USED:
	case USFIELD_QUOTA:
	if (type == DATA_TYPE_UINT64) {
	if (parsable) {
	(void) sprintf(valstr, "%llu",
	(u_longlong_t)val64);
	strval = valstr;
	} else if (field == USFIELD_QUOTA &&
	val64 == 0) {
	strval = "none";
	} else {
	zfs_nicebytes(val64, valstr,
	sizeof (valstr));
	strval = valstr;
	}
	}
	break;
	case USFIELD_OBJUSED:
	case USFIELD_OBJQUOTA:
	if (type == DATA_TYPE_UINT64) {
	if (parsable) {
	(void) sprintf(valstr, "%llu",
	(u_longlong_t)val64);
	strval = valstr;
	} else if (field == USFIELD_OBJQUOTA &&
	val64 == 0) {
	strval = "none";
	} else {
	zfs_nicenum(val64, valstr,
	sizeof (valstr));
	strval = valstr;
	}
	}
	break;
	}

	if (!first) {
	if (scripted)
	(void) putchar('\t');
	else
	(void) fputs(" ", stdout);
	}
	if (scripted)
	(void) fputs(strval, stdout);
	else if (field == USFIELD_TYPE \|\| field == USFIELD_NAME)
	(void) printf("%-*s", (int)width[field], strval);
	else
	(void) printf("%*s", (int)width[field], strval);

	first = B_FALSE;
	cfield++;
	}

	(void) putchar('\n');
	}

	static void
	print_us(boolean_t scripted, boolean_t parsable, int *fields, int types,
	size_t width, boolean_t rmnode, uu_avl_t avl)
	{
	us_node_t *node;
	const char *col;
	int cfield = 0;
	int field;

	if (!scripted) {
	boolean_t first = B_TRUE;

	while ((field = fields[cfield]) != USFIELD_LAST) {
	col = gettext(us_field_hdr[field]);
	if (field == USFIELD_TYPE \|\| field == USFIELD_NAME) {
	(void) printf(first ? "%-s" : " %-s",
	(int)width[field], col);
	} else {
	(void) printf(first ? "%s" : " %s",
	(int)width[field], col);
	}
	first = B_FALSE;
	cfield++;
	}
	(void) printf("\n");
	}

	for (node = uu_avl_first(avl); node; node = uu_avl_next(avl, node)) {
	print_us_node(scripted, parsable, fields, types, width, node);
	if (rmnode)
	nvlist_free(node->usn_nvl);
	}
	}

	static int
	zfs_do_userspace(int argc, char **argv)
	{
	zfs_handle_t *zhp;
	zfs_userquota_prop_t p;
	uu_avl_pool_t *avl_pool;
	uu_avl_t *avl_tree;
	uu_avl_walk_t *walk;
	char *delim;
	char deffields[] = "type,name,used,quota,objused,objquota";
	char *ofield = NULL;
	char *tfield = NULL;
	int cfield = 0;
	int fields[256];
	int i;
	boolean_t scripted = B_FALSE;
	boolean_t prtnum = B_FALSE;
	boolean_t parsable = B_FALSE;
	boolean_t sid2posix = B_FALSE;
	int ret = 0;
	int c;
	zfs_sort_column_t *sortcol = NULL;
	int types = USTYPE_PSX_USR \| USTYPE_SMB_USR;
	us_cbdata_t cb;
	us_node_t *node;
	us_node_t *rmnode;
	uu_list_pool_t *listpool;
	uu_list_t *list;
	uu_avl_index_t idx = 0;
	uu_list_index_t idx2 = 0;

	if (argc < 2)
	usage(B_FALSE);

	if (strcmp(argv[0], "groupspace") == 0) {
	/* Toggle default group types */
	types = USTYPE_PSX_GRP \| USTYPE_SMB_GRP;
	} else if (strcmp(argv[0], "projectspace") == 0) {
	types = USTYPE_PROJ;
	prtnum = B_TRUE;
	}

	while ((c = getopt(argc, argv, "nHpo:s:S:t:i")) != -1) {
	switch (c) {
	case 'n':
	if (types == USTYPE_PROJ) {
	(void) fprintf(stderr,
	gettext("invalid option 'n'\n"));
	usage(B_FALSE);
	}
	prtnum = B_TRUE;
	break;
	case 'H':
	scripted = B_TRUE;
	break;
	case 'p':
	parsable = B_TRUE;
	break;
	case 'o':
	ofield = optarg;
	break;
	case 's':
	case 'S':
	if (zfs_add_sort_column(&sortcol, optarg,
	c == 's' ? B_FALSE : B_TRUE) != 0) {
	(void) fprintf(stderr,
	gettext("invalid field '%s'\n"), optarg);
	usage(B_FALSE);
	}
	break;
	case 't':
	if (types == USTYPE_PROJ) {
	(void) fprintf(stderr,
	gettext("invalid option 't'\n"));
	usage(B_FALSE);
	}
	tfield = optarg;
	break;
	case 'i':
	if (types == USTYPE_PROJ) {
	(void) fprintf(stderr,
	gettext("invalid option 'i'\n"));
	usage(B_FALSE);
	}
	sid2posix = B_TRUE;
	break;
	case ':':
	(void) fprintf(stderr, gettext("missing argument for "
	"'%c' option\n"), optopt);
	usage(B_FALSE);
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing dataset name\n"));
	usage(B_FALSE);
	}
	if (argc > 1) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	/* Use default output fields if not specified using -o */
	if (ofield == NULL)
	ofield = deffields;
	do {
	if ((delim = strchr(ofield, ',')) != NULL)
	*delim = '\0';
	if ((fields[cfield++] = us_field_index(ofield)) == -1) {
	(void) fprintf(stderr, gettext("invalid type '%s' "
	"for -o option\n"), ofield);
	return (-1);
	}
	if (delim != NULL)
	ofield = delim + 1;
	} while (delim != NULL);
	fields[cfield] = USFIELD_LAST;

	/* Override output types (-t option) */
	if (tfield != NULL) {
	types = 0;

	do {
	boolean_t found = B_FALSE;

	if ((delim = strchr(tfield, ',')) != NULL)
	*delim = '\0';
	for (i = 0; i < sizeof (us_type_bits) / sizeof (int);
	i++) {
	if (strcmp(tfield, us_type_names[i]) == 0) {
	found = B_TRUE;
	types \|= us_type_bits[i];
	break;
	}
	}
	if (!found) {
	(void) fprintf(stderr, gettext("invalid type "
	"'%s' for -t option\n"), tfield);
	return (-1);
	}
	if (delim != NULL)
	tfield = delim + 1;
	} while (delim != NULL);
	}

	if ((zhp = zfs_path_to_zhandle(g_zfs, argv[0], ZFS_TYPE_FILESYSTEM \|
	ZFS_TYPE_SNAPSHOT)) == NULL)
	return (1);
	if (zfs_get_underlying_type(zhp) != ZFS_TYPE_FILESYSTEM) {
	(void) fprintf(stderr, gettext("operation is only applicable "
	"to filesystems and their snapshots\n"));
	zfs_close(zhp);
	return (1);
	}

	if ((avl_pool = uu_avl_pool_create("us_avl_pool", sizeof (us_node_t),
	offsetof(us_node_t, usn_avlnode), us_compare, UU_DEFAULT)) == NULL)
	nomem();
	if ((avl_tree = uu_avl_create(avl_pool, NULL, UU_DEFAULT)) == NULL)
	nomem();

	/* Always add default sorting columns */
	(void) zfs_add_sort_column(&sortcol, "type", B_FALSE);
	(void) zfs_add_sort_column(&sortcol, "name", B_FALSE);

	cb.cb_sortcol = sortcol;
	cb.cb_numname = prtnum;
	cb.cb_nicenum = !parsable;
	cb.cb_avl_pool = avl_pool;
	cb.cb_avl = avl_tree;
	cb.cb_sid2posix = sid2posix;

	for (i = 0; i < USFIELD_LAST; i++)
	cb.cb_width[i] = strlen(gettext(us_field_hdr[i]));

	for (p = 0; p < ZFS_NUM_USERQUOTA_PROPS; p++) {
	if ((zfs_prop_is_user(p) &&
	!(types & (USTYPE_PSX_USR \| USTYPE_SMB_USR))) \|\|
	(zfs_prop_is_group(p) &&
	!(types & (USTYPE_PSX_GRP \| USTYPE_SMB_GRP))) \|\|
	(zfs_prop_is_project(p) && types != USTYPE_PROJ))
	continue;

	cb.cb_prop = p;
	if ((ret = zfs_userspace(zhp, p, userspace_cb, &cb)) != 0) {
	zfs_close(zhp);
	return (ret);
	}
	}
	zfs_close(zhp);

	/* Sort the list */
	if ((node = uu_avl_first(avl_tree)) == NULL)
	return (0);

	us_populated = B_TRUE;

	listpool = uu_list_pool_create("tmplist", sizeof (us_node_t),
	offsetof(us_node_t, usn_listnode), NULL, UU_DEFAULT);
	list = uu_list_create(listpool, NULL, UU_DEFAULT);
	uu_list_node_init(node, &node->usn_listnode, listpool);

	while (node != NULL) {
	rmnode = node;
	node = uu_avl_next(avl_tree, node);
	uu_avl_remove(avl_tree, rmnode);
	if (uu_list_find(list, rmnode, NULL, &idx2) == NULL)
	uu_list_insert(list, rmnode, idx2);
	}

	for (node = uu_list_first(list); node != NULL;
	node = uu_list_next(list, node)) {
	us_sort_info_t sortinfo = { sortcol, cb.cb_numname };

	if (uu_avl_find(avl_tree, node, &sortinfo, &idx) == NULL)
	uu_avl_insert(avl_tree, node, idx);
	}

	uu_list_destroy(list);
	uu_list_pool_destroy(listpool);

	/* Print and free node nvlist memory */
	print_us(scripted, parsable, fields, types, cb.cb_width, B_TRUE,
	cb.cb_avl);

	zfs_free_sort_columns(sortcol);

	/* Clean up the AVL tree */
	if ((walk = uu_avl_walk_start(cb.cb_avl, UU_WALK_ROBUST)) == NULL)
	nomem();

	while ((node = uu_avl_walk_next(walk)) != NULL) {
	uu_avl_remove(cb.cb_avl, node);
	free(node);
	}

	uu_avl_walk_end(walk);
	uu_avl_destroy(avl_tree);
	uu_avl_pool_destroy(avl_pool);

	return (ret);
	}

	/*
	* list [-Hp][-r\|-d max] [-o property[,...]] [-s property] ... [-S property]
	* [-t type[,...]] [filesystem\|volume\|snapshot] ...
	*
	* -H Scripted mode; elide headers and separate columns by tabs
	* -p Display values in parsable (literal) format.
	* -r Recurse over all children
	* -d Limit recursion by depth.
	* -o Control which fields to display.
	* -s Specify sort columns, descending order.
	* -S Specify sort columns, ascending order.
	* -t Control which object types to display.
	*
	* When given no arguments, list all filesystems in the system.
	* Otherwise, list the specified datasets, optionally recursing down them if
	* '-r' is specified.
	*/
	typedef struct list_cbdata {
	boolean_t cb_first;
	boolean_t cb_literal;
	boolean_t cb_scripted;
	zprop_list_t *cb_proplist;
	} list_cbdata_t;

	/*
	* Given a list of columns to display, output appropriate headers for each one.
	*/
	static void
	print_header(list_cbdata_t *cb)
	{
	zprop_list_t *pl = cb->cb_proplist;
	char headerbuf[ZFS_MAXPROPLEN];
	const char *header;
	int i;
	boolean_t first = B_TRUE;
	boolean_t right_justify;

	color_start(ANSI_BOLD);

	for (; pl != NULL; pl = pl->pl_next) {
	if (!first) {
	(void) printf(" ");
	} else {
	first = B_FALSE;
	}

	right_justify = B_FALSE;
	if (pl->pl_prop != ZPROP_USERPROP) {
	header = zfs_prop_column_name(pl->pl_prop);
	right_justify = zfs_prop_align_right(pl->pl_prop);
	} else {
	for (i = 0; pl->pl_user_prop[i] != '\0'; i++)
	headerbuf[i] = toupper(pl->pl_user_prop[i]);
	headerbuf[i] = '\0';
	header = headerbuf;
	}

	if (pl->pl_next == NULL && !right_justify)
	(void) printf("%s", header);
	else if (right_justify)
	(void) printf("%*s", (int)pl->pl_width, header);
	else
	(void) printf("%-*s", (int)pl->pl_width, header);
	}

	color_end();

	(void) printf("\n");
	}

	/*
	* Decides on the color that the avail value should be printed in.
	* > 80% used = yellow
	* > 90% used = red
	*/
	static const char *
	zfs_list_avail_color(zfs_handle_t *zhp)
	{
	uint64_t used = zfs_prop_get_int(zhp, ZFS_PROP_USED);
	uint64_t avail = zfs_prop_get_int(zhp, ZFS_PROP_AVAILABLE);
	int percentage = (int)((double)avail / MAX(avail + used, 1) * 100);

	if (percentage > 20)
	return (NULL);
	else if (percentage > 10)
	return (ANSI_YELLOW);
	else
	return (ANSI_RED);
	}

	/*
	* Given a dataset and a list of fields, print out all the properties according
	* to the described layout.
	*/
	static void
	print_dataset(zfs_handle_t zhp, list_cbdata_t cb)
	{
	zprop_list_t *pl = cb->cb_proplist;
	boolean_t first = B_TRUE;
	char property[ZFS_MAXPROPLEN];
	nvlist_t *userprops = zfs_get_user_props(zhp);
	nvlist_t *propval;
	const char *propstr;
	boolean_t right_justify;

	for (; pl != NULL; pl = pl->pl_next) {
	if (!first) {
	if (cb->cb_scripted)
	(void) putchar('\t');
	else
	(void) fputs(" ", stdout);
	} else {
	first = B_FALSE;
	}

	if (pl->pl_prop == ZFS_PROP_NAME) {
	(void) strlcpy(property, zfs_get_name(zhp),
	sizeof (property));
	propstr = property;
	right_justify = zfs_prop_align_right(pl->pl_prop);
	} else if (pl->pl_prop != ZPROP_USERPROP) {
	if (zfs_prop_get(zhp, pl->pl_prop, property,
	sizeof (property), NULL, NULL, 0,
	cb->cb_literal) != 0)
	propstr = "-";
	else
	propstr = property;
	right_justify = zfs_prop_align_right(pl->pl_prop);
	} else if (zfs_prop_userquota(pl->pl_user_prop)) {
	if (zfs_prop_get_userquota(zhp, pl->pl_user_prop,
	property, sizeof (property), cb->cb_literal) != 0)
	propstr = "-";
	else
	propstr = property;
	right_justify = B_TRUE;
	} else if (zfs_prop_written(pl->pl_user_prop)) {
	if (zfs_prop_get_written(zhp, pl->pl_user_prop,
	property, sizeof (property), cb->cb_literal) != 0)
	propstr = "-";
	else
	propstr = property;
	right_justify = B_TRUE;
	} else {
	if (nvlist_lookup_nvlist(userprops,
	pl->pl_user_prop, &propval) != 0)
	propstr = "-";
	else
	propstr = fnvlist_lookup_string(propval,
	ZPROP_VALUE);
	right_justify = B_FALSE;
	}

	/*
	* zfs_list_avail_color() needs ZFS_PROP_AVAILABLE + USED
	* - so we need another for() search for the USED part
	* - when no colors wanted, we can skip the whole thing
	*/
	if (use_color() && pl->pl_prop == ZFS_PROP_AVAILABLE) {
	zprop_list_t *pl2 = cb->cb_proplist;
	for (; pl2 != NULL; pl2 = pl2->pl_next) {
	if (pl2->pl_prop == ZFS_PROP_USED) {
	color_start(zfs_list_avail_color(zhp));
	/* found it, no need for more loops */
	break;
	}
	}
	}

	/*
	* If this is being called in scripted mode, or if this is the
	* last column and it is left-justified, don't include a width
	* format specifier.
	*/
	if (cb->cb_scripted \|\| (pl->pl_next == NULL && !right_justify))
	(void) fputs(propstr, stdout);
	else if (right_justify)
	(void) printf("%*s", (int)pl->pl_width, propstr);
	else
	(void) printf("%-*s", (int)pl->pl_width, propstr);

	if (pl->pl_prop == ZFS_PROP_AVAILABLE)
	color_end();
	}

	(void) putchar('\n');
	}

	/*
	* Generic callback function to list a dataset or snapshot.
	*/
	static int
	list_callback(zfs_handle_t zhp, void data)
	{
	list_cbdata_t *cbp = data;

	if (cbp->cb_first) {
	if (!cbp->cb_scripted)
	print_header(cbp);
	cbp->cb_first = B_FALSE;
	}

	print_dataset(zhp, cbp);

	return (0);
	}

	static int
	zfs_do_list(int argc, char **argv)
	{
	int c;
	char default_fields[] =
	"name,used,available,referenced,mountpoint";
	int types = ZFS_TYPE_DATASET;
	boolean_t types_specified = B_FALSE;
	char *fields = default_fields;
	list_cbdata_t cb = { 0 };
	int limit = 0;
	int ret = 0;
	zfs_sort_column_t *sortcol = NULL;
	int flags = ZFS_ITER_PROP_LISTSNAPS \| ZFS_ITER_ARGS_CAN_BE_PATHS;

	/* check options */
	while ((c = getopt(argc, argv, "HS:d:o:prs:t:")) != -1) {
	switch (c) {
	case 'o':
	fields = optarg;
	break;
	case 'p':
	cb.cb_literal = B_TRUE;
	flags \|= ZFS_ITER_LITERAL_PROPS;
	break;
	case 'd':
	limit = parse_depth(optarg, &flags);
	break;
	case 'r':
	flags \|= ZFS_ITER_RECURSE;
	break;
	case 'H':
	cb.cb_scripted = B_TRUE;
	break;
	case 's':
	if (zfs_add_sort_column(&sortcol, optarg,
	B_FALSE) != 0) {
	(void) fprintf(stderr,
	gettext("invalid property '%s'\n"), optarg);
	usage(B_FALSE);
	}
	break;
	case 'S':
	if (zfs_add_sort_column(&sortcol, optarg,
	B_TRUE) != 0) {
	(void) fprintf(stderr,
	gettext("invalid property '%s'\n"), optarg);
	usage(B_FALSE);
	}
	break;
	case 't':
	types = 0;
	types_specified = B_TRUE;
	flags &= ~ZFS_ITER_PROP_LISTSNAPS;

	for (char *tok; (tok = strsep(&optarg, ",")); ) {
	static const char *const type_subopts[] = {
	"filesystem",
	"fs",
	"volume",
	"vol",
	"snapshot",
	"snap",
	"bookmark",
	"all"
	};
	static const int type_types[] = {
	ZFS_TYPE_FILESYSTEM,
	ZFS_TYPE_FILESYSTEM,
	ZFS_TYPE_VOLUME,
	ZFS_TYPE_VOLUME,
	ZFS_TYPE_SNAPSHOT,
	ZFS_TYPE_SNAPSHOT,
	ZFS_TYPE_BOOKMARK,
	ZFS_TYPE_DATASET \| ZFS_TYPE_BOOKMARK
	};

	for (c = 0; c < ARRAY_SIZE(type_subopts); ++c)
	if (strcmp(tok, type_subopts[c]) == 0) {
	types \|= type_types[c];
	goto found3;
	}

	(void) fprintf(stderr,
	gettext("invalid type '%s'\n"), tok);
	usage(B_FALSE);
	found3:;
	}
	break;
	case ':':
	(void) fprintf(stderr, gettext("missing argument for "
	"'%c' option\n"), optopt);
	usage(B_FALSE);
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/*
	* If "-o space" and no types were specified, don't display snapshots.
	*/
	if (strcmp(fields, "space") == 0 && types_specified == B_FALSE)
	types &= ~ZFS_TYPE_SNAPSHOT;

	/*
	* Handle users who want to list all snapshots or bookmarks
	* of the current dataset (ex. 'zfs list -t snapshot <dataset>').
	*/
	if ((types == ZFS_TYPE_SNAPSHOT \|\| types == ZFS_TYPE_BOOKMARK) &&
	argc > 0 && (flags & ZFS_ITER_RECURSE) == 0 && limit == 0) {
	flags \|= (ZFS_ITER_DEPTH_LIMIT \| ZFS_ITER_RECURSE);
	limit = 1;
	}

	/*
	* If the user specifies '-o all', the zprop_get_list() doesn't
	* normally include the name of the dataset. For 'zfs list', we always
	* want this property to be first.
	*/
	if (zprop_get_list(g_zfs, fields, &cb.cb_proplist, ZFS_TYPE_DATASET)
	!= 0)
	usage(B_FALSE);

	cb.cb_first = B_TRUE;

	/*
	* If we are only going to list and sort by properties that are "fast"
	* then we can use "simple" mode and avoid populating the properties
	* nvlist.
	*/
	if (zfs_list_only_by_fast(cb.cb_proplist) &&
	zfs_sort_only_by_fast(sortcol))
	flags \|= ZFS_ITER_SIMPLE;

	ret = zfs_for_each(argc, argv, flags, types, sortcol, &cb.cb_proplist,
	limit, list_callback, &cb);

	zprop_free_list(cb.cb_proplist);
	zfs_free_sort_columns(sortcol);

	if (ret == 0 && cb.cb_first && !cb.cb_scripted)
	(void) fprintf(stderr, gettext("no datasets available\n"));

	return (ret);
	}

	/*
	* zfs rename [-fu] <fs \| snap \| vol> <fs \| snap \| vol>
	* zfs rename [-f] -p <fs \| vol> <fs \| vol>
	* zfs rename [-u] -r <snap> <snap>
	*
	* Renames the given dataset to another of the same type.
	*
	* The '-p' flag creates all the non-existing ancestors of the target first.
	* The '-u' flag prevents file systems from being remounted during rename.
	*/
	static int
	zfs_do_rename(int argc, char **argv)
	{
	zfs_handle_t *zhp;
	renameflags_t flags = { 0 };
	int c;
	int ret = 0;
	int types;
	boolean_t parents = B_FALSE;

	/* check options */
	while ((c = getopt(argc, argv, "pruf")) != -1) {
	switch (c) {
	case 'p':
	parents = B_TRUE;
	break;
	case 'r':
	flags.recursive = B_TRUE;
	break;
	case 'u':
	flags.nounmount = B_TRUE;
	break;
	case 'f':
	flags.forceunmount = B_TRUE;
	break;
	case '?':
	default:
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* check number of arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing source dataset "
	"argument\n"));
	usage(B_FALSE);
	}
	if (argc < 2) {
	(void) fprintf(stderr, gettext("missing target dataset "
	"argument\n"));
	usage(B_FALSE);
	}
	if (argc > 2) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	if (flags.recursive && parents) {
	(void) fprintf(stderr, gettext("-p and -r options are mutually "
	"exclusive\n"));
	usage(B_FALSE);
	}

	if (flags.nounmount && parents) {
	(void) fprintf(stderr, gettext("-u and -p options are mutually "
	"exclusive\n"));
	usage(B_FALSE);
	}

	if (flags.recursive && strchr(argv[0], '@') == 0) {
	(void) fprintf(stderr, gettext("source dataset for recursive "
	"rename must be a snapshot\n"));
	usage(B_FALSE);
	}

	if (flags.nounmount)
	types = ZFS_TYPE_FILESYSTEM;
	else if (parents)
	types = ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME;
	else
	types = ZFS_TYPE_DATASET;

	if ((zhp = zfs_open(g_zfs, argv[0], types)) == NULL)
	return (1);

	/* If we were asked and the name looks good, try to create ancestors. */
	if (parents && zfs_name_valid(argv[1], zfs_get_type(zhp)) &&
	zfs_create_ancestors(g_zfs, argv[1]) != 0) {
	zfs_close(zhp);
	return (1);
	}

	ret = (zfs_rename(zhp, argv[1], flags) != 0);

	zfs_close(zhp);
	return (ret);
	}

	/*
	* zfs promote <fs>
	*
	* Promotes the given clone fs to be the parent
	*/
	static int
	zfs_do_promote(int argc, char **argv)
	{
	zfs_handle_t *zhp;
	int ret = 0;

	/* check options */
	if (argc > 1 && argv[1][0] == '-') {
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	argv[1][1]);
	usage(B_FALSE);
	}

	/* check number of arguments */
	if (argc < 2) {
	(void) fprintf(stderr, gettext("missing clone filesystem"
	" argument\n"));
	usage(B_FALSE);
	}
	if (argc > 2) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	zhp = zfs_open(g_zfs, argv[1], ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME);
	if (zhp == NULL)
	return (1);

	ret = (zfs_promote(zhp) != 0);


	zfs_close(zhp);
	return (ret);
	}

	static int
	zfs_do_redact(int argc, char **argv)
	{
	char *snap = NULL;
	char *bookname = NULL;
	char **rsnaps = NULL;
	int numrsnaps = 0;
	argv++;
	argc--;
	if (argc < 3) {
	(void) fprintf(stderr, gettext("too few arguments\n"));
	usage(B_FALSE);
	}

	snap = argv[0];
	bookname = argv[1];
	rsnaps = argv + 2;
	numrsnaps = argc - 2;

	nvlist_t *rsnapnv = fnvlist_alloc();

	for (int i = 0; i < numrsnaps; i++) {
	fnvlist_add_boolean(rsnapnv, rsnaps[i]);
	}

	int err = lzc_redact(snap, bookname, rsnapnv);
	fnvlist_free(rsnapnv);

	switch (err) {
	case 0:
	break;
	case ENOENT: {
	zfs_handle_t *zhp = zfs_open(g_zfs, snap, ZFS_TYPE_SNAPSHOT);
	if (zhp == NULL) {
	(void) fprintf(stderr, gettext("provided snapshot %s "
	"does not exist\n"), snap);
	} else {
	zfs_close(zhp);
	}
	for (int i = 0; i < numrsnaps; i++) {
	zhp = zfs_open(g_zfs, rsnaps[i], ZFS_TYPE_SNAPSHOT);
	if (zhp == NULL) {
	(void) fprintf(stderr, gettext("provided "
	"snapshot %s does not exist\n"), rsnaps[i]);
	} else {
	zfs_close(zhp);
	}
	}
	break;
	}
	case EEXIST:
	(void) fprintf(stderr, gettext("specified redaction bookmark "
	"(%s) provided already exists\n"), bookname);
	break;
	case ENAMETOOLONG:
	(void) fprintf(stderr, gettext("provided bookmark name cannot "
	"be used, final name would be too long\n"));
	break;
	case E2BIG:
	(void) fprintf(stderr, gettext("too many redaction snapshots "
	"specified\n"));
	break;
	case EINVAL:
	if (strchr(bookname, '#') != NULL)
	(void) fprintf(stderr, gettext(
	"redaction bookmark name must not contain '#'\n"));
	else
	(void) fprintf(stderr, gettext(
	"redaction snapshot must be descendent of "
	"snapshot being redacted\n"));
	break;
	case EALREADY:
	(void) fprintf(stderr, gettext("attempted to redact redacted "
	"dataset or with respect to redacted dataset\n"));
	break;
	case ENOTSUP:
	(void) fprintf(stderr, gettext("redaction bookmarks feature "
	"not enabled\n"));
	break;
	case EXDEV:
	(void) fprintf(stderr, gettext("potentially invalid redaction "
	"snapshot; full dataset names required\n"));
	break;
	default:
	(void) fprintf(stderr, gettext("internal error: %s\n"),
	strerror(errno));
	}

	return (err);
	}

	/*
	* zfs rollback [-rRf] <snapshot>
	*
	* -r Delete any intervening snapshots before doing rollback
	* -R Delete any snapshots and their clones
	* -f ignored for backwards compatibility
	*
	* Given a filesystem, rollback to a specific snapshot, discarding any changes
	* since then and making it the active dataset. If more recent snapshots exist,
	* the command will complain unless the '-r' flag is given.
	*/
	typedef struct rollback_cbdata {
	uint64_t cb_create;
	uint8_t cb_younger_ds_printed;
	boolean_t cb_first;
	int cb_doclones;
	char *cb_target;
	int cb_error;
	boolean_t cb_recurse;
	} rollback_cbdata_t;

	static int
	rollback_check_dependent(zfs_handle_t zhp, void data)
	{
	rollback_cbdata_t *cbp = data;

	if (cbp->cb_first && cbp->cb_recurse) {
	(void) fprintf(stderr, gettext("cannot rollback to "
	"'%s': clones of previous snapshots exist\n"),
	cbp->cb_target);
	(void) fprintf(stderr, gettext("use '-R' to "
	"force deletion of the following clones and "
	"dependents:\n"));
	cbp->cb_first = 0;
	cbp->cb_error = 1;
	}

	(void) fprintf(stderr, "%s\n", zfs_get_name(zhp));

	zfs_close(zhp);
	return (0);
	}


	/*
	* Report some snapshots/bookmarks more recent than the one specified.
	* Used when '-r' is not specified. We reuse this same callback for the
	* snapshot dependents - if 'cb_dependent' is set, then this is a
	* dependent and we should report it without checking the transaction group.
	*/
	static int
	rollback_check(zfs_handle_t zhp, void data)
	{
	rollback_cbdata_t *cbp = data;
	/*
	* Max number of younger snapshots and/or bookmarks to display before
	* we stop the iteration.
	*/
	const uint8_t max_younger = 32;

	if (cbp->cb_doclones) {
	zfs_close(zhp);
	return (0);
	}

	if (zfs_prop_get_int(zhp, ZFS_PROP_CREATETXG) > cbp->cb_create) {
	if (cbp->cb_first && !cbp->cb_recurse) {
	(void) fprintf(stderr, gettext("cannot "
	"rollback to '%s': more recent snapshots "
	"or bookmarks exist\n"),
	cbp->cb_target);
	(void) fprintf(stderr, gettext("use '-r' to "
	"force deletion of the following "
	"snapshots and bookmarks:\n"));
	cbp->cb_first = 0;
	cbp->cb_error = 1;
	}

	if (cbp->cb_recurse) {
	if (zfs_iter_dependents_v2(zhp, 0, B_TRUE,
	rollback_check_dependent, cbp) != 0) {
	zfs_close(zhp);
	return (-1);
	}
	} else {
	(void) fprintf(stderr, "%s\n",
	zfs_get_name(zhp));
	cbp->cb_younger_ds_printed++;
	}
	}
	zfs_close(zhp);

	if (cbp->cb_younger_ds_printed == max_younger) {
	/*
	* This non-recursive rollback is going to fail due to the
	* presence of snapshots and/or bookmarks that are younger than
	* the rollback target.
	* We printed some of the offending objects, now we stop
	* zfs_iter_snapshot/bookmark iteration so we can fail fast and
	* avoid iterating over the rest of the younger objects
	*/
	(void) fprintf(stderr, gettext("Output limited to %d "
	"snapshots/bookmarks\n"), max_younger);
	return (-1);
	}
	return (0);
	}

	static int
	zfs_do_rollback(int argc, char **argv)
	{
	int ret = 0;
	int c;
	boolean_t force = B_FALSE;
	rollback_cbdata_t cb = { 0 };
	zfs_handle_t zhp, snap;
	char parentname[ZFS_MAX_DATASET_NAME_LEN];
	char *delim;
	uint64_t min_txg = 0;

	/* check options */
	while ((c = getopt(argc, argv, "rRf")) != -1) {
	switch (c) {
	case 'r':
	cb.cb_recurse = 1;
	break;
	case 'R':
	cb.cb_recurse = 1;
	cb.cb_doclones = 1;
	break;
	case 'f':
	force = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* check number of arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing dataset argument\n"));
	usage(B_FALSE);
	}
	if (argc > 1) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	/* open the snapshot */
	if ((snap = zfs_open(g_zfs, argv[0], ZFS_TYPE_SNAPSHOT)) == NULL)
	return (1);

	/* open the parent dataset */
	(void) strlcpy(parentname, argv[0], sizeof (parentname));
	verify((delim = strrchr(parentname, '@')) != NULL);
	*delim = '\0';
	if ((zhp = zfs_open(g_zfs, parentname, ZFS_TYPE_DATASET)) == NULL) {
	zfs_close(snap);
	return (1);
	}

	/*
	* Check for more recent snapshots and/or clones based on the presence
	* of '-r' and '-R'.
	*/
	cb.cb_target = argv[0];
	cb.cb_create = zfs_prop_get_int(snap, ZFS_PROP_CREATETXG);
	cb.cb_first = B_TRUE;
	cb.cb_error = 0;

	if (cb.cb_create > 0)
	min_txg = cb.cb_create;

	if ((ret = zfs_iter_snapshots_v2(zhp, 0, rollback_check, &cb,
	min_txg, 0)) != 0)
	goto out;
	if ((ret = zfs_iter_bookmarks_v2(zhp, 0, rollback_check, &cb)) != 0)
	goto out;

	if ((ret = cb.cb_error) != 0)
	goto out;

	/*
	* Rollback parent to the given snapshot.
	*/
	ret = zfs_rollback(zhp, snap, force);

	out:
	zfs_close(snap);
	zfs_close(zhp);

	if (ret == 0)
	return (0);
	else
	return (1);
	}

	/*
	* zfs set property=value ... { fs \| snap \| vol } ...
	*
	* Sets the given properties for all datasets specified on the command line.
	*/

	static int
	set_callback(zfs_handle_t zhp, void data)
	{
	zprop_set_cbdata_t *cb = data;
	int ret = zfs_prop_set_list_flags(zhp, cb->cb_proplist, cb->cb_flags);

	if (ret != 0 \|\| libzfs_errno(g_zfs) != EZFS_SUCCESS) {
	switch (libzfs_errno(g_zfs)) {
	case EZFS_MOUNTFAILED:
	(void) fprintf(stderr, gettext("property may be set "
	"but unable to remount filesystem\n"));
	break;
	case EZFS_SHARENFSFAILED:
	(void) fprintf(stderr, gettext("property may be set "
	"but unable to reshare filesystem\n"));
	break;
	}
	}
	return (ret);
	}

	static int
	zfs_do_set(int argc, char **argv)
	{
	zprop_set_cbdata_t cb = { 0 };
	int ds_start = -1; /* argv idx of first dataset arg */
	int ret = 0;
	int i, c;

	/* check options */
	while ((c = getopt(argc, argv, "u")) != -1) {
	switch (c) {
	case 'u':
	cb.cb_flags \|= ZFS_SET_NOMOUNT;
	break;
	case '?':
	default:
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* check number of arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing arguments\n"));
	usage(B_FALSE);
	}
	if (argc < 2) {
	if (strchr(argv[0], '=') == NULL) {
	(void) fprintf(stderr, gettext("missing property=value "
	"argument(s)\n"));
	} else {
	(void) fprintf(stderr, gettext("missing dataset "
	"name(s)\n"));
	}
	usage(B_FALSE);
	}

	/* validate argument order: prop=val args followed by dataset args */
	for (i = 0; i < argc; i++) {
	if (strchr(argv[i], '=') != NULL) {
	if (ds_start > 0) {
	/* out-of-order prop=val argument */
	(void) fprintf(stderr, gettext("invalid "
	"argument order\n"));
	usage(B_FALSE);
	}
	} else if (ds_start < 0) {
	ds_start = i;
	}
	}
	if (ds_start < 0) {
	(void) fprintf(stderr, gettext("missing dataset name(s)\n"));
	usage(B_FALSE);
	}

	/* Populate a list of property settings */
	if (nvlist_alloc(&cb.cb_proplist, NV_UNIQUE_NAME, 0) != 0)
	nomem();
	for (i = 0; i < ds_start; i++) {
	if (!parseprop(cb.cb_proplist, argv[i])) {
	ret = -1;
	goto error;
	}
	}

	ret = zfs_for_each(argc - ds_start, argv + ds_start, 0,
	ZFS_TYPE_DATASET, NULL, NULL, 0, set_callback, &cb);

	error:
	nvlist_free(cb.cb_proplist);
	return (ret);
	}

	typedef struct snap_cbdata {
	nvlist_t *sd_nvl;
	boolean_t sd_recursive;
	const char *sd_snapname;
	} snap_cbdata_t;

	static int
	zfs_snapshot_cb(zfs_handle_t zhp, void arg)
	{
	snap_cbdata_t *sd = arg;
	char *name;
	int rv = 0;
	int error;

	if (sd->sd_recursive &&
	zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT) != 0) {
	zfs_close(zhp);
	return (0);
	}

	error = asprintf(&name, "%s@%s", zfs_get_name(zhp), sd->sd_snapname);
	if (error == -1)
	nomem();
	fnvlist_add_boolean(sd->sd_nvl, name);
	free(name);

	if (sd->sd_recursive)
	rv = zfs_iter_filesystems_v2(zhp, 0, zfs_snapshot_cb, sd);
	zfs_close(zhp);
	return (rv);
	}

	/*
	* zfs snapshot [-r] [-o prop=value] ... <fs@snap>
	*
	* Creates a snapshot with the given name. While functionally equivalent to
	* 'zfs create', it is a separate command to differentiate intent.
	*/
	static int
	zfs_do_snapshot(int argc, char **argv)
	{
	int ret = 0;
	int c;
	nvlist_t *props;
	snap_cbdata_t sd = { 0 };
	boolean_t multiple_snaps = B_FALSE;

	if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0)
	nomem();
	if (nvlist_alloc(&sd.sd_nvl, NV_UNIQUE_NAME, 0) != 0)
	nomem();

	/* check options */
	while ((c = getopt(argc, argv, "ro:")) != -1) {
	switch (c) {
	case 'o':
	if (!parseprop(props, optarg)) {
	nvlist_free(sd.sd_nvl);
	nvlist_free(props);
	return (1);
	}
	break;
	case 'r':
	sd.sd_recursive = B_TRUE;
	multiple_snaps = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	goto usage;
	}
	}

	argc -= optind;
	argv += optind;

	/* check number of arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing snapshot argument\n"));
	goto usage;
	}

	if (argc > 1)
	multiple_snaps = B_TRUE;
	for (; argc > 0; argc--, argv++) {
	char *atp;
	zfs_handle_t *zhp;

	atp = strchr(argv[0], '@');
	if (atp == NULL)
	goto usage;
	*atp = '\0';
	sd.sd_snapname = atp + 1;
	zhp = zfs_open(g_zfs, argv[0],
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME);
	if (zhp == NULL)
	goto usage;
	if (zfs_snapshot_cb(zhp, &sd) != 0)
	goto usage;
	}

	ret = zfs_snapshot_nvl(g_zfs, sd.sd_nvl, props);
	nvlist_free(sd.sd_nvl);
	nvlist_free(props);
	if (ret != 0 && multiple_snaps)
	(void) fprintf(stderr, gettext("no snapshots were created\n"));
	return (ret != 0);

	usage:
	nvlist_free(sd.sd_nvl);
	nvlist_free(props);
	usage(B_FALSE);
	return (-1);
	}

	/*
	* Array of prefixes to exclude –
	* a linear search, even if executed for each dataset,
	* is plenty good enough.
	*/
	typedef struct zfs_send_exclude_arg {
	size_t count;
	const char **list;
	} zfs_send_exclude_arg_t;

	static boolean_t
	zfs_do_send_exclude(zfs_handle_t zhp, void context)
	{
	zfs_send_exclude_arg_t *excludes = context;
	const char *name = zfs_get_name(zhp);

	for (size_t i = 0; i < excludes->count; ++i) {
	size_t len = strlen(excludes->list[i]);
	if (strncmp(name, excludes->list[i], len) == 0 &&
	memchr("/@", name[len], sizeof ("/@")))
	return (B_FALSE);
	}

	return (B_TRUE);
	}

	/*
	* Send a backup stream to stdout.
	*/
	static int
	zfs_do_send(int argc, char **argv)
	{
	char *fromname = NULL;
	char *toname = NULL;
	char *resume_token = NULL;
	char *cp;
	zfs_handle_t *zhp;
	sendflags_t flags = { 0 };
	int c, err;
	nvlist_t *dbgnv = NULL;
	char *redactbook = NULL;
	zfs_send_exclude_arg_t excludes = { 0 };

	struct option long_options[] = {
	{"replicate", no_argument, NULL, 'R'},
	{"skip-missing", no_argument, NULL, 's'},
	{"redact", required_argument, NULL, 'd'},
	{"props", no_argument, NULL, 'p'},
	{"parsable", no_argument, NULL, 'P'},
	{"dedup", no_argument, NULL, 'D'},
	{"proctitle", no_argument, NULL, 'V'},
	{"verbose", no_argument, NULL, 'v'},
	{"dryrun", no_argument, NULL, 'n'},
	{"large-block", no_argument, NULL, 'L'},
	{"embed", no_argument, NULL, 'e'},
	{"resume", required_argument, NULL, 't'},
	{"compressed", no_argument, NULL, 'c'},
	{"raw", no_argument, NULL, 'w'},
	{"backup", no_argument, NULL, 'b'},
	{"holds", no_argument, NULL, 'h'},
	{"saved", no_argument, NULL, 'S'},
	{"exclude", required_argument, NULL, 'X'},
	{0, 0, 0, 0}
	};

	/* check options */
	while ((c = getopt_long(argc, argv, ":i:I:RsDpVvnPLeht:cwbd:SX:",
	long_options, NULL)) != -1) {
	switch (c) {
	case 'X':
	for (char *ds; (ds = strsep(&optarg, ",")) != NULL; ) {
	if (!zfs_name_valid(ds, ZFS_TYPE_DATASET) \|\|
	strchr(ds, '/') == NULL) {
	(void) fprintf(stderr, gettext("-X %s: "
	"not a valid non-root dataset name"
	".\n"), ds);
	usage(B_FALSE);
	}
	excludes.list = safe_realloc(excludes.list,
	sizeof (char ) (excludes.count + 1));
	excludes.list[excludes.count++] = ds;
	}
	break;
	case 'i':
	if (fromname)
	usage(B_FALSE);
	fromname = optarg;
	break;
	case 'I':
	if (fromname)
	usage(B_FALSE);
	fromname = optarg;
	flags.doall = B_TRUE;
	break;
	case 'R':
	flags.replicate = B_TRUE;
	break;
	case 's':
	flags.skipmissing = B_TRUE;
	break;
	case 'd':
	redactbook = optarg;
	break;
	case 'p':
	flags.props = B_TRUE;
	break;
	case 'b':
	flags.backup = B_TRUE;
	break;
	case 'h':
	flags.holds = B_TRUE;
	break;
	case 'P':
	flags.parsable = B_TRUE;
	break;
	case 'V':
	flags.progressastitle = B_TRUE;
	break;
	case 'v':
	flags.verbosity++;
	flags.progress = B_TRUE;
	break;
	case 'D':
	(void) fprintf(stderr,
	gettext("WARNING: deduplicated send is no "
	"longer supported. A regular,\n"
	"non-deduplicated stream will be generated.\n\n"));
	break;
	case 'n':
	flags.dryrun = B_TRUE;
	break;
	case 'L':
	flags.largeblock = B_TRUE;
	break;
	case 'e':
	flags.embed_data = B_TRUE;
	break;
	case 't':
	resume_token = optarg;
	break;
	case 'c':
	flags.compress = B_TRUE;
	break;
	case 'w':
	flags.raw = B_TRUE;
	flags.compress = B_TRUE;
	flags.embed_data = B_TRUE;
	flags.largeblock = B_TRUE;
	break;
	case 'S':
	flags.saved = B_TRUE;
	break;
	case ':':
	/*
	* If a parameter was not passed, optopt contains the
	* value that would normally lead us into the
	* appropriate case statement. If it's > 256, then this
	* must be a longopt and we should look at argv to get
	* the string. Otherwise it's just the character, so we
	* should use it directly.
	*/
	if (optopt <= UINT8_MAX) {
	(void) fprintf(stderr,
	gettext("missing argument for '%c' "
	"option\n"), optopt);
	} else {
	(void) fprintf(stderr,
	gettext("missing argument for '%s' "
	"option\n"), argv[optind - 1]);
	}
	free(excludes.list);
	usage(B_FALSE);
	break;
	case '?':
	default:
	/*
	* If an invalid flag was passed, optopt contains the
	* character if it was a short flag, or 0 if it was a
	* longopt.
	*/
	if (optopt != 0) {
	(void) fprintf(stderr,
	gettext("invalid option '%c'\n"), optopt);
	} else {
	(void) fprintf(stderr,
	gettext("invalid option '%s'\n"),
	argv[optind - 1]);

	}
	free(excludes.list);
	usage(B_FALSE);
	}
	}

	if ((flags.parsable \|\| flags.progressastitle) && flags.verbosity == 0)
	flags.verbosity = 1;

	if (excludes.count > 0 && !flags.replicate) {
	free(excludes.list);
	(void) fprintf(stderr, gettext("Cannot specify "
	"dataset exclusion (-X) on a non-recursive "
	"send.\n"));
	return (1);
	}

	argc -= optind;
	argv += optind;

	if (resume_token != NULL) {
	if (fromname != NULL \|\| flags.replicate \|\| flags.props \|\|
	flags.backup \|\| flags.holds \|\|
	flags.saved \|\| redactbook != NULL) {
	free(excludes.list);
	(void) fprintf(stderr,
	gettext("invalid flags combined with -t\n"));
	usage(B_FALSE);
	}
	if (argc > 0) {
	free(excludes.list);
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}
	} else {
	if (argc < 1) {
	free(excludes.list);
	(void) fprintf(stderr,
	gettext("missing snapshot argument\n"));
	usage(B_FALSE);
	}
	if (argc > 1) {
	free(excludes.list);
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}
	}

	if (flags.saved) {
	if (fromname != NULL \|\| flags.replicate \|\| flags.props \|\|
	flags.doall \|\| flags.backup \|\|
	flags.holds \|\| flags.largeblock \|\| flags.embed_data \|\|
	flags.compress \|\| flags.raw \|\| redactbook != NULL) {
	free(excludes.list);

	(void) fprintf(stderr, gettext("incompatible flags "
	"combined with saved send flag\n"));
	usage(B_FALSE);
	}
	if (strchr(argv[0], '@') != NULL) {
	free(excludes.list);

	(void) fprintf(stderr, gettext("saved send must "
	"specify the dataset with partially-received "
	"state\n"));
	usage(B_FALSE);
	}
	}

	if (flags.raw && redactbook != NULL) {
	free(excludes.list);
	(void) fprintf(stderr,
	gettext("Error: raw sends may not be redacted.\n"));
	return (1);
	}

	if (!flags.dryrun && isatty(STDOUT_FILENO)) {
	free(excludes.list);
	(void) fprintf(stderr,
	gettext("Error: Stream can not be written to a terminal.\n"
	"You must redirect standard output.\n"));
	return (1);
	}

	if (flags.saved) {
	zhp = zfs_open(g_zfs, argv[0], ZFS_TYPE_DATASET);
	if (zhp == NULL) {
	free(excludes.list);
	return (1);
	}

	err = zfs_send_saved(zhp, &flags, STDOUT_FILENO,
	resume_token);
	free(excludes.list);
	zfs_close(zhp);
	return (err != 0);
	} else if (resume_token != NULL) {
	free(excludes.list);
	return (zfs_send_resume(g_zfs, &flags, STDOUT_FILENO,
	resume_token));
	}

	if (flags.skipmissing && !flags.replicate) {
	free(excludes.list);
	(void) fprintf(stderr,
	gettext("skip-missing flag can only be used in "
	"conjunction with replicate\n"));
	usage(B_FALSE);
	}

	/*
	* For everything except -R and -I, use the new, cleaner code path.
	*/
	if (!(flags.replicate \|\| flags.doall)) {
	char frombuf[ZFS_MAX_DATASET_NAME_LEN];

	if (fromname != NULL && (strchr(fromname, '#') == NULL &&
	strchr(fromname, '@') == NULL)) {
	/*
	* Neither bookmark or snapshot was specified. Print a
	* warning, and assume snapshot.
	*/
	(void) fprintf(stderr, "Warning: incremental source "
	"didn't specify type, assuming snapshot. Use '@' "
	"or '#' prefix to avoid ambiguity.\n");
	(void) snprintf(frombuf, sizeof (frombuf), "@%s",
	fromname);
	fromname = frombuf;
	}
	if (fromname != NULL &&
	(fromname[0] == '#' \|\| fromname[0] == '@')) {
	/*
	* Incremental source name begins with # or @.
	* Default to same fs as target.
	*/
	char tmpbuf[ZFS_MAX_DATASET_NAME_LEN];
	(void) strlcpy(tmpbuf, fromname, sizeof (tmpbuf));
	(void) strlcpy(frombuf, argv[0], sizeof (frombuf));
	cp = strchr(frombuf, '@');
	if (cp != NULL)
	*cp = '\0';
	(void) strlcat(frombuf, tmpbuf, sizeof (frombuf));
	fromname = frombuf;
	}

	zhp = zfs_open(g_zfs, argv[0], ZFS_TYPE_DATASET);
	if (zhp == NULL) {
	free(excludes.list);
	return (1);
	}
	err = zfs_send_one(zhp, fromname, STDOUT_FILENO, &flags,
	redactbook);

	free(excludes.list);
	zfs_close(zhp);
	return (err != 0);
	}

	if (fromname != NULL && strchr(fromname, '#')) {
	(void) fprintf(stderr,
	gettext("Error: multiple snapshots cannot be "
	"sent from a bookmark.\n"));
	free(excludes.list);
	return (1);
	}

	if (redactbook != NULL) {
	(void) fprintf(stderr, gettext("Error: multiple snapshots "
	"cannot be sent redacted.\n"));
	free(excludes.list);
	return (1);
	}

	if ((cp = strchr(argv[0], '@')) == NULL) {
	(void) fprintf(stderr, gettext("Error: "
	"Unsupported flag with filesystem or bookmark.\n"));
	free(excludes.list);
	return (1);
	}
	*cp = '\0';
	toname = cp + 1;
	zhp = zfs_open(g_zfs, argv[0], ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME);
	if (zhp == NULL) {
	free(excludes.list);
	return (1);
	}

	/*
	* If they specified the full path to the snapshot, chop off
	* everything except the short name of the snapshot, but special
	* case if they specify the origin.
	*/
	if (fromname && (cp = strchr(fromname, '@')) != NULL) {
	char origin[ZFS_MAX_DATASET_NAME_LEN];
	zprop_source_t src;

	(void) zfs_prop_get(zhp, ZFS_PROP_ORIGIN,
	origin, sizeof (origin), &src, NULL, 0, B_FALSE);

	if (strcmp(origin, fromname) == 0) {
	fromname = NULL;
	flags.fromorigin = B_TRUE;
	} else {
	*cp = '\0';
	if (cp != fromname && strcmp(argv[0], fromname)) {
	zfs_close(zhp);
	free(excludes.list);
	(void) fprintf(stderr,
	gettext("incremental source must be "
	"in same filesystem\n"));
	usage(B_FALSE);
	}
	fromname = cp + 1;
	if (strchr(fromname, '@') \|\| strchr(fromname, '/')) {
	zfs_close(zhp);
	free(excludes.list);
	(void) fprintf(stderr,
	gettext("invalid incremental source\n"));
	usage(B_FALSE);
	}
	}
	}

	if (flags.replicate && fromname == NULL)
	flags.doall = B_TRUE;

	err = zfs_send(zhp, fromname, toname, &flags, STDOUT_FILENO,
	excludes.count > 0 ? zfs_do_send_exclude : NULL,
	&excludes, flags.verbosity >= 3 ? &dbgnv : NULL);

	if (flags.verbosity >= 3 && dbgnv != NULL) {
	/*
	* dump_nvlist prints to stdout, but that's been
	* redirected to a file. Make it print to stderr
	* instead.
	*/
	(void) dup2(STDERR_FILENO, STDOUT_FILENO);
	dump_nvlist(dbgnv, 0);
	nvlist_free(dbgnv);
	}

	zfs_close(zhp);
	free(excludes.list);
	return (err != 0);
	}

	/*
	* Restore a backup stream from stdin.
	*/
	static int
	zfs_do_receive(int argc, char **argv)
	{
	int c, err = 0;
	recvflags_t flags = { 0 };
	boolean_t abort_resumable = B_FALSE;
	nvlist_t *props;

	if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0)
	nomem();

	/* check options */
	while ((c = getopt(argc, argv, ":o:x:dehMnuvFsAc")) != -1) {
	switch (c) {
	case 'o':
	if (!parseprop(props, optarg)) {
	nvlist_free(props);
	usage(B_FALSE);
	}
	break;
	case 'x':
	if (!parsepropname(props, optarg)) {
	nvlist_free(props);
	usage(B_FALSE);
	}
	break;
	case 'd':
	if (flags.istail) {
	(void) fprintf(stderr, gettext("invalid option "
	"combination: -d and -e are mutually "
	"exclusive\n"));
	usage(B_FALSE);
	}
	flags.isprefix = B_TRUE;
	break;
	case 'e':
	if (flags.isprefix) {
	(void) fprintf(stderr, gettext("invalid option "
	"combination: -d and -e are mutually "
	"exclusive\n"));
	usage(B_FALSE);
	}
	flags.istail = B_TRUE;
	break;
	case 'h':
	flags.skipholds = B_TRUE;
	break;
	case 'M':
	flags.forceunmount = B_TRUE;
	break;
	case 'n':
	flags.dryrun = B_TRUE;
	break;
	case 'u':
	flags.nomount = B_TRUE;
	break;
	case 'v':
	flags.verbose = B_TRUE;
	break;
	case 's':
	flags.resumable = B_TRUE;
	break;
	case 'F':
	flags.force = B_TRUE;
	break;
	case 'A':
	abort_resumable = B_TRUE;
	break;
	case 'c':
	flags.heal = B_TRUE;
	break;
	case ':':
	(void) fprintf(stderr, gettext("missing argument for "
	"'%c' option\n"), optopt);
	usage(B_FALSE);
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* zfs recv -e (use "tail" name) implies -d (remove dataset "head") */
	if (flags.istail)
	flags.isprefix = B_TRUE;

	/* check number of arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing snapshot argument\n"));
	usage(B_FALSE);
	}
	if (argc > 1) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	if (abort_resumable) {
	if (flags.isprefix \|\| flags.istail \|\| flags.dryrun \|\|
	flags.resumable \|\| flags.nomount) {
	(void) fprintf(stderr, gettext("invalid option\n"));
	usage(B_FALSE);
	}

	char namebuf[ZFS_MAX_DATASET_NAME_LEN];
	(void) snprintf(namebuf, sizeof (namebuf),
	"%s/%%recv", argv[0]);

	if (zfs_dataset_exists(g_zfs, namebuf,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME)) {
	zfs_handle_t *zhp = zfs_open(g_zfs,
	namebuf, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME);
	if (zhp == NULL) {
	nvlist_free(props);
	return (1);
	}
	err = zfs_destroy(zhp, B_FALSE);
	zfs_close(zhp);
	} else {
	zfs_handle_t *zhp = zfs_open(g_zfs,
	argv[0], ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME);
	if (zhp == NULL)
	usage(B_FALSE);
	if (!zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT) \|\|
	zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN,
	NULL, 0, NULL, NULL, 0, B_TRUE) == -1) {
	(void) fprintf(stderr,
	gettext("'%s' does not have any "
	"resumable receive state to abort\n"),
	argv[0]);
	nvlist_free(props);
	zfs_close(zhp);
	return (1);
	}
	err = zfs_destroy(zhp, B_FALSE);
	zfs_close(zhp);
	}
	nvlist_free(props);
	return (err != 0);
	}

	if (isatty(STDIN_FILENO)) {
	(void) fprintf(stderr,
	gettext("Error: Backup stream can not be read "
	"from a terminal.\n"
	"You must redirect standard input.\n"));
	nvlist_free(props);
	return (1);
	}
	err = zfs_receive(g_zfs, argv[0], props, &flags, STDIN_FILENO, NULL);
	nvlist_free(props);

	return (err != 0);
	}

	/*
	* allow/unallow stuff
	*/
	/* copied from zfs/sys/dsl_deleg.h */
	#define ZFS_DELEG_PERM_CREATE "create"
	#define ZFS_DELEG_PERM_DESTROY "destroy"
	#define ZFS_DELEG_PERM_SNAPSHOT "snapshot"
	#define ZFS_DELEG_PERM_ROLLBACK "rollback"
	#define ZFS_DELEG_PERM_CLONE "clone"
	#define ZFS_DELEG_PERM_PROMOTE "promote"
	#define ZFS_DELEG_PERM_RENAME "rename"
	#define ZFS_DELEG_PERM_MOUNT "mount"
	#define ZFS_DELEG_PERM_SHARE "share"
	#define ZFS_DELEG_PERM_SEND "send"
	#define ZFS_DELEG_PERM_RECEIVE "receive"
	#define ZFS_DELEG_PERM_ALLOW "allow"
	#define ZFS_DELEG_PERM_USERPROP "userprop"
	#define ZFS_DELEG_PERM_VSCAN "vscan" /* ??? */
	#define ZFS_DELEG_PERM_USERQUOTA "userquota"
	#define ZFS_DELEG_PERM_GROUPQUOTA "groupquota"
	#define ZFS_DELEG_PERM_USERUSED "userused"
	#define ZFS_DELEG_PERM_GROUPUSED "groupused"
	#define ZFS_DELEG_PERM_USEROBJQUOTA "userobjquota"
	#define ZFS_DELEG_PERM_GROUPOBJQUOTA "groupobjquota"
	#define ZFS_DELEG_PERM_USEROBJUSED "userobjused"
	#define ZFS_DELEG_PERM_GROUPOBJUSED "groupobjused"

	#define ZFS_DELEG_PERM_HOLD "hold"
	#define ZFS_DELEG_PERM_RELEASE "release"
	#define ZFS_DELEG_PERM_DIFF "diff"
	#define ZFS_DELEG_PERM_BOOKMARK "bookmark"
	#define ZFS_DELEG_PERM_LOAD_KEY "load-key"
	#define ZFS_DELEG_PERM_CHANGE_KEY "change-key"

	#define ZFS_DELEG_PERM_PROJECTUSED "projectused"
	#define ZFS_DELEG_PERM_PROJECTQUOTA "projectquota"
	#define ZFS_DELEG_PERM_PROJECTOBJUSED "projectobjused"
	#define ZFS_DELEG_PERM_PROJECTOBJQUOTA "projectobjquota"

	#define ZFS_NUM_DELEG_NOTES ZFS_DELEG_NOTE_NONE

	static zfs_deleg_perm_tab_t zfs_deleg_perm_tbl[] = {
	{ ZFS_DELEG_PERM_ALLOW, ZFS_DELEG_NOTE_ALLOW },
	{ ZFS_DELEG_PERM_CLONE, ZFS_DELEG_NOTE_CLONE },
	{ ZFS_DELEG_PERM_CREATE, ZFS_DELEG_NOTE_CREATE },
	{ ZFS_DELEG_PERM_DESTROY, ZFS_DELEG_NOTE_DESTROY },
	{ ZFS_DELEG_PERM_DIFF, ZFS_DELEG_NOTE_DIFF},
	{ ZFS_DELEG_PERM_HOLD, ZFS_DELEG_NOTE_HOLD },
	{ ZFS_DELEG_PERM_MOUNT, ZFS_DELEG_NOTE_MOUNT },
	{ ZFS_DELEG_PERM_PROMOTE, ZFS_DELEG_NOTE_PROMOTE },
	{ ZFS_DELEG_PERM_RECEIVE, ZFS_DELEG_NOTE_RECEIVE },
	{ ZFS_DELEG_PERM_RELEASE, ZFS_DELEG_NOTE_RELEASE },
	{ ZFS_DELEG_PERM_RENAME, ZFS_DELEG_NOTE_RENAME },
	{ ZFS_DELEG_PERM_ROLLBACK, ZFS_DELEG_NOTE_ROLLBACK },
	{ ZFS_DELEG_PERM_SEND, ZFS_DELEG_NOTE_SEND },
	{ ZFS_DELEG_PERM_SHARE, ZFS_DELEG_NOTE_SHARE },
	{ ZFS_DELEG_PERM_SNAPSHOT, ZFS_DELEG_NOTE_SNAPSHOT },
	{ ZFS_DELEG_PERM_BOOKMARK, ZFS_DELEG_NOTE_BOOKMARK },
	{ ZFS_DELEG_PERM_LOAD_KEY, ZFS_DELEG_NOTE_LOAD_KEY },
	{ ZFS_DELEG_PERM_CHANGE_KEY, ZFS_DELEG_NOTE_CHANGE_KEY },

	{ ZFS_DELEG_PERM_GROUPQUOTA, ZFS_DELEG_NOTE_GROUPQUOTA },
	{ ZFS_DELEG_PERM_GROUPUSED, ZFS_DELEG_NOTE_GROUPUSED },
	{ ZFS_DELEG_PERM_USERPROP, ZFS_DELEG_NOTE_USERPROP },
	{ ZFS_DELEG_PERM_USERQUOTA, ZFS_DELEG_NOTE_USERQUOTA },
	{ ZFS_DELEG_PERM_USERUSED, ZFS_DELEG_NOTE_USERUSED },
	{ ZFS_DELEG_PERM_USEROBJQUOTA, ZFS_DELEG_NOTE_USEROBJQUOTA },
	{ ZFS_DELEG_PERM_USEROBJUSED, ZFS_DELEG_NOTE_USEROBJUSED },
	{ ZFS_DELEG_PERM_GROUPOBJQUOTA, ZFS_DELEG_NOTE_GROUPOBJQUOTA },
	{ ZFS_DELEG_PERM_GROUPOBJUSED, ZFS_DELEG_NOTE_GROUPOBJUSED },
	{ ZFS_DELEG_PERM_PROJECTUSED, ZFS_DELEG_NOTE_PROJECTUSED },
	{ ZFS_DELEG_PERM_PROJECTQUOTA, ZFS_DELEG_NOTE_PROJECTQUOTA },
	{ ZFS_DELEG_PERM_PROJECTOBJUSED, ZFS_DELEG_NOTE_PROJECTOBJUSED },
	{ ZFS_DELEG_PERM_PROJECTOBJQUOTA, ZFS_DELEG_NOTE_PROJECTOBJQUOTA },
	{ NULL, ZFS_DELEG_NOTE_NONE }
	};

	/* permission structure */
	typedef struct deleg_perm {
	zfs_deleg_who_type_t dp_who_type;
	const char *dp_name;
	boolean_t dp_local;
	boolean_t dp_descend;
	} deleg_perm_t;

	/* */
	typedef struct deleg_perm_node {
	deleg_perm_t dpn_perm;

	uu_avl_node_t dpn_avl_node;
	} deleg_perm_node_t;

	typedef struct fs_perm fs_perm_t;

	/* permissions set */
	typedef struct who_perm {
	zfs_deleg_who_type_t who_type;
	const char who_name; / id */
	char who_ug_name[256]; /* user/group name */
	fs_perm_t who_fsperm; / uplink */

	uu_avl_t who_deleg_perm_avl; / permissions */
	} who_perm_t;

	/* */
	typedef struct who_perm_node {
	who_perm_t who_perm;
	uu_avl_node_t who_avl_node;
	} who_perm_node_t;

	typedef struct fs_perm_set fs_perm_set_t;
	/* fs permissions */
	struct fs_perm {
	const char *fsp_name;

	uu_avl_t fsp_sc_avl; / sets,create */
	uu_avl_t fsp_uge_avl; / user,group,everyone */

	fs_perm_set_t fsp_set; / uplink */
	};

	/* */
	typedef struct fs_perm_node {
	fs_perm_t fspn_fsperm;
	uu_avl_t *fspn_avl;

	uu_list_node_t fspn_list_node;
	} fs_perm_node_t;

	/* top level structure */
	struct fs_perm_set {
	uu_list_pool_t *fsps_list_pool;
	uu_list_t fsps_list; / list of fs_perms */

	uu_avl_pool_t *fsps_named_set_avl_pool;
	uu_avl_pool_t *fsps_who_perm_avl_pool;
	uu_avl_pool_t *fsps_deleg_perm_avl_pool;
	};

	static inline const char *
	deleg_perm_type(zfs_deleg_note_t note)
	{
	/* subcommands */
	switch (note) {
	/* SUBCOMMANDS */
	/* OTHER */
	case ZFS_DELEG_NOTE_GROUPQUOTA:
	case ZFS_DELEG_NOTE_GROUPUSED:
	case ZFS_DELEG_NOTE_USERPROP:
	case ZFS_DELEG_NOTE_USERQUOTA:
	case ZFS_DELEG_NOTE_USERUSED:
	case ZFS_DELEG_NOTE_USEROBJQUOTA:
	case ZFS_DELEG_NOTE_USEROBJUSED:
	case ZFS_DELEG_NOTE_GROUPOBJQUOTA:
	case ZFS_DELEG_NOTE_GROUPOBJUSED:
	case ZFS_DELEG_NOTE_PROJECTUSED:
	case ZFS_DELEG_NOTE_PROJECTQUOTA:
	case ZFS_DELEG_NOTE_PROJECTOBJUSED:
	case ZFS_DELEG_NOTE_PROJECTOBJQUOTA:
	/* other */
	return (gettext("other"));
	default:
	return (gettext("subcommand"));
	}
	}

	static int
	who_type2weight(zfs_deleg_who_type_t who_type)
	{
	int res;
	switch (who_type) {
	case ZFS_DELEG_NAMED_SET_SETS:
	case ZFS_DELEG_NAMED_SET:
	res = 0;
	break;
	case ZFS_DELEG_CREATE_SETS:
	case ZFS_DELEG_CREATE:
	res = 1;
	break;
	case ZFS_DELEG_USER_SETS:
	case ZFS_DELEG_USER:
	res = 2;
	break;
	case ZFS_DELEG_GROUP_SETS:
	case ZFS_DELEG_GROUP:
	res = 3;
	break;
	case ZFS_DELEG_EVERYONE_SETS:
	case ZFS_DELEG_EVERYONE:
	res = 4;
	break;
	default:
	res = -1;
	}

	return (res);
	}

	static int
	who_perm_compare(const void larg, const void rarg, void *unused)
	{
	(void) unused;
	const who_perm_node_t *l = larg;
	const who_perm_node_t *r = rarg;
	zfs_deleg_who_type_t ltype = l->who_perm.who_type;
	zfs_deleg_who_type_t rtype = r->who_perm.who_type;
	int lweight = who_type2weight(ltype);
	int rweight = who_type2weight(rtype);
	int res = lweight - rweight;
	if (res == 0)
	res = strncmp(l->who_perm.who_name, r->who_perm.who_name,
	ZFS_MAX_DELEG_NAME-1);

	if (res == 0)
	return (0);
	if (res > 0)
	return (1);
	else
	return (-1);
	}

	static int
	deleg_perm_compare(const void larg, const void rarg, void *unused)
	{
	(void) unused;
	const deleg_perm_node_t *l = larg;
	const deleg_perm_node_t *r = rarg;
	int res = strncmp(l->dpn_perm.dp_name, r->dpn_perm.dp_name,
	ZFS_MAX_DELEG_NAME-1);

	if (res == 0)
	return (0);

	if (res > 0)
	return (1);
	else
	return (-1);
	}

	static inline void
	fs_perm_set_init(fs_perm_set_t *fspset)
	{
	memset(fspset, 0, sizeof (fs_perm_set_t));

	if ((fspset->fsps_list_pool = uu_list_pool_create("fsps_list_pool",
	sizeof (fs_perm_node_t), offsetof(fs_perm_node_t, fspn_list_node),
	NULL, UU_DEFAULT)) == NULL)
	nomem();
	if ((fspset->fsps_list = uu_list_create(fspset->fsps_list_pool, NULL,
	UU_DEFAULT)) == NULL)
	nomem();

	if ((fspset->fsps_named_set_avl_pool = uu_avl_pool_create(
	"named_set_avl_pool", sizeof (who_perm_node_t), offsetof(
	who_perm_node_t, who_avl_node), who_perm_compare,
	UU_DEFAULT)) == NULL)
	nomem();

	if ((fspset->fsps_who_perm_avl_pool = uu_avl_pool_create(
	"who_perm_avl_pool", sizeof (who_perm_node_t), offsetof(
	who_perm_node_t, who_avl_node), who_perm_compare,
	UU_DEFAULT)) == NULL)
	nomem();

	if ((fspset->fsps_deleg_perm_avl_pool = uu_avl_pool_create(
	"deleg_perm_avl_pool", sizeof (deleg_perm_node_t), offsetof(
	deleg_perm_node_t, dpn_avl_node), deleg_perm_compare, UU_DEFAULT))
	== NULL)
	nomem();
	}

	static inline void fs_perm_fini(fs_perm_t *);
	static inline void who_perm_fini(who_perm_t *);

	static inline void
	fs_perm_set_fini(fs_perm_set_t *fspset)
	{
	fs_perm_node_t *node = uu_list_first(fspset->fsps_list);

	while (node != NULL) {
	fs_perm_node_t *next_node =
	uu_list_next(fspset->fsps_list, node);
	fs_perm_t *fsperm = &node->fspn_fsperm;
	fs_perm_fini(fsperm);
	uu_list_remove(fspset->fsps_list, node);
	free(node);
	node = next_node;
	}

	uu_avl_pool_destroy(fspset->fsps_named_set_avl_pool);
	uu_avl_pool_destroy(fspset->fsps_who_perm_avl_pool);
	uu_avl_pool_destroy(fspset->fsps_deleg_perm_avl_pool);
	}

	static inline void
	deleg_perm_init(deleg_perm_t *deleg_perm, zfs_deleg_who_type_t type,
	const char *name)
	{
	deleg_perm->dp_who_type = type;
	deleg_perm->dp_name = name;
	}

	static inline void
	who_perm_init(who_perm_t who_perm, fs_perm_t fsperm,
	zfs_deleg_who_type_t type, const char *name)
	{
	uu_avl_pool_t *pool;
	pool = fsperm->fsp_set->fsps_deleg_perm_avl_pool;

	memset(who_perm, 0, sizeof (who_perm_t));

	if ((who_perm->who_deleg_perm_avl = uu_avl_create(pool, NULL,
	UU_DEFAULT)) == NULL)
	nomem();

	who_perm->who_type = type;
	who_perm->who_name = name;
	who_perm->who_fsperm = fsperm;
	}

	static inline void
	who_perm_fini(who_perm_t *who_perm)
	{
	deleg_perm_node_t *node = uu_avl_first(who_perm->who_deleg_perm_avl);

	while (node != NULL) {
	deleg_perm_node_t *next_node =
	uu_avl_next(who_perm->who_deleg_perm_avl, node);

	uu_avl_remove(who_perm->who_deleg_perm_avl, node);
	free(node);
	node = next_node;
	}

	uu_avl_destroy(who_perm->who_deleg_perm_avl);
	}

	static inline void
	fs_perm_init(fs_perm_t fsperm, fs_perm_set_t fspset, const char *fsname)
	{
	uu_avl_pool_t *nset_pool = fspset->fsps_named_set_avl_pool;
	uu_avl_pool_t *who_pool = fspset->fsps_who_perm_avl_pool;

	memset(fsperm, 0, sizeof (fs_perm_t));

	if ((fsperm->fsp_sc_avl = uu_avl_create(nset_pool, NULL, UU_DEFAULT))
	== NULL)
	nomem();

	if ((fsperm->fsp_uge_avl = uu_avl_create(who_pool, NULL, UU_DEFAULT))
	== NULL)
	nomem();

	fsperm->fsp_set = fspset;
	fsperm->fsp_name = fsname;
	}

	static inline void
	fs_perm_fini(fs_perm_t *fsperm)
	{
	who_perm_node_t *node = uu_avl_first(fsperm->fsp_sc_avl);
	while (node != NULL) {
	who_perm_node_t *next_node = uu_avl_next(fsperm->fsp_sc_avl,
	node);
	who_perm_t *who_perm = &node->who_perm;
	who_perm_fini(who_perm);
	uu_avl_remove(fsperm->fsp_sc_avl, node);
	free(node);
	node = next_node;
	}

	node = uu_avl_first(fsperm->fsp_uge_avl);
	while (node != NULL) {
	who_perm_node_t *next_node = uu_avl_next(fsperm->fsp_uge_avl,
	node);
	who_perm_t *who_perm = &node->who_perm;
	who_perm_fini(who_perm);
	uu_avl_remove(fsperm->fsp_uge_avl, node);
	free(node);
	node = next_node;
	}

	uu_avl_destroy(fsperm->fsp_sc_avl);
	uu_avl_destroy(fsperm->fsp_uge_avl);
	}

	static void
	set_deleg_perm_node(uu_avl_t avl, deleg_perm_node_t node,
	zfs_deleg_who_type_t who_type, const char *name, char locality)
	{
	uu_avl_index_t idx = 0;

	deleg_perm_node_t *found_node = NULL;
	deleg_perm_t *deleg_perm = &node->dpn_perm;

	deleg_perm_init(deleg_perm, who_type, name);

	if ((found_node = uu_avl_find(avl, node, NULL, &idx))
	== NULL)
	uu_avl_insert(avl, node, idx);
	else {
	node = found_node;
	deleg_perm = &node->dpn_perm;
	}


	switch (locality) {
	case ZFS_DELEG_LOCAL:
	deleg_perm->dp_local = B_TRUE;
	break;
	case ZFS_DELEG_DESCENDENT:
	deleg_perm->dp_descend = B_TRUE;
	break;
	case ZFS_DELEG_NA:
	break;
	default:
	assert(B_FALSE); /* invalid locality */
	}
	}

	static inline int
	parse_who_perm(who_perm_t who_perm, nvlist_t nvl, char locality)
	{
	nvpair_t *nvp = NULL;
	fs_perm_set_t *fspset = who_perm->who_fsperm->fsp_set;
	uu_avl_t *avl = who_perm->who_deleg_perm_avl;
	zfs_deleg_who_type_t who_type = who_perm->who_type;

	while ((nvp = nvlist_next_nvpair(nvl, nvp)) != NULL) {
	const char *name = nvpair_name(nvp);
	data_type_t type = nvpair_type(nvp);
	uu_avl_pool_t *avl_pool = fspset->fsps_deleg_perm_avl_pool;
	deleg_perm_node_t *node =
	safe_malloc(sizeof (deleg_perm_node_t));

	VERIFY(type == DATA_TYPE_BOOLEAN);

	uu_avl_node_init(node, &node->dpn_avl_node, avl_pool);
	set_deleg_perm_node(avl, node, who_type, name, locality);
	}

	return (0);
	}

	static inline int
	parse_fs_perm(fs_perm_t fsperm, nvlist_t nvl)
	{
	nvpair_t *nvp = NULL;
	fs_perm_set_t *fspset = fsperm->fsp_set;

	while ((nvp = nvlist_next_nvpair(nvl, nvp)) != NULL) {
	nvlist_t *nvl2 = NULL;
	const char *name = nvpair_name(nvp);
	uu_avl_t *avl = NULL;
	uu_avl_pool_t *avl_pool = NULL;
	zfs_deleg_who_type_t perm_type = name[0];
	char perm_locality = name[1];
	const char *perm_name = name + 3;
	who_perm_t *who_perm = NULL;

	assert('$' == name[2]);

	if (nvpair_value_nvlist(nvp, &nvl2) != 0)
	return (-1);

	switch (perm_type) {
	case ZFS_DELEG_CREATE:
	case ZFS_DELEG_CREATE_SETS:
	case ZFS_DELEG_NAMED_SET:
	case ZFS_DELEG_NAMED_SET_SETS:
	avl_pool = fspset->fsps_named_set_avl_pool;
	avl = fsperm->fsp_sc_avl;
	break;
	case ZFS_DELEG_USER:
	case ZFS_DELEG_USER_SETS:
	case ZFS_DELEG_GROUP:
	case ZFS_DELEG_GROUP_SETS:
	case ZFS_DELEG_EVERYONE:
	case ZFS_DELEG_EVERYONE_SETS:
	avl_pool = fspset->fsps_who_perm_avl_pool;
	avl = fsperm->fsp_uge_avl;
	break;

	default:
	assert(!"unhandled zfs_deleg_who_type_t");
	}

	who_perm_node_t *found_node = NULL;
	who_perm_node_t *node = safe_malloc(
	sizeof (who_perm_node_t));
	who_perm = &node->who_perm;
	uu_avl_index_t idx = 0;

	uu_avl_node_init(node, &node->who_avl_node, avl_pool);
	who_perm_init(who_perm, fsperm, perm_type, perm_name);

	if ((found_node = uu_avl_find(avl, node, NULL, &idx))
	== NULL) {
	if (avl == fsperm->fsp_uge_avl) {
	uid_t rid = 0;
	struct passwd *p = NULL;
	struct group *g = NULL;
	const char *nice_name = NULL;

	switch (perm_type) {
	case ZFS_DELEG_USER_SETS:
	case ZFS_DELEG_USER:
	rid = atoi(perm_name);
	p = getpwuid(rid);
	if (p)
	nice_name = p->pw_name;
	break;
	case ZFS_DELEG_GROUP_SETS:
	case ZFS_DELEG_GROUP:
	rid = atoi(perm_name);
	g = getgrgid(rid);
	if (g)
	nice_name = g->gr_name;
	break;

	default:
	break;
	}

	if (nice_name != NULL) {
	(void) strlcpy(
	node->who_perm.who_ug_name,
	nice_name, 256);
	} else {
	/* User or group unknown */
	(void) snprintf(
	node->who_perm.who_ug_name,
	sizeof (node->who_perm.who_ug_name),
	"(unknown: %d)", rid);
	}
	}

	uu_avl_insert(avl, node, idx);
	} else {
	node = found_node;
	who_perm = &node->who_perm;
	}

	assert(who_perm != NULL);
	(void) parse_who_perm(who_perm, nvl2, perm_locality);
	}

	return (0);
	}

	static inline int
	parse_fs_perm_set(fs_perm_set_t fspset, nvlist_t nvl)
	{
	nvpair_t *nvp = NULL;
	uu_avl_index_t idx = 0;

	while ((nvp = nvlist_next_nvpair(nvl, nvp)) != NULL) {
	nvlist_t *nvl2 = NULL;
	const char *fsname = nvpair_name(nvp);
	data_type_t type = nvpair_type(nvp);
	fs_perm_t *fsperm = NULL;
	fs_perm_node_t *node = safe_malloc(sizeof (fs_perm_node_t));

	fsperm = &node->fspn_fsperm;

	VERIFY(DATA_TYPE_NVLIST == type);

	uu_list_node_init(node, &node->fspn_list_node,
	fspset->fsps_list_pool);

	idx = uu_list_numnodes(fspset->fsps_list);
	fs_perm_init(fsperm, fspset, fsname);

	if (nvpair_value_nvlist(nvp, &nvl2) != 0)
	return (-1);

	(void) parse_fs_perm(fsperm, nvl2);

	uu_list_insert(fspset->fsps_list, node, idx);
	}

	return (0);
	}

	static inline const char *
	deleg_perm_comment(zfs_deleg_note_t note)
	{
	const char *str = "";

	/* subcommands */
	switch (note) {
	/* SUBCOMMANDS */
	case ZFS_DELEG_NOTE_ALLOW:
	str = gettext("Must also have the permission that is being"
	"\n\t\t\t\tallowed");
	break;
	case ZFS_DELEG_NOTE_CLONE:
	str = gettext("Must also have the 'create' ability and 'mount'"
	"\n\t\t\t\tability in the origin file system");
	break;
	case ZFS_DELEG_NOTE_CREATE:
	str = gettext("Must also have the 'mount' ability");
	break;
	case ZFS_DELEG_NOTE_DESTROY:
	str = gettext("Must also have the 'mount' ability");
	break;
	case ZFS_DELEG_NOTE_DIFF:
	str = gettext("Allows lookup of paths within a dataset;"
	"\n\t\t\t\tgiven an object number. Ordinary users need this"
	"\n\t\t\t\tin order to use zfs diff");
	break;
	case ZFS_DELEG_NOTE_HOLD:
	str = gettext("Allows adding a user hold to a snapshot");
	break;
	case ZFS_DELEG_NOTE_MOUNT:
	str = gettext("Allows mount/umount of ZFS datasets");
	break;
	case ZFS_DELEG_NOTE_PROMOTE:
	str = gettext("Must also have the 'mount'\n\t\t\t\tand"
	" 'promote' ability in the origin file system");
	break;
	case ZFS_DELEG_NOTE_RECEIVE:
	str = gettext("Must also have the 'mount' and 'create'"
	" ability");
	break;
	case ZFS_DELEG_NOTE_RELEASE:
	str = gettext("Allows releasing a user hold which\n\t\t\t\t"
	"might destroy the snapshot");
	break;
	case ZFS_DELEG_NOTE_RENAME:
	str = gettext("Must also have the 'mount' and 'create'"
	"\n\t\t\t\tability in the new parent");
	break;
	case ZFS_DELEG_NOTE_ROLLBACK:
	str = gettext("");
	break;
	case ZFS_DELEG_NOTE_SEND:
	str = gettext("");
	break;
	case ZFS_DELEG_NOTE_SHARE:
	str = gettext("Allows sharing file systems over NFS or SMB"
	"\n\t\t\t\tprotocols");
	break;
	case ZFS_DELEG_NOTE_SNAPSHOT:
	str = gettext("");
	break;
	case ZFS_DELEG_NOTE_LOAD_KEY:
	str = gettext("Allows loading or unloading an encryption key");
	break;
	case ZFS_DELEG_NOTE_CHANGE_KEY:
	str = gettext("Allows changing or adding an encryption key");
	break;
	/*
	* case ZFS_DELEG_NOTE_VSCAN:
	* str = gettext("");
	* break;
	*/
	/* OTHER */
	case ZFS_DELEG_NOTE_GROUPQUOTA:
	str = gettext("Allows accessing any groupquota@... property");
	break;
	case ZFS_DELEG_NOTE_GROUPUSED:
	str = gettext("Allows reading any groupused@... property");
	break;
	case ZFS_DELEG_NOTE_USERPROP:
	str = gettext("Allows changing any user property");
	break;
	case ZFS_DELEG_NOTE_USERQUOTA:
	str = gettext("Allows accessing any userquota@... property");
	break;
	case ZFS_DELEG_NOTE_USERUSED:
	str = gettext("Allows reading any userused@... property");
	break;
	case ZFS_DELEG_NOTE_USEROBJQUOTA:
	str = gettext("Allows accessing any userobjquota@... property");
	break;
	case ZFS_DELEG_NOTE_GROUPOBJQUOTA:
	str = gettext("Allows accessing any \n\t\t\t\t"
	"groupobjquota@... property");
	break;
	case ZFS_DELEG_NOTE_GROUPOBJUSED:
	str = gettext("Allows reading any groupobjused@... property");
	break;
	case ZFS_DELEG_NOTE_USEROBJUSED:
	str = gettext("Allows reading any userobjused@... property");
	break;
	case ZFS_DELEG_NOTE_PROJECTQUOTA:
	str = gettext("Allows accessing any projectquota@... property");
	break;
	case ZFS_DELEG_NOTE_PROJECTOBJQUOTA:
	str = gettext("Allows accessing any \n\t\t\t\t"
	"projectobjquota@... property");
	break;
	case ZFS_DELEG_NOTE_PROJECTUSED:
	str = gettext("Allows reading any projectused@... property");
	break;
	case ZFS_DELEG_NOTE_PROJECTOBJUSED:
	str = gettext("Allows accessing any \n\t\t\t\t"
	"projectobjused@... property");
	break;
	/* other */
	default:
	str = "";
	}

	return (str);
	}

	struct allow_opts {
	boolean_t local;
	boolean_t descend;
	boolean_t user;
	boolean_t group;
	boolean_t everyone;
	boolean_t create;
	boolean_t set;
	boolean_t recursive; /* unallow only */
	boolean_t prt_usage;

	boolean_t prt_perms;
	char *who;
	char *perms;
	const char *dataset;
	};

	static inline int
	prop_cmp(const void a, const void b)
	{
	const char str1 = (const char **)a;
	const char str2 = (const char **)b;
	return (strcmp(str1, str2));
	}

	static void
	allow_usage(boolean_t un, boolean_t requested, const char *msg)
	{
	const char *opt_desc[] = {
	"-h", gettext("show this help message and exit"),
	"-l", gettext("set permission locally"),
	"-d", gettext("set permission for descents"),
	"-u", gettext("set permission for user"),
	"-g", gettext("set permission for group"),
	"-e", gettext("set permission for everyone"),
	"-c", gettext("set create time permission"),
	"-s", gettext("define permission set"),
	/* unallow only */
	"-r", gettext("remove permissions recursively"),
	};
	size_t unallow_size = sizeof (opt_desc) / sizeof (char *);
	size_t allow_size = unallow_size - 2;
	const char *props[ZFS_NUM_PROPS];
	int i;
	size_t count = 0;
	FILE *fp = requested ? stdout : stderr;
	zprop_desc_t *pdtbl = zfs_prop_get_table();
	const char *fmt = gettext("%-16s %-14s\t%s\n");

	(void) fprintf(fp, gettext("Usage: %s\n"), get_usage(un ? HELP_UNALLOW :
	HELP_ALLOW));
	(void) fprintf(fp, gettext("Options:\n"));
	for (i = 0; i < (un ? unallow_size : allow_size); i += 2) {
	const char *opt = opt_desc[i];
	const char *optdsc = opt_desc[i + 1];
	(void) fprintf(fp, gettext(" %-10s %s\n"), opt, optdsc);
	}

	(void) fprintf(fp, gettext("\nThe following permissions are "
	"supported:\n\n"));
	(void) fprintf(fp, fmt, gettext("NAME"), gettext("TYPE"),
	gettext("NOTES"));
	for (i = 0; i < ZFS_NUM_DELEG_NOTES; i++) {
	const char *perm_name = zfs_deleg_perm_tbl[i].z_perm;
	zfs_deleg_note_t perm_note = zfs_deleg_perm_tbl[i].z_note;
	const char *perm_type = deleg_perm_type(perm_note);
	const char *perm_comment = deleg_perm_comment(perm_note);
	(void) fprintf(fp, fmt, perm_name, perm_type, perm_comment);
	}

	for (i = 0; i < ZFS_NUM_PROPS; i++) {
	zprop_desc_t *pd = &pdtbl[i];
	if (pd->pd_visible != B_TRUE)
	continue;

	if (pd->pd_attr == PROP_READONLY)
	continue;

	props[count++] = pd->pd_name;
	}
	props[count] = NULL;

	qsort(props, count, sizeof (char *), prop_cmp);

	for (i = 0; i < count; i++)
	(void) fprintf(fp, fmt, props[i], gettext("property"), "");

	if (msg != NULL)
	(void) fprintf(fp, gettext("\nzfs: error: %s"), msg);

	exit(requested ? 0 : 2);
	}

	static inline const char *
	munge_args(int argc, char **argv, boolean_t un, size_t expected_argc,
	char **permsp)
	{
	if (un && argc == expected_argc - 1)
	*permsp = NULL;
	else if (argc == expected_argc)
	*permsp = argv[argc - 2];
	else
	allow_usage(un, B_FALSE,
	gettext("wrong number of parameters\n"));

	return (argv[argc - 1]);
	}

	static void
	parse_allow_args(int argc, char *argv, boolean_t un, struct allow_opts opts)
	{
	int uge_sum = opts->user + opts->group + opts->everyone;
	int csuge_sum = opts->create + opts->set + uge_sum;
	int ldcsuge_sum = csuge_sum + opts->local + opts->descend;
	int all_sum = un ? ldcsuge_sum + opts->recursive : ldcsuge_sum;

	if (uge_sum > 1)
	allow_usage(un, B_FALSE,
	gettext("-u, -g, and -e are mutually exclusive\n"));

	if (opts->prt_usage) {
	if (argc == 0 && all_sum == 0)
	allow_usage(un, B_TRUE, NULL);
	else
	usage(B_FALSE);
	}

	if (opts->set) {
	if (csuge_sum > 1)
	allow_usage(un, B_FALSE,
	gettext("invalid options combined with -s\n"));

	opts->dataset = munge_args(argc, argv, un, 3, &opts->perms);
	if (argv[0][0] != '@')
	allow_usage(un, B_FALSE,
	gettext("invalid set name: missing '@' prefix\n"));
	opts->who = argv[0];
	} else if (opts->create) {
	if (ldcsuge_sum > 1)
	allow_usage(un, B_FALSE,
	gettext("invalid options combined with -c\n"));
	opts->dataset = munge_args(argc, argv, un, 2, &opts->perms);
	} else if (opts->everyone) {
	if (csuge_sum > 1)
	allow_usage(un, B_FALSE,
	gettext("invalid options combined with -e\n"));
	opts->dataset = munge_args(argc, argv, un, 2, &opts->perms);
	} else if (uge_sum == 0 && argc > 0 && strcmp(argv[0], "everyone")
	== 0) {
	opts->everyone = B_TRUE;
	argc--;
	argv++;
	opts->dataset = munge_args(argc, argv, un, 2, &opts->perms);
	} else if (argc == 1 && !un) {
	opts->prt_perms = B_TRUE;
	opts->dataset = argv[argc-1];
	} else {
	opts->dataset = munge_args(argc, argv, un, 3, &opts->perms);
	opts->who = argv[0];
	}

	if (!opts->local && !opts->descend) {
	opts->local = B_TRUE;
	opts->descend = B_TRUE;
	}
	}

	static void
	store_allow_perm(zfs_deleg_who_type_t type, boolean_t local, boolean_t descend,
	const char who, char perms, nvlist_t *top_nvl)
	{
	int i;
	char ld[2] = { '\0', '\0' };
	char who_buf[MAXNAMELEN + 32];
	char base_type = '\0';
	char set_type = '\0';
	nvlist_t *base_nvl = NULL;
	nvlist_t *set_nvl = NULL;
	nvlist_t *nvl;

	if (nvlist_alloc(&base_nvl, NV_UNIQUE_NAME, 0) != 0)
	nomem();
	if (nvlist_alloc(&set_nvl, NV_UNIQUE_NAME, 0) != 0)
	nomem();

	switch (type) {
	case ZFS_DELEG_NAMED_SET_SETS:
	case ZFS_DELEG_NAMED_SET:
	set_type = ZFS_DELEG_NAMED_SET_SETS;
	base_type = ZFS_DELEG_NAMED_SET;
	ld[0] = ZFS_DELEG_NA;
	break;
	case ZFS_DELEG_CREATE_SETS:
	case ZFS_DELEG_CREATE:
	set_type = ZFS_DELEG_CREATE_SETS;
	base_type = ZFS_DELEG_CREATE;
	ld[0] = ZFS_DELEG_NA;
	break;
	case ZFS_DELEG_USER_SETS:
	case ZFS_DELEG_USER:
	set_type = ZFS_DELEG_USER_SETS;
	base_type = ZFS_DELEG_USER;
	if (local)
	ld[0] = ZFS_DELEG_LOCAL;
	if (descend)
	ld[1] = ZFS_DELEG_DESCENDENT;
	break;
	case ZFS_DELEG_GROUP_SETS:
	case ZFS_DELEG_GROUP:
	set_type = ZFS_DELEG_GROUP_SETS;
	base_type = ZFS_DELEG_GROUP;
	if (local)
	ld[0] = ZFS_DELEG_LOCAL;
	if (descend)
	ld[1] = ZFS_DELEG_DESCENDENT;
	break;
	case ZFS_DELEG_EVERYONE_SETS:
	case ZFS_DELEG_EVERYONE:
	set_type = ZFS_DELEG_EVERYONE_SETS;
	base_type = ZFS_DELEG_EVERYONE;
	if (local)
	ld[0] = ZFS_DELEG_LOCAL;
	if (descend)
	ld[1] = ZFS_DELEG_DESCENDENT;
	break;

	default:
	assert(set_type != '\0' && base_type != '\0');
	}

	if (perms != NULL) {
	char *curr = perms;
	char *end = curr + strlen(perms);

	while (curr < end) {
	char *delim = strchr(curr, ',');
	if (delim == NULL)
	delim = end;
	else
	*delim = '\0';

	if (curr[0] == '@')
	nvl = set_nvl;
	else
	nvl = base_nvl;

	(void) nvlist_add_boolean(nvl, curr);
	if (delim != end)
	*delim = ',';
	curr = delim + 1;
	}

	for (i = 0; i < 2; i++) {
	char locality = ld[i];
	if (locality == 0)
	continue;

	if (!nvlist_empty(base_nvl)) {
	if (who != NULL)
	(void) snprintf(who_buf,
	sizeof (who_buf), "%c%c$%s",
	base_type, locality, who);
	else
	(void) snprintf(who_buf,
	sizeof (who_buf), "%c%c$",
	base_type, locality);

	(void) nvlist_add_nvlist(top_nvl, who_buf,
	base_nvl);
	}


	if (!nvlist_empty(set_nvl)) {
	if (who != NULL)
	(void) snprintf(who_buf,
	sizeof (who_buf), "%c%c$%s",
	set_type, locality, who);
	else
	(void) snprintf(who_buf,
	sizeof (who_buf), "%c%c$",
	set_type, locality);

	(void) nvlist_add_nvlist(top_nvl, who_buf,
	set_nvl);
	}
	}
	} else {
	for (i = 0; i < 2; i++) {
	char locality = ld[i];
	if (locality == 0)
	continue;

	if (who != NULL)
	(void) snprintf(who_buf, sizeof (who_buf),
	"%c%c$%s", base_type, locality, who);
	else
	(void) snprintf(who_buf, sizeof (who_buf),
	"%c%c$", base_type, locality);
	(void) nvlist_add_boolean(top_nvl, who_buf);

	if (who != NULL)
	(void) snprintf(who_buf, sizeof (who_buf),
	"%c%c$%s", set_type, locality, who);
	else
	(void) snprintf(who_buf, sizeof (who_buf),
	"%c%c$", set_type, locality);
	(void) nvlist_add_boolean(top_nvl, who_buf);
	}
	}
	}

	static int
	construct_fsacl_list(boolean_t un, struct allow_opts opts, nvlist_t *nvlp)
	{
	if (nvlist_alloc(nvlp, NV_UNIQUE_NAME, 0) != 0)
	nomem();

	if (opts->set) {
	store_allow_perm(ZFS_DELEG_NAMED_SET, opts->local,
	opts->descend, opts->who, opts->perms, *nvlp);
	} else if (opts->create) {
	store_allow_perm(ZFS_DELEG_CREATE, opts->local,
	opts->descend, NULL, opts->perms, *nvlp);
	} else if (opts->everyone) {
	store_allow_perm(ZFS_DELEG_EVERYONE, opts->local,
	opts->descend, NULL, opts->perms, *nvlp);
	} else {
	char *curr = opts->who;
	char *end = curr + strlen(curr);

	while (curr < end) {
	const char *who;
	zfs_deleg_who_type_t who_type = ZFS_DELEG_WHO_UNKNOWN;
	char *endch;
	char *delim = strchr(curr, ',');
	char errbuf[256];
	char id[64];
	struct passwd *p = NULL;
	struct group *g = NULL;

	uid_t rid;
	if (delim == NULL)
	delim = end;
	else
	*delim = '\0';

	rid = (uid_t)strtol(curr, &endch, 0);
	if (opts->user) {
	who_type = ZFS_DELEG_USER;
	if (*endch != '\0')
	p = getpwnam(curr);
	else
	p = getpwuid(rid);

	if (p != NULL)
	rid = p->pw_uid;
	else if (*endch != '\0') {
	(void) snprintf(errbuf, sizeof (errbuf),
	gettext("invalid user %s\n"), curr);
	allow_usage(un, B_TRUE, errbuf);
	}
	} else if (opts->group) {
	who_type = ZFS_DELEG_GROUP;
	if (*endch != '\0')
	g = getgrnam(curr);
	else
	g = getgrgid(rid);

	if (g != NULL)
	rid = g->gr_gid;
	else if (*endch != '\0') {
	(void) snprintf(errbuf, sizeof (errbuf),
	gettext("invalid group %s\n"),
	curr);
	allow_usage(un, B_TRUE, errbuf);
	}
	} else {
	if (*endch != '\0') {
	p = getpwnam(curr);
	} else {
	p = getpwuid(rid);
	}

	if (p == NULL) {
	if (*endch != '\0') {
	g = getgrnam(curr);
	} else {
	g = getgrgid(rid);
	}
	}

	if (p != NULL) {
	who_type = ZFS_DELEG_USER;
	rid = p->pw_uid;
	} else if (g != NULL) {
	who_type = ZFS_DELEG_GROUP;
	rid = g->gr_gid;
	} else {
	(void) snprintf(errbuf, sizeof (errbuf),
	gettext("invalid user/group %s\n"),
	curr);
	allow_usage(un, B_TRUE, errbuf);
	}
	}

	(void) sprintf(id, "%u", rid);
	who = id;

	store_allow_perm(who_type, opts->local,
	opts->descend, who, opts->perms, *nvlp);
	curr = delim + 1;
	}
	}

	return (0);
	}

	static void
	print_set_creat_perms(uu_avl_t *who_avl)
	{
	const char *sc_title[] = {
	gettext("Permission sets:\n"),
	gettext("Create time permissions:\n"),
	NULL
	};
	who_perm_node_t *who_node = NULL;
	int prev_weight = -1;

	for (who_node = uu_avl_first(who_avl); who_node != NULL;
	who_node = uu_avl_next(who_avl, who_node)) {
	uu_avl_t *avl = who_node->who_perm.who_deleg_perm_avl;
	zfs_deleg_who_type_t who_type = who_node->who_perm.who_type;
	const char *who_name = who_node->who_perm.who_name;
	int weight = who_type2weight(who_type);
	boolean_t first = B_TRUE;
	deleg_perm_node_t *deleg_node;

	if (prev_weight != weight) {
	(void) printf("%s", sc_title[weight]);
	prev_weight = weight;
	}

	if (who_name == NULL \|\| strnlen(who_name, 1) == 0)
	(void) printf("\t");
	else
	(void) printf("\t%s ", who_name);

	for (deleg_node = uu_avl_first(avl); deleg_node != NULL;
	deleg_node = uu_avl_next(avl, deleg_node)) {
	if (first) {
	(void) printf("%s",
	deleg_node->dpn_perm.dp_name);
	first = B_FALSE;
	} else
	(void) printf(",%s",
	deleg_node->dpn_perm.dp_name);
	}

	(void) printf("\n");
	}
	}

	static void
	print_uge_deleg_perms(uu_avl_t *who_avl, boolean_t local, boolean_t descend,
	const char *title)
	{
	who_perm_node_t *who_node = NULL;
	boolean_t prt_title = B_TRUE;
	uu_avl_walk_t *walk;

	if ((walk = uu_avl_walk_start(who_avl, UU_WALK_ROBUST)) == NULL)
	nomem();

	while ((who_node = uu_avl_walk_next(walk)) != NULL) {
	const char *who_name = who_node->who_perm.who_name;
	const char *nice_who_name = who_node->who_perm.who_ug_name;
	uu_avl_t *avl = who_node->who_perm.who_deleg_perm_avl;
	zfs_deleg_who_type_t who_type = who_node->who_perm.who_type;
	char delim = ' ';
	deleg_perm_node_t *deleg_node;
	boolean_t prt_who = B_TRUE;

	for (deleg_node = uu_avl_first(avl);
	deleg_node != NULL;
	deleg_node = uu_avl_next(avl, deleg_node)) {
	if (local != deleg_node->dpn_perm.dp_local \|\|
	descend != deleg_node->dpn_perm.dp_descend)
	continue;

	if (prt_who) {
	const char *who = NULL;
	if (prt_title) {
	prt_title = B_FALSE;
	(void) printf("%s", title);
	}

	switch (who_type) {
	case ZFS_DELEG_USER_SETS:
	case ZFS_DELEG_USER:
	who = gettext("user");
	if (nice_who_name)
	who_name = nice_who_name;
	break;
	case ZFS_DELEG_GROUP_SETS:
	case ZFS_DELEG_GROUP:
	who = gettext("group");
	if (nice_who_name)
	who_name = nice_who_name;
	break;
	case ZFS_DELEG_EVERYONE_SETS:
	case ZFS_DELEG_EVERYONE:
	who = gettext("everyone");
	who_name = NULL;
	break;

	default:
	assert(who != NULL);
	}

	prt_who = B_FALSE;
	if (who_name == NULL)
	(void) printf("\t%s", who);
	else
	(void) printf("\t%s %s", who, who_name);
	}

	(void) printf("%c%s", delim,
	deleg_node->dpn_perm.dp_name);
	delim = ',';
	}

	if (!prt_who)
	(void) printf("\n");
	}

	uu_avl_walk_end(walk);
	}

	static void
	print_fs_perms(fs_perm_set_t *fspset)
	{
	fs_perm_node_t *node = NULL;
	char buf[MAXNAMELEN + 32];
	const char *dsname = buf;

	for (node = uu_list_first(fspset->fsps_list); node != NULL;
	node = uu_list_next(fspset->fsps_list, node)) {
	uu_avl_t *sc_avl = node->fspn_fsperm.fsp_sc_avl;
	uu_avl_t *uge_avl = node->fspn_fsperm.fsp_uge_avl;
	int left = 0;

	(void) snprintf(buf, sizeof (buf),
	gettext("---- Permissions on %s "),
	node->fspn_fsperm.fsp_name);
	(void) printf("%s", dsname);
	left = 70 - strlen(buf);
	while (left-- > 0)
	(void) printf("-");
	(void) printf("\n");

	print_set_creat_perms(sc_avl);
	print_uge_deleg_perms(uge_avl, B_TRUE, B_FALSE,
	gettext("Local permissions:\n"));
	print_uge_deleg_perms(uge_avl, B_FALSE, B_TRUE,
	gettext("Descendent permissions:\n"));
	print_uge_deleg_perms(uge_avl, B_TRUE, B_TRUE,
	gettext("Local+Descendent permissions:\n"));
	}
	}

	static fs_perm_set_t fs_perm_set = { NULL, NULL, NULL, NULL };

	struct deleg_perms {
	boolean_t un;
	nvlist_t *nvl;
	};

	static int
	set_deleg_perms(zfs_handle_t zhp, void data)
	{
	struct deleg_perms perms = (struct deleg_perms )data;
	zfs_type_t zfs_type = zfs_get_type(zhp);

	if (zfs_type != ZFS_TYPE_FILESYSTEM && zfs_type != ZFS_TYPE_VOLUME)
	return (0);

	return (zfs_set_fsacl(zhp, perms->un, perms->nvl));
	}

	static int
	zfs_do_allow_unallow_impl(int argc, char **argv, boolean_t un)
	{
	zfs_handle_t *zhp;
	nvlist_t *perm_nvl = NULL;
	nvlist_t *update_perm_nvl = NULL;
	int error = 1;
	int c;
	struct allow_opts opts = { 0 };

	const char *optstr = un ? "ldugecsrh" : "ldugecsh";

	/* check opts */
	while ((c = getopt(argc, argv, optstr)) != -1) {
	switch (c) {
	case 'l':
	opts.local = B_TRUE;
	break;
	case 'd':
	opts.descend = B_TRUE;
	break;
	case 'u':
	opts.user = B_TRUE;
	break;
	case 'g':
	opts.group = B_TRUE;
	break;
	case 'e':
	opts.everyone = B_TRUE;
	break;
	case 's':
	opts.set = B_TRUE;
	break;
	case 'c':
	opts.create = B_TRUE;
	break;
	case 'r':
	opts.recursive = B_TRUE;
	break;
	case ':':
	(void) fprintf(stderr, gettext("missing argument for "
	"'%c' option\n"), optopt);
	usage(B_FALSE);
	break;
	case 'h':
	opts.prt_usage = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* check arguments */
	parse_allow_args(argc, argv, un, &opts);

	/* try to open the dataset */
	if ((zhp = zfs_open(g_zfs, opts.dataset, ZFS_TYPE_FILESYSTEM \|
	ZFS_TYPE_VOLUME)) == NULL) {
	(void) fprintf(stderr, "Failed to open dataset: %s\n",
	opts.dataset);
	return (-1);
	}

	if (zfs_get_fsacl(zhp, &perm_nvl) != 0)
	goto cleanup2;

	fs_perm_set_init(&fs_perm_set);
	if (parse_fs_perm_set(&fs_perm_set, perm_nvl) != 0) {
	(void) fprintf(stderr, "Failed to parse fsacl permissions\n");
	goto cleanup1;
	}

	if (opts.prt_perms)
	print_fs_perms(&fs_perm_set);
	else {
	(void) construct_fsacl_list(un, &opts, &update_perm_nvl);
	if (zfs_set_fsacl(zhp, un, update_perm_nvl) != 0)
	goto cleanup0;

	if (un && opts.recursive) {
	struct deleg_perms data = { un, update_perm_nvl };
	if (zfs_iter_filesystems_v2(zhp, 0, set_deleg_perms,
	&data) != 0)
	goto cleanup0;
	}
	}

	error = 0;

	cleanup0:
	nvlist_free(perm_nvl);
	nvlist_free(update_perm_nvl);
	cleanup1:
	fs_perm_set_fini(&fs_perm_set);
	cleanup2:
	zfs_close(zhp);

	return (error);
	}

	static int
	zfs_do_allow(int argc, char **argv)
	{
	return (zfs_do_allow_unallow_impl(argc, argv, B_FALSE));
	}

	static int
	zfs_do_unallow(int argc, char **argv)
	{
	return (zfs_do_allow_unallow_impl(argc, argv, B_TRUE));
	}

	static int
	zfs_do_hold_rele_impl(int argc, char **argv, boolean_t holding)
	{
	int errors = 0;
	int i;
	const char *tag;
	boolean_t recursive = B_FALSE;
	const char *opts = holding ? "rt" : "r";
	int c;

	/* check options */
	while ((c = getopt(argc, argv, opts)) != -1) {
	switch (c) {
	case 'r':
	recursive = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* check number of arguments */
	if (argc < 2)
	usage(B_FALSE);

	tag = argv[0];
	--argc;
	++argv;

	if (holding && tag[0] == '.') {
	/* tags starting with '.' are reserved for libzfs */
	(void) fprintf(stderr, gettext("tag may not start with '.'\n"));
	usage(B_FALSE);
	}

	for (i = 0; i < argc; ++i) {
	zfs_handle_t *zhp;
	char parent[ZFS_MAX_DATASET_NAME_LEN];
	const char *delim;
	char *path = argv[i];

	delim = strchr(path, '@');
	if (delim == NULL) {
	(void) fprintf(stderr,
	gettext("'%s' is not a snapshot\n"), path);
	++errors;
	continue;
	}
	(void) strlcpy(parent, path, MIN(sizeof (parent),
	delim - path + 1));

	zhp = zfs_open(g_zfs, parent,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME);
	if (zhp == NULL) {
	++errors;
	continue;
	}
	if (holding) {
	if (zfs_hold(zhp, delim+1, tag, recursive, -1) != 0)
	++errors;
	} else {
	if (zfs_release(zhp, delim+1, tag, recursive) != 0)
	++errors;
	}
	zfs_close(zhp);
	}

	return (errors != 0);
	}

	/*
	* zfs hold [-r] [-t] <tag> <snap> ...
	*
	* -r Recursively hold
	*
	* Apply a user-hold with the given tag to the list of snapshots.
	*/
	static int
	zfs_do_hold(int argc, char **argv)
	{
	return (zfs_do_hold_rele_impl(argc, argv, B_TRUE));
	}

	/*
	* zfs release [-r] <tag> <snap> ...
	*
	* -r Recursively release
	*
	* Release a user-hold with the given tag from the list of snapshots.
	*/
	static int
	zfs_do_release(int argc, char **argv)
	{
	return (zfs_do_hold_rele_impl(argc, argv, B_FALSE));
	}

	typedef struct holds_cbdata {
	boolean_t cb_recursive;
	const char *cb_snapname;
	nvlist_t **cb_nvlp;
	size_t cb_max_namelen;
	size_t cb_max_taglen;
	} holds_cbdata_t;

	#define STRFTIME_FMT_STR "%a %b %e %H:%M %Y"
	#define DATETIME_BUF_LEN (32)
	/*
	*
	*/
	static void
	print_holds(boolean_t scripted, int nwidth, int tagwidth, nvlist_t *nvl,
	boolean_t parsable)
	{
	int i;
	nvpair_t *nvp = NULL;
	const char *const hdr_cols[] = { "NAME", "TAG", "TIMESTAMP" };
	const char *col;

	if (!scripted) {
	for (i = 0; i < 3; i++) {
	col = gettext(hdr_cols[i]);
	if (i < 2)
	(void) printf("%-*s ", i ? tagwidth : nwidth,
	col);
	else
	(void) printf("%s\n", col);
	}
	}

	while ((nvp = nvlist_next_nvpair(nvl, nvp)) != NULL) {
	const char *zname = nvpair_name(nvp);
	nvlist_t *nvl2;
	nvpair_t *nvp2 = NULL;
	(void) nvpair_value_nvlist(nvp, &nvl2);
	while ((nvp2 = nvlist_next_nvpair(nvl2, nvp2)) != NULL) {
	char tsbuf[DATETIME_BUF_LEN];
	const char *tagname = nvpair_name(nvp2);
	uint64_t val = 0;
	time_t time;
	struct tm t;

	(void) nvpair_value_uint64(nvp2, &val);
	time = (time_t)val;
	(void) localtime_r(&time, &t);
	(void) strftime(tsbuf, DATETIME_BUF_LEN,
	gettext(STRFTIME_FMT_STR), &t);

	if (scripted) {
	if (parsable) {
	(void) printf("%s\t%s\t%ld\n", zname,
	tagname, (unsigned long)time);
	} else {
	(void) printf("%s\t%s\t%s\n", zname,
	tagname, tsbuf);
	}
	} else {
	if (parsable) {
	(void) printf("%-s %-s %ld\n",
	nwidth, zname, tagwidth,
	tagname, (unsigned long)time);
	} else {
	(void) printf("%-s %-s %s\n",
	nwidth, zname, tagwidth,
	tagname, tsbuf);
	}
	}
	}
	}
	}

	/*
	* Generic callback function to list a dataset or snapshot.
	*/
	static int
	holds_callback(zfs_handle_t zhp, void data)
	{
	holds_cbdata_t *cbp = data;
	nvlist_t top_nvl = cbp->cb_nvlp;
	nvlist_t *nvl = NULL;
	nvpair_t *nvp = NULL;
	const char *zname = zfs_get_name(zhp);
	size_t znamelen = strlen(zname);

	if (cbp->cb_recursive) {
	const char *snapname;
	char *delim = strchr(zname, '@');
	if (delim == NULL)
	return (0);

	snapname = delim + 1;
	if (strcmp(cbp->cb_snapname, snapname))
	return (0);
	}

	if (zfs_get_holds(zhp, &nvl) != 0)
	return (-1);

	if (znamelen > cbp->cb_max_namelen)
	cbp->cb_max_namelen = znamelen;

	while ((nvp = nvlist_next_nvpair(nvl, nvp)) != NULL) {
	const char *tag = nvpair_name(nvp);
	size_t taglen = strlen(tag);
	if (taglen > cbp->cb_max_taglen)
	cbp->cb_max_taglen = taglen;
	}

	return (nvlist_add_nvlist(top_nvl, zname, nvl));
	}

	/*
	* zfs holds [-rHp] <snap> ...
	*
	* -r Lists holds that are set on the named snapshots recursively.
	* -H Scripted mode; elide headers and separate columns by tabs.
	* -p Display values in parsable (literal) format.
	*/
	static int
	zfs_do_holds(int argc, char **argv)
	{
	int c;
	boolean_t errors = B_FALSE;
	boolean_t scripted = B_FALSE;
	boolean_t recursive = B_FALSE;
	boolean_t parsable = B_FALSE;

	int types = ZFS_TYPE_SNAPSHOT;
	holds_cbdata_t cb = { 0 };

	int limit = 0;
	int ret = 0;
	int flags = 0;

	/* check options */
	while ((c = getopt(argc, argv, "rHp")) != -1) {
	switch (c) {
	case 'r':
	recursive = B_TRUE;
	break;
	case 'H':
	scripted = B_TRUE;
	break;
	case 'p':
	parsable = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	if (recursive) {
	types \|= ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME;
	flags \|= ZFS_ITER_RECURSE;
	}

	argc -= optind;
	argv += optind;

	/* check number of arguments */
	if (argc < 1)
	usage(B_FALSE);

	nvlist_t *nvl = fnvlist_alloc();

	for (int i = 0; i < argc; ++i) {
	char *snapshot = argv[i];
	const char *delim;
	const char *snapname;

	delim = strchr(snapshot, '@');
	if (delim == NULL) {
	(void) fprintf(stderr,
	gettext("'%s' is not a snapshot\n"), snapshot);
	errors = B_TRUE;
	continue;
	}
	snapname = delim + 1;
	if (recursive)
	snapshot[delim - snapshot] = '\0';

	cb.cb_recursive = recursive;
	cb.cb_snapname = snapname;
	cb.cb_nvlp = &nvl;

	/*
	* 1. collect holds data, set format options
	*/
	ret = zfs_for_each(1, argv + i, flags, types, NULL, NULL, limit,
	holds_callback, &cb);
	if (ret != 0)
	errors = B_TRUE;
	}

	/*
	* 2. print holds data
	*/
	print_holds(scripted, cb.cb_max_namelen, cb.cb_max_taglen, nvl,
	parsable);

	if (nvlist_empty(nvl))
	(void) fprintf(stderr, gettext("no datasets available\n"));

	nvlist_free(nvl);

	return (errors);
	}

	#define CHECK_SPINNER 30
	#define SPINNER_TIME 3 /* seconds */
	#define MOUNT_TIME 1 /* seconds */

	typedef struct get_all_state {
	+ char **ga_datasets;
	+ int ga_count;
	boolean_t ga_verbose;
	get_all_cb_t *ga_cbp;
	} get_all_state_t;

	static int
	get_one_dataset(zfs_handle_t zhp, void data)
	{
	static const char *const spin[] = { "-", "\\", "\|", "/" };
	static int spinval = 0;
	static int spincheck = 0;
	static time_t last_spin_time = (time_t)0;
	get_all_state_t *state = data;
	zfs_type_t type = zfs_get_type(zhp);

	if (state->ga_verbose) {
	if (--spincheck < 0) {
	time_t now = time(NULL);
	if (last_spin_time + SPINNER_TIME < now) {
	update_progress(spin[spinval++ % 4]);
	last_spin_time = now;
	}
	spincheck = CHECK_SPINNER;
	}
	}

	/*
	* Iterate over any nested datasets.
	*/
	if (zfs_iter_filesystems_v2(zhp, 0, get_one_dataset, data) != 0) {
	zfs_close(zhp);
	return (1);
	}

	/*
	* Skip any datasets whose type does not match.
	*/
	if ((type & ZFS_TYPE_FILESYSTEM) == 0) {
	zfs_close(zhp);
	return (0);
	}
	libzfs_add_handle(state->ga_cbp, zhp);
	assert(state->ga_cbp->cb_used <= state->ga_cbp->cb_alloc);

	return (0);
	}

	-static void
	-get_all_datasets(get_all_cb_t *cbp, boolean_t verbose)
	+static int
	+get_recursive_datasets(zfs_handle_t zhp, void data)
	{
	- get_all_state_t state = {
	- .ga_verbose = verbose,
	- .ga_cbp = cbp
	- };
	+ get_all_state_t *state = data;
	+ int len = strlen(zfs_get_name(zhp));
	+ for (int i = 0; i < state->ga_count; ++i) {
	+ if (strcmp(state->ga_datasets[i], zfs_get_name(zhp)) == 0)
	+ return (get_one_dataset(zhp, data));
	+ else if ((strncmp(state->ga_datasets[i], zfs_get_name(zhp),
	+ len) == 0) && state->ga_datasets[i][len] == '/') {
	+ (void) zfs_iter_filesystems_v2(zhp, 0,
	+ get_recursive_datasets, data);
	+ }
	+ }
	+ zfs_close(zhp);
	+ return (0);
	+}

	- if (verbose)
	+static void
	+get_all_datasets(get_all_state_t *state)
	+{
	+ if (state->ga_verbose)
	set_progress_header(gettext("Reading ZFS config"));
	- (void) zfs_iter_root(g_zfs, get_one_dataset, &state);
	+ if (state->ga_datasets == NULL)
	+ (void) zfs_iter_root(g_zfs, get_one_dataset, state);
	+ else
	+ (void) zfs_iter_root(g_zfs, get_recursive_datasets, state);

	- if (verbose)
	+ if (state->ga_verbose)
	finish_progress(gettext("done."));
	}

	/*
	* Generic callback for sharing or mounting filesystems. Because the code is so
	* similar, we have a common function with an extra parameter to determine which
	* mode we are using.
	*/
	typedef enum { OP_SHARE, OP_MOUNT } share_mount_op_t;

	typedef struct share_mount_state {
	share_mount_op_t sm_op;
	boolean_t sm_verbose;
	int sm_flags;
	char *sm_options;
	enum sa_protocol sm_proto; /* only valid for OP_SHARE */
	pthread_mutex_t sm_lock; /* protects the remaining fields */
	uint_t sm_total; /* number of filesystems to process */
	uint_t sm_done; /* number of filesystems processed */
	int sm_status; /* -1 if any of the share/mount operations failed */
	} share_mount_state_t;

	/*
	* Share or mount a dataset.
	*/
	static int
	share_mount_one(zfs_handle_t *zhp, int op, int flags, enum sa_protocol protocol,
	boolean_t explicit, const char *options)
	{
	char mountpoint[ZFS_MAXPROPLEN];
	char shareopts[ZFS_MAXPROPLEN];
	char smbshareopts[ZFS_MAXPROPLEN];
	const char *cmdname = op == OP_SHARE ? "share" : "mount";
	struct mnttab mnt;
	uint64_t zoned, canmount;
	boolean_t shared_nfs, shared_smb;

	assert(zfs_get_type(zhp) & ZFS_TYPE_FILESYSTEM);

	/*
	* Check to make sure we can mount/share this dataset. If we
	* are in the global zone and the filesystem is exported to a
	* local zone, or if we are in a local zone and the
	* filesystem is not exported, then it is an error.
	*/
	zoned = zfs_prop_get_int(zhp, ZFS_PROP_ZONED);

	if (zoned && getzoneid() == GLOBAL_ZONEID) {
	if (!explicit)
	return (0);

	(void) fprintf(stderr, gettext("cannot %s '%s': "
	"dataset is exported to a local zone\n"), cmdname,
	zfs_get_name(zhp));
	return (1);

	} else if (!zoned && getzoneid() != GLOBAL_ZONEID) {
	if (!explicit)
	return (0);

	(void) fprintf(stderr, gettext("cannot %s '%s': "
	"permission denied\n"), cmdname,
	zfs_get_name(zhp));
	return (1);
	}

	/*
	* Ignore any filesystems which don't apply to us. This
	* includes those with a legacy mountpoint, or those with
	* legacy share options.
	*/
	verify(zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT, mountpoint,
	sizeof (mountpoint), NULL, NULL, 0, B_FALSE) == 0);
	verify(zfs_prop_get(zhp, ZFS_PROP_SHARENFS, shareopts,
	sizeof (shareopts), NULL, NULL, 0, B_FALSE) == 0);
	verify(zfs_prop_get(zhp, ZFS_PROP_SHARESMB, smbshareopts,
	sizeof (smbshareopts), NULL, NULL, 0, B_FALSE) == 0);

	if (op == OP_SHARE && strcmp(shareopts, "off") == 0 &&
	strcmp(smbshareopts, "off") == 0) {
	if (!explicit)
	return (0);

	(void) fprintf(stderr, gettext("cannot share '%s': "
	"legacy share\n"), zfs_get_name(zhp));
	(void) fprintf(stderr, gettext("use exports(5) or "
	"smb.conf(5) to share this filesystem, or set "
	"the sharenfs or sharesmb property\n"));
	return (1);
	}

	/*
	* We cannot share or mount legacy filesystems. If the
	* shareopts is non-legacy but the mountpoint is legacy, we
	* treat it as a legacy share.
	*/
	if (strcmp(mountpoint, "legacy") == 0) {
	if (!explicit)
	return (0);

	(void) fprintf(stderr, gettext("cannot %s '%s': "
	"legacy mountpoint\n"), cmdname, zfs_get_name(zhp));
	(void) fprintf(stderr, gettext("use %s(8) to "
	"%s this filesystem\n"), cmdname, cmdname);
	return (1);
	}

	if (strcmp(mountpoint, "none") == 0) {
	if (!explicit)
	return (0);

	(void) fprintf(stderr, gettext("cannot %s '%s': no "
	"mountpoint set\n"), cmdname, zfs_get_name(zhp));
	return (1);
	}

	/*
	* canmount explicit outcome
	* on no pass through
	* on yes pass through
	* off no return 0
	* off yes display error, return 1
	* noauto no return 0
	* noauto yes pass through
	*/
	canmount = zfs_prop_get_int(zhp, ZFS_PROP_CANMOUNT);
	if (canmount == ZFS_CANMOUNT_OFF) {
	if (!explicit)
	return (0);

	(void) fprintf(stderr, gettext("cannot %s '%s': "
	"'canmount' property is set to 'off'\n"), cmdname,
	zfs_get_name(zhp));
	return (1);
	} else if (canmount == ZFS_CANMOUNT_NOAUTO && !explicit) {
	/*
	* When performing a 'zfs mount -a', we skip any mounts for
	* datasets that have 'noauto' set. Sharing a dataset with
	* 'noauto' set is only allowed if it's mounted.
	*/
	if (op == OP_MOUNT)
	return (0);
	if (op == OP_SHARE && !zfs_is_mounted(zhp, NULL)) {
	/* also purge it from existing exports */
	zfs_unshare(zhp, mountpoint, NULL);
	return (0);
	}
	}

	/*
	* If this filesystem is encrypted and does not have
	* a loaded key, we can not mount it.
	*/
	if ((flags & MS_CRYPT) == 0 &&
	zfs_prop_get_int(zhp, ZFS_PROP_ENCRYPTION) != ZIO_CRYPT_OFF &&
	zfs_prop_get_int(zhp, ZFS_PROP_KEYSTATUS) ==
	ZFS_KEYSTATUS_UNAVAILABLE) {
	if (!explicit)
	return (0);

	(void) fprintf(stderr, gettext("cannot %s '%s': "
	"encryption key not loaded\n"), cmdname, zfs_get_name(zhp));
	return (1);
	}

	/*
	* If this filesystem is inconsistent and has a receive resume
	* token, we can not mount it.
	*/
	if (zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT) &&
	zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN,
	NULL, 0, NULL, NULL, 0, B_TRUE) == 0) {
	if (!explicit)
	return (0);

	(void) fprintf(stderr, gettext("cannot %s '%s': "
	"Contains partially-completed state from "
	"\"zfs receive -s\", which can be resumed with "
	"\"zfs send -t\"\n"),
	cmdname, zfs_get_name(zhp));
	return (1);
	}

	if (zfs_prop_get_int(zhp, ZFS_PROP_REDACTED) && !(flags & MS_FORCE)) {
	if (!explicit)
	return (0);

	(void) fprintf(stderr, gettext("cannot %s '%s': "
	"Dataset is not complete, was created by receiving "
	"a redacted zfs send stream.\n"), cmdname,
	zfs_get_name(zhp));
	return (1);
	}

	/*
	* At this point, we have verified that the mountpoint and/or
	* shareopts are appropriate for auto management. If the
	* filesystem is already mounted or shared, return (failing
	* for explicit requests); otherwise mount or share the
	* filesystem.
	*/
	switch (op) {
	case OP_SHARE: {
	enum sa_protocol prot[] = {SA_PROTOCOL_NFS, SA_NO_PROTOCOL};
	shared_nfs = zfs_is_shared(zhp, NULL, prot);
	*prot = SA_PROTOCOL_SMB;
	shared_smb = zfs_is_shared(zhp, NULL, prot);

	if ((shared_nfs && shared_smb) \|\|
	(shared_nfs && strcmp(shareopts, "on") == 0 &&
	strcmp(smbshareopts, "off") == 0) \|\|
	(shared_smb && strcmp(smbshareopts, "on") == 0 &&
	strcmp(shareopts, "off") == 0)) {
	if (!explicit)
	return (0);

	(void) fprintf(stderr, gettext("cannot share "
	"'%s': filesystem already shared\n"),
	zfs_get_name(zhp));
	return (1);
	}

	if (!zfs_is_mounted(zhp, NULL) &&
	zfs_mount(zhp, NULL, flags) != 0)
	return (1);

	*prot = protocol;
	if (zfs_share(zhp, protocol == SA_NO_PROTOCOL ? NULL : prot))
	return (1);

	}
	break;

	case OP_MOUNT:
	mnt.mnt_mntopts = (char *)(options ?: "");

	if (!hasmntopt(&mnt, MNTOPT_REMOUNT) &&
	zfs_is_mounted(zhp, NULL)) {
	if (!explicit)
	return (0);

	(void) fprintf(stderr, gettext("cannot mount "
	"'%s': filesystem already mounted\n"),
	zfs_get_name(zhp));
	return (1);
	}

	if (zfs_mount(zhp, options, flags) != 0)
	return (1);
	break;
	}

	return (0);
	}

	/*
	* Reports progress in the form "(current/total)". Not thread-safe.
	*/
	static void
	report_mount_progress(int current, int total)
	{
	static time_t last_progress_time = 0;
	time_t now = time(NULL);
	char info[32];

	/* display header if we're here for the first time */
	if (current == 1) {
	set_progress_header(gettext("Mounting ZFS filesystems"));
	} else if (current != total && last_progress_time + MOUNT_TIME >= now) {
	/* too soon to report again */
	return;
	}

	last_progress_time = now;

	(void) sprintf(info, "(%d/%d)", current, total);

	if (current == total)
	finish_progress(info);
	else
	update_progress(info);
	}

	/*
	* zfs_foreach_mountpoint() callback that mounts or shares one filesystem and
	* updates the progress meter.
	*/
	static int
	share_mount_one_cb(zfs_handle_t zhp, void arg)
	{
	share_mount_state_t *sms = arg;
	int ret;

	ret = share_mount_one(zhp, sms->sm_op, sms->sm_flags, sms->sm_proto,
	B_FALSE, sms->sm_options);

	pthread_mutex_lock(&sms->sm_lock);
	if (ret != 0)
	sms->sm_status = ret;
	sms->sm_done++;
	if (sms->sm_verbose)
	report_mount_progress(sms->sm_done, sms->sm_total);
	pthread_mutex_unlock(&sms->sm_lock);
	return (ret);
	}

	static void
	append_options(char mntopts, char newopts)
	{
	int len = strlen(mntopts);

	/* original length plus new string to append plus 1 for the comma */
	if (len + 1 + strlen(newopts) >= MNT_LINE_MAX) {
	(void) fprintf(stderr, gettext("the opts argument for "
	"'%s' option is too long (more than %d chars)\n"),
	"-o", MNT_LINE_MAX);
	usage(B_FALSE);
	}

	if (*mntopts)
	mntopts[len++] = ',';

	(void) strcpy(&mntopts[len], newopts);
	}

	static enum sa_protocol
	sa_protocol_decode(const char *protocol)
	{
	for (enum sa_protocol i = 0; i < ARRAY_SIZE(sa_protocol_names); ++i)
	if (strcmp(protocol, sa_protocol_names[i]) == 0)
	return (i);

	(void) fputs(gettext("share type must be one of: "), stderr);
	for (enum sa_protocol i = 0;
	i < ARRAY_SIZE(sa_protocol_names); ++i)
	(void) fprintf(stderr, "%s%s",
	i != 0 ? ", " : "", sa_protocol_names[i]);
	(void) fputc('\n', stderr);
	usage(B_FALSE);
	}

	static int
	share_mount(int op, int argc, char **argv)
	{
	int do_all = 0;
	+ int recursive = 0;
	boolean_t verbose = B_FALSE;
	int c, ret = 0;
	char *options = NULL;
	int flags = 0;

	/* check options */
	- while ((c = getopt(argc, argv, op == OP_MOUNT ? ":alvo:Of" : "al"))
	+ while ((c = getopt(argc, argv, op == OP_MOUNT ? ":aRlvo:Of" : "al"))
	!= -1) {
	switch (c) {
	case 'a':
	do_all = 1;
	break;
	+ case 'R':
	+ recursive = 1;
	+ break;
	case 'v':
	verbose = B_TRUE;
	break;
	case 'l':
	flags \|= MS_CRYPT;
	break;
	case 'o':
	if (*optarg == '\0') {
	(void) fprintf(stderr, gettext("empty mount "
	"options (-o) specified\n"));
	usage(B_FALSE);
	}

	if (options == NULL)
	options = safe_malloc(MNT_LINE_MAX + 1);

	/* option validation is done later */
	append_options(options, optarg);
	break;
	case 'O':
	flags \|= MS_OVERLAY;
	break;
	case 'f':
	flags \|= MS_FORCE;
	break;
	case ':':
	(void) fprintf(stderr, gettext("missing argument for "
	"'%c' option\n"), optopt);
	usage(B_FALSE);
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* check number of arguments */
	- if (do_all) {
	+ if (do_all \|\| recursive) {
	enum sa_protocol protocol = SA_NO_PROTOCOL;

	if (op == OP_SHARE && argc > 0) {
	protocol = sa_protocol_decode(argv[0]);
	argc--;
	argv++;
	}

	- if (argc != 0) {
	+ if (argc != 0 && do_all) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	+ if (argc == 0 && recursive) {
	+ (void) fprintf(stderr,
	+ gettext("no dataset provided\n"));
	+ usage(B_FALSE);
	+ }
	+
	start_progress_timer();
	get_all_cb_t cb = { 0 };
	- get_all_datasets(&cb, verbose);
	+ get_all_state_t state = { 0 };
	+ if (argc == 0) {
	+ state.ga_datasets = NULL;
	+ state.ga_count = -1;
	+ } else {
	+ zfs_handle_t *zhp;
	+ for (int i = 0; i < argc; i++) {
	+ zhp = zfs_open(g_zfs, argv[i],
	+ ZFS_TYPE_FILESYSTEM);
	+ if (zhp == NULL)
	+ usage(B_FALSE);
	+ zfs_close(zhp);
	+ }
	+ state.ga_datasets = argv;
	+ state.ga_count = argc;
	+ }
	+ state.ga_verbose = verbose;
	+ state.ga_cbp = &cb;
	+ get_all_datasets(&state);

	if (cb.cb_used == 0) {
	free(options);
	return (0);
	}

	share_mount_state_t share_mount_state = { 0 };
	share_mount_state.sm_op = op;
	share_mount_state.sm_verbose = verbose;
	share_mount_state.sm_flags = flags;
	share_mount_state.sm_options = options;
	share_mount_state.sm_proto = protocol;
	share_mount_state.sm_total = cb.cb_used;
	pthread_mutex_init(&share_mount_state.sm_lock, NULL);

	/* For a 'zfs share -a' operation start with a clean slate. */
	if (op == OP_SHARE)
	zfs_truncate_shares(NULL);

	/*
	* libshare isn't mt-safe, so only do the operation in parallel
	* if we're mounting. Additionally, the key-loading option must
	* be serialized so that we can prompt the user for their keys
	* in a consistent manner.
	*/
	zfs_foreach_mountpoint(g_zfs, cb.cb_handles, cb.cb_used,
	share_mount_one_cb, &share_mount_state,
	op == OP_MOUNT && !(flags & MS_CRYPT));
	zfs_commit_shares(NULL);

	ret = share_mount_state.sm_status;

	for (int i = 0; i < cb.cb_used; i++)
	zfs_close(cb.cb_handles[i]);
	free(cb.cb_handles);
	} else if (argc == 0) {
	FILE *mnttab;
	struct mnttab entry;

	if ((op == OP_SHARE) \|\| (options != NULL)) {
	(void) fprintf(stderr, gettext("missing filesystem "
	"argument (specify -a for all)\n"));
	usage(B_FALSE);
	}

	/*
	* When mount is given no arguments, go through
	* /proc/self/mounts and display any active ZFS mounts.
	* We hide any snapshots, since they are controlled
	* automatically.
	*/

	if ((mnttab = fopen(MNTTAB, "re")) == NULL) {
	free(options);
	return (ENOENT);
	}

	while (getmntent(mnttab, &entry) == 0) {
	if (strcmp(entry.mnt_fstype, MNTTYPE_ZFS) != 0 \|\|
	strchr(entry.mnt_special, '@') != NULL)
	continue;

	(void) printf("%-30s %s\n", entry.mnt_special,
	entry.mnt_mountp);
	}

	(void) fclose(mnttab);
	} else {
	zfs_handle_t *zhp;

	if (argc > 1) {
	(void) fprintf(stderr,
	gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	if ((zhp = zfs_open(g_zfs, argv[0],
	ZFS_TYPE_FILESYSTEM)) == NULL) {
	ret = 1;
	} else {
	ret = share_mount_one(zhp, op, flags, SA_NO_PROTOCOL,
	B_TRUE, options);
	zfs_commit_shares(NULL);
	zfs_close(zhp);
	}
	}

	free(options);
	return (ret);
	}

	/*
	* zfs mount -a
	* zfs mount filesystem
	*
	* Mount all filesystems, or mount the given filesystem.
	*/
	static int
	zfs_do_mount(int argc, char **argv)
	{
	return (share_mount(OP_MOUNT, argc, argv));
	}

	/*
	* zfs share -a [nfs \| smb]
	* zfs share filesystem
	*
	* Share all filesystems, or share the given filesystem.
	*/
	static int
	zfs_do_share(int argc, char **argv)
	{
	return (share_mount(OP_SHARE, argc, argv));
	}

	typedef struct unshare_unmount_node {
	zfs_handle_t *un_zhp;
	char *un_mountp;
	uu_avl_node_t un_avlnode;
	} unshare_unmount_node_t;

	static int
	unshare_unmount_compare(const void larg, const void rarg, void *unused)
	{
	(void) unused;
	const unshare_unmount_node_t *l = larg;
	const unshare_unmount_node_t *r = rarg;

	return (strcmp(l->un_mountp, r->un_mountp));
	}

	/*
	* Convenience routine used by zfs_do_umount() and manual_unmount(). Given an
	* absolute path, find the entry /proc/self/mounts, verify that it's a
	* ZFS filesystem, and unmount it appropriately.
	*/
	static int
	unshare_unmount_path(int op, char *path, int flags, boolean_t is_manual)
	{
	zfs_handle_t *zhp;
	int ret = 0;
	struct stat64 statbuf;
	struct extmnttab entry;
	const char *cmdname = (op == OP_SHARE) ? "unshare" : "unmount";
	ino_t path_inode;

	/*
	* Search for the given (major,minor) pair in the mount table.
	*/

	if (getextmntent(path, &entry, &statbuf) != 0) {
	if (op == OP_SHARE) {
	(void) fprintf(stderr, gettext("cannot %s '%s': not "
	"currently mounted\n"), cmdname, path);
	return (1);
	}
	(void) fprintf(stderr, gettext("warning: %s not in"
	"/proc/self/mounts\n"), path);
	if ((ret = umount2(path, flags)) != 0)
	(void) fprintf(stderr, gettext("%s: %s\n"), path,
	strerror(errno));
	return (ret != 0);
	}
	path_inode = statbuf.st_ino;

	if (strcmp(entry.mnt_fstype, MNTTYPE_ZFS) != 0) {
	(void) fprintf(stderr, gettext("cannot %s '%s': not a ZFS "
	"filesystem\n"), cmdname, path);
	return (1);
	}

	if ((zhp = zfs_open(g_zfs, entry.mnt_special,
	ZFS_TYPE_FILESYSTEM)) == NULL)
	return (1);

	ret = 1;
	if (stat64(entry.mnt_mountp, &statbuf) != 0) {
	(void) fprintf(stderr, gettext("cannot %s '%s': %s\n"),
	cmdname, path, strerror(errno));
	goto out;
	} else if (statbuf.st_ino != path_inode) {
	(void) fprintf(stderr, gettext("cannot "
	"%s '%s': not a mountpoint\n"), cmdname, path);
	goto out;
	}

	if (op == OP_SHARE) {
	char nfs_mnt_prop[ZFS_MAXPROPLEN];
	char smbshare_prop[ZFS_MAXPROPLEN];

	verify(zfs_prop_get(zhp, ZFS_PROP_SHARENFS, nfs_mnt_prop,
	sizeof (nfs_mnt_prop), NULL, NULL, 0, B_FALSE) == 0);
	verify(zfs_prop_get(zhp, ZFS_PROP_SHARESMB, smbshare_prop,
	sizeof (smbshare_prop), NULL, NULL, 0, B_FALSE) == 0);

	if (strcmp(nfs_mnt_prop, "off") == 0 &&
	strcmp(smbshare_prop, "off") == 0) {
	(void) fprintf(stderr, gettext("cannot unshare "
	"'%s': legacy share\n"), path);
	(void) fprintf(stderr, gettext("use exportfs(8) "
	"or smbcontrol(1) to unshare this filesystem\n"));
	} else if (!zfs_is_shared(zhp, NULL, NULL)) {
	(void) fprintf(stderr, gettext("cannot unshare '%s': "
	"not currently shared\n"), path);
	} else {
	ret = zfs_unshare(zhp, path, NULL);
	zfs_commit_shares(NULL);
	}
	} else {
	char mtpt_prop[ZFS_MAXPROPLEN];

	verify(zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT, mtpt_prop,
	sizeof (mtpt_prop), NULL, NULL, 0, B_FALSE) == 0);

	if (is_manual) {
	ret = zfs_unmount(zhp, NULL, flags);
	} else if (strcmp(mtpt_prop, "legacy") == 0) {
	(void) fprintf(stderr, gettext("cannot unmount "
	"'%s': legacy mountpoint\n"),
	zfs_get_name(zhp));
	(void) fprintf(stderr, gettext("use umount(8) "
	"to unmount this filesystem\n"));
	} else {
	ret = zfs_unmountall(zhp, flags);
	}
	}

	out:
	zfs_close(zhp);

	return (ret != 0);
	}

	/*
	* Generic callback for unsharing or unmounting a filesystem.
	*/
	static int
	unshare_unmount(int op, int argc, char **argv)
	{
	int do_all = 0;
	int flags = 0;
	int ret = 0;
	int c;
	zfs_handle_t *zhp;
	char nfs_mnt_prop[ZFS_MAXPROPLEN];
	char sharesmb[ZFS_MAXPROPLEN];

	/* check options */
	while ((c = getopt(argc, argv, op == OP_SHARE ? ":a" : "afu")) != -1) {
	switch (c) {
	case 'a':
	do_all = 1;
	break;
	case 'f':
	flags \|= MS_FORCE;
	break;
	case 'u':
	flags \|= MS_CRYPT;
	break;
	case ':':
	(void) fprintf(stderr, gettext("missing argument for "
	"'%c' option\n"), optopt);
	usage(B_FALSE);
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	if (do_all) {
	/*
	* We could make use of zfs_for_each() to walk all datasets in
	* the system, but this would be very inefficient, especially
	* since we would have to linearly search /proc/self/mounts for
	* each one. Instead, do one pass through /proc/self/mounts
	* looking for zfs entries and call zfs_unmount() for each one.
	*
	* Things get a little tricky if the administrator has created
	* mountpoints beneath other ZFS filesystems. In this case, we
	* have to unmount the deepest filesystems first. To accomplish
	* this, we place all the mountpoints in an AVL tree sorted by
	* the special type (dataset name), and walk the result in
	* reverse to make sure to get any snapshots first.
	*/
	FILE *mnttab;
	struct mnttab entry;
	uu_avl_pool_t *pool;
	uu_avl_t *tree = NULL;
	unshare_unmount_node_t *node;
	uu_avl_index_t idx;
	uu_avl_walk_t *walk;
	enum sa_protocol *protocol = NULL,
	single_protocol[] = {SA_NO_PROTOCOL, SA_NO_PROTOCOL};

	if (op == OP_SHARE && argc > 0) {
	*single_protocol = sa_protocol_decode(argv[0]);
	protocol = single_protocol;
	argc--;
	argv++;
	}

	if (argc != 0) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	if (((pool = uu_avl_pool_create("unmount_pool",
	sizeof (unshare_unmount_node_t),
	offsetof(unshare_unmount_node_t, un_avlnode),
	unshare_unmount_compare, UU_DEFAULT)) == NULL) \|\|
	((tree = uu_avl_create(pool, NULL, UU_DEFAULT)) == NULL))
	nomem();

	if ((mnttab = fopen(MNTTAB, "re")) == NULL) {
	uu_avl_destroy(tree);
	uu_avl_pool_destroy(pool);
	return (ENOENT);
	}

	while (getmntent(mnttab, &entry) == 0) {

	/* ignore non-ZFS entries */
	if (strcmp(entry.mnt_fstype, MNTTYPE_ZFS) != 0)
	continue;

	/* ignore snapshots */
	if (strchr(entry.mnt_special, '@') != NULL)
	continue;

	if ((zhp = zfs_open(g_zfs, entry.mnt_special,
	ZFS_TYPE_FILESYSTEM)) == NULL) {
	ret = 1;
	continue;
	}

	/*
	* Ignore datasets that are excluded/restricted by
	* parent pool name.
	*/
	if (zpool_skip_pool(zfs_get_pool_name(zhp))) {
	zfs_close(zhp);
	continue;
	}

	switch (op) {
	case OP_SHARE:
	verify(zfs_prop_get(zhp, ZFS_PROP_SHARENFS,
	nfs_mnt_prop,
	sizeof (nfs_mnt_prop),
	NULL, NULL, 0, B_FALSE) == 0);
	if (strcmp(nfs_mnt_prop, "off") != 0)
	break;
	verify(zfs_prop_get(zhp, ZFS_PROP_SHARESMB,
	nfs_mnt_prop,
	sizeof (nfs_mnt_prop),
	NULL, NULL, 0, B_FALSE) == 0);
	if (strcmp(nfs_mnt_prop, "off") == 0)
	continue;
	break;
	case OP_MOUNT:
	/* Ignore legacy mounts */
	verify(zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT,
	nfs_mnt_prop,
	sizeof (nfs_mnt_prop),
	NULL, NULL, 0, B_FALSE) == 0);
	if (strcmp(nfs_mnt_prop, "legacy") == 0)
	continue;
	/* Ignore canmount=noauto mounts */
	if (zfs_prop_get_int(zhp, ZFS_PROP_CANMOUNT) ==
	ZFS_CANMOUNT_NOAUTO)
	continue;
	break;
	default:
	break;
	}

	node = safe_malloc(sizeof (unshare_unmount_node_t));
	node->un_zhp = zhp;
	node->un_mountp = safe_strdup(entry.mnt_mountp);

	uu_avl_node_init(node, &node->un_avlnode, pool);

	if (uu_avl_find(tree, node, NULL, &idx) == NULL) {
	uu_avl_insert(tree, node, idx);
	} else {
	zfs_close(node->un_zhp);
	free(node->un_mountp);
	free(node);
	}
	}
	(void) fclose(mnttab);

	/*
	* Walk the AVL tree in reverse, unmounting each filesystem and
	* removing it from the AVL tree in the process.
	*/
	if ((walk = uu_avl_walk_start(tree,
	UU_WALK_REVERSE \| UU_WALK_ROBUST)) == NULL)
	nomem();

	while ((node = uu_avl_walk_next(walk)) != NULL) {
	const char *mntarg = NULL;

	uu_avl_remove(tree, node);
	switch (op) {
	case OP_SHARE:
	if (zfs_unshare(node->un_zhp,
	node->un_mountp, protocol) != 0)
	ret = 1;
	break;

	case OP_MOUNT:
	if (zfs_unmount(node->un_zhp,
	mntarg, flags) != 0)
	ret = 1;
	break;
	}

	zfs_close(node->un_zhp);
	free(node->un_mountp);
	free(node);
	}

	if (op == OP_SHARE)
	zfs_commit_shares(protocol);

	uu_avl_walk_end(walk);
	uu_avl_destroy(tree);
	uu_avl_pool_destroy(pool);

	} else {
	if (argc != 1) {
	if (argc == 0)
	(void) fprintf(stderr,
	gettext("missing filesystem argument\n"));
	else
	(void) fprintf(stderr,
	gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	/*
	* We have an argument, but it may be a full path or a ZFS
	* filesystem. Pass full paths off to unmount_path() (shared by
	* manual_unmount), otherwise open the filesystem and pass to
	* zfs_unmount().
	*/
	if (argv[0][0] == '/')
	return (unshare_unmount_path(op, argv[0],
	flags, B_FALSE));

	if ((zhp = zfs_open(g_zfs, argv[0],
	ZFS_TYPE_FILESYSTEM)) == NULL)
	return (1);

	verify(zfs_prop_get(zhp, op == OP_SHARE ?
	ZFS_PROP_SHARENFS : ZFS_PROP_MOUNTPOINT,
	nfs_mnt_prop, sizeof (nfs_mnt_prop), NULL,
	NULL, 0, B_FALSE) == 0);

	switch (op) {
	case OP_SHARE:
	verify(zfs_prop_get(zhp, ZFS_PROP_SHARENFS,
	nfs_mnt_prop,
	sizeof (nfs_mnt_prop),
	NULL, NULL, 0, B_FALSE) == 0);
	verify(zfs_prop_get(zhp, ZFS_PROP_SHARESMB,
	sharesmb, sizeof (sharesmb), NULL, NULL,
	0, B_FALSE) == 0);

	if (strcmp(nfs_mnt_prop, "off") == 0 &&
	strcmp(sharesmb, "off") == 0) {
	(void) fprintf(stderr, gettext("cannot "
	"unshare '%s': legacy share\n"),
	zfs_get_name(zhp));
	(void) fprintf(stderr, gettext("use "
	"exports(5) or smb.conf(5) to unshare "
	"this filesystem\n"));
	ret = 1;
	} else if (!zfs_is_shared(zhp, NULL, NULL)) {
	(void) fprintf(stderr, gettext("cannot "
	"unshare '%s': not currently "
	"shared\n"), zfs_get_name(zhp));
	ret = 1;
	} else if (zfs_unshareall(zhp, NULL) != 0) {
	ret = 1;
	}
	break;

	case OP_MOUNT:
	if (strcmp(nfs_mnt_prop, "legacy") == 0) {
	(void) fprintf(stderr, gettext("cannot "
	"unmount '%s': legacy "
	"mountpoint\n"), zfs_get_name(zhp));
	(void) fprintf(stderr, gettext("use "
	"umount(8) to unmount this "
	"filesystem\n"));
	ret = 1;
	} else if (!zfs_is_mounted(zhp, NULL)) {
	(void) fprintf(stderr, gettext("cannot "
	"unmount '%s': not currently "
	"mounted\n"),
	zfs_get_name(zhp));
	ret = 1;
	} else if (zfs_unmountall(zhp, flags) != 0) {
	ret = 1;
	}
	break;
	}

	zfs_close(zhp);
	}

	return (ret);
	}

	/*
	* zfs unmount [-fu] -a
	* zfs unmount [-fu] filesystem
	*
	* Unmount all filesystems, or a specific ZFS filesystem.
	*/
	static int
	zfs_do_unmount(int argc, char **argv)
	{
	return (unshare_unmount(OP_MOUNT, argc, argv));
	}

	/*
	* zfs unshare -a
	* zfs unshare filesystem
	*
	* Unshare all filesystems, or a specific ZFS filesystem.
	*/
	static int
	zfs_do_unshare(int argc, char **argv)
	{
	return (unshare_unmount(OP_SHARE, argc, argv));
	}

	static int
	find_command_idx(const char command, int idx)
	{
	int i;

	for (i = 0; i < NCOMMAND; i++) {
	if (command_table[i].name == NULL)
	continue;

	if (strcmp(command, command_table[i].name) == 0) {
	*idx = i;
	return (0);
	}
	}
	return (1);
	}

	static int
	zfs_do_diff(int argc, char **argv)
	{
	zfs_handle_t *zhp;
	int flags = 0;
	char *tosnap = NULL;
	char *fromsnap = NULL;
	char atp, copy;
	int err = 0;
	int c;
	struct sigaction sa;

	while ((c = getopt(argc, argv, "FHth")) != -1) {
	switch (c) {
	case 'F':
	flags \|= ZFS_DIFF_CLASSIFY;
	break;
	case 'H':
	flags \|= ZFS_DIFF_PARSEABLE;
	break;
	case 't':
	flags \|= ZFS_DIFF_TIMESTAMP;
	break;
	case 'h':
	flags \|= ZFS_DIFF_NO_MANGLE;
	break;
	default:
	(void) fprintf(stderr,
	gettext("invalid option '%c'\n"), optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	if (argc < 1) {
	(void) fprintf(stderr,
	gettext("must provide at least one snapshot name\n"));
	usage(B_FALSE);
	}

	if (argc > 2) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	fromsnap = argv[0];
	tosnap = (argc == 2) ? argv[1] : NULL;

	copy = NULL;
	if (*fromsnap != '@')
	copy = strdup(fromsnap);
	else if (tosnap)
	copy = strdup(tosnap);
	if (copy == NULL)
	usage(B_FALSE);

	if ((atp = strchr(copy, '@')) != NULL)
	*atp = '\0';

	if ((zhp = zfs_open(g_zfs, copy, ZFS_TYPE_FILESYSTEM)) == NULL) {
	free(copy);
	return (1);
	}
	free(copy);

	/*
	* Ignore SIGPIPE so that the library can give us
	* information on any failure
	*/
	if (sigemptyset(&sa.sa_mask) == -1) {
	err = errno;
	goto out;
	}
	sa.sa_flags = 0;
	sa.sa_handler = SIG_IGN;
	if (sigaction(SIGPIPE, &sa, NULL) == -1) {
	err = errno;
	goto out;
	}

	err = zfs_show_diffs(zhp, STDOUT_FILENO, fromsnap, tosnap, flags);
	out:
	zfs_close(zhp);

	return (err != 0);
	}

	/*
	* zfs bookmark <fs@source>\|<fs#source> <fs#bookmark>
	*
	* Creates a bookmark with the given name from the source snapshot
	* or creates a copy of an existing source bookmark.
	*/
	static int
	zfs_do_bookmark(int argc, char **argv)
	{
	char source, bookname;
	char expbuf[ZFS_MAX_DATASET_NAME_LEN];
	int source_type;
	nvlist_t *nvl;
	int ret = 0;
	int c;

	/* check options */
	while ((c = getopt(argc, argv, "")) != -1) {
	switch (c) {
	case '?':
	(void) fprintf(stderr,
	gettext("invalid option '%c'\n"), optopt);
	goto usage;
	}
	}

	argc -= optind;
	argv += optind;

	/* check number of arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing source argument\n"));
	goto usage;
	}
	if (argc < 2) {
	(void) fprintf(stderr, gettext("missing bookmark argument\n"));
	goto usage;
	}

	source = argv[0];
	bookname = argv[1];

	if (strchr(source, '@') == NULL && strchr(source, '#') == NULL) {
	(void) fprintf(stderr,
	gettext("invalid source name '%s': "
	"must contain a '@' or '#'\n"), source);
	goto usage;
	}
	if (strchr(bookname, '#') == NULL) {
	(void) fprintf(stderr,
	gettext("invalid bookmark name '%s': "
	"must contain a '#'\n"), bookname);
	goto usage;
	}

	/*
	* expand source or bookname to full path:
	* one of them may be specified as short name
	*/
	{
	char **expand;
	char source_short, bookname_short;
	source_short = strpbrk(source, "@#");
	bookname_short = strpbrk(bookname, "#");
	if (source_short == source &&
	bookname_short == bookname) {
	(void) fprintf(stderr, gettext(
	"either source or bookmark must be specified as "
	"full dataset paths"));
	goto usage;
	} else if (source_short != source &&
	bookname_short != bookname) {
	expand = NULL;
	} else if (source_short != source) {
	strlcpy(expbuf, source, sizeof (expbuf));
	expand = &bookname;
	} else if (bookname_short != bookname) {
	strlcpy(expbuf, bookname, sizeof (expbuf));
	expand = &source;
	} else {
	abort();
	}
	if (expand != NULL) {
	strpbrk(expbuf, "@#") = '\0'; / dataset name in buf */
	(void) strlcat(expbuf, *expand, sizeof (expbuf));
	*expand = expbuf;
	}
	}

	/* determine source type */
	switch (*strpbrk(source, "@#")) {
	case '@': source_type = ZFS_TYPE_SNAPSHOT; break;
	case '#': source_type = ZFS_TYPE_BOOKMARK; break;
	default: abort();
	}

	/* test the source exists */
	zfs_handle_t *zhp;
	zhp = zfs_open(g_zfs, source, source_type);
	if (zhp == NULL)
	goto usage;
	zfs_close(zhp);

	nvl = fnvlist_alloc();
	fnvlist_add_string(nvl, bookname, source);
	ret = lzc_bookmark(nvl, NULL);
	fnvlist_free(nvl);

	if (ret != 0) {
	const char *err_msg = NULL;
	char errbuf[1024];

	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN,
	"cannot create bookmark '%s'"), bookname);

	switch (ret) {
	case EXDEV:
	err_msg = "bookmark is in a different pool";
	break;
	case ZFS_ERR_BOOKMARK_SOURCE_NOT_ANCESTOR:
	err_msg = "source is not an ancestor of the "
	"new bookmark's dataset";
	break;
	case EEXIST:
	err_msg = "bookmark exists";
	break;
	case EINVAL:
	err_msg = "invalid argument";
	break;
	case ENOTSUP:
	err_msg = "bookmark feature not enabled";
	break;
	case ENOSPC:
	err_msg = "out of space";
	break;
	case ENOENT:
	err_msg = "dataset does not exist";
	break;
	default:
	(void) zfs_standard_error(g_zfs, ret, errbuf);
	break;
	}
	if (err_msg != NULL) {
	(void) fprintf(stderr, "%s: %s\n", errbuf,
	dgettext(TEXT_DOMAIN, err_msg));
	}
	}

	return (ret != 0);

	usage:
	usage(B_FALSE);
	return (-1);
	}

	static int
	zfs_do_channel_program(int argc, char **argv)
	{
	int ret, fd, c;
	size_t progsize, progread;
	nvlist_t *outnvl = NULL;
	uint64_t instrlimit = ZCP_DEFAULT_INSTRLIMIT;
	uint64_t memlimit = ZCP_DEFAULT_MEMLIMIT;
	boolean_t sync_flag = B_TRUE, json_output = B_FALSE;
	zpool_handle_t *zhp;

	/* check options */
	while ((c = getopt(argc, argv, "nt:m:j")) != -1) {
	switch (c) {
	case 't':
	case 'm': {
	uint64_t arg;
	char *endp;

	errno = 0;
	arg = strtoull(optarg, &endp, 0);
	if (errno != 0 \|\| *endp != '\0') {
	(void) fprintf(stderr, gettext(
	"invalid argument "
	"'%s': expected integer\n"), optarg);
	goto usage;
	}

	if (c == 't') {
	instrlimit = arg;
	} else {
	ASSERT3U(c, ==, 'm');
	memlimit = arg;
	}
	break;
	}
	case 'n': {
	sync_flag = B_FALSE;
	break;
	}
	case 'j': {
	json_output = B_TRUE;
	break;
	}
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	goto usage;
	}
	}

	argc -= optind;
	argv += optind;

	if (argc < 2) {
	(void) fprintf(stderr,
	gettext("invalid number of arguments\n"));
	goto usage;
	}

	const char *poolname = argv[0];
	const char *filename = argv[1];
	if (strcmp(filename, "-") == 0) {
	fd = 0;
	filename = "standard input";
	} else if ((fd = open(filename, O_RDONLY)) < 0) {
	(void) fprintf(stderr, gettext("cannot open '%s': %s\n"),
	filename, strerror(errno));
	return (1);
	}

	if ((zhp = zpool_open(g_zfs, poolname)) == NULL) {
	(void) fprintf(stderr, gettext("cannot open pool '%s'\n"),
	poolname);
	if (fd != 0)
	(void) close(fd);
	return (1);
	}
	zpool_close(zhp);

	/*
	* Read in the channel program, expanding the program buffer as
	* necessary.
	*/
	progread = 0;
	progsize = 1024;
	char *progbuf = safe_malloc(progsize);
	do {
	ret = read(fd, progbuf + progread, progsize - progread);
	progread += ret;
	if (progread == progsize && ret > 0) {
	progsize *= 2;
	progbuf = safe_realloc(progbuf, progsize);
	}
	} while (ret > 0);

	if (fd != 0)
	(void) close(fd);
	if (ret < 0) {
	free(progbuf);
	(void) fprintf(stderr,
	gettext("cannot read '%s': %s\n"),
	filename, strerror(errno));
	return (1);
	}
	progbuf[progread] = '\0';

	/*
	* Any remaining arguments are passed as arguments to the lua script as
	* a string array:
	* {
	* "argv" -> [ "arg 1", ... "arg n" ],
	* }
	*/
	nvlist_t *argnvl = fnvlist_alloc();
	fnvlist_add_string_array(argnvl, ZCP_ARG_CLIARGV,
	(const char **)argv + 2, argc - 2);

	if (sync_flag) {
	ret = lzc_channel_program(poolname, progbuf,
	instrlimit, memlimit, argnvl, &outnvl);
	} else {
	ret = lzc_channel_program_nosync(poolname, progbuf,
	instrlimit, memlimit, argnvl, &outnvl);
	}

	if (ret != 0) {
	/*
	* On error, report the error message handed back by lua if one
	* exists. Otherwise, generate an appropriate error message,
	* falling back on strerror() for an unexpected return code.
	*/
	const char *errstring = NULL;
	const char *msg = gettext("Channel program execution failed");
	uint64_t instructions = 0;
	if (outnvl != NULL && nvlist_exists(outnvl, ZCP_RET_ERROR)) {
	const char *es = NULL;
	(void) nvlist_lookup_string(outnvl,
	ZCP_RET_ERROR, &es);
	if (es == NULL)
	errstring = strerror(ret);
	else
	errstring = es;
	if (ret == ETIME) {
	(void) nvlist_lookup_uint64(outnvl,
	ZCP_ARG_INSTRLIMIT, &instructions);
	}
	} else {
	switch (ret) {
	case EINVAL:
	errstring =
	"Invalid instruction or memory limit.";
	break;
	case ENOMEM:
	errstring = "Return value too large.";
	break;
	case ENOSPC:
	errstring = "Memory limit exhausted.";
	break;
	case ETIME:
	errstring = "Timed out.";
	break;
	case EPERM:
	errstring = "Permission denied. Channel "
	"programs must be run as root.";
	break;
	default:
	(void) zfs_standard_error(g_zfs, ret, msg);
	}
	}
	if (errstring != NULL)
	(void) fprintf(stderr, "%s:\n%s\n", msg, errstring);

	if (ret == ETIME && instructions != 0)
	(void) fprintf(stderr,
	gettext("%llu Lua instructions\n"),
	(u_longlong_t)instructions);
	} else {
	if (json_output) {
	(void) nvlist_print_json(stdout, outnvl);
	} else if (nvlist_empty(outnvl)) {
	(void) fprintf(stdout, gettext("Channel program fully "
	"executed and did not produce output.\n"));
	} else {
	(void) fprintf(stdout, gettext("Channel program fully "
	"executed and produced output:\n"));
	dump_nvlist(outnvl, 4);
	}
	}

	free(progbuf);
	fnvlist_free(outnvl);
	fnvlist_free(argnvl);
	return (ret != 0);

	usage:
	usage(B_FALSE);
	return (-1);
	}


	typedef struct loadkey_cbdata {
	boolean_t cb_loadkey;
	boolean_t cb_recursive;
	boolean_t cb_noop;
	char *cb_keylocation;
	uint64_t cb_numfailed;
	uint64_t cb_numattempted;
	} loadkey_cbdata_t;

	static int
	load_key_callback(zfs_handle_t zhp, void data)
	{
	int ret;
	boolean_t is_encroot;
	loadkey_cbdata_t *cb = data;
	uint64_t keystatus = zfs_prop_get_int(zhp, ZFS_PROP_KEYSTATUS);

	/*
	* If we are working recursively, we want to skip loading / unloading
	* keys for non-encryption roots and datasets whose keys are already
	* in the desired end-state.
	*/
	if (cb->cb_recursive) {
	ret = zfs_crypto_get_encryption_root(zhp, &is_encroot, NULL);
	if (ret != 0)
	return (ret);
	if (!is_encroot)
	return (0);

	if ((cb->cb_loadkey && keystatus == ZFS_KEYSTATUS_AVAILABLE) \|\|
	(!cb->cb_loadkey && keystatus == ZFS_KEYSTATUS_UNAVAILABLE))
	return (0);
	}

	cb->cb_numattempted++;

	if (cb->cb_loadkey)
	ret = zfs_crypto_load_key(zhp, cb->cb_noop, cb->cb_keylocation);
	else
	ret = zfs_crypto_unload_key(zhp);

	if (ret != 0) {
	cb->cb_numfailed++;
	return (ret);
	}

	return (0);
	}

	static int
	load_unload_keys(int argc, char **argv, boolean_t loadkey)
	{
	int c, ret = 0, flags = 0;
	boolean_t do_all = B_FALSE;
	loadkey_cbdata_t cb = { 0 };

	cb.cb_loadkey = loadkey;

	while ((c = getopt(argc, argv, "anrL:")) != -1) {
	/* noop and alternate keylocations only apply to zfs load-key */
	if (loadkey) {
	switch (c) {
	case 'n':
	cb.cb_noop = B_TRUE;
	continue;
	case 'L':
	cb.cb_keylocation = optarg;
	continue;
	default:
	break;
	}
	}

	switch (c) {
	case 'a':
	do_all = B_TRUE;
	cb.cb_recursive = B_TRUE;
	break;
	case 'r':
	flags \|= ZFS_ITER_RECURSE;
	cb.cb_recursive = B_TRUE;
	break;
	default:
	(void) fprintf(stderr,
	gettext("invalid option '%c'\n"), optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	if (!do_all && argc == 0) {
	(void) fprintf(stderr,
	gettext("Missing dataset argument or -a option\n"));
	usage(B_FALSE);
	}

	if (do_all && argc != 0) {
	(void) fprintf(stderr,
	gettext("Cannot specify dataset with -a option\n"));
	usage(B_FALSE);
	}

	if (cb.cb_recursive && cb.cb_keylocation != NULL &&
	strcmp(cb.cb_keylocation, "prompt") != 0) {
	(void) fprintf(stderr, gettext("alternate keylocation may only "
	"be 'prompt' with -r or -a\n"));
	usage(B_FALSE);
	}

	ret = zfs_for_each(argc, argv, flags,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME, NULL, NULL, 0,
	load_key_callback, &cb);

	if (cb.cb_noop \|\| (cb.cb_recursive && cb.cb_numattempted != 0)) {
	(void) printf(gettext("%llu / %llu key(s) successfully %s\n"),
	(u_longlong_t)(cb.cb_numattempted - cb.cb_numfailed),
	(u_longlong_t)cb.cb_numattempted,
	loadkey ? (cb.cb_noop ? "verified" : "loaded") :
	"unloaded");
	}

	if (cb.cb_numfailed != 0)
	ret = -1;

	return (ret);
	}

	static int
	zfs_do_load_key(int argc, char **argv)
	{
	return (load_unload_keys(argc, argv, B_TRUE));
	}


	static int
	zfs_do_unload_key(int argc, char **argv)
	{
	return (load_unload_keys(argc, argv, B_FALSE));
	}

	static int
	zfs_do_change_key(int argc, char **argv)
	{
	int c, ret;
	uint64_t keystatus;
	boolean_t loadkey = B_FALSE, inheritkey = B_FALSE;
	zfs_handle_t *zhp = NULL;
	nvlist_t *props = fnvlist_alloc();

	while ((c = getopt(argc, argv, "lio:")) != -1) {
	switch (c) {
	case 'l':
	loadkey = B_TRUE;
	break;
	case 'i':
	inheritkey = B_TRUE;
	break;
	case 'o':
	if (!parseprop(props, optarg)) {
	nvlist_free(props);
	return (1);
	}
	break;
	default:
	(void) fprintf(stderr,
	gettext("invalid option '%c'\n"), optopt);
	usage(B_FALSE);
	}
	}

	if (inheritkey && !nvlist_empty(props)) {
	(void) fprintf(stderr,
	gettext("Properties not allowed for inheriting\n"));
	usage(B_FALSE);
	}

	argc -= optind;
	argv += optind;

	if (argc < 1) {
	(void) fprintf(stderr, gettext("Missing dataset argument\n"));
	usage(B_FALSE);
	}

	if (argc > 1) {
	(void) fprintf(stderr, gettext("Too many arguments\n"));
	usage(B_FALSE);
	}

	zhp = zfs_open(g_zfs, argv[argc - 1],
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME);
	if (zhp == NULL)
	usage(B_FALSE);

	if (loadkey) {
	keystatus = zfs_prop_get_int(zhp, ZFS_PROP_KEYSTATUS);
	if (keystatus != ZFS_KEYSTATUS_AVAILABLE) {
	ret = zfs_crypto_load_key(zhp, B_FALSE, NULL);
	if (ret != 0) {
	nvlist_free(props);
	zfs_close(zhp);
	return (-1);
	}
	}

	/* refresh the properties so the new keystatus is visible */
	zfs_refresh_properties(zhp);
	}

	ret = zfs_crypto_rewrap(zhp, props, inheritkey);
	if (ret != 0) {
	nvlist_free(props);
	zfs_close(zhp);
	return (-1);
	}

	nvlist_free(props);
	zfs_close(zhp);
	return (0);
	}

	/*
	* 1) zfs project [-d\|-r] <file\|directory ...>
	* List project ID and inherit flag of file(s) or directories.
	* -d: List the directory itself, not its children.
	* -r: List subdirectories recursively.
	*
	* 2) zfs project -C [-k] [-r] <file\|directory ...>
	* Clear project inherit flag and/or ID on the file(s) or directories.
	* -k: Keep the project ID unchanged. If not specified, the project ID
	* will be reset as zero.
	* -r: Clear on subdirectories recursively.
	*
	* 3) zfs project -c [-0] [-d\|-r] [-p id] <file\|directory ...>
	* Check project ID and inherit flag on the file(s) or directories,
	* report the outliers.
	* -0: Print file name followed by a NUL instead of newline.
	* -d: Check the directory itself, not its children.
	* -p: Specify the referenced ID for comparing with the target file(s)
	* or directories' project IDs. If not specified, the target (top)
	* directory's project ID will be used as the referenced one.
	* -r: Check subdirectories recursively.
	*
	* 4) zfs project [-p id] [-r] [-s] <file\|directory ...>
	* Set project ID and/or inherit flag on the file(s) or directories.
	* -p: Set the project ID as the given id.
	* -r: Set on subdirectories recursively. If not specify "-p" option,
	* it will use top-level directory's project ID as the given id,
	* then set both project ID and inherit flag on all descendants
	* of the top-level directory.
	* -s: Set project inherit flag.
	*/
	static int
	zfs_do_project(int argc, char **argv)
	{
	zfs_project_control_t zpc = {
	.zpc_expected_projid = ZFS_INVALID_PROJID,
	.zpc_op = ZFS_PROJECT_OP_DEFAULT,
	.zpc_dironly = B_FALSE,
	.zpc_keep_projid = B_FALSE,
	.zpc_newline = B_TRUE,
	.zpc_recursive = B_FALSE,
	.zpc_set_flag = B_FALSE,
	};
	int ret = 0, c;

	if (argc < 2)
	usage(B_FALSE);

	while ((c = getopt(argc, argv, "0Ccdkp:rs")) != -1) {
	switch (c) {
	case '0':
	zpc.zpc_newline = B_FALSE;
	break;
	case 'C':
	if (zpc.zpc_op != ZFS_PROJECT_OP_DEFAULT) {
	(void) fprintf(stderr, gettext("cannot "
	"specify '-C' '-c' '-s' together\n"));
	usage(B_FALSE);
	}

	zpc.zpc_op = ZFS_PROJECT_OP_CLEAR;
	break;
	case 'c':
	if (zpc.zpc_op != ZFS_PROJECT_OP_DEFAULT) {
	(void) fprintf(stderr, gettext("cannot "
	"specify '-C' '-c' '-s' together\n"));
	usage(B_FALSE);
	}

	zpc.zpc_op = ZFS_PROJECT_OP_CHECK;
	break;
	case 'd':
	zpc.zpc_dironly = B_TRUE;
	/* overwrite "-r" option */
	zpc.zpc_recursive = B_FALSE;
	break;
	case 'k':
	zpc.zpc_keep_projid = B_TRUE;
	break;
	case 'p': {
	char *endptr;

	errno = 0;
	zpc.zpc_expected_projid = strtoull(optarg, &endptr, 0);
	if (errno != 0 \|\| *endptr != '\0') {
	(void) fprintf(stderr,
	gettext("project ID must be less than "
	"%u\n"), UINT32_MAX);
	usage(B_FALSE);
	}
	if (zpc.zpc_expected_projid >= UINT32_MAX) {
	(void) fprintf(stderr,
	gettext("invalid project ID\n"));
	usage(B_FALSE);
	}
	break;
	}
	case 'r':
	zpc.zpc_recursive = B_TRUE;
	/* overwrite "-d" option */
	zpc.zpc_dironly = B_FALSE;
	break;
	case 's':
	if (zpc.zpc_op != ZFS_PROJECT_OP_DEFAULT) {
	(void) fprintf(stderr, gettext("cannot "
	"specify '-C' '-c' '-s' together\n"));
	usage(B_FALSE);
	}

	zpc.zpc_set_flag = B_TRUE;
	zpc.zpc_op = ZFS_PROJECT_OP_SET;
	break;
	default:
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	if (zpc.zpc_op == ZFS_PROJECT_OP_DEFAULT) {
	if (zpc.zpc_expected_projid != ZFS_INVALID_PROJID)
	zpc.zpc_op = ZFS_PROJECT_OP_SET;
	else
	zpc.zpc_op = ZFS_PROJECT_OP_LIST;
	}

	switch (zpc.zpc_op) {
	case ZFS_PROJECT_OP_LIST:
	if (zpc.zpc_keep_projid) {
	(void) fprintf(stderr,
	gettext("'-k' is only valid together with '-C'\n"));
	usage(B_FALSE);
	}
	if (!zpc.zpc_newline) {
	(void) fprintf(stderr,
	gettext("'-0' is only valid together with '-c'\n"));
	usage(B_FALSE);
	}
	break;
	case ZFS_PROJECT_OP_CHECK:
	if (zpc.zpc_keep_projid) {
	(void) fprintf(stderr,
	gettext("'-k' is only valid together with '-C'\n"));
	usage(B_FALSE);
	}
	break;
	case ZFS_PROJECT_OP_CLEAR:
	if (zpc.zpc_dironly) {
	(void) fprintf(stderr,
	gettext("'-d' is useless together with '-C'\n"));
	usage(B_FALSE);
	}
	if (!zpc.zpc_newline) {
	(void) fprintf(stderr,
	gettext("'-0' is only valid together with '-c'\n"));
	usage(B_FALSE);
	}
	if (zpc.zpc_expected_projid != ZFS_INVALID_PROJID) {
	(void) fprintf(stderr,
	gettext("'-p' is useless together with '-C'\n"));
	usage(B_FALSE);
	}
	break;
	case ZFS_PROJECT_OP_SET:
	if (zpc.zpc_dironly) {
	(void) fprintf(stderr,
	gettext("'-d' is useless for set project ID and/or "
	"inherit flag\n"));
	usage(B_FALSE);
	}
	if (zpc.zpc_keep_projid) {
	(void) fprintf(stderr,
	gettext("'-k' is only valid together with '-C'\n"));
	usage(B_FALSE);
	}
	if (!zpc.zpc_newline) {
	(void) fprintf(stderr,
	gettext("'-0' is only valid together with '-c'\n"));
	usage(B_FALSE);
	}
	break;
	default:
	ASSERT(0);
	break;
	}

	argv += optind;
	argc -= optind;
	if (argc == 0) {
	(void) fprintf(stderr,
	gettext("missing file or directory target(s)\n"));
	usage(B_FALSE);
	}

	for (int i = 0; i < argc; i++) {
	int err;

	err = zfs_project_handle(argv[i], &zpc);
	if (err && !ret)
	ret = err;
	}

	return (ret);
	}

	static int
	zfs_do_wait(int argc, char **argv)
	{
	boolean_t enabled[ZFS_WAIT_NUM_ACTIVITIES];
	int error = 0, i;
	int c;

	/* By default, wait for all types of activity. */
	for (i = 0; i < ZFS_WAIT_NUM_ACTIVITIES; i++)
	enabled[i] = B_TRUE;

	while ((c = getopt(argc, argv, "t:")) != -1) {
	switch (c) {
	case 't':
	/* Reset activities array */
	memset(&enabled, 0, sizeof (enabled));

	for (char *tok; (tok = strsep(&optarg, ",")); ) {
	static const char *const col_subopts[
	ZFS_WAIT_NUM_ACTIVITIES] = { "deleteq" };

	for (i = 0; i < ARRAY_SIZE(col_subopts); ++i)
	if (strcmp(tok, col_subopts[i]) == 0) {
	enabled[i] = B_TRUE;
	goto found;
	}

	(void) fprintf(stderr,
	gettext("invalid activity '%s'\n"), tok);
	usage(B_FALSE);
	found:;
	}
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argv += optind;
	argc -= optind;
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing 'filesystem' "
	"argument\n"));
	usage(B_FALSE);
	}
	if (argc > 1) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	zfs_handle_t *zhp = zfs_open(g_zfs, argv[0], ZFS_TYPE_FILESYSTEM);
	if (zhp == NULL)
	return (1);

	for (;;) {
	boolean_t missing = B_FALSE;
	boolean_t any_waited = B_FALSE;

	for (int i = 0; i < ZFS_WAIT_NUM_ACTIVITIES; i++) {
	boolean_t waited;

	if (!enabled[i])
	continue;

	error = zfs_wait_status(zhp, i, &missing, &waited);
	if (error != 0 \|\| missing)
	break;

	any_waited = (any_waited \|\| waited);
	}

	if (error != 0 \|\| missing \|\| !any_waited)
	break;
	}

	zfs_close(zhp);

	return (error);
	}

	/*
	* Display version message
	*/
	static int
	zfs_do_version(int argc, char **argv)
	{
	(void) argc, (void) argv;
	return (zfs_version_print() != 0);
	}

	/* Display documentation */
	static int
	zfs_do_help(int argc, char **argv)
	{
	char page[MAXNAMELEN];
	if (argc < 3 \|\| strcmp(argv[2], "zfs") == 0)
	strcpy(page, "zfs");
	else if (strcmp(argv[2], "concepts") == 0 \|\|
	strcmp(argv[2], "props") == 0)
	snprintf(page, sizeof (page), "zfs%s", argv[2]);
	else
	snprintf(page, sizeof (page), "zfs-%s", argv[2]);

	execlp("man", "man", page, NULL);

	fprintf(stderr, "couldn't run man program: %s", strerror(errno));
	return (-1);
	}

	int
	main(int argc, char **argv)
	{
	int ret = 0;
	int i = 0;
	const char *cmdname;
	char **newargv;

	(void) setlocale(LC_ALL, "");
	(void) setlocale(LC_NUMERIC, "C");
	(void) textdomain(TEXT_DOMAIN);

	opterr = 0;

	/*
	* Make sure the user has specified some command.
	*/
	if (argc < 2) {
	(void) fprintf(stderr, gettext("missing command\n"));
	usage(B_FALSE);
	}

	cmdname = argv[1];

	/*
	* The 'umount' command is an alias for 'unmount'
	*/
	if (strcmp(cmdname, "umount") == 0)
	cmdname = "unmount";

	/*
	* The 'recv' command is an alias for 'receive'
	*/
	if (strcmp(cmdname, "recv") == 0)
	cmdname = "receive";

	/*
	* The 'snap' command is an alias for 'snapshot'
	*/
	if (strcmp(cmdname, "snap") == 0)
	cmdname = "snapshot";

	/*
	* Special case '-?'
	*/
	if ((strcmp(cmdname, "-?") == 0) \|\|
	(strcmp(cmdname, "--help") == 0))
	usage(B_TRUE);

	/*
	* Special case '-V\|--version'
	*/
	if ((strcmp(cmdname, "-V") == 0) \|\| (strcmp(cmdname, "--version") == 0))
	return (zfs_do_version(argc, argv));

	/*
	* Special case 'help'
	*/
	if (strcmp(cmdname, "help") == 0)
	return (zfs_do_help(argc, argv));

	if ((g_zfs = libzfs_init()) == NULL) {
	(void) fprintf(stderr, "%s\n", libzfs_error_init(errno));
	return (1);
	}

	zfs_save_arguments(argc, argv, history_str, sizeof (history_str));

	libzfs_print_on_error(g_zfs, B_TRUE);

	zfs_setproctitle_init(argc, argv, environ);

	/*
	* Many commands modify input strings for string parsing reasons.
	* We create a copy to protect the original argv.
	*/
	newargv = safe_malloc((argc + 1) * sizeof (newargv[0]));
	for (i = 0; i < argc; i++)
	newargv[i] = strdup(argv[i]);
	newargv[argc] = NULL;

	/*
	* Run the appropriate command.
	*/
	libzfs_mnttab_cache(g_zfs, B_TRUE);
	if (find_command_idx(cmdname, &i) == 0) {
	current_command = &command_table[i];
	ret = command_table[i].func(argc - 1, newargv + 1);
	} else if (strchr(cmdname, '=') != NULL) {
	verify(find_command_idx("set", &i) == 0);
	current_command = &command_table[i];
	ret = command_table[i].func(argc, newargv);
	} else {
	(void) fprintf(stderr, gettext("unrecognized "
	"command '%s'\n"), cmdname);
	usage(B_FALSE);
	ret = 1;
	}

	for (i = 0; i < argc; i++)
	free(newargv[i]);
	free(newargv);

	if (ret == 0 && log_history)
	(void) zpool_log_history(g_zfs, history_str);

	libzfs_fini(g_zfs);

	/*
	* The 'ZFS_ABORT' environment variable causes us to dump core on exit
	* for the purposes of running ::findleaks.
	*/
	if (getenv("ZFS_ABORT") != NULL) {
	(void) printf("dumping core by request\n");
	abort();
	}

	return (ret);
	}

	/*
	* zfs zone nsfile filesystem
	*
	* Add or delete the given dataset to/from the namespace.
	*/
	#ifdef __linux__
	static int
	zfs_do_zone_impl(int argc, char **argv, boolean_t attach)
	{
	zfs_handle_t *zhp;
	int ret;

	if (argc < 3) {
	(void) fprintf(stderr, gettext("missing argument(s)\n"));
	usage(B_FALSE);
	}
	if (argc > 3) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	zhp = zfs_open(g_zfs, argv[2], ZFS_TYPE_FILESYSTEM);
	if (zhp == NULL)
	return (1);

	ret = (zfs_userns(zhp, argv[1], attach) != 0);

	zfs_close(zhp);
	return (ret);
	}

	static int
	zfs_do_zone(int argc, char **argv)
	{
	return (zfs_do_zone_impl(argc, argv, B_TRUE));
	}

	static int
	zfs_do_unzone(int argc, char **argv)
	{
	return (zfs_do_zone_impl(argc, argv, B_FALSE));
	}
	#endif

	#ifdef __FreeBSD__
	#include <sys/jail.h>
	#include <jail.h>
	/*
	* Attach/detach the given dataset to/from the given jail
	*/
	static int
	zfs_do_jail_impl(int argc, char **argv, boolean_t attach)
	{
	zfs_handle_t *zhp;
	int jailid, ret;

	/* check number of arguments */
	if (argc < 3) {
	(void) fprintf(stderr, gettext("missing argument(s)\n"));
	usage(B_FALSE);
	}
	if (argc > 3) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	jailid = jail_getid(argv[1]);
	if (jailid < 0) {
	(void) fprintf(stderr, gettext("invalid jail id or name\n"));
	usage(B_FALSE);
	}

	zhp = zfs_open(g_zfs, argv[2], ZFS_TYPE_FILESYSTEM);
	if (zhp == NULL)
	return (1);

	ret = (zfs_jail(zhp, jailid, attach) != 0);

	zfs_close(zhp);
	return (ret);
	}

	/*
	* zfs jail jailid filesystem
	*
	* Attach the given dataset to the given jail
	*/
	static int
	zfs_do_jail(int argc, char **argv)
	{
	return (zfs_do_jail_impl(argc, argv, B_TRUE));
	}

	/*
	* zfs unjail jailid filesystem
	*
	* Detach the given dataset from the given jail
	*/
	static int
	zfs_do_unjail(int argc, char **argv)
	{
	return (zfs_do_jail_impl(argc, argv, B_FALSE));
	}
	#endif
	diff --git a/sys/contrib/openzfs/cmd/zinject/zinject.c b/sys/contrib/openzfs/cmd/zinject/zinject.c
	index f1262ed772de..a11b6d0b7fac 100644
	--- a/sys/contrib/openzfs/cmd/zinject/zinject.c
	+++ b/sys/contrib/openzfs/cmd/zinject/zinject.c
	@@ -1,1288 +1,1304 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2012, 2015 by Delphix. All rights reserved.
	* Copyright (c) 2017, Intel Corporation.
	*/

	/*
	* ZFS Fault Injector
	*
	* This userland component takes a set of options and uses libzpool to translate
	* from a user-visible object type and name to an internal representation.
	* There are two basic types of faults: device faults and data faults.
	*
	*
	* DEVICE FAULTS
	*
	* Errors can be injected into a particular vdev using the '-d' option. This
	* option takes a path or vdev GUID to uniquely identify the device within a
	* pool. There are four types of errors that can be injected, IO, ENXIO,
	* ECHILD, and EILSEQ. These can be controlled through the '-e' option and the
	* default is ENXIO. For EIO failures, any attempt to read data from the device
	* will return EIO, but a subsequent attempt to reopen the device will succeed.
	* For ENXIO failures, any attempt to read from the device will return EIO, but
	* any attempt to reopen the device will also return ENXIO. The EILSEQ failures
	* only apply to read operations (-T read) and will flip a bit after the device
	* has read the original data.
	*
	* For label faults, the -L option must be specified. This allows faults
	* to be injected into either the nvlist, uberblock, pad1, or pad2 region
	* of all the labels for the specified device.
	*
	* This form of the command looks like:
	*
	* zinject -d device [-e errno] [-L <uber \| nvlist \| pad1 \| pad2>] pool
	*
	*
	* DATA FAULTS
	*
	* We begin with a tuple of the form:
	*
	* <type,level,range,object>
	*
	* type A string describing the type of data to target. Each type
	* implicitly describes how to interpret 'object'. Currently,
	* the following values are supported:
	*
	* data User data for a file
	* dnode Dnode for a file or directory
	*
	* The following MOS objects are special. Instead of injecting
	* errors on a particular object or blkid, we inject errors across
	* all objects of the given type.
	*
	* mos Any data in the MOS
	* mosdir object directory
	* config pool configuration
	* bpobj blkptr list
	* spacemap spacemap
	* metaslab metaslab
	* errlog persistent error log
	*
	* level Object level. Defaults to '0', not applicable to all types. If
	* a range is given, this corresponds to the indirect block
	* corresponding to the specific range.
	*
	* range A numerical range [start,end) within the object. Defaults to
	* the full size of the file.
	*
	* object A string describing the logical location of the object. For
	* files and directories (currently the only supported types),
	* this is the path of the object on disk.
	*
	* This is translated, via libzpool, into the following internal representation:
	*
	* <type,objset,object,level,range>
	*
	* These types should be self-explanatory. This tuple is then passed to the
	* kernel via a special ioctl() to initiate fault injection for the given
	* object. Note that 'type' is not strictly necessary for fault injection, but
	* is used when translating existing faults into a human-readable string.
	*
	*
	* The command itself takes one of the forms:
	*
	* zinject
	* zinject <-a \| -u pool>
	* zinject -c <id\|all>
	* zinject [-q] <-t type> [-f freq] [-u] [-a] [-m] [-e errno] [-l level]
	* [-r range] <object>
	* zinject [-f freq] [-a] [-m] [-u] -b objset:object:level:start:end pool
	*
	* With no arguments, the command prints all currently registered injection
	* handlers, with their numeric identifiers.
	*
	* The '-c' option will clear the given handler, or all handlers if 'all' is
	* specified.
	*
	* The '-e' option takes a string describing the errno to simulate. This must
	* be one of 'io', 'checksum', 'decompress', or 'decrypt'. In most cases this
	* will result in the same behavior, but RAID-Z will produce a different set of
	* ereports for this situation.
	*
	* The '-a', '-u', and '-m' flags toggle internal flush behavior. If '-a' is
	* specified, then the ARC cache is flushed appropriately. If '-u' is
	* specified, then the underlying SPA is unloaded. Either of these flags can be
	* specified independently of any other handlers. The '-m' flag automatically
	* does an unmount and remount of the underlying dataset to aid in flushing the
	* cache.
	*
	* The '-f' flag controls the frequency of errors injected, expressed as a
	* real number percentage between 0.0001 and 100. The default is 100.
	*
	* The this form is responsible for actually injecting the handler into the
	* framework. It takes the arguments described above, translates them to the
	* internal tuple using libzpool, and then issues an ioctl() to register the
	* handler.
	*
	* The final form can target a specific bookmark, regardless of whether a
	* human-readable interface has been designed. It allows developers to specify
	* a particular block by number.
	*/

	#include <errno.h>
	#include <fcntl.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <strings.h>
	#include <unistd.h>

	#include <sys/fs/zfs.h>
	#include <sys/mount.h>

	#include <libzfs.h>

	#undef verify /* both libzfs.h and zfs_context.h want to define this */

	#include "zinject.h"

	libzfs_handle_t *g_zfs;
	int zfs_fd;

	static const char *const errtable[TYPE_INVAL] = {
	"data",
	"dnode",
	"mos",
	"mosdir",
	"metaslab",
	"config",
	"bpobj",
	"spacemap",
	"errlog",
	"uber",
	"nvlist",
	"pad1",
	"pad2"
	};

	static err_type_t
	name_to_type(const char *arg)
	{
	int i;
	for (i = 0; i < TYPE_INVAL; i++)
	if (strcmp(errtable[i], arg) == 0)
	return (i);

	return (TYPE_INVAL);
	}

	static const char *
	type_to_name(uint64_t type)
	{
	switch (type) {
	case DMU_OT_OBJECT_DIRECTORY:
	return ("mosdir");
	case DMU_OT_OBJECT_ARRAY:
	return ("metaslab");
	case DMU_OT_PACKED_NVLIST:
	return ("config");
	case DMU_OT_BPOBJ:
	return ("bpobj");
	case DMU_OT_SPACE_MAP:
	return ("spacemap");
	case DMU_OT_ERROR_LOG:
	return ("errlog");
	default:
	return ("-");
	}
	}


	/*
	* Print usage message.
	*/
	void
	usage(void)
	{
	(void) printf(
	"usage:\n"
	"\n"
	"\tzinject\n"
	"\n"
	"\t\tList all active injection records.\n"
	"\n"
	"\tzinject -c <id\|all>\n"
	"\n"
	"\t\tClear the particular record (if given a numeric ID), or\n"
	"\t\tall records if 'all' is specified.\n"
	"\n"
	"\tzinject -p <function name> pool\n"
	"\t\tInject a panic fault at the specified function. Only \n"
	"\t\tfunctions which call spa_vdev_config_exit(), or \n"
	"\t\tspa_vdev_exit() will trigger a panic.\n"
	"\n"
	"\tzinject -d device [-e errno] [-L <nvlist\|uber\|pad1\|pad2>] [-F]\n"
	"\t\t[-T <read\|write\|free\|claim\|all>] [-f frequency] pool\n\n"
	"\t\tInject a fault into a particular device or the device's\n"
	"\t\tlabel. Label injection can either be 'nvlist', 'uber',\n "
	"\t\t'pad1', or 'pad2'.\n"
	"\t\t'errno' can be 'nxio' (the default), 'io', 'dtl', or\n"
	"\t\t'corrupt' (bit flip).\n"
	"\t\t'frequency' is a value between 0.0001 and 100.0 that limits\n"
	"\t\tdevice error injection to a percentage of the IOs.\n"
	"\n"
	"\tzinject -d device -A <degrade\|fault> -D <delay secs> pool\n"
	"\t\tPerform a specific action on a particular device.\n"
	"\n"
	"\tzinject -d device -D latency:lanes pool\n"
	"\n"
	"\t\tAdd an artificial delay to IO requests on a particular\n"
	"\t\tdevice, such that the requests take a minimum of 'latency'\n"
	"\t\tmilliseconds to complete. Each delay has an associated\n"
	"\t\tnumber of 'lanes' which defines the number of concurrent\n"
	"\t\tIO requests that can be processed.\n"
	"\n"
	"\t\tFor example, with a single lane delay of 10 ms (-D 10:1),\n"
	"\t\tthe device will only be able to service a single IO request\n"
	"\t\tat a time with each request taking 10 ms to complete. So,\n"
	"\t\tif only a single request is submitted every 10 ms, the\n"
	"\t\taverage latency will be 10 ms; but if more than one request\n"
	"\t\tis submitted every 10 ms, the average latency will be more\n"
	"\t\tthan 10 ms.\n"
	"\n"
	"\t\tSimilarly, if a delay of 10 ms is specified to have two\n"
	"\t\tlanes (-D 10:2), then the device will be able to service\n"
	"\t\ttwo requests at a time, each with a minimum latency of\n"
	"\t\t10 ms. So, if two requests are submitted every 10 ms, then\n"
	"\t\tthe average latency will be 10 ms; but if more than two\n"
	"\t\trequests are submitted every 10 ms, the average latency\n"
	"\t\twill be more than 10 ms.\n"
	"\n"
	"\t\tAlso note, these delays are additive. So two invocations\n"
	"\t\tof '-D 10:1', is roughly equivalent to a single invocation\n"
	"\t\tof '-D 10:2'. This also means, one can specify multiple\n"
	"\t\tlanes with differing target latencies. For example, an\n"
	"\t\tinvocation of '-D 10:1' followed by '-D 25:2' will\n"
	"\t\tcreate 3 lanes on the device; one lane with a latency\n"
	"\t\tof 10 ms and two lanes with a 25 ms latency.\n"
	"\n"
	"\tzinject -I [-s <seconds> \| -g <txgs>] pool\n"
	"\t\tCause the pool to stop writing blocks yet not\n"
	"\t\treport errors for a duration. Simulates buggy hardware\n"
	"\t\tthat fails to honor cache flush requests.\n"
	"\t\tDefault duration is 30 seconds. The machine is panicked\n"
	"\t\tat the end of the duration.\n"
	"\n"
	"\tzinject -b objset:object:level:blkid pool\n"
	"\n"
	"\t\tInject an error into pool 'pool' with the numeric bookmark\n"
	"\t\tspecified by the remaining tuple. Each number is in\n"
	"\t\thexadecimal, and only one block can be specified.\n"
	"\n"
	"\tzinject [-q] <-t type> [-C dvas] [-e errno] [-l level]\n"
	"\t\t[-r range] [-a] [-m] [-u] [-f freq] <object>\n"
	"\n"
	"\t\tInject an error into the object specified by the '-t' option\n"
	"\t\tand the object descriptor. The 'object' parameter is\n"
	"\t\tinterpreted depending on the '-t' option.\n"
	"\n"
	"\t\t-q\tQuiet mode. Only print out the handler number added.\n"
	"\t\t-e\tInject a specific error. Must be one of 'io',\n"
	"\t\t\t'checksum', 'decompress', or 'decrypt'. Default is 'io'.\n"
	"\t\t-C\tInject the given error only into specific DVAs. The\n"
	"\t\t\tDVAs should be specified as a list of 0-indexed DVAs\n"
	"\t\t\tseparated by commas (ex. '0,2').\n"
	"\t\t-l\tInject error at a particular block level. Default is "
	"0.\n"
	"\t\t-m\tAutomatically remount underlying filesystem.\n"
	"\t\t-r\tInject error over a particular logical range of an\n"
	"\t\t\tobject. Will be translated to the appropriate blkid\n"
	"\t\t\trange according to the object's properties.\n"
	"\t\t-a\tFlush the ARC cache. Can be specified without any\n"
	"\t\t\tassociated object.\n"
	"\t\t-u\tUnload the associated pool. Can be specified with only\n"
	"\t\t\ta pool object.\n"
	"\t\t-f\tOnly inject errors a fraction of the time. Expressed as\n"
	"\t\t\ta percentage between 0.0001 and 100.\n"
	"\n"
	"\t-t data\t\tInject an error into the plain file contents of a\n"
	"\t\t\tfile. The object must be specified as a complete path\n"
	"\t\t\tto a file on a ZFS filesystem.\n"
	"\n"
	"\t-t dnode\tInject an error into the metadnode in the block\n"
	"\t\t\tcorresponding to the dnode for a file or directory. The\n"
	"\t\t\t'-r' option is incompatible with this mode. The object\n"
	"\t\t\tis specified as a complete path to a file or directory\n"
	"\t\t\ton a ZFS filesystem.\n"
	"\n"
	"\t-t <mos>\tInject errors into the MOS for objects of the given\n"
	"\t\t\ttype. Valid types are: mos, mosdir, config, bpobj,\n"
	"\t\t\tspacemap, metaslab, errlog. The only valid <object> is\n"
	"\t\t\tthe poolname.\n");
	}

	static int
	iter_handlers(int (func)(int, const char , zinject_record_t , void ),
	void *data)
	{
	zfs_cmd_t zc = {"\0"};
	int ret;

	while (zfs_ioctl(g_zfs, ZFS_IOC_INJECT_LIST_NEXT, &zc) == 0)
	if ((ret = func((int)zc.zc_guid, zc.zc_name,
	&zc.zc_inject_record, data)) != 0)
	return (ret);

	if (errno != ENOENT) {
	(void) fprintf(stderr, "Unable to list handlers: %s\n",
	strerror(errno));
	return (-1);
	}

	return (0);
	}

	static int
	print_data_handler(int id, const char pool, zinject_record_t record,
	void *data)
	{
	int *count = data;

	if (record->zi_guid != 0 \|\| record->zi_func[0] != '\0')
	return (0);

	if (*count == 0) {
	(void) printf("%3s %-15s %-6s %-6s %-8s %3s %-4s "
	"%-15s\n", "ID", "POOL", "OBJSET", "OBJECT", "TYPE",
	"LVL", "DVAs", "RANGE");
	(void) printf("--- --------------- ------ "
	"------ -------- --- ---- ---------------\n");
	}

	*count += 1;

	(void) printf("%3d %-15s %-6llu %-6llu %-8s %-3d 0x%02x ",
	id, pool, (u_longlong_t)record->zi_objset,
	(u_longlong_t)record->zi_object, type_to_name(record->zi_type),
	record->zi_level, record->zi_dvas);


	if (record->zi_start == 0 &&
	record->zi_end == -1ULL)
	(void) printf("all\n");
	else
	(void) printf("[%llu, %llu]\n", (u_longlong_t)record->zi_start,
	(u_longlong_t)record->zi_end);

	return (0);
	}

	static int
	print_device_handler(int id, const char pool, zinject_record_t record,
	void *data)
	{
	int *count = data;

	if (record->zi_guid == 0 \|\| record->zi_func[0] != '\0')
	return (0);

	if (record->zi_cmd == ZINJECT_DELAY_IO)
	return (0);

	if (*count == 0) {
	(void) printf("%3s %-15s %s\n", "ID", "POOL", "GUID");
	(void) printf("--- --------------- ----------------\n");
	}

	*count += 1;

	(void) printf("%3d %-15s %llx\n", id, pool,
	(u_longlong_t)record->zi_guid);

	return (0);
	}

	static int
	print_delay_handler(int id, const char pool, zinject_record_t record,
	void *data)
	{
	int *count = data;

	if (record->zi_guid == 0 \|\| record->zi_func[0] != '\0')
	return (0);

	if (record->zi_cmd != ZINJECT_DELAY_IO)
	return (0);

	if (*count == 0) {
	(void) printf("%3s %-15s %-15s %-15s %s\n",
	"ID", "POOL", "DELAY (ms)", "LANES", "GUID");
	(void) printf("--- --------------- --------------- "
	"--------------- ----------------\n");
	}

	*count += 1;

	(void) printf("%3d %-15s %-15llu %-15llu %llx\n", id, pool,
	(u_longlong_t)NSEC2MSEC(record->zi_timer),
	(u_longlong_t)record->zi_nlanes,
	(u_longlong_t)record->zi_guid);

	return (0);
	}

	static int
	print_panic_handler(int id, const char pool, zinject_record_t record,
	void *data)
	{
	int *count = data;

	if (record->zi_func[0] == '\0')
	return (0);

	if (*count == 0) {
	(void) printf("%3s %-15s %s\n", "ID", "POOL", "FUNCTION");
	(void) printf("--- --------------- ----------------\n");
	}

	*count += 1;

	(void) printf("%3d %-15s %s\n", id, pool, record->zi_func);

	return (0);
	}

	/*
	* Print all registered error handlers. Returns the number of handlers
	* registered.
	*/
	static int
	print_all_handlers(void)
	{
	int count = 0, total = 0;

	(void) iter_handlers(print_device_handler, &count);
	if (count > 0) {
	total += count;
	(void) printf("\n");
	count = 0;
	}

	(void) iter_handlers(print_delay_handler, &count);
	if (count > 0) {
	total += count;
	(void) printf("\n");
	count = 0;
	}

	(void) iter_handlers(print_data_handler, &count);
	if (count > 0) {
	total += count;
	(void) printf("\n");
	count = 0;
	}

	(void) iter_handlers(print_panic_handler, &count);

	return (count + total);
	}

	static int
	cancel_one_handler(int id, const char pool, zinject_record_t record,
	void *data)
	{
	(void) pool, (void) record, (void) data;
	zfs_cmd_t zc = {"\0"};

	zc.zc_guid = (uint64_t)id;

	if (zfs_ioctl(g_zfs, ZFS_IOC_CLEAR_FAULT, &zc) != 0) {
	(void) fprintf(stderr, "failed to remove handler %d: %s\n",
	id, strerror(errno));
	return (1);
	}

	return (0);
	}

	/*
	* Remove all fault injection handlers.
	*/
	static int
	cancel_all_handlers(void)
	{
	int ret = iter_handlers(cancel_one_handler, NULL);

	if (ret == 0)
	(void) printf("removed all registered handlers\n");

	return (ret);
	}

	/*
	* Remove a specific fault injection handler.
	*/
	static int
	cancel_handler(int id)
	{
	zfs_cmd_t zc = {"\0"};

	zc.zc_guid = (uint64_t)id;

	if (zfs_ioctl(g_zfs, ZFS_IOC_CLEAR_FAULT, &zc) != 0) {
	(void) fprintf(stderr, "failed to remove handler %d: %s\n",
	id, strerror(errno));
	return (1);
	}

	(void) printf("removed handler %d\n", id);

	return (0);
	}

	/*
	* Register a new fault injection handler.
	*/
	static int
	register_handler(const char pool, int flags, zinject_record_t record,
	int quiet)
	{
	zfs_cmd_t zc = {"\0"};

	(void) strlcpy(zc.zc_name, pool, sizeof (zc.zc_name));
	zc.zc_inject_record = *record;
	zc.zc_guid = flags;

	if (zfs_ioctl(g_zfs, ZFS_IOC_INJECT_FAULT, &zc) != 0) {
	(void) fprintf(stderr, "failed to add handler: %s\n",
	errno == EDOM ? "block level exceeds max level of object" :
	strerror(errno));
	return (1);
	}

	if (flags & ZINJECT_NULL)
	return (0);

	if (quiet) {
	(void) printf("%llu\n", (u_longlong_t)zc.zc_guid);
	} else {
	(void) printf("Added handler %llu with the following "
	"properties:\n", (u_longlong_t)zc.zc_guid);
	(void) printf(" pool: %s\n", pool);
	if (record->zi_guid) {
	(void) printf(" vdev: %llx\n",
	(u_longlong_t)record->zi_guid);
	} else if (record->zi_func[0] != '\0') {
	(void) printf(" panic function: %s\n",
	record->zi_func);
	} else if (record->zi_duration > 0) {
	(void) printf(" time: %lld seconds\n",
	(u_longlong_t)record->zi_duration);
	} else if (record->zi_duration < 0) {
	(void) printf(" txgs: %lld \n",
	(u_longlong_t)-record->zi_duration);
	} else {
	(void) printf("objset: %llu\n",
	(u_longlong_t)record->zi_objset);
	(void) printf("object: %llu\n",
	(u_longlong_t)record->zi_object);
	(void) printf(" type: %llu\n",
	(u_longlong_t)record->zi_type);
	(void) printf(" level: %d\n", record->zi_level);
	if (record->zi_start == 0 &&
	record->zi_end == -1ULL)
	(void) printf(" range: all\n");
	else
	(void) printf(" range: [%llu, %llu)\n",
	(u_longlong_t)record->zi_start,
	(u_longlong_t)record->zi_end);
	(void) printf(" dvas: 0x%x\n", record->zi_dvas);
	}
	}

	return (0);
	}

	static int
	perform_action(const char pool, zinject_record_t record, int cmd)
	{
	zfs_cmd_t zc = {"\0"};

	ASSERT(cmd == VDEV_STATE_DEGRADED \|\| cmd == VDEV_STATE_FAULTED);
	(void) strlcpy(zc.zc_name, pool, sizeof (zc.zc_name));
	zc.zc_guid = record->zi_guid;
	zc.zc_cookie = cmd;

	if (zfs_ioctl(g_zfs, ZFS_IOC_VDEV_SET_STATE, &zc) == 0)
	return (0);

	return (1);
	}

	static int
	parse_delay(char str, uint64_t delay, uint64_t *nlanes)
	{
	unsigned long scan_delay;
	unsigned long scan_nlanes;

	if (sscanf(str, "%lu:%lu", &scan_delay, &scan_nlanes) != 2)
	return (1);

	/*
	* We explicitly disallow a delay of zero here, because we key
	* off this value being non-zero in translate_device(), to
	* determine if the fault is a ZINJECT_DELAY_IO fault or not.
	*/
	if (scan_delay == 0)
	return (1);

	/*
	* The units for the CLI delay parameter is milliseconds, but
	* the data passed to the kernel is interpreted as nanoseconds.
	* Thus we scale the milliseconds to nanoseconds here, and this
	* nanosecond value is used to pass the delay to the kernel.
	*/
	*delay = MSEC2NSEC(scan_delay);
	*nlanes = scan_nlanes;

	return (0);
	}

	static int
	parse_frequency(const char str, uint32_t percent)
	{
	double val;
	char *post;

	val = strtod(str, &post);
	if (post == NULL \|\| *post != '\0')
	return (EINVAL);

	/* valid range is [0.0001, 100.0] */
	val /= 100.0f;
	if (val < 0.000001f \|\| val > 1.0f)
	return (ERANGE);

	/* convert to an integer for use by kernel */
	percent = ((uint32_t)(val ZI_PERCENTAGE_MAX));

	return (0);
	}

	/*
	* This function converts a string specifier for DVAs into a bit mask.
	* The dva's provided by the user should be 0 indexed and separated by
	* a comma. For example:
	* "1" -> 0b0010 (0x2)
	* "0,1" -> 0b0011 (0x3)
	* "0,1,2" -> 0b0111 (0x7)
	*/
	static int
	parse_dvas(const char str, uint32_t dvas_out)
	{
	const char *c = str;
	uint32_t mask = 0;
	boolean_t need_delim = B_FALSE;

	/* max string length is 5 ("0,1,2") */
	if (strlen(str) > 5 \|\| strlen(str) == 0)
	return (EINVAL);

	while (*c != '\0') {
	switch (*c) {
	case '0':
	case '1':
	case '2':
	/* check for pipe between DVAs */
	if (need_delim)
	return (EINVAL);

	/* check if this DVA has been set already */
	if (mask & (1 << ((*c) - '0')))
	return (EINVAL);

	mask \|= (1 << ((*c) - '0'));
	need_delim = B_TRUE;
	break;
	case ',':
	need_delim = B_FALSE;
	break;
	default:
	/* check for invalid character */
	return (EINVAL);
	}
	c++;
	}

	/* check for dangling delimiter */
	if (!need_delim)
	return (EINVAL);

	*dvas_out = mask;
	return (0);
	}

	int
	main(int argc, char **argv)
	{
	int c;
	char *range = NULL;
	char *cancel = NULL;
	char *end;
	char *raw = NULL;
	char *device = NULL;
	int level = 0;
	int quiet = 0;
	int error = 0;
	int domount = 0;
	int io_type = ZIO_TYPES;
	int action = VDEV_STATE_UNKNOWN;
	err_type_t type = TYPE_INVAL;
	err_type_t label = TYPE_INVAL;
	zinject_record_t record = { 0 };
	char pool[MAXNAMELEN] = "";
	char dataset[MAXNAMELEN] = "";
	zfs_handle_t *zhp = NULL;
	int nowrites = 0;
	int dur_txg = 0;
	int dur_secs = 0;
	int ret;
	int flags = 0;
	uint32_t dvas = 0;

	if ((g_zfs = libzfs_init()) == NULL) {
	(void) fprintf(stderr, "%s\n", libzfs_error_init(errno));
	return (1);
	}

	libzfs_print_on_error(g_zfs, B_TRUE);

	if ((zfs_fd = open(ZFS_DEV, O_RDWR)) < 0) {
	(void) fprintf(stderr, "failed to open ZFS device\n");
	libzfs_fini(g_zfs);
	return (1);
	}

	if (argc == 1) {
	/*
	* No arguments. Print the available handlers. If there are no
	* available handlers, direct the user to '-h' for help
	* information.
	*/
	if (print_all_handlers() == 0) {
	(void) printf("No handlers registered.\n");
	(void) printf("Run 'zinject -h' for usage "
	"information.\n");
	}
	libzfs_fini(g_zfs);
	return (0);
	}

	while ((c = getopt(argc, argv,
	":aA:b:C:d:D:f:Fg:qhIc:t:T:l:mr:s:e:uL:p:")) != -1) {
	switch (c) {
	case 'a':
	flags \|= ZINJECT_FLUSH_ARC;
	break;
	case 'A':
	if (strcasecmp(optarg, "degrade") == 0) {
	action = VDEV_STATE_DEGRADED;
	} else if (strcasecmp(optarg, "fault") == 0) {
	action = VDEV_STATE_FAULTED;
	} else {
	(void) fprintf(stderr, "invalid action '%s': "
	"must be 'degrade' or 'fault'\n", optarg);
	usage();
	libzfs_fini(g_zfs);
	return (1);
	}
	break;
	case 'b':
	raw = optarg;
	break;
	case 'c':
	cancel = optarg;
	break;
	case 'C':
	ret = parse_dvas(optarg, &dvas);
	if (ret != 0) {
	(void) fprintf(stderr, "invalid DVA list '%s': "
	"DVAs should be 0 indexed and separated by "
	"commas.\n", optarg);
	usage();
	libzfs_fini(g_zfs);
	return (1);
	}
	break;
	case 'd':
	device = optarg;
	break;
	case 'D':
	errno = 0;
	ret = parse_delay(optarg, &record.zi_timer,
	&record.zi_nlanes);
	if (ret != 0) {

	(void) fprintf(stderr, "invalid i/o delay "
	"value: '%s'\n", optarg);
	usage();
	libzfs_fini(g_zfs);
	return (1);
	}
	break;
	case 'e':
	if (strcasecmp(optarg, "io") == 0) {
	error = EIO;
	} else if (strcasecmp(optarg, "checksum") == 0) {
	error = ECKSUM;
	} else if (strcasecmp(optarg, "decompress") == 0) {
	error = EINVAL;
	} else if (strcasecmp(optarg, "decrypt") == 0) {
	error = EACCES;
	} else if (strcasecmp(optarg, "nxio") == 0) {
	error = ENXIO;
	} else if (strcasecmp(optarg, "dtl") == 0) {
	error = ECHILD;
	} else if (strcasecmp(optarg, "corrupt") == 0) {
	error = EILSEQ;
	} else {
	(void) fprintf(stderr, "invalid error type "
	"'%s': must be 'io', 'checksum' or "
	"'nxio'\n", optarg);
	usage();
	libzfs_fini(g_zfs);
	return (1);
	}
	break;
	case 'f':
	ret = parse_frequency(optarg, &record.zi_freq);
	if (ret != 0) {
	(void) fprintf(stderr, "%sfrequency value must "
	"be in the range [0.0001, 100.0]\n",
	ret == EINVAL ? "invalid value: " :
	ret == ERANGE ? "out of range: " : "");
	libzfs_fini(g_zfs);
	return (1);
	}
	break;
	case 'F':
	record.zi_failfast = B_TRUE;
	break;
	case 'g':
	dur_txg = 1;
	record.zi_duration = (int)strtol(optarg, &end, 10);
	if (record.zi_duration <= 0 \|\| *end != '\0') {
	(void) fprintf(stderr, "invalid duration '%s': "
	"must be a positive integer\n", optarg);
	usage();
	libzfs_fini(g_zfs);
	return (1);
	}
	/* store duration of txgs as its negative */
	record.zi_duration *= -1;
	break;
	case 'h':
	usage();
	libzfs_fini(g_zfs);
	return (0);
	case 'I':
	/* default duration, if one hasn't yet been defined */
	nowrites = 1;
	if (dur_secs == 0 && dur_txg == 0)
	record.zi_duration = 30;
	break;
	case 'l':
	level = (int)strtol(optarg, &end, 10);
	if (*end != '\0') {
	(void) fprintf(stderr, "invalid level '%s': "
	"must be an integer\n", optarg);
	usage();
	libzfs_fini(g_zfs);
	return (1);
	}
	break;
	case 'm':
	domount = 1;
	break;
	case 'p':
	(void) strlcpy(record.zi_func, optarg,
	sizeof (record.zi_func));
	record.zi_cmd = ZINJECT_PANIC;
	break;
	case 'q':
	quiet = 1;
	break;
	case 'r':
	range = optarg;
	flags \|= ZINJECT_CALC_RANGE;
	break;
	case 's':
	dur_secs = 1;
	record.zi_duration = (int)strtol(optarg, &end, 10);
	if (record.zi_duration <= 0 \|\| *end != '\0') {
	(void) fprintf(stderr, "invalid duration '%s': "
	"must be a positive integer\n", optarg);
	usage();
	libzfs_fini(g_zfs);
	return (1);
	}
	break;
	case 'T':
	if (strcasecmp(optarg, "read") == 0) {
	io_type = ZIO_TYPE_READ;
	} else if (strcasecmp(optarg, "write") == 0) {
	io_type = ZIO_TYPE_WRITE;
	} else if (strcasecmp(optarg, "free") == 0) {
	io_type = ZIO_TYPE_FREE;
	} else if (strcasecmp(optarg, "claim") == 0) {
	io_type = ZIO_TYPE_CLAIM;
	} else if (strcasecmp(optarg, "all") == 0) {
	io_type = ZIO_TYPES;
	} else {
	(void) fprintf(stderr, "invalid I/O type "
	"'%s': must be 'read', 'write', 'free', "
	"'claim' or 'all'\n", optarg);
	usage();
	libzfs_fini(g_zfs);
	return (1);
	}
	break;
	case 't':
	if ((type = name_to_type(optarg)) == TYPE_INVAL &&
	!MOS_TYPE(type)) {
	(void) fprintf(stderr, "invalid type '%s'\n",
	optarg);
	usage();
	libzfs_fini(g_zfs);
	return (1);
	}
	break;
	case 'u':
	flags \|= ZINJECT_UNLOAD_SPA;
	break;
	case 'L':
	if ((label = name_to_type(optarg)) == TYPE_INVAL &&
	!LABEL_TYPE(type)) {
	(void) fprintf(stderr, "invalid label type "
	"'%s'\n", optarg);
	usage();
	libzfs_fini(g_zfs);
	return (1);
	}
	break;
	case ':':
	(void) fprintf(stderr, "option -%c requires an "
	"operand\n", optopt);
	usage();
	libzfs_fini(g_zfs);
	return (1);
	case '?':
	(void) fprintf(stderr, "invalid option '%c'\n",
	optopt);
	usage();
	libzfs_fini(g_zfs);
	return (2);
	}
	}

	argc -= optind;
	argv += optind;

	if (record.zi_duration != 0)
	record.zi_cmd = ZINJECT_IGNORED_WRITES;

	if (cancel != NULL) {
	/*
	* '-c' is invalid with any other options.
	*/
	if (raw != NULL \|\| range != NULL \|\| type != TYPE_INVAL \|\|
	level != 0 \|\| record.zi_cmd != ZINJECT_UNINITIALIZED \|\|
	record.zi_freq > 0 \|\| dvas != 0) {
	(void) fprintf(stderr, "cancel (-c) incompatible with "
	"any other options\n");
	usage();
	libzfs_fini(g_zfs);
	return (2);
	}
	if (argc != 0) {
	(void) fprintf(stderr, "extraneous argument to '-c'\n");
	usage();
	libzfs_fini(g_zfs);
	return (2);
	}

	if (strcmp(cancel, "all") == 0) {
	return (cancel_all_handlers());
	} else {
	int id = (int)strtol(cancel, &end, 10);
	if (*end != '\0') {
	(void) fprintf(stderr, "invalid handle id '%s':"
	" must be an integer or 'all'\n", cancel);
	usage();
	libzfs_fini(g_zfs);
	return (1);
	}
	return (cancel_handler(id));
	}
	}

	if (device != NULL) {
	/*
	* Device (-d) injection uses a completely different mechanism
	* for doing injection, so handle it separately here.
	*/
	if (raw != NULL \|\| range != NULL \|\| type != TYPE_INVAL \|\|
	level != 0 \|\| record.zi_cmd != ZINJECT_UNINITIALIZED \|\|
	dvas != 0) {
	(void) fprintf(stderr, "device (-d) incompatible with "
	"data error injection\n");
	usage();
	libzfs_fini(g_zfs);
	return (2);
	}

	if (argc != 1) {
	(void) fprintf(stderr, "device (-d) injection requires "
	"a single pool name\n");
	usage();
	libzfs_fini(g_zfs);
	return (2);
	}

	(void) strlcpy(pool, argv[0], sizeof (pool));
	dataset[0] = '\0';

	if (error == ECKSUM) {
	(void) fprintf(stderr, "device error type must be "
	"'io', 'nxio' or 'corrupt'\n");
	libzfs_fini(g_zfs);
	return (1);
	}

	if (error == EILSEQ &&
	(record.zi_freq == 0 \|\| io_type != ZIO_TYPE_READ)) {
	(void) fprintf(stderr, "device corrupt errors require "
	"io type read and a frequency value\n");
	libzfs_fini(g_zfs);
	return (1);
	}

	record.zi_iotype = io_type;
	if (translate_device(pool, device, label, &record) != 0) {
	libzfs_fini(g_zfs);
	return (1);
	}
	+
	+ if (record.zi_nlanes) {
	+ switch (io_type) {
	+ case ZIO_TYPE_READ:
	+ case ZIO_TYPE_WRITE:
	+ case ZIO_TYPES:
	+ break;
	+ default:
	+ (void) fprintf(stderr, "I/O type for a delay "
	+ "must be 'read' or 'write'\n");
	+ usage();
	+ libzfs_fini(g_zfs);
	+ return (1);
	+ }
	+ }
	+
	if (!error)
	error = ENXIO;

	if (action != VDEV_STATE_UNKNOWN)
	return (perform_action(pool, &record, action));

	} else if (raw != NULL) {
	if (range != NULL \|\| type != TYPE_INVAL \|\| level != 0 \|\|
	record.zi_cmd != ZINJECT_UNINITIALIZED \|\|
	record.zi_freq > 0 \|\| dvas != 0) {
	(void) fprintf(stderr, "raw (-b) format with "
	"any other options\n");
	usage();
	libzfs_fini(g_zfs);
	return (2);
	}

	if (argc != 1) {
	(void) fprintf(stderr, "raw (-b) format expects a "
	"single pool name\n");
	usage();
	libzfs_fini(g_zfs);
	return (2);
	}

	(void) strlcpy(pool, argv[0], sizeof (pool));
	dataset[0] = '\0';

	if (error == ENXIO) {
	(void) fprintf(stderr, "data error type must be "
	"'checksum' or 'io'\n");
	libzfs_fini(g_zfs);
	return (1);
	}

	record.zi_cmd = ZINJECT_DATA_FAULT;
	if (translate_raw(raw, &record) != 0) {
	libzfs_fini(g_zfs);
	return (1);
	}
	if (!error)
	error = EIO;
	} else if (record.zi_cmd == ZINJECT_PANIC) {
	if (raw != NULL \|\| range != NULL \|\| type != TYPE_INVAL \|\|
	level != 0 \|\| device != NULL \|\| record.zi_freq > 0 \|\|
	dvas != 0) {
	(void) fprintf(stderr, "panic (-p) incompatible with "
	"other options\n");
	usage();
	libzfs_fini(g_zfs);
	return (2);
	}

	if (argc < 1 \|\| argc > 2) {
	(void) fprintf(stderr, "panic (-p) injection requires "
	"a single pool name and an optional id\n");
	usage();
	libzfs_fini(g_zfs);
	return (2);
	}

	(void) strlcpy(pool, argv[0], sizeof (pool));
	if (argv[1] != NULL)
	record.zi_type = atoi(argv[1]);
	dataset[0] = '\0';
	} else if (record.zi_cmd == ZINJECT_IGNORED_WRITES) {
	if (raw != NULL \|\| range != NULL \|\| type != TYPE_INVAL \|\|
	level != 0 \|\| record.zi_freq > 0 \|\| dvas != 0) {
	(void) fprintf(stderr, "hardware failure (-I) "
	"incompatible with other options\n");
	usage();
	libzfs_fini(g_zfs);
	return (2);
	}

	if (nowrites == 0) {
	(void) fprintf(stderr, "-s or -g meaningless "
	"without -I (ignore writes)\n");
	usage();
	libzfs_fini(g_zfs);
	return (2);
	} else if (dur_secs && dur_txg) {
	(void) fprintf(stderr, "choose a duration either "
	"in seconds (-s) or a number of txgs (-g) "
	"but not both\n");
	usage();
	libzfs_fini(g_zfs);
	return (2);
	} else if (argc != 1) {
	(void) fprintf(stderr, "ignore writes (-I) "
	"injection requires a single pool name\n");
	usage();
	libzfs_fini(g_zfs);
	return (2);
	}

	(void) strlcpy(pool, argv[0], sizeof (pool));
	dataset[0] = '\0';
	} else if (type == TYPE_INVAL) {
	if (flags == 0) {
	(void) fprintf(stderr, "at least one of '-b', '-d', "
	"'-t', '-a', '-p', '-I' or '-u' "
	"must be specified\n");
	usage();
	libzfs_fini(g_zfs);
	return (2);
	}

	if (argc == 1 && (flags & ZINJECT_UNLOAD_SPA)) {
	(void) strlcpy(pool, argv[0], sizeof (pool));
	dataset[0] = '\0';
	} else if (argc != 0) {
	(void) fprintf(stderr, "extraneous argument for "
	"'-f'\n");
	usage();
	libzfs_fini(g_zfs);
	return (2);
	}

	flags \|= ZINJECT_NULL;
	} else {
	if (argc != 1) {
	(void) fprintf(stderr, "missing object\n");
	usage();
	libzfs_fini(g_zfs);
	return (2);
	}

	if (error == ENXIO \|\| error == EILSEQ) {
	(void) fprintf(stderr, "data error type must be "
	"'checksum' or 'io'\n");
	libzfs_fini(g_zfs);
	return (1);
	}

	if (dvas != 0) {
	if (error == EACCES \|\| error == EINVAL) {
	(void) fprintf(stderr, "the '-C' option may "
	"not be used with logical data errors "
	"'decrypt' and 'decompress'\n");
	libzfs_fini(g_zfs);
	return (1);
	}

	record.zi_dvas = dvas;
	}

	if (error == EACCES) {
	if (type != TYPE_DATA) {
	(void) fprintf(stderr, "decryption errors "
	"may only be injected for 'data' types\n");
	libzfs_fini(g_zfs);
	return (1);
	}

	record.zi_cmd = ZINJECT_DECRYPT_FAULT;
	/*
	* Internally, ZFS actually uses ECKSUM for decryption
	* errors since EACCES is used to indicate the key was
	* not found.
	*/
	error = ECKSUM;
	} else {
	record.zi_cmd = ZINJECT_DATA_FAULT;
	}

	if (translate_record(type, argv[0], range, level, &record, pool,
	dataset) != 0) {
	libzfs_fini(g_zfs);
	return (1);
	}
	if (!error)
	error = EIO;
	}

	/*
	* If this is pool-wide metadata, unmount everything. The ioctl() will
	* unload the pool, so that we trigger spa-wide reopen of metadata next
	* time we access the pool.
	*/
	if (dataset[0] != '\0' && domount) {
	if ((zhp = zfs_open(g_zfs, dataset,
	ZFS_TYPE_DATASET)) == NULL) {
	libzfs_fini(g_zfs);
	return (1);
	}
	if (zfs_unmount(zhp, NULL, 0) != 0) {
	libzfs_fini(g_zfs);
	return (1);
	}
	}

	record.zi_error = error;

	ret = register_handler(pool, flags, &record, quiet);

	if (dataset[0] != '\0' && domount)
	ret = (zfs_mount(zhp, NULL, 0) != 0);

	libzfs_fini(g_zfs);

	return (ret);
	}
	diff --git a/sys/contrib/openzfs/cmd/zpool/os/linux/zpool_vdev_os.c b/sys/contrib/openzfs/cmd/zpool/os/linux/zpool_vdev_os.c
	index 006a3a7d8e01..f194d28c55a9 100644
	--- a/sys/contrib/openzfs/cmd/zpool/os/linux/zpool_vdev_os.c
	+++ b/sys/contrib/openzfs/cmd/zpool/os/linux/zpool_vdev_os.c
	@@ -1,673 +1,673 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2013, 2018 by Delphix. All rights reserved.
	* Copyright (c) 2016, 2017 Intel Corporation.
	* Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
	*/

	/*
	* Functions to convert between a list of vdevs and an nvlist representing the
	* configuration. Each entry in the list can be one of:
	*
	* Device vdevs
	* disk=(path=..., devid=...)
	* file=(path=...)
	*
	* Group vdevs
	* raidz[1\|2]=(...)
	* mirror=(...)
	*
	* Hot spares
	*
	* While the underlying implementation supports it, group vdevs cannot contain
	* other group vdevs. All userland verification of devices is contained within
	* this file. If successful, the nvlist returned can be passed directly to the
	* kernel; we've done as much verification as possible in userland.
	*
	* Hot spares are a special case, and passed down as an array of disk vdevs, at
	* the same level as the root of the vdev tree.
	*
	* The only function exported by this file is 'make_root_vdev'. The
	* function performs several passes:
	*
	* 1. Construct the vdev specification. Performs syntax validation and
	* makes sure each device is valid.
	* 2. Check for devices in use. Using libblkid to make sure that no
	* devices are also in use. Some can be overridden using the 'force'
	* flag, others cannot.
	* 3. Check for replication errors if the 'force' flag is not specified.
	* validates that the replication level is consistent across the
	* entire pool.
	* 4. Call libzfs to label any whole disks with an EFI label.
	*/

	#include <assert.h>
	#include <ctype.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libintl.h>
	#include <libnvpair.h>
	#include <libzutil.h>
	#include <limits.h>
	#include <sys/spa.h>
	#include <stdio.h>
	#include <string.h>
	#include <unistd.h>
	#include "zpool_util.h"
	#include <sys/zfs_context.h>

	#include <scsi/scsi.h>
	#include <scsi/sg.h>
	#include <sys/efi_partition.h>
	#include <sys/stat.h>
	#include <sys/mntent.h>
	#include <uuid/uuid.h>
	#include <blkid/blkid.h>

	typedef struct vdev_disk_db_entry
	{
	char id[24];
	int sector_size;
	} vdev_disk_db_entry_t;

	/*
	* Database of block devices that lie about physical sector sizes. The
	* identification string must be precisely 24 characters to avoid false
	* negatives
	*/
	static vdev_disk_db_entry_t vdev_disk_database[] = {
	{"ATA ADATA SSD S396 3", 8192},
	{"ATA APPLE SSD SM128E", 8192},
	{"ATA APPLE SSD SM256E", 8192},
	{"ATA APPLE SSD SM512E", 8192},
	{"ATA APPLE SSD SM768E", 8192},
	{"ATA C400-MTFDDAC064M", 8192},
	{"ATA C400-MTFDDAC128M", 8192},
	{"ATA C400-MTFDDAC256M", 8192},
	{"ATA C400-MTFDDAC512M", 8192},
	{"ATA Corsair Force 3 ", 8192},
	{"ATA Corsair Force GS", 8192},
	{"ATA INTEL SSDSA2CT04", 8192},
	{"ATA INTEL SSDSA2BZ10", 8192},
	{"ATA INTEL SSDSA2BZ20", 8192},
	{"ATA INTEL SSDSA2BZ30", 8192},
	{"ATA INTEL SSDSA2CW04", 8192},
	{"ATA INTEL SSDSA2CW08", 8192},
	{"ATA INTEL SSDSA2CW12", 8192},
	{"ATA INTEL SSDSA2CW16", 8192},
	{"ATA INTEL SSDSA2CW30", 8192},
	{"ATA INTEL SSDSA2CW60", 8192},
	{"ATA INTEL SSDSC2CT06", 8192},
	{"ATA INTEL SSDSC2CT12", 8192},
	{"ATA INTEL SSDSC2CT18", 8192},
	{"ATA INTEL SSDSC2CT24", 8192},
	{"ATA INTEL SSDSC2CW06", 8192},
	{"ATA INTEL SSDSC2CW12", 8192},
	{"ATA INTEL SSDSC2CW18", 8192},
	{"ATA INTEL SSDSC2CW24", 8192},
	{"ATA INTEL SSDSC2CW48", 8192},
	{"ATA KINGSTON SH100S3", 8192},
	{"ATA KINGSTON SH103S3", 8192},
	{"ATA M4-CT064M4SSD2 ", 8192},
	{"ATA M4-CT128M4SSD2 ", 8192},
	{"ATA M4-CT256M4SSD2 ", 8192},
	{"ATA M4-CT512M4SSD2 ", 8192},
	{"ATA OCZ-AGILITY2 ", 8192},
	{"ATA OCZ-AGILITY3 ", 8192},
	{"ATA OCZ-VERTEX2 3.5 ", 8192},
	{"ATA OCZ-VERTEX3 ", 8192},
	{"ATA OCZ-VERTEX3 LT ", 8192},
	{"ATA OCZ-VERTEX3 MI ", 8192},
	{"ATA OCZ-VERTEX4 ", 8192},
	{"ATA SAMSUNG MZ7WD120", 8192},
	{"ATA SAMSUNG MZ7WD240", 8192},
	{"ATA SAMSUNG MZ7WD480", 8192},
	{"ATA SAMSUNG MZ7WD960", 8192},
	{"ATA SAMSUNG SSD 830 ", 8192},
	{"ATA Samsung SSD 840 ", 8192},
	{"ATA SanDisk SSD U100", 8192},
	{"ATA TOSHIBA THNSNH06", 8192},
	{"ATA TOSHIBA THNSNH12", 8192},
	{"ATA TOSHIBA THNSNH25", 8192},
	{"ATA TOSHIBA THNSNH51", 8192},
	{"ATA APPLE SSD TS064C", 4096},
	{"ATA APPLE SSD TS128C", 4096},
	{"ATA APPLE SSD TS256C", 4096},
	{"ATA APPLE SSD TS512C", 4096},
	{"ATA INTEL SSDSA2M040", 4096},
	{"ATA INTEL SSDSA2M080", 4096},
	{"ATA INTEL SSDSA2M160", 4096},
	{"ATA INTEL SSDSC2MH12", 4096},
	{"ATA INTEL SSDSC2MH25", 4096},
	{"ATA OCZ CORE_SSD ", 4096},
	{"ATA OCZ-VERTEX ", 4096},
	{"ATA SAMSUNG MCCOE32G", 4096},
	{"ATA SAMSUNG MCCOE64G", 4096},
	{"ATA SAMSUNG SSD PM80", 4096},
	/* Flash drives optimized for 4KB IOs on larger pages */
	{"ATA INTEL SSDSC2BA10", 4096},
	{"ATA INTEL SSDSC2BA20", 4096},
	{"ATA INTEL SSDSC2BA40", 4096},
	{"ATA INTEL SSDSC2BA80", 4096},
	{"ATA INTEL SSDSC2BB08", 4096},
	{"ATA INTEL SSDSC2BB12", 4096},
	{"ATA INTEL SSDSC2BB16", 4096},
	{"ATA INTEL SSDSC2BB24", 4096},
	{"ATA INTEL SSDSC2BB30", 4096},
	{"ATA INTEL SSDSC2BB40", 4096},
	{"ATA INTEL SSDSC2BB48", 4096},
	{"ATA INTEL SSDSC2BB60", 4096},
	{"ATA INTEL SSDSC2BB80", 4096},
	{"ATA INTEL SSDSC2BW24", 4096},
	{"ATA INTEL SSDSC2BW48", 4096},
	{"ATA INTEL SSDSC2BP24", 4096},
	{"ATA INTEL SSDSC2BP48", 4096},
	{"NA SmrtStorSDLKAE9W", 4096},
	{"NVMe Amazon EC2 NVMe ", 4096},
	/* Imported from Open Solaris */
	{"ATA MARVELL SD88SA02", 4096},
	/* Advanced format Hard drives */
	{"ATA Hitachi HDS5C303", 4096},
	{"ATA SAMSUNG HD204UI ", 4096},
	{"ATA ST2000DL004 HD20", 4096},
	{"ATA WDC WD10EARS-00M", 4096},
	{"ATA WDC WD10EARS-00S", 4096},
	{"ATA WDC WD10EARS-00Z", 4096},
	{"ATA WDC WD15EARS-00M", 4096},
	{"ATA WDC WD15EARS-00S", 4096},
	{"ATA WDC WD15EARS-00Z", 4096},
	{"ATA WDC WD20EARS-00M", 4096},
	{"ATA WDC WD20EARS-00S", 4096},
	{"ATA WDC WD20EARS-00Z", 4096},
	{"ATA WDC WD1600BEVT-0", 4096},
	{"ATA WDC WD2500BEVT-0", 4096},
	{"ATA WDC WD3200BEVT-0", 4096},
	{"ATA WDC WD5000BEVT-0", 4096},
	};


	#define INQ_REPLY_LEN 96
	#define INQ_CMD_LEN 6

	static const int vdev_disk_database_size =
	sizeof (vdev_disk_database) / sizeof (vdev_disk_database[0]);

	boolean_t
	check_sector_size_database(char path, int sector_size)
	{
	unsigned char inq_buff[INQ_REPLY_LEN];
	unsigned char sense_buffer[32];
	unsigned char inq_cmd_blk[INQ_CMD_LEN] =
	{INQUIRY, 0, 0, 0, INQ_REPLY_LEN, 0};
	sg_io_hdr_t io_hdr;
	int error;
	int fd;
	int i;

	/* Prepare INQUIRY command */
	memset(&io_hdr, 0, sizeof (sg_io_hdr_t));
	io_hdr.interface_id = 'S';
	io_hdr.cmd_len = sizeof (inq_cmd_blk);
	io_hdr.mx_sb_len = sizeof (sense_buffer);
	io_hdr.dxfer_direction = SG_DXFER_FROM_DEV;
	io_hdr.dxfer_len = INQ_REPLY_LEN;
	io_hdr.dxferp = inq_buff;
	io_hdr.cmdp = inq_cmd_blk;
	io_hdr.sbp = sense_buffer;
	io_hdr.timeout = 10; /* 10 milliseconds is ample time */

	if ((fd = open(path, O_RDONLY\|O_DIRECT)) < 0)
	return (B_FALSE);

	error = ioctl(fd, SG_IO, (unsigned long) &io_hdr);

	(void) close(fd);

	if (error < 0)
	return (B_FALSE);

	if ((io_hdr.info & SG_INFO_OK_MASK) != SG_INFO_OK)
	return (B_FALSE);

	for (i = 0; i < vdev_disk_database_size; i++) {
	if (memcmp(inq_buff + 8, vdev_disk_database[i].id, 24))
	continue;

	*sector_size = vdev_disk_database[i].sector_size;
	return (B_TRUE);
	}

	return (B_FALSE);
	}

	static int
	check_slice(const char *path, blkid_cache cache, int force, boolean_t isspare)
	{
	int err;
	char *value;

	/* No valid type detected device is safe to use */
	value = blkid_get_tag_value(cache, "TYPE", path);
	if (value == NULL)
	return (0);

	/*
	* If libblkid detects a ZFS device, we check the device
	* using check_file() to see if it's safe. The one safe
	* case is a spare device shared between multiple pools.
	*/
	if (strcmp(value, "zfs_member") == 0) {
	err = check_file(path, force, isspare);
	} else {
	if (force) {
	err = 0;
	} else {
	err = -1;
	vdev_error(gettext("%s contains a filesystem of "
	"type '%s'\n"), path, value);
	}
	}

	free(value);

	return (err);
	}

	/*
	* Validate that a disk including all partitions are safe to use.
	*
	* For EFI labeled disks this can done relatively easily with the libefi
	* library. The partition numbers are extracted from the label and used
	* to generate the expected /dev/ paths. Each partition can then be
	* checked for conflicts.
	*
	* For non-EFI labeled disks (MBR/EBR/etc) the same process is possible
	* but due to the lack of a readily available libraries this scanning is
	* not implemented. Instead only the device path as given is checked.
	*/
	static int
	check_disk(const char *path, blkid_cache cache, int force,
	boolean_t isspare, boolean_t iswholedisk)
	{
	struct dk_gpt *vtoc;
	char slice_path[MAXPATHLEN];
	int err = 0;
	int fd, i;
	int flags = O_RDONLY\|O_DIRECT;

	if (!iswholedisk)
	return (check_slice(path, cache, force, isspare));

	/* only spares can be shared, other devices require exclusive access */
	if (!isspare)
	flags \|= O_EXCL;

	if ((fd = open(path, flags)) < 0) {
	char *value = blkid_get_tag_value(cache, "TYPE", path);
	(void) fprintf(stderr, gettext("%s is in use and contains "
	"a %s filesystem.\n"), path, value ? value : "unknown");
	free(value);
	return (-1);
	}

	/*
	* Expected to fail for non-EFI labeled disks. Just check the device
	* as given and do not attempt to detect and scan partitions.
	*/
	err = efi_alloc_and_read(fd, &vtoc);
	if (err) {
	(void) close(fd);
	return (check_slice(path, cache, force, isspare));
	}

	/*
	* The primary efi partition label is damaged however the secondary
	* label at the end of the device is intact. Rather than use this
	* label we should play it safe and treat this as a non efi device.
	*/
	if (vtoc->efi_flags & EFI_GPT_PRIMARY_CORRUPT) {
	efi_free(vtoc);
	(void) close(fd);

	if (force) {
	/* Partitions will now be created using the backup */
	return (0);
	} else {
	vdev_error(gettext("%s contains a corrupt primary "
	"EFI label.\n"), path);
	return (-1);
	}
	}

	for (i = 0; i < vtoc->efi_nparts; i++) {

	if (vtoc->efi_parts[i].p_tag == V_UNASSIGNED \|\|
	uuid_is_null((uchar_t *)&vtoc->efi_parts[i].p_guid))
	continue;

	if (strncmp(path, UDISK_ROOT, strlen(UDISK_ROOT)) == 0)
	(void) snprintf(slice_path, sizeof (slice_path),
	"%s%s%d", path, "-part", i+1);
	else
	(void) snprintf(slice_path, sizeof (slice_path),
	"%s%s%d", path, isdigit(path[strlen(path)-1]) ?
	"p" : "", i+1);

	err = check_slice(slice_path, cache, force, isspare);
	if (err)
	break;
	}

	efi_free(vtoc);
	(void) close(fd);

	return (err);
	}

	int
	check_device(const char *path, boolean_t force,
	boolean_t isspare, boolean_t iswholedisk)
	{
	blkid_cache cache;
	int error;

	error = blkid_get_cache(&cache, NULL);
	if (error != 0) {
	(void) fprintf(stderr, gettext("unable to access the blkid "
	"cache.\n"));
	return (-1);
	}

	error = check_disk(path, cache, force, isspare, iswholedisk);
	blkid_put_cache(cache);

	return (error);
	}

	void
	after_zpool_upgrade(zpool_handle_t *zhp)
	{
	(void) zhp;
	}

	int
	check_file(const char *file, boolean_t force, boolean_t isspare)
	{
	return (check_file_generic(file, force, isspare));
	}

	/*
	* Read from a sysfs file and return an allocated string. Removes
	* the newline from the end of the string if there is one.
	*
	* Returns a string on success (which must be freed), or NULL on error.
	*/
	static char zpool_sysfs_gets(char path)
	{
	int fd;
	struct stat statbuf;
	char *buf = NULL;
	ssize_t count = 0;
	fd = open(path, O_RDONLY);
	if (fd < 0)
	return (NULL);

	if (fstat(fd, &statbuf) != 0) {
	close(fd);
	return (NULL);
	}

	- buf = calloc(sizeof (*buf), statbuf.st_size + 1);
	+ buf = calloc(statbuf.st_size + 1, sizeof (*buf));
	if (buf == NULL) {
	close(fd);
	return (NULL);
	}

	/*
	* Note, we can read less bytes than st_size, and that's ok. Sysfs
	* files will report their size is 4k even if they only return a small
	* string.
	*/
	count = read(fd, buf, statbuf.st_size);
	if (count < 0) {
	/* Error doing read() or we overran the buffer */
	close(fd);
	free(buf);
	return (NULL);
	}

	/* Remove trailing newline */
	- if (buf[count - 1] == '\n')
	+ if (count > 0 && buf[count - 1] == '\n')
	buf[count - 1] = 0;

	close(fd);

	return (buf);
	}

	/*
	* Write a string to a sysfs file.
	*
	* Returns 0 on success, non-zero otherwise.
	*/
	static int zpool_sysfs_puts(char path, char str)
	{
	FILE *file;

	file = fopen(path, "w");
	if (!file) {
	return (-1);
	}

	if (fputs(str, file) < 0) {
	fclose(file);
	return (-2);
	}
	fclose(file);
	return (0);
	}

	/* Given a vdev nvlist_t, rescan its enclosure sysfs path */
	static void
	rescan_vdev_config_dev_sysfs_path(nvlist_t *vdev_nv)
	{
	update_vdev_config_dev_sysfs_path(vdev_nv,
	fnvlist_lookup_string(vdev_nv, ZPOOL_CONFIG_PATH),
	ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH);
	}

	/*
	* Given a power string: "on", "off", "1", or "0", return 0 if it's an
	* off value, 1 if it's an on value, and -1 if the value is unrecognized.
	*/
	static int zpool_power_parse_value(char *str)
	{
	if ((strcmp(str, "off") == 0) \|\| (strcmp(str, "0") == 0))
	return (0);

	if ((strcmp(str, "on") == 0) \|\| (strcmp(str, "1") == 0))
	return (1);

	return (-1);
	}

	/*
	* Given a vdev string return an allocated string containing the sysfs path to
	* its power control file. Also do a check if the power control file really
	* exists and has correct permissions.
	*
	* Example returned strings:
	*
	* /sys/class/enclosure/0:0:122:0/10/power_status
	* /sys/bus/pci/slots/10/power
	*
	* Returns allocated string on success (which must be freed), NULL on failure.
	*/
	static char *
	zpool_power_sysfs_path(zpool_handle_t zhp, char vdev)
	{
	const char *enc_sysfs_dir = NULL;
	char *path = NULL;
	nvlist_t *vdev_nv = zpool_find_vdev(zhp, vdev, NULL, NULL, NULL);

	if (vdev_nv == NULL) {
	return (NULL);
	}

	/* Make sure we're getting the updated enclosure sysfs path */
	rescan_vdev_config_dev_sysfs_path(vdev_nv);

	if (nvlist_lookup_string(vdev_nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH,
	&enc_sysfs_dir) != 0) {
	return (NULL);
	}

	if (asprintf(&path, "%s/power_status", enc_sysfs_dir) == -1)
	return (NULL);

	if (access(path, W_OK) != 0) {
	free(path);
	path = NULL;
	/* No HDD 'power_control' file, maybe it's NVMe? */
	if (asprintf(&path, "%s/power", enc_sysfs_dir) == -1) {
	return (NULL);
	}

	if (access(path, R_OK \| W_OK) != 0) {
	/* Not NVMe either */
	free(path);
	return (NULL);
	}
	}

	return (path);
	}

	/*
	* Given a path to a sysfs power control file, return B_TRUE if you should use
	* "on/off" words to control it, or B_FALSE otherwise ("0/1" to control).
	*/
	static boolean_t
	zpool_power_use_word(char *sysfs_path)
	{
	if (strcmp(&sysfs_path[strlen(sysfs_path) - strlen("power_status")],
	"power_status") == 0) {
	return (B_TRUE);
	}
	return (B_FALSE);
	}

	/*
	* Check the sysfs power control value for a vdev.
	*
	* Returns:
	* 0 - Power is off
	* 1 - Power is on
	* -1 - Error or unsupported
	*/
	int
	zpool_power_current_state(zpool_handle_t zhp, char vdev)
	{
	char *val;
	int rc;

	char *path = zpool_power_sysfs_path(zhp, vdev);
	if (path == NULL)
	return (-1);

	val = zpool_sysfs_gets(path);
	if (val == NULL) {
	free(path);
	return (-1);
	}

	rc = zpool_power_parse_value(val);
	free(val);
	free(path);
	return (rc);
	}

	/*
	* Turn on or off the slot to a device
	*
	* Device path is the full path to the device (like /dev/sda or /dev/sda1).
	*
	* Return code:
	* 0: Success
	* ENOTSUP: Power control not supported for OS
	* EBADSLT: Couldn't read current power state
	* ENOENT: No sysfs path to power control
	* EIO: Couldn't write sysfs power value
	* EBADE: Sysfs power value didn't change
	*/
	int
	zpool_power(zpool_handle_t zhp, char vdev, boolean_t turn_on)
	{
	char *sysfs_path;
	const char *val;
	int rc;
	int timeout_ms;

	rc = zpool_power_current_state(zhp, vdev);
	if (rc == -1) {
	return (EBADSLT);
	}

	/* Already correct value? */
	if (rc == (int)turn_on)
	return (0);

	sysfs_path = zpool_power_sysfs_path(zhp, vdev);
	if (sysfs_path == NULL)
	return (ENOENT);

	if (zpool_power_use_word(sysfs_path)) {
	val = turn_on ? "on" : "off";
	} else {
	val = turn_on ? "1" : "0";
	}

	rc = zpool_sysfs_puts(sysfs_path, (char *)val);

	free(sysfs_path);
	if (rc != 0) {
	return (EIO);
	}

	/*
	* Wait up to 30 seconds for sysfs power value to change after
	* writing it.
	*/
	timeout_ms = zpool_getenv_int("ZPOOL_POWER_ON_SLOT_TIMEOUT_MS", 30000);
	for (int i = 0; i < MAX(1, timeout_ms / 200); i++) {
	rc = zpool_power_current_state(zhp, vdev);
	if (rc == (int)turn_on)
	return (0); /* success */

	fsleep(0.200); /* 200ms */
	}

	/* sysfs value never changed */
	return (EBADE);
	}
	diff --git a/sys/contrib/openzfs/cmd/zpool/zpool_main.c b/sys/contrib/openzfs/cmd/zpool/zpool_main.c
	index 69bf9649acf6..ed0b8d7a12d7 100644
	--- a/sys/contrib/openzfs/cmd/zpool/zpool_main.c
	+++ b/sys/contrib/openzfs/cmd/zpool/zpool_main.c
	@@ -1,11468 +1,11512 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright 2011 Nexenta Systems, Inc. All rights reserved.
	- * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
	+ * Copyright (c) 2011, 2024 by Delphix. All rights reserved.
	* Copyright (c) 2012 by Frederik Wessels. All rights reserved.
	* Copyright (c) 2012 by Cyril Plisko. All rights reserved.
	* Copyright (c) 2013 by Prasad Joshi (sTec). All rights reserved.
	* Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
	* Copyright (c) 2017 Datto Inc.
	* Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
	* Copyright (c) 2017, Intel Corporation.
	* Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>
	* Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
	* Copyright (c) 2021, Klara Inc.
	* Copyright [2021] Hewlett Packard Enterprise Development LP
	*/

	#include <assert.h>
	#include <ctype.h>
	#include <dirent.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <getopt.h>
	#include <libgen.h>
	#include <libintl.h>
	#include <libuutil.h>
	#include <locale.h>
	#include <pthread.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <time.h>
	#include <unistd.h>
	#include <pwd.h>
	#include <zone.h>
	#include <sys/wait.h>
	#include <zfs_prop.h>
	#include <sys/fs/zfs.h>
	#include <sys/stat.h>
	#include <sys/systeminfo.h>
	#include <sys/fm/fs/zfs.h>
	#include <sys/fm/util.h>
	#include <sys/fm/protocol.h>
	#include <sys/zfs_ioctl.h>
	#include <sys/mount.h>
	#include <sys/sysmacros.h>

	#include <math.h>

	#include <libzfs.h>
	#include <libzutil.h>

	#include "zpool_util.h"
	#include "zfs_comutil.h"
	#include "zfeature_common.h"

	#include "statcommon.h"

	libzfs_handle_t *g_zfs;

	static int zpool_do_create(int, char **);
	static int zpool_do_destroy(int, char **);

	static int zpool_do_add(int, char **);
	static int zpool_do_remove(int, char **);
	static int zpool_do_labelclear(int, char **);

	static int zpool_do_checkpoint(int, char **);

	static int zpool_do_list(int, char **);
	static int zpool_do_iostat(int, char **);
	static int zpool_do_status(int, char **);

	static int zpool_do_online(int, char **);
	static int zpool_do_offline(int, char **);
	static int zpool_do_clear(int, char **);
	static int zpool_do_reopen(int, char **);

	static int zpool_do_reguid(int, char **);

	static int zpool_do_attach(int, char **);
	static int zpool_do_detach(int, char **);
	static int zpool_do_replace(int, char **);
	static int zpool_do_split(int, char **);

	static int zpool_do_initialize(int, char **);
	static int zpool_do_scrub(int, char **);
	static int zpool_do_resilver(int, char **);
	static int zpool_do_trim(int, char **);

	static int zpool_do_import(int, char **);
	static int zpool_do_export(int, char **);

	static int zpool_do_upgrade(int, char **);

	static int zpool_do_history(int, char **);
	static int zpool_do_events(int, char **);

	static int zpool_do_get(int, char **);
	static int zpool_do_set(int, char **);

	static int zpool_do_sync(int, char **);

	static int zpool_do_version(int, char **);

	static int zpool_do_wait(int, char **);

	static int zpool_do_help(int argc, char **argv);

	static zpool_compat_status_t zpool_do_load_compat(
	const char , boolean_t );

	+enum zpool_options {
	+ ZPOOL_OPTION_POWER = 1024,
	+ ZPOOL_OPTION_ALLOW_INUSE,
	+ ZPOOL_OPTION_ALLOW_REPLICATION_MISMATCH,
	+ ZPOOL_OPTION_ALLOW_ASHIFT_MISMATCH
	+};
	+
	/*
	* These libumem hooks provide a reasonable set of defaults for the allocator's
	* debugging facilities.
	*/

	#ifdef DEBUG
	const char *
	_umem_debug_init(void)
	{
	return ("default,verbose"); /* $UMEM_DEBUG setting */
	}

	const char *
	_umem_logging_init(void)
	{
	return ("fail,contents"); /* $UMEM_LOGGING setting */
	}
	#endif

	typedef enum {
	HELP_ADD,
	HELP_ATTACH,
	HELP_CLEAR,
	HELP_CREATE,
	HELP_CHECKPOINT,
	HELP_DESTROY,
	HELP_DETACH,
	HELP_EXPORT,
	HELP_HISTORY,
	HELP_IMPORT,
	HELP_IOSTAT,
	HELP_LABELCLEAR,
	HELP_LIST,
	HELP_OFFLINE,
	HELP_ONLINE,
	HELP_REPLACE,
	HELP_REMOVE,
	HELP_INITIALIZE,
	HELP_SCRUB,
	HELP_RESILVER,
	HELP_TRIM,
	HELP_STATUS,
	HELP_UPGRADE,
	HELP_EVENTS,
	HELP_GET,
	HELP_SET,
	HELP_SPLIT,
	HELP_SYNC,
	HELP_REGUID,
	HELP_REOPEN,
	HELP_VERSION,
	HELP_WAIT
	} zpool_help_t;


	/*
	* Flags for stats to display with "zpool iostats"
	*/
	enum iostat_type {
	IOS_DEFAULT = 0,
	IOS_LATENCY = 1,
	IOS_QUEUES = 2,
	IOS_L_HISTO = 3,
	IOS_RQ_HISTO = 4,
	IOS_COUNT, /* always last element */
	};

	/* iostat_type entries as bitmasks */
	#define IOS_DEFAULT_M (1ULL << IOS_DEFAULT)
	#define IOS_LATENCY_M (1ULL << IOS_LATENCY)
	#define IOS_QUEUES_M (1ULL << IOS_QUEUES)
	#define IOS_L_HISTO_M (1ULL << IOS_L_HISTO)
	#define IOS_RQ_HISTO_M (1ULL << IOS_RQ_HISTO)

	/* Mask of all the histo bits */
	#define IOS_ANYHISTO_M (IOS_L_HISTO_M \| IOS_RQ_HISTO_M)

	/*
	* Lookup table for iostat flags to nvlist names. Basically a list
	* of all the nvlists a flag requires. Also specifies the order in
	* which data gets printed in zpool iostat.
	*/
	static const char *vsx_type_to_nvlist[IOS_COUNT][15] = {
	[IOS_L_HISTO] = {
	ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_REBUILD_LAT_HISTO,
	NULL},
	[IOS_LATENCY] = {
	ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_REBUILD_LAT_HISTO,
	NULL},
	[IOS_QUEUES] = {
	ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
	ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
	ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
	ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
	ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
	ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE,
	ZPOOL_CONFIG_VDEV_REBUILD_ACTIVE_QUEUE,
	NULL},
	[IOS_RQ_HISTO] = {
	ZPOOL_CONFIG_VDEV_SYNC_IND_R_HISTO,
	ZPOOL_CONFIG_VDEV_SYNC_AGG_R_HISTO,
	ZPOOL_CONFIG_VDEV_SYNC_IND_W_HISTO,
	ZPOOL_CONFIG_VDEV_SYNC_AGG_W_HISTO,
	ZPOOL_CONFIG_VDEV_ASYNC_IND_R_HISTO,
	ZPOOL_CONFIG_VDEV_ASYNC_AGG_R_HISTO,
	ZPOOL_CONFIG_VDEV_ASYNC_IND_W_HISTO,
	ZPOOL_CONFIG_VDEV_ASYNC_AGG_W_HISTO,
	ZPOOL_CONFIG_VDEV_IND_SCRUB_HISTO,
	ZPOOL_CONFIG_VDEV_AGG_SCRUB_HISTO,
	ZPOOL_CONFIG_VDEV_IND_TRIM_HISTO,
	ZPOOL_CONFIG_VDEV_AGG_TRIM_HISTO,
	ZPOOL_CONFIG_VDEV_IND_REBUILD_HISTO,
	ZPOOL_CONFIG_VDEV_AGG_REBUILD_HISTO,
	NULL},
	};


	/*
	* Given a cb->cb_flags with a histogram bit set, return the iostat_type.
	* Right now, only one histo bit is ever set at one time, so we can
	* just do a highbit64(a)
	*/
	#define IOS_HISTO_IDX(a) (highbit64(a & IOS_ANYHISTO_M) - 1)

	typedef struct zpool_command {
	const char *name;
	int (func)(int, char *);
	zpool_help_t usage;
	} zpool_command_t;

	/*
	* Master command table. Each ZFS command has a name, associated function, and
	* usage message. The usage messages need to be internationalized, so we have
	* to have a function to return the usage message based on a command index.
	*
	* These commands are organized according to how they are displayed in the usage
	* message. An empty command (one with a NULL name) indicates an empty line in
	* the generic usage message.
	*/
	static zpool_command_t command_table[] = {
	{ "version", zpool_do_version, HELP_VERSION },
	{ NULL },
	{ "create", zpool_do_create, HELP_CREATE },
	{ "destroy", zpool_do_destroy, HELP_DESTROY },
	{ NULL },
	{ "add", zpool_do_add, HELP_ADD },
	{ "remove", zpool_do_remove, HELP_REMOVE },
	{ NULL },
	{ "labelclear", zpool_do_labelclear, HELP_LABELCLEAR },
	{ NULL },
	{ "checkpoint", zpool_do_checkpoint, HELP_CHECKPOINT },
	{ NULL },
	{ "list", zpool_do_list, HELP_LIST },
	{ "iostat", zpool_do_iostat, HELP_IOSTAT },
	{ "status", zpool_do_status, HELP_STATUS },
	{ NULL },
	{ "online", zpool_do_online, HELP_ONLINE },
	{ "offline", zpool_do_offline, HELP_OFFLINE },
	{ "clear", zpool_do_clear, HELP_CLEAR },
	{ "reopen", zpool_do_reopen, HELP_REOPEN },
	{ NULL },
	{ "attach", zpool_do_attach, HELP_ATTACH },
	{ "detach", zpool_do_detach, HELP_DETACH },
	{ "replace", zpool_do_replace, HELP_REPLACE },
	{ "split", zpool_do_split, HELP_SPLIT },
	{ NULL },
	{ "initialize", zpool_do_initialize, HELP_INITIALIZE },
	{ "resilver", zpool_do_resilver, HELP_RESILVER },
	{ "scrub", zpool_do_scrub, HELP_SCRUB },
	{ "trim", zpool_do_trim, HELP_TRIM },
	{ NULL },
	{ "import", zpool_do_import, HELP_IMPORT },
	{ "export", zpool_do_export, HELP_EXPORT },
	{ "upgrade", zpool_do_upgrade, HELP_UPGRADE },
	{ "reguid", zpool_do_reguid, HELP_REGUID },
	{ NULL },
	{ "history", zpool_do_history, HELP_HISTORY },
	{ "events", zpool_do_events, HELP_EVENTS },
	{ NULL },
	{ "get", zpool_do_get, HELP_GET },
	{ "set", zpool_do_set, HELP_SET },
	{ "sync", zpool_do_sync, HELP_SYNC },
	{ NULL },
	{ "wait", zpool_do_wait, HELP_WAIT },
	};

	#define NCOMMAND (ARRAY_SIZE(command_table))

	#define VDEV_ALLOC_CLASS_LOGS "logs"

	static zpool_command_t *current_command;
	static zfs_type_t current_prop_type = (ZFS_TYPE_POOL \| ZFS_TYPE_VDEV);
	static char history_str[HIS_MAX_RECORD_LEN];
	static boolean_t log_history = B_TRUE;
	static uint_t timestamp_fmt = NODATE;

	static const char *
	get_usage(zpool_help_t idx)
	{
	switch (idx) {
	case HELP_ADD:
	- return (gettext("\tadd [-fgLnP] [-o property=value] "
	+ return (gettext("\tadd [-afgLnP] [-o property=value] "
	"<pool> <vdev> ...\n"));
	case HELP_ATTACH:
	return (gettext("\tattach [-fsw] [-o property=value] "
	"<pool> <device> <new-device>\n"));
	case HELP_CLEAR:
	return (gettext("\tclear [[--power]\|[-nF]] <pool> [device]\n"));
	case HELP_CREATE:
	return (gettext("\tcreate [-fnd] [-o property=value] ... \n"
	"\t [-O file-system-property=value] ... \n"
	"\t [-m mountpoint] [-R root] <pool> <vdev> ...\n"));
	case HELP_CHECKPOINT:
	return (gettext("\tcheckpoint [-d [-w]] <pool> ...\n"));
	case HELP_DESTROY:
	return (gettext("\tdestroy [-f] <pool>\n"));
	case HELP_DETACH:
	return (gettext("\tdetach <pool> <device>\n"));
	case HELP_EXPORT:
	return (gettext("\texport [-af] <pool> ...\n"));
	case HELP_HISTORY:
	return (gettext("\thistory [-il] [<pool>] ...\n"));
	case HELP_IMPORT:
	return (gettext("\timport [-d dir] [-D]\n"
	"\timport [-o mntopts] [-o property=value] ... \n"
	"\t [-d dir \| -c cachefile] [-D] [-l] [-f] [-m] [-N] "
	"[-R root] [-F [-n]] -a\n"
	"\timport [-o mntopts] [-o property=value] ... \n"
	"\t [-d dir \| -c cachefile] [-D] [-l] [-f] [-m] [-N] "
	"[-R root] [-F [-n]]\n"
	"\t [--rewind-to-checkpoint] <pool \| id> [newpool]\n"));
	case HELP_IOSTAT:
	return (gettext("\tiostat [[[-c [script1,script2,...]"
	"[-lq]]\|[-rw]] [-T d \| u] [-ghHLpPvy]\n"
	"\t [[pool ...]\|[pool vdev ...]\|[vdev ...]]"
	" [[-n] interval [count]]\n"));
	case HELP_LABELCLEAR:
	return (gettext("\tlabelclear [-f] <vdev>\n"));
	case HELP_LIST:
	return (gettext("\tlist [-gHLpPv] [-o property[,...]] "
	"[-T d\|u] [pool] ... \n"
	"\t [interval [count]]\n"));
	case HELP_OFFLINE:
	return (gettext("\toffline [--power]\|[[-f][-t]] <pool> "
	"<device> ...\n"));
	case HELP_ONLINE:
	return (gettext("\tonline [--power][-e] <pool> <device> "
	"...\n"));
	case HELP_REPLACE:
	return (gettext("\treplace [-fsw] [-o property=value] "
	"<pool> <device> [new-device]\n"));
	case HELP_REMOVE:
	return (gettext("\tremove [-npsw] <pool> <device> ...\n"));
	case HELP_REOPEN:
	return (gettext("\treopen [-n] <pool>\n"));
	case HELP_INITIALIZE:
	return (gettext("\tinitialize [-c \| -s \| -u] [-w] <pool> "
	"[<device> ...]\n"));
	case HELP_SCRUB:
	return (gettext("\tscrub [-s \| -p] [-w] [-e] <pool> ...\n"));
	case HELP_RESILVER:
	return (gettext("\tresilver <pool> ...\n"));
	case HELP_TRIM:
	return (gettext("\ttrim [-dw] [-r <rate>] [-c \| -s] <pool> "
	"[<device> ...]\n"));
	case HELP_STATUS:
	return (gettext("\tstatus [--power] [-c [script1,script2,...]] "
	- "[-igLpPstvxD] [-T d\|u] [pool] ... \n"
	+ "[-DegiLpPstvx] [-T d\|u] [pool] ...\n"
	"\t [interval [count]]\n"));
	case HELP_UPGRADE:
	return (gettext("\tupgrade\n"
	"\tupgrade -v\n"
	"\tupgrade [-V version] <-a \| pool ...>\n"));
	case HELP_EVENTS:
	return (gettext("\tevents [-vHf [pool] \| -c]\n"));
	case HELP_GET:
	return (gettext("\tget [-Hp] [-o \"all\" \| field[,...]] "
	"<\"all\" \| property[,...]> <pool> ...\n"));
	case HELP_SET:
	return (gettext("\tset <property=value> <pool>\n"
	"\tset <vdev_property=value> <pool> <vdev>\n"));
	case HELP_SPLIT:
	return (gettext("\tsplit [-gLnPl] [-R altroot] [-o mntopts]\n"
	"\t [-o property=value] <pool> <newpool> "
	"[<device> ...]\n"));
	case HELP_REGUID:
	return (gettext("\treguid <pool>\n"));
	case HELP_SYNC:
	return (gettext("\tsync [pool] ...\n"));
	case HELP_VERSION:
	return (gettext("\tversion\n"));
	case HELP_WAIT:
	return (gettext("\twait [-Hp] [-T d\|u] [-t <activity>[,...]] "
	"<pool> [interval]\n"));
	default:
	__builtin_unreachable();
	}
	}

	static void
	zpool_collect_leaves(zpool_handle_t zhp, nvlist_t nvroot, nvlist_t *res)
	{
	uint_t children = 0;
	nvlist_t **child;
	uint_t i;

	(void) nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
	&child, &children);

	if (children == 0) {
	char *path = zpool_vdev_name(g_zfs, zhp, nvroot,
	VDEV_NAME_PATH);

	if (strcmp(path, VDEV_TYPE_INDIRECT) != 0 &&
	strcmp(path, VDEV_TYPE_HOLE) != 0)
	fnvlist_add_boolean(res, path);

	free(path);
	return;
	}

	for (i = 0; i < children; i++) {
	zpool_collect_leaves(zhp, child[i], res);
	}
	}

	/*
	* Callback routine that will print out a pool property value.
	*/
	static int
	print_pool_prop_cb(int prop, void *cb)
	{
	FILE *fp = cb;

	(void) fprintf(fp, "\t%-19s ", zpool_prop_to_name(prop));

	if (zpool_prop_readonly(prop))
	(void) fprintf(fp, " NO ");
	else
	(void) fprintf(fp, " YES ");

	if (zpool_prop_values(prop) == NULL)
	(void) fprintf(fp, "-\n");
	else
	(void) fprintf(fp, "%s\n", zpool_prop_values(prop));

	return (ZPROP_CONT);
	}

	/*
	* Callback routine that will print out a vdev property value.
	*/
	static int
	print_vdev_prop_cb(int prop, void *cb)
	{
	FILE *fp = cb;

	(void) fprintf(fp, "\t%-19s ", vdev_prop_to_name(prop));

	if (vdev_prop_readonly(prop))
	(void) fprintf(fp, " NO ");
	else
	(void) fprintf(fp, " YES ");

	if (vdev_prop_values(prop) == NULL)
	(void) fprintf(fp, "-\n");
	else
	(void) fprintf(fp, "%s\n", vdev_prop_values(prop));

	return (ZPROP_CONT);
	}

	/*
	* Given a leaf vdev name like 'L5' return its VDEV_CONFIG_PATH like
	* '/dev/disk/by-vdev/L5'.
	*/
	static const char *
	vdev_name_to_path(zpool_handle_t zhp, char vdev)
	{
	nvlist_t *vdev_nv = zpool_find_vdev(zhp, vdev, NULL, NULL, NULL);
	if (vdev_nv == NULL) {
	return (NULL);
	}
	return (fnvlist_lookup_string(vdev_nv, ZPOOL_CONFIG_PATH));
	}

	static int
	zpool_power_on(zpool_handle_t zhp, char vdev)
	{
	return (zpool_power(zhp, vdev, B_TRUE));
	}

	static int
	zpool_power_on_and_disk_wait(zpool_handle_t zhp, char vdev)
	{
	int rc;

	rc = zpool_power_on(zhp, vdev);
	if (rc != 0)
	return (rc);

	zpool_disk_wait(vdev_name_to_path(zhp, vdev));

	return (0);
	}

	static int
	zpool_power_on_pool_and_wait_for_devices(zpool_handle_t *zhp)
	{
	nvlist_t *nv;
	const char *path = NULL;
	int rc;

	/* Power up all the devices first */
	FOR_EACH_REAL_LEAF_VDEV(zhp, nv) {
	path = fnvlist_lookup_string(nv, ZPOOL_CONFIG_PATH);
	if (path != NULL) {
	rc = zpool_power_on(zhp, (char *)path);
	if (rc != 0) {
	return (rc);
	}
	}
	}

	/*
	* Wait for their devices to show up. Since we powered them on
	* at roughly the same time, they should all come online around
	* the same time.
	*/
	FOR_EACH_REAL_LEAF_VDEV(zhp, nv) {
	path = fnvlist_lookup_string(nv, ZPOOL_CONFIG_PATH);
	zpool_disk_wait(path);
	}

	return (0);
	}

	static int
	zpool_power_off(zpool_handle_t zhp, char vdev)
	{
	return (zpool_power(zhp, vdev, B_FALSE));
	}

	/*
	* Display usage message. If we're inside a command, display only the usage for
	* that command. Otherwise, iterate over the entire command table and display
	* a complete usage message.
	*/
	static __attribute__((noreturn)) void
	usage(boolean_t requested)
	{
	FILE *fp = requested ? stdout : stderr;

	if (current_command == NULL) {
	int i;

	(void) fprintf(fp, gettext("usage: zpool command args ...\n"));
	(void) fprintf(fp,
	gettext("where 'command' is one of the following:\n\n"));

	for (i = 0; i < NCOMMAND; i++) {
	if (command_table[i].name == NULL)
	(void) fprintf(fp, "\n");
	else
	(void) fprintf(fp, "%s",
	get_usage(command_table[i].usage));
	}

	(void) fprintf(fp,
	gettext("\nFor further help on a command or topic, "
	"run: %s\n"), "zpool help [<topic>]");
	} else {
	(void) fprintf(fp, gettext("usage:\n"));
	(void) fprintf(fp, "%s", get_usage(current_command->usage));
	}

	if (current_command != NULL &&
	current_prop_type != (ZFS_TYPE_POOL \| ZFS_TYPE_VDEV) &&
	((strcmp(current_command->name, "set") == 0) \|\|
	(strcmp(current_command->name, "get") == 0) \|\|
	(strcmp(current_command->name, "list") == 0))) {

	(void) fprintf(fp, "%s",
	gettext("\nthe following properties are supported:\n"));

	(void) fprintf(fp, "\n\t%-19s %s %s\n\n",
	"PROPERTY", "EDIT", "VALUES");

	/* Iterate over all properties */
	if (current_prop_type == ZFS_TYPE_POOL) {
	(void) zprop_iter(print_pool_prop_cb, fp, B_FALSE,
	B_TRUE, current_prop_type);

	(void) fprintf(fp, "\t%-19s ", "feature@...");
	(void) fprintf(fp, "YES "
	"disabled \| enabled \| active\n");

	(void) fprintf(fp, gettext("\nThe feature@ properties "
	"must be appended with a feature name.\n"
	"See zpool-features(7).\n"));
	} else if (current_prop_type == ZFS_TYPE_VDEV) {
	(void) zprop_iter(print_vdev_prop_cb, fp, B_FALSE,
	B_TRUE, current_prop_type);
	}
	}

	/*
	* See comments at end of main().
	*/
	if (getenv("ZFS_ABORT") != NULL) {
	(void) printf("dumping core by request\n");
	abort();
	}

	exit(requested ? 0 : 2);
	}

	/*
	* zpool initialize [-c \| -s \| -u] [-w] <pool> [<vdev> ...]
	* Initialize all unused blocks in the specified vdevs, or all vdevs in the pool
	* if none specified.
	*
	* -c Cancel. Ends active initializing.
	* -s Suspend. Initializing can then be restarted with no flags.
	* -u Uninitialize. Clears initialization state.
	* -w Wait. Blocks until initializing has completed.
	*/
	int
	zpool_do_initialize(int argc, char **argv)
	{
	int c;
	char *poolname;
	zpool_handle_t *zhp;
	nvlist_t *vdevs;
	int err = 0;
	boolean_t wait = B_FALSE;

	struct option long_options[] = {
	{"cancel", no_argument, NULL, 'c'},
	{"suspend", no_argument, NULL, 's'},
	{"uninit", no_argument, NULL, 'u'},
	{"wait", no_argument, NULL, 'w'},
	{0, 0, 0, 0}
	};

	pool_initialize_func_t cmd_type = POOL_INITIALIZE_START;
	while ((c = getopt_long(argc, argv, "csuw", long_options,
	NULL)) != -1) {
	switch (c) {
	case 'c':
	if (cmd_type != POOL_INITIALIZE_START &&
	cmd_type != POOL_INITIALIZE_CANCEL) {
	(void) fprintf(stderr, gettext("-c cannot be "
	"combined with other options\n"));
	usage(B_FALSE);
	}
	cmd_type = POOL_INITIALIZE_CANCEL;
	break;
	case 's':
	if (cmd_type != POOL_INITIALIZE_START &&
	cmd_type != POOL_INITIALIZE_SUSPEND) {
	(void) fprintf(stderr, gettext("-s cannot be "
	"combined with other options\n"));
	usage(B_FALSE);
	}
	cmd_type = POOL_INITIALIZE_SUSPEND;
	break;
	case 'u':
	if (cmd_type != POOL_INITIALIZE_START &&
	cmd_type != POOL_INITIALIZE_UNINIT) {
	(void) fprintf(stderr, gettext("-u cannot be "
	"combined with other options\n"));
	usage(B_FALSE);
	}
	cmd_type = POOL_INITIALIZE_UNINIT;
	break;
	case 'w':
	wait = B_TRUE;
	break;
	case '?':
	if (optopt != 0) {
	(void) fprintf(stderr,
	gettext("invalid option '%c'\n"), optopt);
	} else {
	(void) fprintf(stderr,
	gettext("invalid option '%s'\n"),
	argv[optind - 1]);
	}
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing pool name argument\n"));
	usage(B_FALSE);
	return (-1);
	}

	if (wait && (cmd_type != POOL_INITIALIZE_START)) {
	(void) fprintf(stderr, gettext("-w cannot be used with -c, -s"
	"or -u\n"));
	usage(B_FALSE);
	}

	poolname = argv[0];
	zhp = zpool_open(g_zfs, poolname);
	if (zhp == NULL)
	return (-1);

	vdevs = fnvlist_alloc();
	if (argc == 1) {
	/* no individual leaf vdevs specified, so add them all */
	nvlist_t *config = zpool_get_config(zhp, NULL);
	nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
	ZPOOL_CONFIG_VDEV_TREE);
	zpool_collect_leaves(zhp, nvroot, vdevs);
	} else {
	for (int i = 1; i < argc; i++) {
	fnvlist_add_boolean(vdevs, argv[i]);
	}
	}

	if (wait)
	err = zpool_initialize_wait(zhp, cmd_type, vdevs);
	else
	err = zpool_initialize(zhp, cmd_type, vdevs);

	fnvlist_free(vdevs);
	zpool_close(zhp);

	return (err);
	}

	/*
	* print a pool vdev config for dry runs
	*/
	static void
	print_vdev_tree(zpool_handle_t zhp, const char name, nvlist_t *nv, int indent,
	const char *match, int name_flags)
	{
	nvlist_t **child;
	uint_t c, children;
	char *vname;
	boolean_t printed = B_FALSE;

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
	&child, &children) != 0) {
	if (name != NULL)
	(void) printf("\t%*s%s\n", indent, "", name);
	return;
	}

	for (c = 0; c < children; c++) {
	uint64_t is_log = B_FALSE, is_hole = B_FALSE;
	const char *class = "";

	(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
	&is_hole);

	if (is_hole == B_TRUE) {
	continue;
	}

	(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
	&is_log);
	if (is_log)
	class = VDEV_ALLOC_BIAS_LOG;
	(void) nvlist_lookup_string(child[c],
	ZPOOL_CONFIG_ALLOCATION_BIAS, &class);
	if (strcmp(match, class) != 0)
	continue;

	if (!printed && name != NULL) {
	(void) printf("\t%*s%s\n", indent, "", name);
	printed = B_TRUE;
	}
	vname = zpool_vdev_name(g_zfs, zhp, child[c], name_flags);
	print_vdev_tree(zhp, vname, child[c], indent + 2, "",
	name_flags);
	free(vname);
	}
	}

	/*
	* Print the list of l2cache devices for dry runs.
	*/
	static void
	print_cache_list(nvlist_t *nv, int indent)
	{
	nvlist_t **child;
	uint_t c, children;

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
	&child, &children) == 0 && children > 0) {
	(void) printf("\t%*s%s\n", indent, "", "cache");
	} else {
	return;
	}
	for (c = 0; c < children; c++) {
	char *vname;

	vname = zpool_vdev_name(g_zfs, NULL, child[c], 0);
	(void) printf("\t%*s%s\n", indent + 2, "", vname);
	free(vname);
	}
	}

	/*
	* Print the list of spares for dry runs.
	*/
	static void
	print_spare_list(nvlist_t *nv, int indent)
	{
	nvlist_t **child;
	uint_t c, children;

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
	&child, &children) == 0 && children > 0) {
	(void) printf("\t%*s%s\n", indent, "", "spares");
	} else {
	return;
	}
	for (c = 0; c < children; c++) {
	char *vname;

	vname = zpool_vdev_name(g_zfs, NULL, child[c], 0);
	(void) printf("\t%*s%s\n", indent + 2, "", vname);
	free(vname);
	}
	}

	static boolean_t
	prop_list_contains_feature(nvlist_t *proplist)
	{
	nvpair_t *nvp;
	for (nvp = nvlist_next_nvpair(proplist, NULL); NULL != nvp;
	nvp = nvlist_next_nvpair(proplist, nvp)) {
	if (zpool_prop_feature(nvpair_name(nvp)))
	return (B_TRUE);
	}
	return (B_FALSE);
	}

	/*
	* Add a property pair (name, string-value) into a property nvlist.
	*/
	static int
	add_prop_list(const char propname, const char propval, nvlist_t **props,
	boolean_t poolprop)
	{
	zpool_prop_t prop = ZPOOL_PROP_INVAL;
	nvlist_t *proplist;
	const char *normnm;
	const char *strval;

	if (*props == NULL &&
	nvlist_alloc(props, NV_UNIQUE_NAME, 0) != 0) {
	(void) fprintf(stderr,
	gettext("internal error: out of memory\n"));
	return (1);
	}

	proplist = *props;

	if (poolprop) {
	const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
	const char *cname =
	zpool_prop_to_name(ZPOOL_PROP_COMPATIBILITY);

	if ((prop = zpool_name_to_prop(propname)) == ZPOOL_PROP_INVAL &&
	(!zpool_prop_feature(propname) &&
	!zpool_prop_vdev(propname))) {
	(void) fprintf(stderr, gettext("property '%s' is "
	"not a valid pool or vdev property\n"), propname);
	return (2);
	}

	/*
	* feature@ properties and version should not be specified
	* at the same time.
	*/
	if ((prop == ZPOOL_PROP_INVAL && zpool_prop_feature(propname) &&
	nvlist_exists(proplist, vname)) \|\|
	(prop == ZPOOL_PROP_VERSION &&
	prop_list_contains_feature(proplist))) {
	(void) fprintf(stderr, gettext("'feature@' and "
	"'version' properties cannot be specified "
	"together\n"));
	return (2);
	}

	/*
	* if version is specified, only "legacy" compatibility
	* may be requested
	*/
	if ((prop == ZPOOL_PROP_COMPATIBILITY &&
	strcmp(propval, ZPOOL_COMPAT_LEGACY) != 0 &&
	nvlist_exists(proplist, vname)) \|\|
	(prop == ZPOOL_PROP_VERSION &&
	nvlist_exists(proplist, cname) &&
	strcmp(fnvlist_lookup_string(proplist, cname),
	ZPOOL_COMPAT_LEGACY) != 0)) {
	(void) fprintf(stderr, gettext("when 'version' is "
	"specified, the 'compatibility' feature may only "
	"be set to '" ZPOOL_COMPAT_LEGACY "'\n"));
	return (2);
	}

	if (zpool_prop_feature(propname) \|\| zpool_prop_vdev(propname))
	normnm = propname;
	else
	normnm = zpool_prop_to_name(prop);
	} else {
	zfs_prop_t fsprop = zfs_name_to_prop(propname);

	if (zfs_prop_valid_for_type(fsprop, ZFS_TYPE_FILESYSTEM,
	B_FALSE)) {
	normnm = zfs_prop_to_name(fsprop);
	} else if (zfs_prop_user(propname) \|\|
	zfs_prop_userquota(propname)) {
	normnm = propname;
	} else {
	(void) fprintf(stderr, gettext("property '%s' is "
	"not a valid filesystem property\n"), propname);
	return (2);
	}
	}

	if (nvlist_lookup_string(proplist, normnm, &strval) == 0 &&
	prop != ZPOOL_PROP_CACHEFILE) {
	(void) fprintf(stderr, gettext("property '%s' "
	"specified multiple times\n"), propname);
	return (2);
	}

	if (nvlist_add_string(proplist, normnm, propval) != 0) {
	(void) fprintf(stderr, gettext("internal "
	"error: out of memory\n"));
	return (1);
	}

	return (0);
	}

	/*
	* Set a default property pair (name, string-value) in a property nvlist
	*/
	static int
	add_prop_list_default(const char propname, const char propval,
	nvlist_t **props)
	{
	const char *pval;

	if (nvlist_lookup_string(*props, propname, &pval) == 0)
	return (0);

	return (add_prop_list(propname, propval, props, B_TRUE));
	}

	/*
	- * zpool add [-fgLnP] [-o property=value] <pool> <vdev> ...
	+ * zpool add [-afgLnP] [-o property=value] <pool> <vdev> ...
	*
	+ * -a Disable the ashift validation checks
	* -f Force addition of devices, even if they appear in use
	* -g Display guid for individual vdev name.
	* -L Follow links when resolving vdev path name.
	* -n Do not add the devices, but display the resulting layout if
	* they were to be added.
	* -o Set property=value.
	* -P Display full path for vdev name.
	*
	* Adds the given vdevs to 'pool'. As with create, the bulk of this work is
	* handled by make_root_vdev(), which constructs the nvlist needed to pass to
	* libzfs.
	*/
	int
	zpool_do_add(int argc, char **argv)
	{
	- boolean_t force = B_FALSE;
	+ boolean_t check_replication = B_TRUE;
	+ boolean_t check_inuse = B_TRUE;
	boolean_t dryrun = B_FALSE;
	+ boolean_t check_ashift = B_TRUE;
	+ boolean_t force = B_FALSE;
	int name_flags = 0;
	int c;
	nvlist_t *nvroot;
	char *poolname;
	int ret;
	zpool_handle_t *zhp;
	nvlist_t *config;
	nvlist_t *props = NULL;
	char *propval;

	+ struct option long_options[] = {
	+ {"allow-in-use", no_argument, NULL, ZPOOL_OPTION_ALLOW_INUSE},
	+ {"allow-replication-mismatch", no_argument, NULL,
	+ ZPOOL_OPTION_ALLOW_REPLICATION_MISMATCH},
	+ {"allow-ashift-mismatch", no_argument, NULL,
	+ ZPOOL_OPTION_ALLOW_ASHIFT_MISMATCH},
	+ {0, 0, 0, 0}
	+ };
	+
	/* check options */
	- while ((c = getopt(argc, argv, "fgLno:P")) != -1) {
	+ while ((c = getopt_long(argc, argv, "fgLno:P", long_options, NULL))
	+ != -1) {
	switch (c) {
	case 'f':
	force = B_TRUE;
	break;
	case 'g':
	name_flags \|= VDEV_NAME_GUID;
	break;
	case 'L':
	name_flags \|= VDEV_NAME_FOLLOW_LINKS;
	break;
	case 'n':
	dryrun = B_TRUE;
	break;
	case 'o':
	if ((propval = strchr(optarg, '=')) == NULL) {
	(void) fprintf(stderr, gettext("missing "
	"'=' for -o option\n"));
	usage(B_FALSE);
	}
	*propval = '\0';
	propval++;

	if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) \|\|
	(add_prop_list(optarg, propval, &props, B_TRUE)))
	usage(B_FALSE);
	break;
	case 'P':
	name_flags \|= VDEV_NAME_PATH;
	break;
	+ case ZPOOL_OPTION_ALLOW_INUSE:
	+ check_inuse = B_FALSE;
	+ break;
	+ case ZPOOL_OPTION_ALLOW_REPLICATION_MISMATCH:
	+ check_replication = B_FALSE;
	+ break;
	+ case ZPOOL_OPTION_ALLOW_ASHIFT_MISMATCH:
	+ check_ashift = B_FALSE;
	+ break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* get pool name and check number of arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing pool name argument\n"));
	usage(B_FALSE);
	}
	if (argc < 2) {
	(void) fprintf(stderr, gettext("missing vdev specification\n"));
	usage(B_FALSE);
	}

	+ if (force) {
	+ if (!check_inuse \|\| !check_replication \|\| !check_ashift) {
	+ (void) fprintf(stderr, gettext("'-f' option is not "
	+ "allowed with '--allow-replication-mismatch', "
	+ "'--allow-ashift-mismatch', or "
	+ "'--allow-in-use'\n"));
	+ usage(B_FALSE);
	+ }
	+ check_inuse = B_FALSE;
	+ check_replication = B_FALSE;
	+ check_ashift = B_FALSE;
	+ }
	+
	poolname = argv[0];

	argc--;
	argv++;

	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
	return (1);

	if ((config = zpool_get_config(zhp, NULL)) == NULL) {
	(void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
	poolname);
	zpool_close(zhp);
	return (1);
	}

	/* unless manually specified use "ashift" pool property (if set) */
	if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
	int intval;
	zprop_source_t src;
	char strval[ZPOOL_MAXPROPLEN];

	intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
	if (src != ZPROP_SRC_DEFAULT) {
	(void) sprintf(strval, "%" PRId32, intval);
	verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
	&props, B_TRUE) == 0);
	}
	}

	/* pass off to make_root_vdev for processing */
	- nvroot = make_root_vdev(zhp, props, force, !force, B_FALSE, dryrun,
	- argc, argv);
	+ nvroot = make_root_vdev(zhp, props, !check_inuse,
	+ check_replication, B_FALSE, dryrun, argc, argv);
	if (nvroot == NULL) {
	zpool_close(zhp);
	return (1);
	}

	if (dryrun) {
	nvlist_t *poolnvroot;
	nvlist_t l2child, sparechild;
	uint_t l2children, sparechildren, c;
	char *vname;
	boolean_t hadcache = B_FALSE, hadspare = B_FALSE;

	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
	&poolnvroot) == 0);

	(void) printf(gettext("would update '%s' to the following "
	"configuration:\n\n"), zpool_get_name(zhp));

	/* print original main pool and new tree */
	print_vdev_tree(zhp, poolname, poolnvroot, 0, "",
	name_flags \| VDEV_NAME_TYPE_ID);
	print_vdev_tree(zhp, NULL, nvroot, 0, "", name_flags);

	/* print other classes: 'dedup', 'special', and 'log' */
	if (zfs_special_devs(poolnvroot, VDEV_ALLOC_BIAS_DEDUP)) {
	print_vdev_tree(zhp, "dedup", poolnvroot, 0,
	VDEV_ALLOC_BIAS_DEDUP, name_flags);
	print_vdev_tree(zhp, NULL, nvroot, 0,
	VDEV_ALLOC_BIAS_DEDUP, name_flags);
	} else if (zfs_special_devs(nvroot, VDEV_ALLOC_BIAS_DEDUP)) {
	print_vdev_tree(zhp, "dedup", nvroot, 0,
	VDEV_ALLOC_BIAS_DEDUP, name_flags);
	}

	if (zfs_special_devs(poolnvroot, VDEV_ALLOC_BIAS_SPECIAL)) {
	print_vdev_tree(zhp, "special", poolnvroot, 0,
	VDEV_ALLOC_BIAS_SPECIAL, name_flags);
	print_vdev_tree(zhp, NULL, nvroot, 0,
	VDEV_ALLOC_BIAS_SPECIAL, name_flags);
	} else if (zfs_special_devs(nvroot, VDEV_ALLOC_BIAS_SPECIAL)) {
	print_vdev_tree(zhp, "special", nvroot, 0,
	VDEV_ALLOC_BIAS_SPECIAL, name_flags);
	}

	if (num_logs(poolnvroot) > 0) {
	print_vdev_tree(zhp, "logs", poolnvroot, 0,
	VDEV_ALLOC_BIAS_LOG, name_flags);
	print_vdev_tree(zhp, NULL, nvroot, 0,
	VDEV_ALLOC_BIAS_LOG, name_flags);
	} else if (num_logs(nvroot) > 0) {
	print_vdev_tree(zhp, "logs", nvroot, 0,
	VDEV_ALLOC_BIAS_LOG, name_flags);
	}

	/* Do the same for the caches */
	if (nvlist_lookup_nvlist_array(poolnvroot, ZPOOL_CONFIG_L2CACHE,
	&l2child, &l2children) == 0 && l2children) {
	hadcache = B_TRUE;
	(void) printf(gettext("\tcache\n"));
	for (c = 0; c < l2children; c++) {
	vname = zpool_vdev_name(g_zfs, NULL,
	l2child[c], name_flags);
	(void) printf("\t %s\n", vname);
	free(vname);
	}
	}
	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
	&l2child, &l2children) == 0 && l2children) {
	if (!hadcache)
	(void) printf(gettext("\tcache\n"));
	for (c = 0; c < l2children; c++) {
	vname = zpool_vdev_name(g_zfs, NULL,
	l2child[c], name_flags);
	(void) printf("\t %s\n", vname);
	free(vname);
	}
	}
	/* And finally the spares */
	if (nvlist_lookup_nvlist_array(poolnvroot, ZPOOL_CONFIG_SPARES,
	&sparechild, &sparechildren) == 0 && sparechildren > 0) {
	hadspare = B_TRUE;
	(void) printf(gettext("\tspares\n"));
	for (c = 0; c < sparechildren; c++) {
	vname = zpool_vdev_name(g_zfs, NULL,
	sparechild[c], name_flags);
	(void) printf("\t %s\n", vname);
	free(vname);
	}
	}
	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
	&sparechild, &sparechildren) == 0 && sparechildren > 0) {
	if (!hadspare)
	(void) printf(gettext("\tspares\n"));
	for (c = 0; c < sparechildren; c++) {
	vname = zpool_vdev_name(g_zfs, NULL,
	sparechild[c], name_flags);
	(void) printf("\t %s\n", vname);
	free(vname);
	}
	}

	ret = 0;
	} else {
	- ret = (zpool_add(zhp, nvroot) != 0);
	+ ret = (zpool_add(zhp, nvroot, check_ashift) != 0);
	}

	nvlist_free(props);
	nvlist_free(nvroot);
	zpool_close(zhp);

	return (ret);
	}

	/*
	* zpool remove [-npsw] <pool> <vdev> ...
	*
	* Removes the given vdev from the pool.
	*/
	int
	zpool_do_remove(int argc, char **argv)
	{
	char *poolname;
	int i, ret = 0;
	zpool_handle_t *zhp = NULL;
	boolean_t stop = B_FALSE;
	int c;
	boolean_t noop = B_FALSE;
	boolean_t parsable = B_FALSE;
	boolean_t wait = B_FALSE;

	/* check options */
	while ((c = getopt(argc, argv, "npsw")) != -1) {
	switch (c) {
	case 'n':
	noop = B_TRUE;
	break;
	case 'p':
	parsable = B_TRUE;
	break;
	case 's':
	stop = B_TRUE;
	break;
	case 'w':
	wait = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* get pool name and check number of arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing pool name argument\n"));
	usage(B_FALSE);
	}

	poolname = argv[0];

	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
	return (1);

	if (stop && noop) {
	zpool_close(zhp);
	(void) fprintf(stderr, gettext("stop request ignored\n"));
	return (0);
	}

	if (stop) {
	if (argc > 1) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}
	if (zpool_vdev_remove_cancel(zhp) != 0)
	ret = 1;
	if (wait) {
	(void) fprintf(stderr, gettext("invalid option "
	"combination: -w cannot be used with -s\n"));
	usage(B_FALSE);
	}
	} else {
	if (argc < 2) {
	(void) fprintf(stderr, gettext("missing device\n"));
	usage(B_FALSE);
	}

	for (i = 1; i < argc; i++) {
	if (noop) {
	uint64_t size;

	if (zpool_vdev_indirect_size(zhp, argv[i],
	&size) != 0) {
	ret = 1;
	break;
	}
	if (parsable) {
	(void) printf("%s %llu\n",
	argv[i], (unsigned long long)size);
	} else {
	char valstr[32];
	zfs_nicenum(size, valstr,
	sizeof (valstr));
	(void) printf("Memory that will be "
	"used after removing %s: %s\n",
	argv[i], valstr);
	}
	} else {
	if (zpool_vdev_remove(zhp, argv[i]) != 0)
	ret = 1;
	}
	}

	if (ret == 0 && wait)
	ret = zpool_wait(zhp, ZPOOL_WAIT_REMOVE);
	}
	zpool_close(zhp);

	return (ret);
	}

	/*
	* Return 1 if a vdev is active (being used in a pool)
	* Return 0 if a vdev is inactive (offlined or faulted, or not in active pool)
	*
	* This is useful for checking if a disk in an active pool is offlined or
	* faulted.
	*/
	static int
	vdev_is_active(char *vdev_path)
	{
	int fd;
	fd = open(vdev_path, O_EXCL);
	if (fd < 0) {
	return (1); /* cant open O_EXCL - disk is active */
	}

	close(fd);
	return (0); /* disk is inactive in the pool */
	}

	/*
	* zpool labelclear [-f] <vdev>
	*
	* -f Force clearing the label for the vdevs which are members of
	* the exported or foreign pools.
	*
	* Verifies that the vdev is not active and zeros out the label information
	* on the device.
	*/
	int
	zpool_do_labelclear(int argc, char **argv)
	{
	char vdev[MAXPATHLEN];
	char *name = NULL;
	int c, fd = -1, ret = 0;
	nvlist_t *config;
	pool_state_t state;
	boolean_t inuse = B_FALSE;
	boolean_t force = B_FALSE;

	/* check options */
	while ((c = getopt(argc, argv, "f")) != -1) {
	switch (c) {
	case 'f':
	force = B_TRUE;
	break;
	default:
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* get vdev name */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing vdev name\n"));
	usage(B_FALSE);
	}
	if (argc > 1) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	(void) strlcpy(vdev, argv[0], sizeof (vdev));

	/*
	* If we cannot open an absolute path, we quit.
	* Otherwise if the provided vdev name doesn't point to a file,
	* try prepending expected disk paths and partition numbers.
	*/
	if ((fd = open(vdev, O_RDWR)) < 0) {
	int error;
	if (vdev[0] == '/') {
	(void) fprintf(stderr, gettext("failed to open "
	"%s: %s\n"), vdev, strerror(errno));
	return (1);
	}

	error = zfs_resolve_shortname(argv[0], vdev, MAXPATHLEN);
	if (error == 0 && zfs_dev_is_whole_disk(vdev)) {
	if (zfs_append_partition(vdev, MAXPATHLEN) == -1)
	error = ENOENT;
	}

	if (error \|\| ((fd = open(vdev, O_RDWR)) < 0)) {
	if (errno == ENOENT) {
	(void) fprintf(stderr, gettext(
	"failed to find device %s, try "
	"specifying absolute path instead\n"),
	argv[0]);
	return (1);
	}

	(void) fprintf(stderr, gettext("failed to open %s:"
	" %s\n"), vdev, strerror(errno));
	return (1);
	}
	}

	/*
	* Flush all dirty pages for the block device. This should not be
	* fatal when the device does not support BLKFLSBUF as would be the
	* case for a file vdev.
	*/
	if ((zfs_dev_flush(fd) != 0) && (errno != ENOTTY))
	(void) fprintf(stderr, gettext("failed to invalidate "
	"cache for %s: %s\n"), vdev, strerror(errno));

	if (zpool_read_label(fd, &config, NULL) != 0) {
	(void) fprintf(stderr,
	gettext("failed to read label from %s\n"), vdev);
	ret = 1;
	goto errout;
	}
	nvlist_free(config);

	ret = zpool_in_use(g_zfs, fd, &state, &name, &inuse);
	if (ret != 0) {
	(void) fprintf(stderr,
	gettext("failed to check state for %s\n"), vdev);
	ret = 1;
	goto errout;
	}

	if (!inuse)
	goto wipe_label;

	switch (state) {
	default:
	case POOL_STATE_ACTIVE:
	case POOL_STATE_SPARE:
	case POOL_STATE_L2CACHE:
	/*
	* We allow the user to call 'zpool offline -f'
	* on an offlined disk in an active pool. We can check if
	* the disk is online by calling vdev_is_active().
	*/
	if (force && !vdev_is_active(vdev))
	break;

	(void) fprintf(stderr, gettext(
	"%s is a member (%s) of pool \"%s\""),
	vdev, zpool_pool_state_to_name(state), name);

	if (force) {
	(void) fprintf(stderr, gettext(
	". Offline the disk first to clear its label."));
	}
	printf("\n");
	ret = 1;
	goto errout;

	case POOL_STATE_EXPORTED:
	if (force)
	break;
	(void) fprintf(stderr, gettext(
	"use '-f' to override the following error:\n"
	"%s is a member of exported pool \"%s\"\n"),
	vdev, name);
	ret = 1;
	goto errout;

	case POOL_STATE_POTENTIALLY_ACTIVE:
	if (force)
	break;
	(void) fprintf(stderr, gettext(
	"use '-f' to override the following error:\n"
	"%s is a member of potentially active pool \"%s\"\n"),
	vdev, name);
	ret = 1;
	goto errout;

	case POOL_STATE_DESTROYED:
	/* inuse should never be set for a destroyed pool */
	assert(0);
	break;
	}

	wipe_label:
	ret = zpool_clear_label(fd);
	if (ret != 0) {
	(void) fprintf(stderr,
	gettext("failed to clear label for %s\n"), vdev);
	}

	errout:
	free(name);
	(void) close(fd);

	return (ret);
	}

	/*
	* zpool create [-fnd] [-o property=value] ...
	* [-O file-system-property=value] ...
	* [-R root] [-m mountpoint] <pool> <dev> ...
	*
	* -f Force creation, even if devices appear in use
	* -n Do not create the pool, but display the resulting layout if it
	* were to be created.
	* -R Create a pool under an alternate root
	* -m Set default mountpoint for the root dataset. By default it's
	* '/<pool>'
	* -o Set property=value.
	* -o Set feature@feature=enabled\|disabled.
	* -d Don't automatically enable all supported pool features
	* (individual features can be enabled with -o).
	* -O Set fsproperty=value in the pool's root file system
	*
	* Creates the named pool according to the given vdev specification. The
	* bulk of the vdev processing is done in make_root_vdev() in zpool_vdev.c.
	* Once we get the nvlist back from make_root_vdev(), we either print out the
	* contents (if '-n' was specified), or pass it to libzfs to do the creation.
	*/
	int
	zpool_do_create(int argc, char **argv)
	{
	boolean_t force = B_FALSE;
	boolean_t dryrun = B_FALSE;
	boolean_t enable_pool_features = B_TRUE;

	int c;
	nvlist_t *nvroot = NULL;
	char *poolname;
	char *tname = NULL;
	int ret = 1;
	char *altroot = NULL;
	char *compat = NULL;
	char *mountpoint = NULL;
	nvlist_t *fsprops = NULL;
	nvlist_t *props = NULL;
	char *propval;

	/* check options */
	while ((c = getopt(argc, argv, ":fndR:m:o:O:t:")) != -1) {
	switch (c) {
	case 'f':
	force = B_TRUE;
	break;
	case 'n':
	dryrun = B_TRUE;
	break;
	case 'd':
	enable_pool_features = B_FALSE;
	break;
	case 'R':
	altroot = optarg;
	if (add_prop_list(zpool_prop_to_name(
	ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
	goto errout;
	if (add_prop_list_default(zpool_prop_to_name(
	ZPOOL_PROP_CACHEFILE), "none", &props))
	goto errout;
	break;
	case 'm':
	/* Equivalent to -O mountpoint=optarg */
	mountpoint = optarg;
	break;
	case 'o':
	if ((propval = strchr(optarg, '=')) == NULL) {
	(void) fprintf(stderr, gettext("missing "
	"'=' for -o option\n"));
	goto errout;
	}
	*propval = '\0';
	propval++;

	if (add_prop_list(optarg, propval, &props, B_TRUE))
	goto errout;

	/*
	* If the user is creating a pool that doesn't support
	* feature flags, don't enable any features.
	*/
	if (zpool_name_to_prop(optarg) == ZPOOL_PROP_VERSION) {
	char *end;
	u_longlong_t ver;

	ver = strtoull(propval, &end, 10);
	if (*end == '\0' &&
	ver < SPA_VERSION_FEATURES) {
	enable_pool_features = B_FALSE;
	}
	}
	if (zpool_name_to_prop(optarg) == ZPOOL_PROP_ALTROOT)
	altroot = propval;
	if (zpool_name_to_prop(optarg) ==
	ZPOOL_PROP_COMPATIBILITY)
	compat = propval;
	break;
	case 'O':
	if ((propval = strchr(optarg, '=')) == NULL) {
	(void) fprintf(stderr, gettext("missing "
	"'=' for -O option\n"));
	goto errout;
	}
	*propval = '\0';
	propval++;

	/*
	* Mountpoints are checked and then added later.
	* Uniquely among properties, they can be specified
	* more than once, to avoid conflict with -m.
	*/
	if (0 == strcmp(optarg,
	zfs_prop_to_name(ZFS_PROP_MOUNTPOINT))) {
	mountpoint = propval;
	} else if (add_prop_list(optarg, propval, &fsprops,
	B_FALSE)) {
	goto errout;
	}
	break;
	case 't':
	/*
	* Sanity check temporary pool name.
	*/
	if (strchr(optarg, '/') != NULL) {
	(void) fprintf(stderr, gettext("cannot create "
	"'%s': invalid character '/' in temporary "
	"name\n"), optarg);
	(void) fprintf(stderr, gettext("use 'zfs "
	"create' to create a dataset\n"));
	goto errout;
	}

	if (add_prop_list(zpool_prop_to_name(
	ZPOOL_PROP_TNAME), optarg, &props, B_TRUE))
	goto errout;
	if (add_prop_list_default(zpool_prop_to_name(
	ZPOOL_PROP_CACHEFILE), "none", &props))
	goto errout;
	tname = optarg;
	break;
	case ':':
	(void) fprintf(stderr, gettext("missing argument for "
	"'%c' option\n"), optopt);
	goto badusage;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	goto badusage;
	}
	}

	argc -= optind;
	argv += optind;

	/* get pool name and check number of arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing pool name argument\n"));
	goto badusage;
	}
	if (argc < 2) {
	(void) fprintf(stderr, gettext("missing vdev specification\n"));
	goto badusage;
	}

	poolname = argv[0];

	/*
	* As a special case, check for use of '/' in the name, and direct the
	* user to use 'zfs create' instead.
	*/
	if (strchr(poolname, '/') != NULL) {
	(void) fprintf(stderr, gettext("cannot create '%s': invalid "
	"character '/' in pool name\n"), poolname);
	(void) fprintf(stderr, gettext("use 'zfs create' to "
	"create a dataset\n"));
	goto errout;
	}

	/* pass off to make_root_vdev for bulk processing */
	nvroot = make_root_vdev(NULL, props, force, !force, B_FALSE, dryrun,
	argc - 1, argv + 1);
	if (nvroot == NULL)
	goto errout;

	/* make_root_vdev() allows 0 toplevel children if there are spares */
	if (!zfs_allocatable_devs(nvroot)) {
	(void) fprintf(stderr, gettext("invalid vdev "
	"specification: at least one toplevel vdev must be "
	"specified\n"));
	goto errout;
	}

	if (altroot != NULL && altroot[0] != '/') {
	(void) fprintf(stderr, gettext("invalid alternate root '%s': "
	"must be an absolute path\n"), altroot);
	goto errout;
	}

	/*
	* Check the validity of the mountpoint and direct the user to use the
	* '-m' mountpoint option if it looks like its in use.
	*/
	if (mountpoint == NULL \|\|
	(strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) != 0 &&
	strcmp(mountpoint, ZFS_MOUNTPOINT_NONE) != 0)) {
	char buf[MAXPATHLEN];
	DIR *dirp;

	if (mountpoint && mountpoint[0] != '/') {
	(void) fprintf(stderr, gettext("invalid mountpoint "
	"'%s': must be an absolute path, 'legacy', or "
	"'none'\n"), mountpoint);
	goto errout;
	}

	if (mountpoint == NULL) {
	if (altroot != NULL)
	(void) snprintf(buf, sizeof (buf), "%s/%s",
	altroot, poolname);
	else
	(void) snprintf(buf, sizeof (buf), "/%s",
	poolname);
	} else {
	if (altroot != NULL)
	(void) snprintf(buf, sizeof (buf), "%s%s",
	altroot, mountpoint);
	else
	(void) snprintf(buf, sizeof (buf), "%s",
	mountpoint);
	}

	if ((dirp = opendir(buf)) == NULL && errno != ENOENT) {
	(void) fprintf(stderr, gettext("mountpoint '%s' : "
	"%s\n"), buf, strerror(errno));
	(void) fprintf(stderr, gettext("use '-m' "
	"option to provide a different default\n"));
	goto errout;
	} else if (dirp) {
	int count = 0;

	while (count < 3 && readdir(dirp) != NULL)
	count++;
	(void) closedir(dirp);

	if (count > 2) {
	(void) fprintf(stderr, gettext("mountpoint "
	"'%s' exists and is not empty\n"), buf);
	(void) fprintf(stderr, gettext("use '-m' "
	"option to provide a "
	"different default\n"));
	goto errout;
	}
	}
	}

	/*
	* Now that the mountpoint's validity has been checked, ensure that
	* the property is set appropriately prior to creating the pool.
	*/
	if (mountpoint != NULL) {
	ret = add_prop_list(zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
	mountpoint, &fsprops, B_FALSE);
	if (ret != 0)
	goto errout;
	}

	ret = 1;
	if (dryrun) {
	/*
	* For a dry run invocation, print out a basic message and run
	* through all the vdevs in the list and print out in an
	* appropriate hierarchy.
	*/
	(void) printf(gettext("would create '%s' with the "
	"following layout:\n\n"), poolname);

	print_vdev_tree(NULL, poolname, nvroot, 0, "", 0);
	print_vdev_tree(NULL, "dedup", nvroot, 0,
	VDEV_ALLOC_BIAS_DEDUP, 0);
	print_vdev_tree(NULL, "special", nvroot, 0,
	VDEV_ALLOC_BIAS_SPECIAL, 0);
	print_vdev_tree(NULL, "logs", nvroot, 0,
	VDEV_ALLOC_BIAS_LOG, 0);
	print_cache_list(nvroot, 0);
	print_spare_list(nvroot, 0);

	ret = 0;
	} else {
	/*
	* Load in feature set.
	* Note: if compatibility property not given, we'll have
	* NULL, which means 'all features'.
	*/
	boolean_t requested_features[SPA_FEATURES];
	if (zpool_do_load_compat(compat, requested_features) !=
	ZPOOL_COMPATIBILITY_OK)
	goto errout;

	/*
	* props contains list of features to enable.
	* For each feature:
	* - remove it if feature@name=disabled
	* - leave it there if feature@name=enabled
	* - add it if:
	* - enable_pool_features (ie: no '-d' or '-o version')
	* - it's supported by the kernel module
	* - it's in the requested feature set
	* - warn if it's enabled but not in compat
	*/
	for (spa_feature_t i = 0; i < SPA_FEATURES; i++) {
	char propname[MAXPATHLEN];
	const char *propval;
	zfeature_info_t *feat = &spa_feature_table[i];

	(void) snprintf(propname, sizeof (propname),
	"feature@%s", feat->fi_uname);

	if (!nvlist_lookup_string(props, propname, &propval)) {
	if (strcmp(propval,
	ZFS_FEATURE_DISABLED) == 0) {
	(void) nvlist_remove_all(props,
	propname);
	} else if (strcmp(propval,
	ZFS_FEATURE_ENABLED) == 0 &&
	!requested_features[i]) {
	(void) fprintf(stderr, gettext(
	"Warning: feature \"%s\" enabled "
	"but is not in specified "
	"'compatibility' feature set.\n"),
	feat->fi_uname);
	}
	} else if (
	enable_pool_features &&
	feat->fi_zfs_mod_supported &&
	requested_features[i]) {
	ret = add_prop_list(propname,
	ZFS_FEATURE_ENABLED, &props, B_TRUE);
	if (ret != 0)
	goto errout;
	}
	}

	ret = 1;
	if (zpool_create(g_zfs, poolname,
	nvroot, props, fsprops) == 0) {
	zfs_handle_t *pool = zfs_open(g_zfs,
	tname ? tname : poolname, ZFS_TYPE_FILESYSTEM);
	if (pool != NULL) {
	if (zfs_mount(pool, NULL, 0) == 0) {
	ret = zfs_share(pool, NULL);
	zfs_commit_shares(NULL);
	}
	zfs_close(pool);
	}
	} else if (libzfs_errno(g_zfs) == EZFS_INVALIDNAME) {
	(void) fprintf(stderr, gettext("pool name may have "
	"been omitted\n"));
	}
	}

	errout:
	nvlist_free(nvroot);
	nvlist_free(fsprops);
	nvlist_free(props);
	return (ret);
	badusage:
	nvlist_free(fsprops);
	nvlist_free(props);
	usage(B_FALSE);
	return (2);
	}

	/*
	* zpool destroy <pool>
	*
	* -f Forcefully unmount any datasets
	*
	* Destroy the given pool. Automatically unmounts any datasets in the pool.
	*/
	int
	zpool_do_destroy(int argc, char **argv)
	{
	boolean_t force = B_FALSE;
	int c;
	char *pool;
	zpool_handle_t *zhp;
	int ret;

	/* check options */
	while ((c = getopt(argc, argv, "f")) != -1) {
	switch (c) {
	case 'f':
	force = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* check arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing pool argument\n"));
	usage(B_FALSE);
	}
	if (argc > 1) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	pool = argv[0];

	if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
	/*
	* As a special case, check for use of '/' in the name, and
	* direct the user to use 'zfs destroy' instead.
	*/
	if (strchr(pool, '/') != NULL)
	(void) fprintf(stderr, gettext("use 'zfs destroy' to "
	"destroy a dataset\n"));
	return (1);
	}

	if (zpool_disable_datasets(zhp, force) != 0) {
	(void) fprintf(stderr, gettext("could not destroy '%s': "
	"could not unmount datasets\n"), zpool_get_name(zhp));
	zpool_close(zhp);
	return (1);
	}

	/* The history must be logged as part of the export */
	log_history = B_FALSE;

	ret = (zpool_destroy(zhp, history_str) != 0);

	zpool_close(zhp);

	return (ret);
	}

	typedef struct export_cbdata {
	boolean_t force;
	boolean_t hardforce;
	} export_cbdata_t;

	/*
	* Export one pool
	*/
	static int
	zpool_export_one(zpool_handle_t zhp, void data)
	{
	export_cbdata_t *cb = data;

	if (zpool_disable_datasets(zhp, cb->force) != 0)
	return (1);

	/* The history must be logged as part of the export */
	log_history = B_FALSE;

	if (cb->hardforce) {
	if (zpool_export_force(zhp, history_str) != 0)
	return (1);
	} else if (zpool_export(zhp, cb->force, history_str) != 0) {
	return (1);
	}

	return (0);
	}

	/*
	* zpool export [-f] <pool> ...
	*
	* -a Export all pools
	* -f Forcefully unmount datasets
	*
	* Export the given pools. By default, the command will attempt to cleanly
	* unmount any active datasets within the pool. If the '-f' flag is specified,
	* then the datasets will be forcefully unmounted.
	*/
	int
	zpool_do_export(int argc, char **argv)
	{
	export_cbdata_t cb;
	boolean_t do_all = B_FALSE;
	boolean_t force = B_FALSE;
	boolean_t hardforce = B_FALSE;
	int c, ret;

	/* check options */
	while ((c = getopt(argc, argv, "afF")) != -1) {
	switch (c) {
	case 'a':
	do_all = B_TRUE;
	break;
	case 'f':
	force = B_TRUE;
	break;
	case 'F':
	hardforce = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	cb.force = force;
	cb.hardforce = hardforce;
	argc -= optind;
	argv += optind;

	if (do_all) {
	if (argc != 0) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	return (for_each_pool(argc, argv, B_TRUE, NULL,
	ZFS_TYPE_POOL, B_FALSE, zpool_export_one, &cb));
	}

	/* check arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing pool argument\n"));
	usage(B_FALSE);
	}

	ret = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
	B_FALSE, zpool_export_one, &cb);

	return (ret);
	}

	/*
	* Given a vdev configuration, determine the maximum width needed for the device
	* name column.
	*/
	static int
	max_width(zpool_handle_t zhp, nvlist_t nv, int depth, int max,
	int name_flags)
	{
	static const char *const subtypes[] =
	{ZPOOL_CONFIG_SPARES, ZPOOL_CONFIG_L2CACHE, ZPOOL_CONFIG_CHILDREN};

	char *name = zpool_vdev_name(g_zfs, zhp, nv, name_flags);
	max = MAX(strlen(name) + depth, max);
	free(name);

	nvlist_t **child;
	uint_t children;
	for (size_t i = 0; i < ARRAY_SIZE(subtypes); ++i)
	if (nvlist_lookup_nvlist_array(nv, subtypes[i],
	&child, &children) == 0)
	for (uint_t c = 0; c < children; ++c)
	max = MAX(max_width(zhp, child[c], depth + 2,
	max, name_flags), max);

	return (max);
	}

	typedef struct spare_cbdata {
	uint64_t cb_guid;
	zpool_handle_t *cb_zhp;
	} spare_cbdata_t;

	static boolean_t
	find_vdev(nvlist_t *nv, uint64_t search)
	{
	uint64_t guid;
	nvlist_t **child;
	uint_t c, children;

	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) == 0 &&
	search == guid)
	return (B_TRUE);

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
	&child, &children) == 0) {
	for (c = 0; c < children; c++)
	if (find_vdev(child[c], search))
	return (B_TRUE);
	}

	return (B_FALSE);
	}

	static int
	find_spare(zpool_handle_t zhp, void data)
	{
	spare_cbdata_t *cbp = data;
	nvlist_t config, nvroot;

	config = zpool_get_config(zhp, NULL);
	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
	&nvroot) == 0);

	if (find_vdev(nvroot, cbp->cb_guid)) {
	cbp->cb_zhp = zhp;
	return (1);
	}

	zpool_close(zhp);
	return (0);
	}

	typedef struct status_cbdata {
	int cb_count;
	int cb_name_flags;
	int cb_namewidth;
	boolean_t cb_allpools;
	boolean_t cb_verbose;
	boolean_t cb_literal;
	boolean_t cb_explain;
	boolean_t cb_first;
	boolean_t cb_dedup_stats;
	boolean_t cb_print_unhealthy;
	boolean_t cb_print_status;
	boolean_t cb_print_slow_ios;
	boolean_t cb_print_vdev_init;
	boolean_t cb_print_vdev_trim;
	vdev_cmd_data_list_t *vcdl;
	boolean_t cb_print_power;
	} status_cbdata_t;

	/* Return 1 if string is NULL, empty, or whitespace; return 0 otherwise. */
	static boolean_t
	is_blank_str(const char *str)
	{
	for (; str != NULL && *str != '\0'; ++str)
	if (!isblank(*str))
	return (B_FALSE);
	return (B_TRUE);
	}

	/* Print command output lines for specific vdev in a specific pool */
	static void
	zpool_print_cmd(vdev_cmd_data_list_t vcdl, const char pool, const char *path)
	{
	vdev_cmd_data_t *data;
	int i, j;
	const char *val;

	for (i = 0; i < vcdl->count; i++) {
	if ((strcmp(vcdl->data[i].path, path) != 0) \|\|
	(strcmp(vcdl->data[i].pool, pool) != 0)) {
	/* Not the vdev we're looking for */
	continue;
	}

	data = &vcdl->data[i];
	/* Print out all the output values for this vdev */
	for (j = 0; j < vcdl->uniq_cols_cnt; j++) {
	val = NULL;
	/* Does this vdev have values for this column? */
	for (int k = 0; k < data->cols_cnt; k++) {
	if (strcmp(data->cols[k],
	vcdl->uniq_cols[j]) == 0) {
	/* yes it does, record the value */
	val = data->lines[k];
	break;
	}
	}
	/*
	* Mark empty values with dashes to make output
	* awk-able.
	*/
	if (val == NULL \|\| is_blank_str(val))
	val = "-";

	printf("%*s", vcdl->uniq_cols_width[j], val);
	if (j < vcdl->uniq_cols_cnt - 1)
	fputs(" ", stdout);
	}

	/* Print out any values that aren't in a column at the end */
	for (j = data->cols_cnt; j < data->lines_cnt; j++) {
	/* Did we have any columns? If so print a spacer. */
	if (vcdl->uniq_cols_cnt > 0)
	fputs(" ", stdout);

	val = data->lines[j];
	fputs(val ?: "", stdout);
	}
	break;
	}
	}

	/*
	* Print vdev initialization status for leaves
	*/
	static void
	print_status_initialize(vdev_stat_t *vs, boolean_t verbose)
	{
	if (verbose) {
	if ((vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE \|\|
	vs->vs_initialize_state == VDEV_INITIALIZE_SUSPENDED \|\|
	vs->vs_initialize_state == VDEV_INITIALIZE_COMPLETE) &&
	!vs->vs_scan_removing) {
	char zbuf[1024];
	char tbuf[256];
	- struct tm zaction_ts;

	time_t t = vs->vs_initialize_action_time;
	int initialize_pct = 100;
	if (vs->vs_initialize_state !=
	VDEV_INITIALIZE_COMPLETE) {
	initialize_pct = (vs->vs_initialize_bytes_done *
	100 / (vs->vs_initialize_bytes_est + 1));
	}

	- (void) localtime_r(&t, &zaction_ts);
	- (void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
	+ (void) ctime_r(&t, tbuf);
	+ tbuf[24] = 0;

	switch (vs->vs_initialize_state) {
	case VDEV_INITIALIZE_SUSPENDED:
	(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
	gettext("suspended, started at"), tbuf);
	break;
	case VDEV_INITIALIZE_ACTIVE:
	(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
	gettext("started at"), tbuf);
	break;
	case VDEV_INITIALIZE_COMPLETE:
	(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
	gettext("completed at"), tbuf);
	break;
	}

	(void) printf(gettext(" (%d%% initialized%s)"),
	initialize_pct, zbuf);
	} else {
	(void) printf(gettext(" (uninitialized)"));
	}
	} else if (vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE) {
	(void) printf(gettext(" (initializing)"));
	}
	}

	/*
	* Print vdev TRIM status for leaves
	*/
	static void
	print_status_trim(vdev_stat_t *vs, boolean_t verbose)
	{
	if (verbose) {
	if ((vs->vs_trim_state == VDEV_TRIM_ACTIVE \|\|
	vs->vs_trim_state == VDEV_TRIM_SUSPENDED \|\|
	vs->vs_trim_state == VDEV_TRIM_COMPLETE) &&
	!vs->vs_scan_removing) {
	char zbuf[1024];
	char tbuf[256];
	- struct tm zaction_ts;

	time_t t = vs->vs_trim_action_time;
	int trim_pct = 100;
	if (vs->vs_trim_state != VDEV_TRIM_COMPLETE) {
	trim_pct = (vs->vs_trim_bytes_done *
	100 / (vs->vs_trim_bytes_est + 1));
	}

	- (void) localtime_r(&t, &zaction_ts);
	- (void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
	+ (void) ctime_r(&t, tbuf);
	+ tbuf[24] = 0;

	switch (vs->vs_trim_state) {
	case VDEV_TRIM_SUSPENDED:
	(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
	gettext("suspended, started at"), tbuf);
	break;
	case VDEV_TRIM_ACTIVE:
	(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
	gettext("started at"), tbuf);
	break;
	case VDEV_TRIM_COMPLETE:
	(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
	gettext("completed at"), tbuf);
	break;
	}

	(void) printf(gettext(" (%d%% trimmed%s)"),
	trim_pct, zbuf);
	} else if (vs->vs_trim_notsup) {
	(void) printf(gettext(" (trim unsupported)"));
	} else {
	(void) printf(gettext(" (untrimmed)"));
	}
	} else if (vs->vs_trim_state == VDEV_TRIM_ACTIVE) {
	(void) printf(gettext(" (trimming)"));
	}
	}

	/*
	* Return the color associated with a health string. This includes returning
	* NULL for no color change.
	*/
	static const char *
	health_str_to_color(const char *health)
	{
	if (strcmp(health, gettext("FAULTED")) == 0 \|\|
	strcmp(health, gettext("SUSPENDED")) == 0 \|\|
	strcmp(health, gettext("UNAVAIL")) == 0) {
	return (ANSI_RED);
	}

	if (strcmp(health, gettext("OFFLINE")) == 0 \|\|
	strcmp(health, gettext("DEGRADED")) == 0 \|\|
	strcmp(health, gettext("REMOVED")) == 0) {
	return (ANSI_YELLOW);
	}

	return (NULL);
	}

	/*
	* Called for each leaf vdev. Returns 0 if the vdev is healthy.
	* A vdev is unhealthy if any of the following are true:
	* 1) there are read, write, or checksum errors,
	* 2) its state is not ONLINE, or
	* 3) slow IO reporting was requested (-s) and there are slow IOs.
	*/
	static int
	vdev_health_check_cb(void hdl_data, nvlist_t nv, void *data)
	{
	status_cbdata_t *cb = data;
	vdev_stat_t *vs;
	uint_t vsc;
	(void) hdl_data;

	if (nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
	(uint64_t **)&vs, &vsc) != 0)
	return (1);

	if (vs->vs_checksum_errors \|\| vs->vs_read_errors \|\|
	vs->vs_write_errors \|\| vs->vs_state != VDEV_STATE_HEALTHY)
	return (1);

	if (cb->cb_print_slow_ios && vs->vs_slow_ios)
	return (1);

	return (0);
	}

	/*
	* Print out configuration state as requested by status_callback.
	*/
	static void
	print_status_config(zpool_handle_t zhp, status_cbdata_t cb, const char *name,
	nvlist_t nv, int depth, boolean_t isspare, vdev_rebuild_stat_t vrs)
	{
	nvlist_t *child, root;
	uint_t c, i, vsc, children;
	pool_scan_stat_t *ps = NULL;
	vdev_stat_t *vs;
	char rbuf[6], wbuf[6], cbuf[6];
	char *vname;
	uint64_t notpresent;
	spare_cbdata_t spare_cb;
	const char *state;
	const char *type;
	const char *path = NULL;
	const char rcolor = NULL, wcolor = NULL, *ccolor = NULL,
	*scolor = NULL;

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
	&child, &children) != 0)
	children = 0;

	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
	(uint64_t **)&vs, &vsc) == 0);

	verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);

	if (strcmp(type, VDEV_TYPE_INDIRECT) == 0)
	return;

	state = zpool_state_to_name(vs->vs_state, vs->vs_aux);

	if (isspare) {
	/*
	* For hot spares, we use the terms 'INUSE' and 'AVAILABLE' for
	* online drives.
	*/
	if (vs->vs_aux == VDEV_AUX_SPARED)
	state = gettext("INUSE");
	else if (vs->vs_state == VDEV_STATE_HEALTHY)
	state = gettext("AVAIL");
	}

	/*
	* If '-e' is specified then top-level vdevs and their children
	* can be pruned if all of their leaves are healthy.
	*/
	if (cb->cb_print_unhealthy && depth > 0 &&
	for_each_vdev_in_nvlist(nv, vdev_health_check_cb, cb) == 0) {
	return;
	}

	printf_color(health_str_to_color(state),
	"\t%s%-s %-8s", depth, "", cb->cb_namewidth - depth,
	name, state);

	if (!isspare) {
	if (vs->vs_read_errors)
	rcolor = ANSI_RED;

	if (vs->vs_write_errors)
	wcolor = ANSI_RED;

	if (vs->vs_checksum_errors)
	ccolor = ANSI_RED;

	if (vs->vs_slow_ios)
	scolor = ANSI_BLUE;

	if (cb->cb_literal) {
	fputc(' ', stdout);
	printf_color(rcolor, "%5llu",
	(u_longlong_t)vs->vs_read_errors);
	fputc(' ', stdout);
	printf_color(wcolor, "%5llu",
	(u_longlong_t)vs->vs_write_errors);
	fputc(' ', stdout);
	printf_color(ccolor, "%5llu",
	(u_longlong_t)vs->vs_checksum_errors);
	} else {
	zfs_nicenum(vs->vs_read_errors, rbuf, sizeof (rbuf));
	zfs_nicenum(vs->vs_write_errors, wbuf, sizeof (wbuf));
	zfs_nicenum(vs->vs_checksum_errors, cbuf,
	sizeof (cbuf));
	fputc(' ', stdout);
	printf_color(rcolor, "%5s", rbuf);
	fputc(' ', stdout);
	printf_color(wcolor, "%5s", wbuf);
	fputc(' ', stdout);
	printf_color(ccolor, "%5s", cbuf);
	}
	if (cb->cb_print_slow_ios) {
	if (children == 0) {
	/* Only leafs vdevs have slow IOs */
	zfs_nicenum(vs->vs_slow_ios, rbuf,
	sizeof (rbuf));
	} else {
	snprintf(rbuf, sizeof (rbuf), "-");
	}

	if (cb->cb_literal)
	printf_color(scolor, " %5llu",
	(u_longlong_t)vs->vs_slow_ios);
	else
	printf_color(scolor, " %5s", rbuf);
	}
	if (cb->cb_print_power) {
	if (children == 0) {
	/* Only leaf vdevs have physical slots */
	switch (zpool_power_current_state(zhp, (char *)
	fnvlist_lookup_string(nv,
	ZPOOL_CONFIG_PATH))) {
	case 0:
	printf_color(ANSI_RED, " %5s",
	gettext("off"));
	break;
	case 1:
	printf(" %5s", gettext("on"));
	break;
	default:
	printf(" %5s", "-");
	}
	} else {
	printf(" %5s", "-");
	}
	}
	}

	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
	&notpresent) == 0) {
	verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0);
	(void) printf(" %s %s", gettext("was"), path);
	} else if (vs->vs_aux != 0) {
	(void) printf(" ");
	color_start(ANSI_RED);
	switch (vs->vs_aux) {
	case VDEV_AUX_OPEN_FAILED:
	(void) printf(gettext("cannot open"));
	break;

	case VDEV_AUX_BAD_GUID_SUM:
	(void) printf(gettext("missing device"));
	break;

	case VDEV_AUX_NO_REPLICAS:
	(void) printf(gettext("insufficient replicas"));
	break;

	case VDEV_AUX_VERSION_NEWER:
	(void) printf(gettext("newer version"));
	break;

	case VDEV_AUX_UNSUP_FEAT:
	(void) printf(gettext("unsupported feature(s)"));
	break;

	case VDEV_AUX_ASHIFT_TOO_BIG:
	(void) printf(gettext("unsupported minimum blocksize"));
	break;

	case VDEV_AUX_SPARED:
	verify(nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID,
	&spare_cb.cb_guid) == 0);
	if (zpool_iter(g_zfs, find_spare, &spare_cb) == 1) {
	if (strcmp(zpool_get_name(spare_cb.cb_zhp),
	zpool_get_name(zhp)) == 0)
	(void) printf(gettext("currently in "
	"use"));
	else
	(void) printf(gettext("in use by "
	"pool '%s'"),
	zpool_get_name(spare_cb.cb_zhp));
	zpool_close(spare_cb.cb_zhp);
	} else {
	(void) printf(gettext("currently in use"));
	}
	break;

	case VDEV_AUX_ERR_EXCEEDED:
	- (void) printf(gettext("too many errors"));
	+ if (vs->vs_read_errors + vs->vs_write_errors +
	+ vs->vs_checksum_errors == 0 && children == 0 &&
	+ vs->vs_slow_ios > 0) {
	+ (void) printf(gettext("too many slow I/Os"));
	+ } else {
	+ (void) printf(gettext("too many errors"));
	+ }
	break;

	case VDEV_AUX_IO_FAILURE:
	(void) printf(gettext("experienced I/O failures"));
	break;

	case VDEV_AUX_BAD_LOG:
	(void) printf(gettext("bad intent log"));
	break;

	case VDEV_AUX_EXTERNAL:
	(void) printf(gettext("external device fault"));
	break;

	case VDEV_AUX_SPLIT_POOL:
	(void) printf(gettext("split into new pool"));
	break;

	case VDEV_AUX_ACTIVE:
	(void) printf(gettext("currently in use"));
	break;

	case VDEV_AUX_CHILDREN_OFFLINE:
	(void) printf(gettext("all children offline"));
	break;

	case VDEV_AUX_BAD_LABEL:
	(void) printf(gettext("invalid label"));
	break;

	default:
	(void) printf(gettext("corrupted data"));
	break;
	}
	color_end();
	} else if (children == 0 && !isspare &&
	getenv("ZPOOL_STATUS_NON_NATIVE_ASHIFT_IGNORE") == NULL &&
	VDEV_STAT_VALID(vs_physical_ashift, vsc) &&
	vs->vs_configured_ashift < vs->vs_physical_ashift) {
	(void) printf(
	gettext(" block size: %dB configured, %dB native"),
	1 << vs->vs_configured_ashift, 1 << vs->vs_physical_ashift);
	}

	if (vs->vs_scan_removing != 0) {
	(void) printf(gettext(" (removing)"));
	} else if (VDEV_STAT_VALID(vs_noalloc, vsc) && vs->vs_noalloc != 0) {
	(void) printf(gettext(" (non-allocating)"));
	}

	/* The root vdev has the scrub/resilver stats */
	root = fnvlist_lookup_nvlist(zpool_get_config(zhp, NULL),
	ZPOOL_CONFIG_VDEV_TREE);
	(void) nvlist_lookup_uint64_array(root, ZPOOL_CONFIG_SCAN_STATS,
	(uint64_t **)&ps, &c);

	/*
	* If you force fault a drive that's resilvering, its scan stats can
	* get frozen in time, giving the false impression that it's
	* being resilvered. That's why we check the state to see if the vdev
	* is healthy before reporting "resilvering" or "repairing".
	*/
	if (ps != NULL && ps->pss_state == DSS_SCANNING && children == 0 &&
	vs->vs_state == VDEV_STATE_HEALTHY) {
	if (vs->vs_scan_processed != 0) {
	(void) printf(gettext(" (%s)"),
	(ps->pss_func == POOL_SCAN_RESILVER) ?
	"resilvering" : "repairing");
	} else if (vs->vs_resilver_deferred) {
	(void) printf(gettext(" (awaiting resilver)"));
	}
	}

	/* The top-level vdevs have the rebuild stats */
	if (vrs != NULL && vrs->vrs_state == VDEV_REBUILD_ACTIVE &&
	children == 0 && vs->vs_state == VDEV_STATE_HEALTHY) {
	if (vs->vs_rebuild_processed != 0) {
	(void) printf(gettext(" (resilvering)"));
	}
	}

	if (cb->vcdl != NULL) {
	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0) {
	printf(" ");
	zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
	}
	}

	/* Display vdev initialization and trim status for leaves. */
	if (children == 0) {
	print_status_initialize(vs, cb->cb_print_vdev_init);
	print_status_trim(vs, cb->cb_print_vdev_trim);
	}

	(void) printf("\n");

	for (c = 0; c < children; c++) {
	uint64_t islog = B_FALSE, ishole = B_FALSE;

	/* Don't print logs or holes here */
	(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
	&islog);
	(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
	&ishole);
	if (islog \|\| ishole)
	continue;
	/* Only print normal classes here */
	if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
	continue;

	/* Provide vdev_rebuild_stats to children if available */
	if (vrs == NULL) {
	(void) nvlist_lookup_uint64_array(nv,
	ZPOOL_CONFIG_REBUILD_STATS,
	(uint64_t **)&vrs, &i);
	}

	vname = zpool_vdev_name(g_zfs, zhp, child[c],
	cb->cb_name_flags \| VDEV_NAME_TYPE_ID);
	print_status_config(zhp, cb, vname, child[c], depth + 2,
	isspare, vrs);
	free(vname);
	}
	}

	/*
	* Print the configuration of an exported pool. Iterate over all vdevs in the
	* pool, printing out the name and status for each one.
	*/
	static void
	print_import_config(status_cbdata_t cb, const char name, nvlist_t *nv,
	int depth)
	{
	nvlist_t **child;
	uint_t c, children;
	vdev_stat_t *vs;
	const char *type;
	char *vname;

	verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
	if (strcmp(type, VDEV_TYPE_MISSING) == 0 \|\|
	strcmp(type, VDEV_TYPE_HOLE) == 0)
	return;

	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
	(uint64_t **)&vs, &c) == 0);

	(void) printf("\t%s%-s", depth, "", cb->cb_namewidth - depth, name);
	(void) printf(" %s", zpool_state_to_name(vs->vs_state, vs->vs_aux));

	if (vs->vs_aux != 0) {
	(void) printf(" ");

	switch (vs->vs_aux) {
	case VDEV_AUX_OPEN_FAILED:
	(void) printf(gettext("cannot open"));
	break;

	case VDEV_AUX_BAD_GUID_SUM:
	(void) printf(gettext("missing device"));
	break;

	case VDEV_AUX_NO_REPLICAS:
	(void) printf(gettext("insufficient replicas"));
	break;

	case VDEV_AUX_VERSION_NEWER:
	(void) printf(gettext("newer version"));
	break;

	case VDEV_AUX_UNSUP_FEAT:
	(void) printf(gettext("unsupported feature(s)"));
	break;

	case VDEV_AUX_ERR_EXCEEDED:
	(void) printf(gettext("too many errors"));
	break;

	case VDEV_AUX_ACTIVE:
	(void) printf(gettext("currently in use"));
	break;

	case VDEV_AUX_CHILDREN_OFFLINE:
	(void) printf(gettext("all children offline"));
	break;

	case VDEV_AUX_BAD_LABEL:
	(void) printf(gettext("invalid label"));
	break;

	default:
	(void) printf(gettext("corrupted data"));
	break;
	}
	}
	(void) printf("\n");

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
	&child, &children) != 0)
	return;

	for (c = 0; c < children; c++) {
	uint64_t is_log = B_FALSE;

	(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
	&is_log);
	if (is_log)
	continue;
	if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
	continue;

	vname = zpool_vdev_name(g_zfs, NULL, child[c],
	cb->cb_name_flags \| VDEV_NAME_TYPE_ID);
	print_import_config(cb, vname, child[c], depth + 2);
	free(vname);
	}

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
	&child, &children) == 0) {
	(void) printf(gettext("\tcache\n"));
	for (c = 0; c < children; c++) {
	vname = zpool_vdev_name(g_zfs, NULL, child[c],
	cb->cb_name_flags);
	(void) printf("\t %s\n", vname);
	free(vname);
	}
	}

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
	&child, &children) == 0) {
	(void) printf(gettext("\tspares\n"));
	for (c = 0; c < children; c++) {
	vname = zpool_vdev_name(g_zfs, NULL, child[c],
	cb->cb_name_flags);
	(void) printf("\t %s\n", vname);
	free(vname);
	}
	}
	}

	/*
	* Print specialized class vdevs.
	*
	* These are recorded as top level vdevs in the main pool child array
	* but with "is_log" set to 1 or an "alloc_bias" string. We use either
	* print_status_config() or print_import_config() to print the top level
	* class vdevs then any of their children (eg mirrored slogs) are printed
	* recursively - which works because only the top level vdev is marked.
	*/
	static void
	print_class_vdevs(zpool_handle_t zhp, status_cbdata_t cb, nvlist_t *nv,
	const char *class)
	{
	uint_t c, children;
	nvlist_t **child;
	boolean_t printed = B_FALSE;

	assert(zhp != NULL \|\| !cb->cb_verbose);

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, &child,
	&children) != 0)
	return;

	for (c = 0; c < children; c++) {
	uint64_t is_log = B_FALSE;
	const char *bias = NULL;
	const char *type = NULL;

	(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
	&is_log);

	if (is_log) {
	bias = (char *)VDEV_ALLOC_CLASS_LOGS;
	} else {
	(void) nvlist_lookup_string(child[c],
	ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
	(void) nvlist_lookup_string(child[c],
	ZPOOL_CONFIG_TYPE, &type);
	}

	if (bias == NULL \|\| strcmp(bias, class) != 0)
	continue;
	if (!is_log && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
	continue;

	if (!printed) {
	(void) printf("\t%s\t\n", gettext(class));
	printed = B_TRUE;
	}

	char *name = zpool_vdev_name(g_zfs, zhp, child[c],
	cb->cb_name_flags \| VDEV_NAME_TYPE_ID);
	if (cb->cb_print_status)
	print_status_config(zhp, cb, name, child[c], 2,
	B_FALSE, NULL);
	else
	print_import_config(cb, name, child[c], 2);
	free(name);
	}
	}

	/*
	* Display the status for the given pool.
	*/
	static int
	show_import(nvlist_t *config, boolean_t report_error)
	{
	uint64_t pool_state;
	vdev_stat_t *vs;
	const char *name;
	uint64_t guid;
	uint64_t hostid = 0;
	const char *msgid;
	const char *hostname = "unknown";
	nvlist_t nvroot, nvinfo;
	zpool_status_t reason;
	zpool_errata_t errata;
	const char *health;
	uint_t vsc;
	const char *comment;
	status_cbdata_t cb = { 0 };

	verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
	&name) == 0);
	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
	&guid) == 0);
	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
	&pool_state) == 0);
	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
	&nvroot) == 0);

	verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
	(uint64_t **)&vs, &vsc) == 0);
	health = zpool_state_to_name(vs->vs_state, vs->vs_aux);

	reason = zpool_import_status(config, &msgid, &errata);

	/*
	* If we're importing using a cachefile, then we won't report any
	* errors unless we are in the scan phase of the import.
	*/
	if (reason != ZPOOL_STATUS_OK && !report_error)
	return (reason);

	(void) printf(gettext(" pool: %s\n"), name);
	(void) printf(gettext(" id: %llu\n"), (u_longlong_t)guid);
	(void) printf(gettext(" state: %s"), health);
	if (pool_state == POOL_STATE_DESTROYED)
	(void) printf(gettext(" (DESTROYED)"));
	(void) printf("\n");

	switch (reason) {
	case ZPOOL_STATUS_MISSING_DEV_R:
	case ZPOOL_STATUS_MISSING_DEV_NR:
	case ZPOOL_STATUS_BAD_GUID_SUM:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices are "
	"missing from the system.\n"));
	break;

	case ZPOOL_STATUS_CORRUPT_LABEL_R:
	case ZPOOL_STATUS_CORRUPT_LABEL_NR:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices contains"
	" corrupted data.\n"));
	break;

	case ZPOOL_STATUS_CORRUPT_DATA:
	(void) printf(
	gettext(" status: The pool data is corrupted.\n"));
	break;

	case ZPOOL_STATUS_OFFLINE_DEV:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices "
	"are offlined.\n"));
	break;

	case ZPOOL_STATUS_CORRUPT_POOL:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("The pool metadata is "
	"corrupted.\n"));
	break;

	case ZPOOL_STATUS_VERSION_OLDER:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
	"a legacy on-disk version.\n"));
	break;

	case ZPOOL_STATUS_VERSION_NEWER:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
	"an incompatible version.\n"));
	break;

	case ZPOOL_STATUS_FEAT_DISABLED:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("Some supported "
	"features are not enabled on the pool.\n\t"
	"(Note that they may be intentionally disabled "
	"if the\n\t'compatibility' property is set.)\n"));
	break;

	case ZPOOL_STATUS_COMPATIBILITY_ERR:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("Error reading or parsing "
	"the file(s) indicated by the 'compatibility'\n"
	"property.\n"));
	break;

	case ZPOOL_STATUS_INCOMPATIBLE_FEAT:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more features "
	"are enabled on the pool despite not being\n"
	"requested by the 'compatibility' property.\n"));
	break;

	case ZPOOL_STATUS_UNSUP_FEAT_READ:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("The pool uses the following "
	"feature(s) not supported on this system:\n"));
	color_start(ANSI_YELLOW);
	zpool_print_unsup_feat(config);
	color_end();
	break;

	case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("The pool can only be "
	"accessed in read-only mode on this system. It\n\tcannot be"
	" accessed in read-write mode because it uses the "
	"following\n\tfeature(s) not supported on this system:\n"));
	color_start(ANSI_YELLOW);
	zpool_print_unsup_feat(config);
	color_end();
	break;

	case ZPOOL_STATUS_HOSTID_ACTIVE:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("The pool is currently "
	"imported by another system.\n"));
	break;

	case ZPOOL_STATUS_HOSTID_REQUIRED:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("The pool has the "
	"multihost property on. It cannot\n\tbe safely imported "
	"when the system hostid is not set.\n"));
	break;

	case ZPOOL_STATUS_HOSTID_MISMATCH:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("The pool was last accessed "
	"by another system.\n"));
	break;

	case ZPOOL_STATUS_FAULTED_DEV_R:
	case ZPOOL_STATUS_FAULTED_DEV_NR:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices are "
	"faulted.\n"));
	break;

	case ZPOOL_STATUS_BAD_LOG:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("An intent log record cannot "
	"be read.\n"));
	break;

	case ZPOOL_STATUS_RESILVERING:
	case ZPOOL_STATUS_REBUILDING:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices were "
	"being resilvered.\n"));
	break;

	case ZPOOL_STATUS_ERRATA:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("Errata #%d detected.\n"),
	errata);
	break;

	case ZPOOL_STATUS_NON_NATIVE_ASHIFT:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices are "
	"configured to use a non-native block size.\n"
	"\tExpect reduced performance.\n"));
	break;

	default:
	/*
	* No other status can be seen when importing pools.
	*/
	assert(reason == ZPOOL_STATUS_OK);
	}

	/*
	* Print out an action according to the overall state of the pool.
	*/
	if (vs->vs_state == VDEV_STATE_HEALTHY) {
	if (reason == ZPOOL_STATUS_VERSION_OLDER \|\|
	reason == ZPOOL_STATUS_FEAT_DISABLED) {
	(void) printf(gettext(" action: The pool can be "
	"imported using its name or numeric identifier, "
	"though\n\tsome features will not be available "
	"without an explicit 'zpool upgrade'.\n"));
	} else if (reason == ZPOOL_STATUS_COMPATIBILITY_ERR) {
	(void) printf(gettext(" action: The pool can be "
	"imported using its name or numeric\n\tidentifier, "
	"though the file(s) indicated by its "
	"'compatibility'\n\tproperty cannot be parsed at "
	"this time.\n"));
	} else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH) {
	(void) printf(gettext(" action: The pool can be "
	"imported using its name or numeric "
	"identifier and\n\tthe '-f' flag.\n"));
	} else if (reason == ZPOOL_STATUS_ERRATA) {
	switch (errata) {
	case ZPOOL_ERRATA_NONE:
	break;

	case ZPOOL_ERRATA_ZOL_2094_SCRUB:
	(void) printf(gettext(" action: The pool can "
	"be imported using its name or numeric "
	"identifier,\n\thowever there is a compat"
	"ibility issue which should be corrected"
	"\n\tby running 'zpool scrub'\n"));
	break;

	case ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY:
	(void) printf(gettext(" action: The pool can"
	"not be imported with this version of ZFS "
	"due to\n\tan active asynchronous destroy. "
	"Revert to an earlier version\n\tand "
	"allow the destroy to complete before "
	"updating.\n"));
	break;

	case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
	(void) printf(gettext(" action: Existing "
	"encrypted datasets contain an on-disk "
	"incompatibility, which\n\tneeds to be "
	"corrected. Backup these datasets to new "
	"encrypted datasets\n\tand destroy the "
	"old ones.\n"));
	break;

	case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
	(void) printf(gettext(" action: Existing "
	"encrypted snapshots and bookmarks contain "
	"an on-disk\n\tincompatibility. This may "
	"cause on-disk corruption if they are used"
	"\n\twith 'zfs recv'. To correct the "
	"issue, enable the bookmark_v2 feature.\n\t"
	"No additional action is needed if there "
	"are no encrypted snapshots or\n\t"
	"bookmarks. If preserving the encrypted "
	"snapshots and bookmarks is\n\trequired, "
	"use a non-raw send to backup and restore "
	"them. Alternately,\n\tthey may be removed"
	" to resolve the incompatibility.\n"));
	break;
	default:
	/*
	* All errata must contain an action message.
	*/
	assert(0);
	}
	} else {
	(void) printf(gettext(" action: The pool can be "
	"imported using its name or numeric "
	"identifier.\n"));
	}
	} else if (vs->vs_state == VDEV_STATE_DEGRADED) {
	(void) printf(gettext(" action: The pool can be imported "
	"despite missing or damaged devices. The\n\tfault "
	"tolerance of the pool may be compromised if imported.\n"));
	} else {
	switch (reason) {
	case ZPOOL_STATUS_VERSION_NEWER:
	(void) printf(gettext(" action: The pool cannot be "
	"imported. Access the pool on a system running "
	"newer\n\tsoftware, or recreate the pool from "
	"backup.\n"));
	break;
	case ZPOOL_STATUS_UNSUP_FEAT_READ:
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("The pool cannot be "
	"imported. Access the pool on a system that "
	"supports\n\tthe required feature(s), or recreate "
	"the pool from backup.\n"));
	break;
	case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("The pool cannot be "
	"imported in read-write mode. Import the pool "
	"with\n"
	"\t\"-o readonly=on\", access the pool on a system "
	"that supports the\n\trequired feature(s), or "
	"recreate the pool from backup.\n"));
	break;
	case ZPOOL_STATUS_MISSING_DEV_R:
	case ZPOOL_STATUS_MISSING_DEV_NR:
	case ZPOOL_STATUS_BAD_GUID_SUM:
	(void) printf(gettext(" action: The pool cannot be "
	"imported. Attach the missing\n\tdevices and try "
	"again.\n"));
	break;
	case ZPOOL_STATUS_HOSTID_ACTIVE:
	VERIFY0(nvlist_lookup_nvlist(config,
	ZPOOL_CONFIG_LOAD_INFO, &nvinfo));

	if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
	hostname = fnvlist_lookup_string(nvinfo,
	ZPOOL_CONFIG_MMP_HOSTNAME);

	if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
	hostid = fnvlist_lookup_uint64(nvinfo,
	ZPOOL_CONFIG_MMP_HOSTID);

	(void) printf(gettext(" action: The pool must be "
	"exported from %s (hostid=%"PRIx64")\n\tbefore it "
	"can be safely imported.\n"), hostname, hostid);
	break;
	case ZPOOL_STATUS_HOSTID_REQUIRED:
	(void) printf(gettext(" action: Set a unique system "
	"hostid with the zgenhostid(8) command.\n"));
	break;
	default:
	(void) printf(gettext(" action: The pool cannot be "
	"imported due to damaged devices or data.\n"));
	}
	}

	/* Print the comment attached to the pool. */
	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
	(void) printf(gettext("comment: %s\n"), comment);

	/*
	* If the state is "closed" or "can't open", and the aux state
	* is "corrupt data":
	*/
	if (((vs->vs_state == VDEV_STATE_CLOSED) \|\|
	(vs->vs_state == VDEV_STATE_CANT_OPEN)) &&
	(vs->vs_aux == VDEV_AUX_CORRUPT_DATA)) {
	if (pool_state == POOL_STATE_DESTROYED)
	(void) printf(gettext("\tThe pool was destroyed, "
	"but can be imported using the '-Df' flags.\n"));
	else if (pool_state != POOL_STATE_EXPORTED)
	(void) printf(gettext("\tThe pool may be active on "
	"another system, but can be imported using\n\t"
	"the '-f' flag.\n"));
	}

	if (msgid != NULL) {
	(void) printf(gettext(
	" see: https://openzfs.github.io/openzfs-docs/msg/%s\n"),
	msgid);
	}

	(void) printf(gettext(" config:\n\n"));

	cb.cb_namewidth = max_width(NULL, nvroot, 0, strlen(name),
	VDEV_NAME_TYPE_ID);
	if (cb.cb_namewidth < 10)
	cb.cb_namewidth = 10;

	print_import_config(&cb, name, nvroot, 0);

	print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_BIAS_DEDUP);
	print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_BIAS_SPECIAL);
	print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_CLASS_LOGS);

	if (reason == ZPOOL_STATUS_BAD_GUID_SUM) {
	(void) printf(gettext("\n\tAdditional devices are known to "
	"be part of this pool, though their\n\texact "
	"configuration cannot be determined.\n"));
	}
	return (0);
	}

	static boolean_t
	zfs_force_import_required(nvlist_t *config)
	{
	uint64_t state;
	uint64_t hostid = 0;
	nvlist_t *nvinfo;

	state = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE);
	nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);

	/*
	* The hostid on LOAD_INFO comes from the MOS label via
	* spa_tryimport(). If its not there then we're likely talking to an
	* older kernel, so use the top one, which will be from the label
	* discovered in zpool_find_import(), or if a cachefile is in use, the
	* local hostid.
	*/
	if (nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_HOSTID, &hostid) != 0)
	nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid);

	if (state != POOL_STATE_EXPORTED && hostid != get_system_hostid())
	return (B_TRUE);

	if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE)) {
	mmp_state_t mmp_state = fnvlist_lookup_uint64(nvinfo,
	ZPOOL_CONFIG_MMP_STATE);

	if (mmp_state != MMP_STATE_INACTIVE)
	return (B_TRUE);
	}

	return (B_FALSE);
	}

	/*
	* Perform the import for the given configuration. This passes the heavy
	* lifting off to zpool_import_props(), and then mounts the datasets contained
	* within the pool.
	*/
	static int
	do_import(nvlist_t config, const char newname, const char *mntopts,
	nvlist_t *props, int flags)
	{
	int ret = 0;
	int ms_status = 0;
	zpool_handle_t *zhp;
	const char *name;
	uint64_t version;

	name = fnvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME);
	version = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION);

	if (!SPA_VERSION_IS_SUPPORTED(version)) {
	(void) fprintf(stderr, gettext("cannot import '%s': pool "
	"is formatted using an unsupported ZFS version\n"), name);
	return (1);
	} else if (zfs_force_import_required(config) &&
	!(flags & ZFS_IMPORT_ANY_HOST)) {
	mmp_state_t mmp_state = MMP_STATE_INACTIVE;
	nvlist_t *nvinfo;

	nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
	if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE))
	mmp_state = fnvlist_lookup_uint64(nvinfo,
	ZPOOL_CONFIG_MMP_STATE);

	if (mmp_state == MMP_STATE_ACTIVE) {
	const char *hostname = "<unknown>";
	uint64_t hostid = 0;

	if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
	hostname = fnvlist_lookup_string(nvinfo,
	ZPOOL_CONFIG_MMP_HOSTNAME);

	if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
	hostid = fnvlist_lookup_uint64(nvinfo,
	ZPOOL_CONFIG_MMP_HOSTID);

	(void) fprintf(stderr, gettext("cannot import '%s': "
	"pool is imported on %s (hostid: "
	"0x%"PRIx64")\nExport the pool on the other "
	"system, then run 'zpool import'.\n"),
	name, hostname, hostid);
	} else if (mmp_state == MMP_STATE_NO_HOSTID) {
	(void) fprintf(stderr, gettext("Cannot import '%s': "
	"pool has the multihost property on and the\n"
	"system's hostid is not set. Set a unique hostid "
	"with the zgenhostid(8) command.\n"), name);
	} else {
	const char *hostname = "<unknown>";
	time_t timestamp = 0;
	uint64_t hostid = 0;

	if (nvlist_exists(nvinfo, ZPOOL_CONFIG_HOSTNAME))
	hostname = fnvlist_lookup_string(nvinfo,
	ZPOOL_CONFIG_HOSTNAME);
	else if (nvlist_exists(config, ZPOOL_CONFIG_HOSTNAME))
	hostname = fnvlist_lookup_string(config,
	ZPOOL_CONFIG_HOSTNAME);

	if (nvlist_exists(config, ZPOOL_CONFIG_TIMESTAMP))
	timestamp = fnvlist_lookup_uint64(config,
	ZPOOL_CONFIG_TIMESTAMP);

	if (nvlist_exists(nvinfo, ZPOOL_CONFIG_HOSTID))
	hostid = fnvlist_lookup_uint64(nvinfo,
	ZPOOL_CONFIG_HOSTID);
	else if (nvlist_exists(config, ZPOOL_CONFIG_HOSTID))
	hostid = fnvlist_lookup_uint64(config,
	ZPOOL_CONFIG_HOSTID);

	(void) fprintf(stderr, gettext("cannot import '%s': "
	"pool was previously in use from another system.\n"
	"Last accessed by %s (hostid=%"PRIx64") at %s"
	"The pool can be imported, use 'zpool import -f' "
	"to import the pool.\n"), name, hostname,
	hostid, ctime(&timestamp));
	}

	return (1);
	}

	if (zpool_import_props(g_zfs, config, newname, props, flags) != 0)
	return (1);

	if (newname != NULL)
	name = newname;

	if ((zhp = zpool_open_canfail(g_zfs, name)) == NULL)
	return (1);

	/*
	* Loading keys is best effort. We don't want to return immediately
	* if it fails but we do want to give the error to the caller.
	*/
	if (flags & ZFS_IMPORT_LOAD_KEYS &&
	zfs_crypto_attempt_load_keys(g_zfs, name) != 0)
	ret = 1;

	if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
	!(flags & ZFS_IMPORT_ONLY)) {
	ms_status = zpool_enable_datasets(zhp, mntopts, 0);
	if (ms_status == EZFS_SHAREFAILED) {
	(void) fprintf(stderr, gettext("Import was "
	- "successful, but unable to share some datasets"));
	+ "successful, but unable to share some datasets\n"));
	} else if (ms_status == EZFS_MOUNTFAILED) {
	(void) fprintf(stderr, gettext("Import was "
	- "successful, but unable to mount some datasets"));
	+ "successful, but unable to mount some datasets\n"));
	}
	}

	zpool_close(zhp);
	return (ret);
	}

	static int
	import_pools(nvlist_t pools, nvlist_t props, char *mntopts, int flags,
	char orig_name, char new_name,
	boolean_t do_destroyed, boolean_t pool_specified, boolean_t do_all,
	importargs_t *import)
	{
	nvlist_t *config = NULL;
	nvlist_t *found_config = NULL;
	uint64_t pool_state;

	/*
	* At this point we have a list of import candidate configs. Even if
	* we were searching by pool name or guid, we still need to
	* post-process the list to deal with pool state and possible
	* duplicate names.
	*/
	int err = 0;
	nvpair_t *elem = NULL;
	boolean_t first = B_TRUE;
	while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) {

	verify(nvpair_value_nvlist(elem, &config) == 0);

	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
	&pool_state) == 0);
	if (!do_destroyed && pool_state == POOL_STATE_DESTROYED)
	continue;
	if (do_destroyed && pool_state != POOL_STATE_DESTROYED)
	continue;

	verify(nvlist_add_nvlist(config, ZPOOL_LOAD_POLICY,
	import->policy) == 0);

	if (!pool_specified) {
	if (first)
	first = B_FALSE;
	else if (!do_all)
	(void) fputc('\n', stdout);

	if (do_all) {
	err \|= do_import(config, NULL, mntopts,
	props, flags);
	} else {
	/*
	* If we're importing from cachefile, then
	* we don't want to report errors until we
	* are in the scan phase of the import. If
	* we get an error, then we return that error
	* to invoke the scan phase.
	*/
	if (import->cachefile && !import->scan)
	err = show_import(config, B_FALSE);
	else
	(void) show_import(config, B_TRUE);
	}
	} else if (import->poolname != NULL) {
	const char *name;

	/*
	* We are searching for a pool based on name.
	*/
	verify(nvlist_lookup_string(config,
	ZPOOL_CONFIG_POOL_NAME, &name) == 0);

	if (strcmp(name, import->poolname) == 0) {
	if (found_config != NULL) {
	(void) fprintf(stderr, gettext(
	"cannot import '%s': more than "
	"one matching pool\n"),
	import->poolname);
	(void) fprintf(stderr, gettext(
	"import by numeric ID instead\n"));
	err = B_TRUE;
	}
	found_config = config;
	}
	} else {
	uint64_t guid;

	/*
	* Search for a pool by guid.
	*/
	verify(nvlist_lookup_uint64(config,
	ZPOOL_CONFIG_POOL_GUID, &guid) == 0);

	if (guid == import->guid)
	found_config = config;
	}
	}

	/*
	* If we were searching for a specific pool, verify that we found a
	* pool, and then do the import.
	*/
	if (pool_specified && err == 0) {
	if (found_config == NULL) {
	(void) fprintf(stderr, gettext("cannot import '%s': "
	"no such pool available\n"), orig_name);
	err = B_TRUE;
	} else {
	err \|= do_import(found_config, new_name,
	mntopts, props, flags);
	}
	}

	/*
	* If we were just looking for pools, report an error if none were
	* found.
	*/
	if (!pool_specified && first)
	(void) fprintf(stderr,
	gettext("no pools available to import\n"));
	return (err);
	}

	typedef struct target_exists_args {
	const char *poolname;
	uint64_t poolguid;
	} target_exists_args_t;

	static int
	name_or_guid_exists(zpool_handle_t zhp, void data)
	{
	target_exists_args_t *args = data;
	nvlist_t *config = zpool_get_config(zhp, NULL);
	int found = 0;

	if (config == NULL)
	return (0);

	if (args->poolname != NULL) {
	const char *pool_name;

	verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
	&pool_name) == 0);
	if (strcmp(pool_name, args->poolname) == 0)
	found = 1;
	} else {
	uint64_t pool_guid;

	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
	&pool_guid) == 0);
	if (pool_guid == args->poolguid)
	found = 1;
	}
	zpool_close(zhp);

	return (found);
	}
	/*
	* zpool checkpoint <pool>
	* checkpoint --discard <pool>
	*
	* -d Discard the checkpoint from a checkpointed
	* --discard pool.
	*
	* -w Wait for discarding a checkpoint to complete.
	* --wait
	*
	* Checkpoints the specified pool, by taking a "snapshot" of its
	* current state. A pool can only have one checkpoint at a time.
	*/
	int
	zpool_do_checkpoint(int argc, char **argv)
	{
	boolean_t discard, wait;
	char *pool;
	zpool_handle_t *zhp;
	int c, err;

	struct option long_options[] = {
	{"discard", no_argument, NULL, 'd'},
	{"wait", no_argument, NULL, 'w'},
	{0, 0, 0, 0}
	};

	discard = B_FALSE;
	wait = B_FALSE;
	while ((c = getopt_long(argc, argv, ":dw", long_options, NULL)) != -1) {
	switch (c) {
	case 'd':
	discard = B_TRUE;
	break;
	case 'w':
	wait = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	if (wait && !discard) {
	(void) fprintf(stderr, gettext("--wait only valid when "
	"--discard also specified\n"));
	usage(B_FALSE);
	}

	argc -= optind;
	argv += optind;

	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing pool argument\n"));
	usage(B_FALSE);
	}

	if (argc > 1) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	pool = argv[0];

	if ((zhp = zpool_open(g_zfs, pool)) == NULL) {
	/* As a special case, check for use of '/' in the name */
	if (strchr(pool, '/') != NULL)
	(void) fprintf(stderr, gettext("'zpool checkpoint' "
	"doesn't work on datasets. To save the state "
	"of a dataset from a specific point in time "
	"please use 'zfs snapshot'\n"));
	return (1);
	}

	if (discard) {
	err = (zpool_discard_checkpoint(zhp) != 0);
	if (err == 0 && wait)
	err = zpool_wait(zhp, ZPOOL_WAIT_CKPT_DISCARD);
	} else {
	err = (zpool_checkpoint(zhp) != 0);
	}

	zpool_close(zhp);

	return (err);
	}

	#define CHECKPOINT_OPT 1024

	/*
	* zpool import [-d dir] [-D]
	* import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
	* [-d dir \| -c cachefile \| -s] [-f] -a
	* import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
	* [-d dir \| -c cachefile \| -s] [-f] [-n] [-F] <pool \| id>
	* [newpool]
	*
	* -c Read pool information from a cachefile instead of searching
	* devices. If importing from a cachefile config fails, then
	* fallback to searching for devices only in the directories that
	* exist in the cachefile.
	*
	* -d Scan in a specific directory, other than /dev/. More than
	* one directory can be specified using multiple '-d' options.
	*
	* -D Scan for previously destroyed pools or import all or only
	* specified destroyed pools.
	*
	* -R Temporarily import the pool, with all mountpoints relative to
	* the given root. The pool will remain exported when the machine
	* is rebooted.
	*
	* -V Import even in the presence of faulted vdevs. This is an
	* intentionally undocumented option for testing purposes, and
	* treats the pool configuration as complete, leaving any bad
	* vdevs in the FAULTED state. In other words, it does verbatim
	* import.
	*
	* -f Force import, even if it appears that the pool is active.
	*
	* -F Attempt rewind if necessary.
	*
	* -n See if rewind would work, but don't actually rewind.
	*
	* -N Import the pool but don't mount datasets.
	*
	* -T Specify a starting txg to use for import. This option is
	* intentionally undocumented option for testing purposes.
	*
	* -a Import all pools found.
	*
	* -l Load encryption keys while importing.
	*
	* -o Set property=value and/or temporary mount options (without '=').
	*
	* -s Scan using the default search path, the libblkid cache will
	* not be consulted.
	*
	* --rewind-to-checkpoint
	* Import the pool and revert back to the checkpoint.
	*
	* The import command scans for pools to import, and import pools based on pool
	* name and GUID. The pool can also be renamed as part of the import process.
	*/
	int
	zpool_do_import(int argc, char **argv)
	{
	char **searchdirs = NULL;
	char env, envdup = NULL;
	int nsearch = 0;
	int c;
	int err = 0;
	nvlist_t *pools = NULL;
	boolean_t do_all = B_FALSE;
	boolean_t do_destroyed = B_FALSE;
	char *mntopts = NULL;
	uint64_t searchguid = 0;
	char *searchname = NULL;
	char *propval;
	nvlist_t *policy = NULL;
	nvlist_t *props = NULL;
	int flags = ZFS_IMPORT_NORMAL;
	uint32_t rewind_policy = ZPOOL_NO_REWIND;
	boolean_t dryrun = B_FALSE;
	boolean_t do_rewind = B_FALSE;
	boolean_t xtreme_rewind = B_FALSE;
	boolean_t do_scan = B_FALSE;
	boolean_t pool_exists = B_FALSE;
	boolean_t pool_specified = B_FALSE;
	uint64_t txg = -1ULL;
	char *cachefile = NULL;
	importargs_t idata = { 0 };
	char *endptr;

	struct option long_options[] = {
	{"rewind-to-checkpoint", no_argument, NULL, CHECKPOINT_OPT},
	{0, 0, 0, 0}
	};

	/* check options */
	while ((c = getopt_long(argc, argv, ":aCc:d:DEfFlmnNo:R:stT:VX",
	long_options, NULL)) != -1) {
	switch (c) {
	case 'a':
	do_all = B_TRUE;
	break;
	case 'c':
	cachefile = optarg;
	break;
	case 'd':
	searchdirs = safe_realloc(searchdirs,
	(nsearch + 1) * sizeof (char *));
	searchdirs[nsearch++] = optarg;
	break;
	case 'D':
	do_destroyed = B_TRUE;
	break;
	case 'f':
	flags \|= ZFS_IMPORT_ANY_HOST;
	break;
	case 'F':
	do_rewind = B_TRUE;
	break;
	case 'l':
	flags \|= ZFS_IMPORT_LOAD_KEYS;
	break;
	case 'm':
	flags \|= ZFS_IMPORT_MISSING_LOG;
	break;
	case 'n':
	dryrun = B_TRUE;
	break;
	case 'N':
	flags \|= ZFS_IMPORT_ONLY;
	break;
	case 'o':
	if ((propval = strchr(optarg, '=')) != NULL) {
	*propval = '\0';
	propval++;
	if (add_prop_list(optarg, propval,
	&props, B_TRUE))
	goto error;
	} else {
	mntopts = optarg;
	}
	break;
	case 'R':
	if (add_prop_list(zpool_prop_to_name(
	ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
	goto error;
	if (add_prop_list_default(zpool_prop_to_name(
	ZPOOL_PROP_CACHEFILE), "none", &props))
	goto error;
	break;
	case 's':
	do_scan = B_TRUE;
	break;
	case 't':
	flags \|= ZFS_IMPORT_TEMP_NAME;
	if (add_prop_list_default(zpool_prop_to_name(
	ZPOOL_PROP_CACHEFILE), "none", &props))
	goto error;
	break;

	case 'T':
	errno = 0;
	txg = strtoull(optarg, &endptr, 0);
	if (errno != 0 \|\| *endptr != '\0') {
	(void) fprintf(stderr,
	gettext("invalid txg value\n"));
	usage(B_FALSE);
	}
	rewind_policy = ZPOOL_DO_REWIND \| ZPOOL_EXTREME_REWIND;
	break;
	case 'V':
	flags \|= ZFS_IMPORT_VERBATIM;
	break;
	case 'X':
	xtreme_rewind = B_TRUE;
	break;
	case CHECKPOINT_OPT:
	flags \|= ZFS_IMPORT_CHECKPOINT;
	break;
	case ':':
	(void) fprintf(stderr, gettext("missing argument for "
	"'%c' option\n"), optopt);
	usage(B_FALSE);
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	if (cachefile && nsearch != 0) {
	(void) fprintf(stderr, gettext("-c is incompatible with -d\n"));
	usage(B_FALSE);
	}

	if (cachefile && do_scan) {
	(void) fprintf(stderr, gettext("-c is incompatible with -s\n"));
	usage(B_FALSE);
	}

	if ((flags & ZFS_IMPORT_LOAD_KEYS) && (flags & ZFS_IMPORT_ONLY)) {
	(void) fprintf(stderr, gettext("-l is incompatible with -N\n"));
	usage(B_FALSE);
	}

	if ((flags & ZFS_IMPORT_LOAD_KEYS) && !do_all && argc == 0) {
	(void) fprintf(stderr, gettext("-l is only meaningful during "
	"an import\n"));
	usage(B_FALSE);
	}

	if ((dryrun \|\| xtreme_rewind) && !do_rewind) {
	(void) fprintf(stderr,
	gettext("-n or -X only meaningful with -F\n"));
	usage(B_FALSE);
	}
	if (dryrun)
	rewind_policy = ZPOOL_TRY_REWIND;
	else if (do_rewind)
	rewind_policy = ZPOOL_DO_REWIND;
	if (xtreme_rewind)
	rewind_policy \|= ZPOOL_EXTREME_REWIND;

	/* In the future, we can capture further policy and include it here */
	if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 \|\|
	nvlist_add_uint64(policy, ZPOOL_LOAD_REQUEST_TXG, txg) != 0 \|\|
	nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
	rewind_policy) != 0)
	goto error;

	/* check argument count */
	if (do_all) {
	if (argc != 0) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}
	} else {
	if (argc > 2) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}
	}

	/*
	* Check for the effective uid. We do this explicitly here because
	* otherwise any attempt to discover pools will silently fail.
	*/
	if (argc == 0 && geteuid() != 0) {
	(void) fprintf(stderr, gettext("cannot "
	"discover pools: permission denied\n"));

	free(searchdirs);
	nvlist_free(props);
	nvlist_free(policy);
	return (1);
	}

	/*
	* Depending on the arguments given, we do one of the following:
	*
	* <none> Iterate through all pools and display information about
	* each one.
	*
	* -a Iterate through all pools and try to import each one.
	*
	* <id> Find the pool that corresponds to the given GUID/pool
	* name and import that one.
	*
	* -D Above options applies only to destroyed pools.
	*/
	if (argc != 0) {
	char *endptr;

	errno = 0;
	searchguid = strtoull(argv[0], &endptr, 10);
	if (errno != 0 \|\| *endptr != '\0') {
	searchname = argv[0];
	searchguid = 0;
	}
	pool_specified = B_TRUE;

	/*
	* User specified a name or guid. Ensure it's unique.
	*/
	target_exists_args_t search = {searchname, searchguid};
	pool_exists = zpool_iter(g_zfs, name_or_guid_exists, &search);
	}

	/*
	* Check the environment for the preferred search path.
	*/
	if ((searchdirs == NULL) && (env = getenv("ZPOOL_IMPORT_PATH"))) {
	char dir, tmp = NULL;

	envdup = strdup(env);

	for (dir = strtok_r(envdup, ":", &tmp);
	dir != NULL;
	dir = strtok_r(NULL, ":", &tmp)) {
	searchdirs = safe_realloc(searchdirs,
	(nsearch + 1) * sizeof (char *));
	searchdirs[nsearch++] = dir;
	}
	}

	idata.path = searchdirs;
	idata.paths = nsearch;
	idata.poolname = searchname;
	idata.guid = searchguid;
	idata.cachefile = cachefile;
	idata.scan = do_scan;
	idata.policy = policy;

	libpc_handle_t lpch = {
	.lpc_lib_handle = g_zfs,
	.lpc_ops = &libzfs_config_ops,
	.lpc_printerr = B_TRUE
	};
	pools = zpool_search_import(&lpch, &idata);

	if (pools != NULL && pool_exists &&
	(argc == 1 \|\| strcmp(argv[0], argv[1]) == 0)) {
	(void) fprintf(stderr, gettext("cannot import '%s': "
	"a pool with that name already exists\n"),
	argv[0]);
	(void) fprintf(stderr, gettext("use the form '%s "
	"<pool \| id> <newpool>' to give it a new name\n"),
	"zpool import");
	err = 1;
	} else if (pools == NULL && pool_exists) {
	(void) fprintf(stderr, gettext("cannot import '%s': "
	"a pool with that name is already created/imported,\n"),
	argv[0]);
	(void) fprintf(stderr, gettext("and no additional pools "
	"with that name were found\n"));
	err = 1;
	} else if (pools == NULL) {
	if (argc != 0) {
	(void) fprintf(stderr, gettext("cannot import '%s': "
	"no such pool available\n"), argv[0]);
	}
	err = 1;
	}

	if (err == 1) {
	free(searchdirs);
	free(envdup);
	nvlist_free(policy);
	nvlist_free(pools);
	nvlist_free(props);
	return (1);
	}

	err = import_pools(pools, props, mntopts, flags,
	argc >= 1 ? argv[0] : NULL,
	argc >= 2 ? argv[1] : NULL,
	do_destroyed, pool_specified, do_all, &idata);

	/*
	* If we're using the cachefile and we failed to import, then
	* fallback to scanning the directory for pools that match
	* those in the cachefile.
	*/
	if (err != 0 && cachefile != NULL) {
	(void) printf(gettext("cachefile import failed, retrying\n"));

	/*
	* We use the scan flag to gather the directories that exist
	* in the cachefile. If we need to fallback to searching for
	* the pool config, we will only search devices in these
	* directories.
	*/
	idata.scan = B_TRUE;
	nvlist_free(pools);
	pools = zpool_search_import(&lpch, &idata);

	err = import_pools(pools, props, mntopts, flags,
	argc >= 1 ? argv[0] : NULL,
	argc >= 2 ? argv[1] : NULL,
	do_destroyed, pool_specified, do_all, &idata);
	}

	error:
	nvlist_free(props);
	nvlist_free(pools);
	nvlist_free(policy);
	free(searchdirs);
	free(envdup);

	return (err ? 1 : 0);
	}

	/*
	* zpool sync [-f] [pool] ...
	*
	* -f (undocumented) force uberblock (and config including zpool cache file)
	* update.
	*
	* Sync the specified pool(s).
	* Without arguments "zpool sync" will sync all pools.
	* This command initiates TXG sync(s) and will return after the TXG(s) commit.
	*
	*/
	static int
	zpool_do_sync(int argc, char **argv)
	{
	int ret;
	boolean_t force = B_FALSE;

	/* check options */
	while ((ret = getopt(argc, argv, "f")) != -1) {
	switch (ret) {
	case 'f':
	force = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* if argc == 0 we will execute zpool_sync_one on all pools */
	ret = for_each_pool(argc, argv, B_FALSE, NULL, ZFS_TYPE_POOL,
	B_FALSE, zpool_sync_one, &force);

	return (ret);
	}

	typedef struct iostat_cbdata {
	uint64_t cb_flags;
	int cb_namewidth;
	int cb_iteration;
	boolean_t cb_verbose;
	boolean_t cb_literal;
	boolean_t cb_scripted;
	zpool_list_t *cb_list;
	vdev_cmd_data_list_t *vcdl;
	vdev_cbdata_t cb_vdevs;
	} iostat_cbdata_t;

	/* iostat labels */
	typedef struct name_and_columns {
	const char name; / Column name */
	unsigned int columns; /* Center name to this number of columns */
	} name_and_columns_t;

	#define IOSTAT_MAX_LABELS 15 /* Max number of labels on one line */

	static const name_and_columns_t iostat_top_labels[][IOSTAT_MAX_LABELS] =
	{
	[IOS_DEFAULT] = {{"capacity", 2}, {"operations", 2}, {"bandwidth", 2},
	{NULL}},
	[IOS_LATENCY] = {{"total_wait", 2}, {"disk_wait", 2}, {"syncq_wait", 2},
	{"asyncq_wait", 2}, {"scrub", 1}, {"trim", 1}, {"rebuild", 1},
	{NULL}},
	[IOS_QUEUES] = {{"syncq_read", 2}, {"syncq_write", 2},
	{"asyncq_read", 2}, {"asyncq_write", 2}, {"scrubq_read", 2},
	{"trimq_write", 2}, {"rebuildq_write", 2}, {NULL}},
	[IOS_L_HISTO] = {{"total_wait", 2}, {"disk_wait", 2}, {"syncq_wait", 2},
	{"asyncq_wait", 2}, {NULL}},
	[IOS_RQ_HISTO] = {{"sync_read", 2}, {"sync_write", 2},
	{"async_read", 2}, {"async_write", 2}, {"scrub", 2},
	{"trim", 2}, {"rebuild", 2}, {NULL}},
	};

	/* Shorthand - if "columns" field not set, default to 1 column */
	static const name_and_columns_t iostat_bottom_labels[][IOSTAT_MAX_LABELS] =
	{
	[IOS_DEFAULT] = {{"alloc"}, {"free"}, {"read"}, {"write"}, {"read"},
	{"write"}, {NULL}},
	[IOS_LATENCY] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
	{"write"}, {"read"}, {"write"}, {"wait"}, {"wait"}, {"wait"},
	{NULL}},
	[IOS_QUEUES] = {{"pend"}, {"activ"}, {"pend"}, {"activ"}, {"pend"},
	{"activ"}, {"pend"}, {"activ"}, {"pend"}, {"activ"},
	{"pend"}, {"activ"}, {"pend"}, {"activ"}, {NULL}},
	[IOS_L_HISTO] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
	{"write"}, {"read"}, {"write"}, {"scrub"}, {"trim"}, {"rebuild"},
	{NULL}},
	[IOS_RQ_HISTO] = {{"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"},
	{"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"},
	{"ind"}, {"agg"}, {NULL}},
	};

	static const char *histo_to_title[] = {
	[IOS_L_HISTO] = "latency",
	[IOS_RQ_HISTO] = "req_size",
	};

	/*
	* Return the number of labels in a null-terminated name_and_columns_t
	* array.
	*
	*/
	static unsigned int
	label_array_len(const name_and_columns_t *labels)
	{
	int i = 0;

	while (labels[i].name)
	i++;

	return (i);
	}

	/*
	* Return the number of strings in a null-terminated string array.
	* For example:
	*
	* const char foo[] = {"bar", "baz", NULL}
	*
	* returns 2
	*/
	static uint64_t
	str_array_len(const char *array[])
	{
	uint64_t i = 0;
	while (array[i])
	i++;

	return (i);
	}


	/*
	* Return a default column width for default/latency/queue columns. This does
	* not include histograms, which have their columns autosized.
	*/
	static unsigned int
	default_column_width(iostat_cbdata_t *cb, enum iostat_type type)
	{
	unsigned long column_width = 5; /* Normal niceprint */
	static unsigned long widths[] = {
	/*
	* Choose some sane default column sizes for printing the
	* raw numbers.
	*/
	[IOS_DEFAULT] = 15, /* 1PB capacity */
	[IOS_LATENCY] = 10, /* 1B ns = 10sec */
	[IOS_QUEUES] = 6, /* 1M queue entries */
	[IOS_L_HISTO] = 10, /* 1B ns = 10sec */
	[IOS_RQ_HISTO] = 6, /* 1M queue entries */
	};

	if (cb->cb_literal)
	column_width = widths[type];

	return (column_width);
	}

	/*
	* Print the column labels, i.e:
	*
	* capacity operations bandwidth
	* alloc free read write read write ...
	*
	* If force_column_width is set, use it for the column width. If not set, use
	* the default column width.
	*/
	static void
	print_iostat_labels(iostat_cbdata_t *cb, unsigned int force_column_width,
	const name_and_columns_t labels[][IOSTAT_MAX_LABELS])
	{
	int i, idx, s;
	int text_start, rw_column_width, spaces_to_end;
	uint64_t flags = cb->cb_flags;
	uint64_t f;
	unsigned int column_width = force_column_width;

	/* For each bit set in flags */
	for (f = flags; f; f &= ~(1ULL << idx)) {
	idx = lowbit64(f) - 1;
	if (!force_column_width)
	column_width = default_column_width(cb, idx);
	/* Print our top labels centered over "read write" label. */
	for (i = 0; i < label_array_len(labels[idx]); i++) {
	const char *name = labels[idx][i].name;
	/*
	* We treat labels[][].columns == 0 as shorthand
	* for one column. It makes writing out the label
	* tables more concise.
	*/
	unsigned int columns = MAX(1, labels[idx][i].columns);
	unsigned int slen = strlen(name);

	rw_column_width = (column_width * columns) +
	(2 * (columns - 1));

	text_start = (int)((rw_column_width) / columns -
	slen / columns);
	if (text_start < 0)
	text_start = 0;

	printf(" "); /* Two spaces between columns */

	/* Space from beginning of column to label */
	for (s = 0; s < text_start; s++)
	printf(" ");

	printf("%s", name);

	/* Print space after label to end of column */
	spaces_to_end = rw_column_width - text_start - slen;
	if (spaces_to_end < 0)
	spaces_to_end = 0;

	for (s = 0; s < spaces_to_end; s++)
	printf(" ");
	}
	}
	}


	/*
	* print_cmd_columns - Print custom column titles from -c
	*
	* If the user specified the "zpool status\|iostat -c" then print their custom
	* column titles in the header. For example, print_cmd_columns() would print
	* the " col1 col2" part of this:
	*
	* $ zpool iostat -vc 'echo col1=val1; echo col2=val2'
	* ...
	* capacity operations bandwidth
	* pool alloc free read write read write col1 col2
	* ---------- ----- ----- ----- ----- ----- ----- ---- ----
	* mypool 269K 1008M 0 0 107 946
	* mirror 269K 1008M 0 0 107 946
	* sdb - - 0 0 102 473 val1 val2
	* sdc - - 0 0 5 473 val1 val2
	* ---------- ----- ----- ----- ----- ----- ----- ---- ----
	*/
	static void
	print_cmd_columns(vdev_cmd_data_list_t *vcdl, int use_dashes)
	{
	int i, j;
	vdev_cmd_data_t *data = &vcdl->data[0];

	if (vcdl->count == 0 \|\| data == NULL)
	return;

	/*
	* Each vdev cmd should have the same column names unless the user did
	* something weird with their cmd. Just take the column names from the
	* first vdev and assume it works for all of them.
	*/
	for (i = 0; i < vcdl->uniq_cols_cnt; i++) {
	printf(" ");
	if (use_dashes) {
	for (j = 0; j < vcdl->uniq_cols_width[i]; j++)
	printf("-");
	} else {
	printf_color(ANSI_BOLD, "%*s", vcdl->uniq_cols_width[i],
	vcdl->uniq_cols[i]);
	}
	}
	}


	/*
	* Utility function to print out a line of dashes like:
	*
	* -------------------------------- ----- ----- ----- ----- -----
	*
	* ...or a dashed named-row line like:
	*
	* logs - - - - -
	*
	* @cb: iostat data
	*
	* @force_column_width If non-zero, use the value as the column width.
	* Otherwise use the default column widths.
	*
	* @name: Print a dashed named-row line starting
	* with @name. Otherwise, print a regular
	* dashed line.
	*/
	static void
	print_iostat_dashes(iostat_cbdata_t *cb, unsigned int force_column_width,
	const char *name)
	{
	int i;
	unsigned int namewidth;
	uint64_t flags = cb->cb_flags;
	uint64_t f;
	int idx;
	const name_and_columns_t *labels;
	const char *title;


	if (cb->cb_flags & IOS_ANYHISTO_M) {
	title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
	} else if (cb->cb_vdevs.cb_names_count) {
	title = "vdev";
	} else {
	title = "pool";
	}

	namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
	name ? strlen(name) : 0);


	if (name) {
	printf("%-*s", namewidth, name);
	} else {
	for (i = 0; i < namewidth; i++)
	(void) printf("-");
	}

	/* For each bit in flags */
	for (f = flags; f; f &= ~(1ULL << idx)) {
	unsigned int column_width;
	idx = lowbit64(f) - 1;
	if (force_column_width)
	column_width = force_column_width;
	else
	column_width = default_column_width(cb, idx);

	labels = iostat_bottom_labels[idx];
	for (i = 0; i < label_array_len(labels); i++) {
	if (name)
	printf(" %*s-", column_width - 1, " ");
	else
	printf(" %.*s", column_width,
	"--------------------");
	}
	}
	}


	static void
	print_iostat_separator_impl(iostat_cbdata_t *cb,
	unsigned int force_column_width)
	{
	print_iostat_dashes(cb, force_column_width, NULL);
	}

	static void
	print_iostat_separator(iostat_cbdata_t *cb)
	{
	print_iostat_separator_impl(cb, 0);
	}

	static void
	print_iostat_header_impl(iostat_cbdata_t *cb, unsigned int force_column_width,
	const char *histo_vdev_name)
	{
	unsigned int namewidth;
	const char *title;

	color_start(ANSI_BOLD);

	if (cb->cb_flags & IOS_ANYHISTO_M) {
	title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
	} else if (cb->cb_vdevs.cb_names_count) {
	title = "vdev";
	} else {
	title = "pool";
	}

	namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
	histo_vdev_name ? strlen(histo_vdev_name) : 0);

	if (histo_vdev_name)
	printf("%-*s", namewidth, histo_vdev_name);
	else
	printf("%*s", namewidth, "");


	print_iostat_labels(cb, force_column_width, iostat_top_labels);
	printf("\n");

	printf("%-*s", namewidth, title);

	print_iostat_labels(cb, force_column_width, iostat_bottom_labels);
	if (cb->vcdl != NULL)
	print_cmd_columns(cb->vcdl, 0);

	printf("\n");

	print_iostat_separator_impl(cb, force_column_width);

	if (cb->vcdl != NULL)
	print_cmd_columns(cb->vcdl, 1);

	color_end();

	printf("\n");
	}

	static void
	print_iostat_header(iostat_cbdata_t *cb)
	{
	print_iostat_header_impl(cb, 0, NULL);
	}

	/*
	* Prints a size string (i.e. 120M) with the suffix ("M") colored
	* by order of magnitude. Uses column_size to add padding.
	*/
	static void
	print_stat_color(const char *statbuf, unsigned int column_size)
	{
	fputs(" ", stdout);
	size_t len = strlen(statbuf);
	while (len < column_size) {
	fputc(' ', stdout);
	column_size--;
	}
	if (*statbuf == '0') {
	color_start(ANSI_GRAY);
	fputc('0', stdout);
	} else {
	for (; *statbuf; statbuf++) {
	if (*statbuf == 'K') color_start(ANSI_GREEN);
	else if (*statbuf == 'M') color_start(ANSI_YELLOW);
	else if (*statbuf == 'G') color_start(ANSI_RED);
	else if (*statbuf == 'T') color_start(ANSI_BOLD_BLUE);
	else if (*statbuf == 'P') color_start(ANSI_MAGENTA);
	else if (*statbuf == 'E') color_start(ANSI_CYAN);
	fputc(*statbuf, stdout);
	if (--column_size <= 0)
	break;
	}
	}
	color_end();
	}

	/*
	* Display a single statistic.
	*/
	static void
	print_one_stat(uint64_t value, enum zfs_nicenum_format format,
	unsigned int column_size, boolean_t scripted)
	{
	char buf[64];

	zfs_nicenum_format(value, buf, sizeof (buf), format);

	if (scripted)
	printf("\t%s", buf);
	else
	print_stat_color(buf, column_size);
	}

	/*
	* Calculate the default vdev stats
	*
	* Subtract oldvs from newvs, apply a scaling factor, and save the resulting
	* stats into calcvs.
	*/
	static void
	calc_default_iostats(vdev_stat_t oldvs, vdev_stat_t newvs,
	vdev_stat_t *calcvs)
	{
	int i;

	memcpy(calcvs, newvs, sizeof (*calcvs));
	for (i = 0; i < ARRAY_SIZE(calcvs->vs_ops); i++)
	calcvs->vs_ops[i] = (newvs->vs_ops[i] - oldvs->vs_ops[i]);

	for (i = 0; i < ARRAY_SIZE(calcvs->vs_bytes); i++)
	calcvs->vs_bytes[i] = (newvs->vs_bytes[i] - oldvs->vs_bytes[i]);
	}

	/*
	* Internal representation of the extended iostats data.
	*
	* The extended iostat stats are exported in nvlists as either uint64_t arrays
	* or single uint64_t's. We make both look like arrays to make them easier
	* to process. In order to make single uint64_t's look like arrays, we set
	* __data to the stat data, and then set *data = &__data with count = 1. Then,
	* we can just use *data and count.
	*/
	struct stat_array {
	uint64_t *data;
	uint_t count; /* Number of entries in data[] */
	uint64_t __data; /* Only used when data is a single uint64_t */
	};

	static uint64_t
	stat_histo_max(struct stat_array *nva, unsigned int len)
	{
	uint64_t max = 0;
	int i;
	for (i = 0; i < len; i++)
	max = MAX(max, array64_max(nva[i].data, nva[i].count));

	return (max);
	}

	/*
	* Helper function to lookup a uint64_t array or uint64_t value and store its
	* data as a stat_array. If the nvpair is a single uint64_t value, then we make
	* it look like a one element array to make it easier to process.
	*/
	static int
	nvpair64_to_stat_array(nvlist_t nvl, const char name,
	struct stat_array *nva)
	{
	nvpair_t *tmp;
	int ret;

	verify(nvlist_lookup_nvpair(nvl, name, &tmp) == 0);
	switch (nvpair_type(tmp)) {
	case DATA_TYPE_UINT64_ARRAY:
	ret = nvpair_value_uint64_array(tmp, &nva->data, &nva->count);
	break;
	case DATA_TYPE_UINT64:
	ret = nvpair_value_uint64(tmp, &nva->__data);
	nva->data = &nva->__data;
	nva->count = 1;
	break;
	default:
	/* Not a uint64_t */
	ret = EINVAL;
	break;
	}

	return (ret);
	}

	/*
	* Given a list of nvlist names, look up the extended stats in newnv and oldnv,
	* subtract them, and return the results in a newly allocated stat_array.
	* You must free the returned array after you are done with it with
	* free_calc_stats().
	*
	* Additionally, you can set "oldnv" to NULL if you simply want the newnv
	* values.
	*/
	static struct stat_array *
	calc_and_alloc_stats_ex(const char *names, unsigned int len, nvlist_t oldnv,
	nvlist_t *newnv)
	{
	nvlist_t oldnvx = NULL, newnvx;
	struct stat_array oldnva, newnva, *calcnva;
	int i, j;
	unsigned int alloc_size = (sizeof (struct stat_array)) * len;

	/* Extract our extended stats nvlist from the main list */
	verify(nvlist_lookup_nvlist(newnv, ZPOOL_CONFIG_VDEV_STATS_EX,
	&newnvx) == 0);
	if (oldnv) {
	verify(nvlist_lookup_nvlist(oldnv, ZPOOL_CONFIG_VDEV_STATS_EX,
	&oldnvx) == 0);
	}

	newnva = safe_malloc(alloc_size);
	oldnva = safe_malloc(alloc_size);
	calcnva = safe_malloc(alloc_size);

	for (j = 0; j < len; j++) {
	verify(nvpair64_to_stat_array(newnvx, names[j],
	&newnva[j]) == 0);
	calcnva[j].count = newnva[j].count;
	alloc_size = calcnva[j].count * sizeof (calcnva[j].data[0]);
	calcnva[j].data = safe_malloc(alloc_size);
	memcpy(calcnva[j].data, newnva[j].data, alloc_size);

	if (oldnvx) {
	verify(nvpair64_to_stat_array(oldnvx, names[j],
	&oldnva[j]) == 0);
	for (i = 0; i < oldnva[j].count; i++)
	calcnva[j].data[i] -= oldnva[j].data[i];
	}
	}
	free(newnva);
	free(oldnva);
	return (calcnva);
	}

	static void
	free_calc_stats(struct stat_array *nva, unsigned int len)
	{
	int i;
	for (i = 0; i < len; i++)
	free(nva[i].data);

	free(nva);
	}

	static void
	print_iostat_histo(struct stat_array *nva, unsigned int len,
	iostat_cbdata_t *cb, unsigned int column_width, unsigned int namewidth,
	double scale)
	{
	int i, j;
	char buf[6];
	uint64_t val;
	enum zfs_nicenum_format format;
	unsigned int buckets;
	unsigned int start_bucket;

	if (cb->cb_literal)
	format = ZFS_NICENUM_RAW;
	else
	format = ZFS_NICENUM_1024;

	/* All these histos are the same size, so just use nva[0].count */
	buckets = nva[0].count;

	if (cb->cb_flags & IOS_RQ_HISTO_M) {
	/* Start at 512 - req size should never be lower than this */
	start_bucket = 9;
	} else {
	start_bucket = 0;
	}

	for (j = start_bucket; j < buckets; j++) {
	/* Print histogram bucket label */
	if (cb->cb_flags & IOS_L_HISTO_M) {
	/* Ending range of this bucket */
	val = (1UL << (j + 1)) - 1;
	zfs_nicetime(val, buf, sizeof (buf));
	} else {
	/* Request size (starting range of bucket) */
	val = (1UL << j);
	zfs_nicenum(val, buf, sizeof (buf));
	}

	if (cb->cb_scripted)
	printf("%llu", (u_longlong_t)val);
	else
	printf("%-*s", namewidth, buf);

	/* Print the values on the line */
	for (i = 0; i < len; i++) {
	print_one_stat(nva[i].data[j] * scale, format,
	column_width, cb->cb_scripted);
	}
	printf("\n");
	}
	}

	static void
	print_solid_separator(unsigned int length)
	{
	while (length--)
	printf("-");
	printf("\n");
	}

	static void
	print_iostat_histos(iostat_cbdata_t cb, nvlist_t oldnv,
	nvlist_t newnv, double scale, const char name)
	{
	unsigned int column_width;
	unsigned int namewidth;
	unsigned int entire_width;
	enum iostat_type type;
	struct stat_array *nva;
	const char **names;
	unsigned int names_len;

	/* What type of histo are we? */
	type = IOS_HISTO_IDX(cb->cb_flags);

	/* Get NULL-terminated array of nvlist names for our histo */
	names = vsx_type_to_nvlist[type];
	names_len = str_array_len(names); /* num of names */

	nva = calc_and_alloc_stats_ex(names, names_len, oldnv, newnv);

	if (cb->cb_literal) {
	column_width = MAX(5,
	(unsigned int) log10(stat_histo_max(nva, names_len)) + 1);
	} else {
	column_width = 5;
	}

	namewidth = MAX(cb->cb_namewidth,
	strlen(histo_to_title[IOS_HISTO_IDX(cb->cb_flags)]));

	/*
	* Calculate the entire line width of what we're printing. The
	* +2 is for the two spaces between columns:
	*/
	/* read write */
	/* ----- ----- */
	/* \|___\| <---------- column_width */
	/* */
	/* \|__________\| <--- entire_width */
	/* */
	entire_width = namewidth + (column_width + 2) *
	label_array_len(iostat_bottom_labels[type]);

	if (cb->cb_scripted)
	printf("%s\n", name);
	else
	print_iostat_header_impl(cb, column_width, name);

	print_iostat_histo(nva, names_len, cb, column_width,
	namewidth, scale);

	free_calc_stats(nva, names_len);
	if (!cb->cb_scripted)
	print_solid_separator(entire_width);
	}

	/*
	* Calculate the average latency of a power-of-two latency histogram
	*/
	static uint64_t
	single_histo_average(uint64_t *histo, unsigned int buckets)
	{
	int i;
	uint64_t count = 0, total = 0;

	for (i = 0; i < buckets; i++) {
	/*
	* Our buckets are power-of-two latency ranges. Use the
	* midpoint latency of each bucket to calculate the average.
	* For example:
	*
	* Bucket Midpoint
	* 8ns-15ns: 12ns
	* 16ns-31ns: 24ns
	* ...
	*/
	if (histo[i] != 0) {
	total += histo[i] * (((1UL << i) + ((1UL << i)/2)));
	count += histo[i];
	}
	}

	/* Prevent divide by zero */
	return (count == 0 ? 0 : total / count);
	}

	static void
	print_iostat_queues(iostat_cbdata_t cb, nvlist_t newnv)
	{
	const char *names[] = {
	ZPOOL_CONFIG_VDEV_SYNC_R_PEND_QUEUE,
	ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
	ZPOOL_CONFIG_VDEV_SYNC_W_PEND_QUEUE,
	ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
	ZPOOL_CONFIG_VDEV_ASYNC_R_PEND_QUEUE,
	ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
	ZPOOL_CONFIG_VDEV_ASYNC_W_PEND_QUEUE,
	ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
	ZPOOL_CONFIG_VDEV_SCRUB_PEND_QUEUE,
	ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
	ZPOOL_CONFIG_VDEV_TRIM_PEND_QUEUE,
	ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE,
	ZPOOL_CONFIG_VDEV_REBUILD_PEND_QUEUE,
	ZPOOL_CONFIG_VDEV_REBUILD_ACTIVE_QUEUE,
	};

	struct stat_array *nva;

	unsigned int column_width = default_column_width(cb, IOS_QUEUES);
	enum zfs_nicenum_format format;

	nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), NULL, newnv);

	if (cb->cb_literal)
	format = ZFS_NICENUM_RAW;
	else
	format = ZFS_NICENUM_1024;

	for (int i = 0; i < ARRAY_SIZE(names); i++) {
	uint64_t val = nva[i].data[0];
	print_one_stat(val, format, column_width, cb->cb_scripted);
	}

	free_calc_stats(nva, ARRAY_SIZE(names));
	}

	static void
	print_iostat_latency(iostat_cbdata_t cb, nvlist_t oldnv,
	nvlist_t *newnv)
	{
	int i;
	uint64_t val;
	const char *names[] = {
	ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
	ZPOOL_CONFIG_VDEV_REBUILD_LAT_HISTO,
	};
	struct stat_array *nva;

	unsigned int column_width = default_column_width(cb, IOS_LATENCY);
	enum zfs_nicenum_format format;

	nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), oldnv, newnv);

	if (cb->cb_literal)
	format = ZFS_NICENUM_RAWTIME;
	else
	format = ZFS_NICENUM_TIME;

	/* Print our avg latencies on the line */
	for (i = 0; i < ARRAY_SIZE(names); i++) {
	/* Compute average latency for a latency histo */
	val = single_histo_average(nva[i].data, nva[i].count);
	print_one_stat(val, format, column_width, cb->cb_scripted);
	}
	free_calc_stats(nva, ARRAY_SIZE(names));
	}

	/*
	* Print default statistics (capacity/operations/bandwidth)
	*/
	static void
	print_iostat_default(vdev_stat_t vs, iostat_cbdata_t cb, double scale)
	{
	unsigned int column_width = default_column_width(cb, IOS_DEFAULT);
	enum zfs_nicenum_format format;
	char na; /* char to print for "not applicable" values */

	if (cb->cb_literal) {
	format = ZFS_NICENUM_RAW;
	na = '0';
	} else {
	format = ZFS_NICENUM_1024;
	na = '-';
	}

	/* only toplevel vdevs have capacity stats */
	if (vs->vs_space == 0) {
	if (cb->cb_scripted)
	printf("\t%c\t%c", na, na);
	else
	printf(" %c %c", column_width, na, column_width,
	na);
	} else {
	print_one_stat(vs->vs_alloc, format, column_width,
	cb->cb_scripted);
	print_one_stat(vs->vs_space - vs->vs_alloc, format,
	column_width, cb->cb_scripted);
	}

	print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_READ] * scale),
	format, column_width, cb->cb_scripted);
	print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_WRITE] * scale),
	format, column_width, cb->cb_scripted);
	print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_READ] * scale),
	format, column_width, cb->cb_scripted);
	print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_WRITE] * scale),
	format, column_width, cb->cb_scripted);
	}

	static const char *const class_name[] = {
	VDEV_ALLOC_BIAS_DEDUP,
	VDEV_ALLOC_BIAS_SPECIAL,
	VDEV_ALLOC_CLASS_LOGS
	};

	/*
	* Print out all the statistics for the given vdev. This can either be the
	* toplevel configuration, or called recursively. If 'name' is NULL, then this
	* is a verbose output, and we don't want to display the toplevel pool stats.
	*
	* Returns the number of stat lines printed.
	*/
	static unsigned int
	print_vdev_stats(zpool_handle_t zhp, const char name, nvlist_t *oldnv,
	nvlist_t newnv, iostat_cbdata_t cb, int depth)
	{
	nvlist_t oldchild, newchild;
	uint_t c, children, oldchildren;
	vdev_stat_t oldvs, newvs, *calcvs;
	vdev_stat_t zerovs = { 0 };
	char *vname;
	int i;
	int ret = 0;
	uint64_t tdelta;
	double scale;

	if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
	return (ret);

	calcvs = safe_malloc(sizeof (*calcvs));

	if (oldnv != NULL) {
	verify(nvlist_lookup_uint64_array(oldnv,
	ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&oldvs, &c) == 0);
	} else {
	oldvs = &zerovs;
	}

	/* Do we only want to see a specific vdev? */
	for (i = 0; i < cb->cb_vdevs.cb_names_count; i++) {
	/* Yes we do. Is this the vdev? */
	if (strcmp(name, cb->cb_vdevs.cb_names[i]) == 0) {
	/*
	* This is our vdev. Since it is the only vdev we
	* will be displaying, make depth = 0 so that it
	* doesn't get indented.
	*/
	depth = 0;
	break;
	}
	}

	if (cb->cb_vdevs.cb_names_count && (i == cb->cb_vdevs.cb_names_count)) {
	/* Couldn't match the name */
	goto children;
	}


	verify(nvlist_lookup_uint64_array(newnv, ZPOOL_CONFIG_VDEV_STATS,
	(uint64_t **)&newvs, &c) == 0);

	/*
	* Print the vdev name unless it's is a histogram. Histograms
	* display the vdev name in the header itself.
	*/
	if (!(cb->cb_flags & IOS_ANYHISTO_M)) {
	if (cb->cb_scripted) {
	printf("%s", name);
	} else {
	if (strlen(name) + depth > cb->cb_namewidth)
	(void) printf("%*s%s", depth, "", name);
	else
	(void) printf("%s%s%s", depth, "", name,
	(int)(cb->cb_namewidth - strlen(name) -
	depth), "");
	}
	}

	/* Calculate our scaling factor */
	tdelta = newvs->vs_timestamp - oldvs->vs_timestamp;
	if ((oldvs->vs_timestamp == 0) && (cb->cb_flags & IOS_ANYHISTO_M)) {
	/*
	* If we specify printing histograms with no time interval, then
	* print the histogram numbers over the entire lifetime of the
	* vdev.
	*/
	scale = 1;
	} else {
	if (tdelta == 0)
	scale = 1.0;
	else
	scale = (double)NANOSEC / tdelta;
	}

	if (cb->cb_flags & IOS_DEFAULT_M) {
	calc_default_iostats(oldvs, newvs, calcvs);
	print_iostat_default(calcvs, cb, scale);
	}
	if (cb->cb_flags & IOS_LATENCY_M)
	print_iostat_latency(cb, oldnv, newnv);
	if (cb->cb_flags & IOS_QUEUES_M)
	print_iostat_queues(cb, newnv);
	if (cb->cb_flags & IOS_ANYHISTO_M) {
	printf("\n");
	print_iostat_histos(cb, oldnv, newnv, scale, name);
	}

	if (cb->vcdl != NULL) {
	const char *path;
	if (nvlist_lookup_string(newnv, ZPOOL_CONFIG_PATH,
	&path) == 0) {
	printf(" ");
	zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
	}
	}

	if (!(cb->cb_flags & IOS_ANYHISTO_M))
	printf("\n");

	ret++;

	children:

	free(calcvs);

	if (!cb->cb_verbose)
	return (ret);

	if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_CHILDREN,
	&newchild, &children) != 0)
	return (ret);

	if (oldnv) {
	if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_CHILDREN,
	&oldchild, &oldchildren) != 0)
	return (ret);

	children = MIN(oldchildren, children);
	}

	/*
	* print normal top-level devices
	*/
	for (c = 0; c < children; c++) {
	uint64_t ishole = B_FALSE, islog = B_FALSE;

	(void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_HOLE,
	&ishole);

	(void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_LOG,
	&islog);

	if (ishole \|\| islog)
	continue;

	if (nvlist_exists(newchild[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
	continue;

	vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
	cb->cb_vdevs.cb_name_flags \| VDEV_NAME_TYPE_ID);
	ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c] : NULL,
	newchild[c], cb, depth + 2);
	free(vname);
	}

	/*
	* print all other top-level devices
	*/
	for (uint_t n = 0; n < ARRAY_SIZE(class_name); n++) {
	boolean_t printed = B_FALSE;

	for (c = 0; c < children; c++) {
	uint64_t islog = B_FALSE;
	const char *bias = NULL;
	const char *type = NULL;

	(void) nvlist_lookup_uint64(newchild[c],
	ZPOOL_CONFIG_IS_LOG, &islog);
	if (islog) {
	bias = VDEV_ALLOC_CLASS_LOGS;
	} else {
	(void) nvlist_lookup_string(newchild[c],
	ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
	(void) nvlist_lookup_string(newchild[c],
	ZPOOL_CONFIG_TYPE, &type);
	}
	if (bias == NULL \|\| strcmp(bias, class_name[n]) != 0)
	continue;
	if (!islog && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
	continue;

	if (!printed) {
	if ((!(cb->cb_flags & IOS_ANYHISTO_M)) &&
	!cb->cb_scripted &&
	!cb->cb_vdevs.cb_names) {
	print_iostat_dashes(cb, 0,
	class_name[n]);
	}
	printf("\n");
	printed = B_TRUE;
	}

	vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
	cb->cb_vdevs.cb_name_flags \| VDEV_NAME_TYPE_ID);
	ret += print_vdev_stats(zhp, vname, oldnv ?
	oldchild[c] : NULL, newchild[c], cb, depth + 2);
	free(vname);
	}
	}

	/*
	* Include level 2 ARC devices in iostat output
	*/
	if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_L2CACHE,
	&newchild, &children) != 0)
	return (ret);

	if (oldnv) {
	if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_L2CACHE,
	&oldchild, &oldchildren) != 0)
	return (ret);

	children = MIN(oldchildren, children);
	}

	if (children > 0) {
	if ((!(cb->cb_flags & IOS_ANYHISTO_M)) && !cb->cb_scripted &&
	!cb->cb_vdevs.cb_names) {
	print_iostat_dashes(cb, 0, "cache");
	}
	printf("\n");

	for (c = 0; c < children; c++) {
	vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
	cb->cb_vdevs.cb_name_flags);
	ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c]
	: NULL, newchild[c], cb, depth + 2);
	free(vname);
	}
	}

	return (ret);
	}

	static int
	refresh_iostat(zpool_handle_t zhp, void data)
	{
	iostat_cbdata_t *cb = data;
	boolean_t missing;

	/*
	* If the pool has disappeared, remove it from the list and continue.
	*/
	if (zpool_refresh_stats(zhp, &missing) != 0)
	return (-1);

	if (missing)
	pool_list_remove(cb->cb_list, zhp);

	return (0);
	}

	/*
	* Callback to print out the iostats for the given pool.
	*/
	static int
	print_iostat(zpool_handle_t zhp, void data)
	{
	iostat_cbdata_t *cb = data;
	nvlist_t oldconfig, newconfig;
	nvlist_t oldnvroot, newnvroot;
	int ret;

	newconfig = zpool_get_config(zhp, &oldconfig);

	if (cb->cb_iteration == 1)
	oldconfig = NULL;

	verify(nvlist_lookup_nvlist(newconfig, ZPOOL_CONFIG_VDEV_TREE,
	&newnvroot) == 0);

	if (oldconfig == NULL)
	oldnvroot = NULL;
	else
	verify(nvlist_lookup_nvlist(oldconfig, ZPOOL_CONFIG_VDEV_TREE,
	&oldnvroot) == 0);

	ret = print_vdev_stats(zhp, zpool_get_name(zhp), oldnvroot, newnvroot,
	cb, 0);
	if ((ret != 0) && !(cb->cb_flags & IOS_ANYHISTO_M) &&
	!cb->cb_scripted && cb->cb_verbose &&
	!cb->cb_vdevs.cb_names_count) {
	print_iostat_separator(cb);
	if (cb->vcdl != NULL) {
	print_cmd_columns(cb->vcdl, 1);
	}
	printf("\n");
	}

	return (ret);
	}

	static int
	get_columns(void)
	{
	struct winsize ws;
	int columns = 80;
	int error;

	if (isatty(STDOUT_FILENO)) {
	error = ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws);
	if (error == 0)
	columns = ws.ws_col;
	} else {
	columns = 999;
	}

	return (columns);
	}

	/*
	* Return the required length of the pool/vdev name column. The minimum
	* allowed width and output formatting flags must be provided.
	*/
	static int
	get_namewidth(zpool_handle_t *zhp, int min_width, int flags, boolean_t verbose)
	{
	nvlist_t config, nvroot;
	int width = min_width;

	if ((config = zpool_get_config(zhp, NULL)) != NULL) {
	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
	&nvroot) == 0);
	size_t poolname_len = strlen(zpool_get_name(zhp));
	if (verbose == B_FALSE) {
	width = MAX(poolname_len, min_width);
	} else {
	width = MAX(poolname_len,
	max_width(zhp, nvroot, 0, min_width, flags));
	}
	}

	return (width);
	}

	/*
	* Parse the input string, get the 'interval' and 'count' value if there is one.
	*/
	static void
	get_interval_count(int argcp, char argv, float iv,
	unsigned long *cnt)
	{
	float interval = 0;
	unsigned long count = 0;
	int argc = *argcp;

	/*
	* Determine if the last argument is an integer or a pool name
	*/
	if (argc > 0 && zfs_isnumber(argv[argc - 1])) {
	char *end;

	errno = 0;
	interval = strtof(argv[argc - 1], &end);

	if (*end == '\0' && errno == 0) {
	if (interval == 0) {
	(void) fprintf(stderr, gettext(
	"interval cannot be zero\n"));
	usage(B_FALSE);
	}
	/*
	* Ignore the last parameter
	*/
	argc--;
	} else {
	/*
	* If this is not a valid number, just plow on. The
	* user will get a more informative error message later
	* on.
	*/
	interval = 0;
	}
	}

	/*
	* If the last argument is also an integer, then we have both a count
	* and an interval.
	*/
	if (argc > 0 && zfs_isnumber(argv[argc - 1])) {
	char *end;

	errno = 0;
	count = interval;
	interval = strtof(argv[argc - 1], &end);

	if (*end == '\0' && errno == 0) {
	if (interval == 0) {
	(void) fprintf(stderr, gettext(
	"interval cannot be zero\n"));
	usage(B_FALSE);
	}

	/*
	* Ignore the last parameter
	*/
	argc--;
	} else {
	interval = 0;
	}
	}

	*iv = interval;
	*cnt = count;
	*argcp = argc;
	}

	static void
	get_timestamp_arg(char c)
	{
	if (c == 'u')
	timestamp_fmt = UDATE;
	else if (c == 'd')
	timestamp_fmt = DDATE;
	else
	usage(B_FALSE);
	}

	/*
	* Return stat flags that are supported by all pools by both the module and
	* zpool iostat. "*data" should be initialized to all 0xFFs before running.
	* It will get ANDed down until only the flags that are supported on all pools
	* remain.
	*/
	static int
	get_stat_flags_cb(zpool_handle_t zhp, void data)
	{
	uint64_t *mask = data;
	nvlist_t config, nvroot, *nvx;
	uint64_t flags = 0;
	int i, j;

	config = zpool_get_config(zhp, NULL);
	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
	&nvroot) == 0);

	/* Default stats are always supported, but for completeness.. */
	if (nvlist_exists(nvroot, ZPOOL_CONFIG_VDEV_STATS))
	flags \|= IOS_DEFAULT_M;

	/* Get our extended stats nvlist from the main list */
	if (nvlist_lookup_nvlist(nvroot, ZPOOL_CONFIG_VDEV_STATS_EX,
	&nvx) != 0) {
	/*
	* No extended stats; they're probably running an older
	* module. No big deal, we support that too.
	*/
	goto end;
	}

	/* For each extended stat, make sure all its nvpairs are supported */
	for (j = 0; j < ARRAY_SIZE(vsx_type_to_nvlist); j++) {
	if (!vsx_type_to_nvlist[j][0])
	continue;

	/* Start off by assuming the flag is supported, then check */
	flags \|= (1ULL << j);
	for (i = 0; vsx_type_to_nvlist[j][i]; i++) {
	if (!nvlist_exists(nvx, vsx_type_to_nvlist[j][i])) {
	/* flag isn't supported */
	flags = flags & ~(1ULL << j);
	break;
	}
	}
	}
	end:
	mask = mask & flags;
	return (0);
	}

	/*
	* Return a bitmask of stats that are supported on all pools by both the module
	* and zpool iostat.
	*/
	static uint64_t
	get_stat_flags(zpool_list_t *list)
	{
	uint64_t mask = -1;

	/*
	* get_stat_flags_cb() will lop off bits from "mask" until only the
	* flags that are supported on all pools remain.
	*/
	pool_list_iter(list, B_FALSE, get_stat_flags_cb, &mask);
	return (mask);
	}

	/*
	* Return 1 if cb_data->cb_names[0] is this vdev's name, 0 otherwise.
	*/
	static int
	is_vdev_cb(void zhp_data, nvlist_t nv, void *cb_data)
	{
	uint64_t guid;
	vdev_cbdata_t *cb = cb_data;
	zpool_handle_t *zhp = zhp_data;

	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
	return (0);

	return (guid == zpool_vdev_path_to_guid(zhp, cb->cb_names[0]));
	}

	/*
	* Returns 1 if cb_data->cb_names[0] is a vdev name, 0 otherwise.
	*/
	static int
	is_vdev(zpool_handle_t zhp, void cb_data)
	{
	return (for_each_vdev(zhp, is_vdev_cb, cb_data));
	}

	/*
	* Check if vdevs are in a pool
	*
	* Return 1 if all argv[] strings are vdev names in pool "pool_name". Otherwise
	* return 0. If pool_name is NULL, then search all pools.
	*/
	static int
	are_vdevs_in_pool(int argc, char *argv, char pool_name,
	vdev_cbdata_t *cb)
	{
	char **tmp_name;
	int ret = 0;
	int i;
	int pool_count = 0;

	if ((argc == 0) \|\| !*argv)
	return (0);

	if (pool_name)
	pool_count = 1;

	/* Temporarily hijack cb_names for a second... */
	tmp_name = cb->cb_names;

	/* Go though our list of prospective vdev names */
	for (i = 0; i < argc; i++) {
	cb->cb_names = argv + i;

	/* Is this name a vdev in our pools? */
	ret = for_each_pool(pool_count, &pool_name, B_TRUE, NULL,
	ZFS_TYPE_POOL, B_FALSE, is_vdev, cb);
	if (!ret) {
	/* No match */
	break;
	}
	}

	cb->cb_names = tmp_name;

	return (ret);
	}

	static int
	is_pool_cb(zpool_handle_t zhp, void data)
	{
	char *name = data;
	if (strcmp(name, zpool_get_name(zhp)) == 0)
	return (1);

	return (0);
	}

	/*
	* Do we have a pool named *name? If so, return 1, otherwise 0.
	*/
	static int
	is_pool(char *name)
	{
	return (for_each_pool(0, NULL, B_TRUE, NULL, ZFS_TYPE_POOL, B_FALSE,
	is_pool_cb, name));
	}

	/* Are all our argv[] strings pool names? If so return 1, 0 otherwise. */
	static int
	are_all_pools(int argc, char **argv)
	{
	if ((argc == 0) \|\| !*argv)
	return (0);

	while (--argc >= 0)
	if (!is_pool(argv[argc]))
	return (0);

	return (1);
	}

	/*
	* Helper function to print out vdev/pool names we can't resolve. Used for an
	* error message.
	*/
	static void
	error_list_unresolved_vdevs(int argc, char *argv, char pool_name,
	vdev_cbdata_t *cb)
	{
	int i;
	char *name;
	char *str;
	for (i = 0; i < argc; i++) {
	name = argv[i];

	if (is_pool(name))
	str = gettext("pool");
	else if (are_vdevs_in_pool(1, &name, pool_name, cb))
	str = gettext("vdev in this pool");
	else if (are_vdevs_in_pool(1, &name, NULL, cb))
	str = gettext("vdev in another pool");
	else
	str = gettext("unknown");

	fprintf(stderr, "\t%s (%s)\n", name, str);
	}
	}

	/*
	* Same as get_interval_count(), but with additional checks to not misinterpret
	* guids as interval/count values. Assumes VDEV_NAME_GUID is set in
	* cb.cb_vdevs.cb_name_flags.
	*/
	static void
	get_interval_count_filter_guids(int argc, char argv, float interval,
	unsigned long count, iostat_cbdata_t cb)
	{
	char **tmpargv = argv;
	int argc_for_interval = 0;

	/* Is the last arg an interval value? Or a guid? */
	if (argc >= 1 && !are_vdevs_in_pool(1, &argv[argc - 1], NULL,
	&cb->cb_vdevs)) {
	/*
	* The last arg is not a guid, so it's probably an
	* interval value.
	*/
	argc_for_interval++;

	if (*argc >= 2 &&
	!are_vdevs_in_pool(1, &argv[*argc - 2], NULL,
	&cb->cb_vdevs)) {
	/*
	* The 2nd to last arg is not a guid, so it's probably
	* an interval value.
	*/
	argc_for_interval++;
	}
	}

	/* Point to our list of possible intervals */
	tmpargv = &argv[*argc - argc_for_interval];

	argc = argc - argc_for_interval;
	get_interval_count(&argc_for_interval, tmpargv,
	interval, count);
	}

	/*
	* Terminal height, in rows. Returns -1 if stdout is not connected to a TTY or
	* if we were unable to determine its size.
	*/
	static int
	terminal_height(void)
	{
	struct winsize win;

	if (isatty(STDOUT_FILENO) == 0)
	return (-1);

	if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &win) != -1 && win.ws_row > 0)
	return (win.ws_row);

	return (-1);
	}

	/*
	* Run one of the zpool status/iostat -c scripts with the help (-h) option and
	* print the result.
	*
	* name: Short name of the script ('iostat').
	* path: Full path to the script ('/usr/local/etc/zfs/zpool.d/iostat');
	*/
	static void
	print_zpool_script_help(char name, char path)
	{
	char argv[] = {path, (char )"-h", NULL};
	char **lines = NULL;
	int lines_cnt = 0;
	int rc;

	rc = libzfs_run_process_get_stdout_nopath(path, argv, NULL, &lines,
	&lines_cnt);
	if (rc != 0 \|\| lines == NULL \|\| lines_cnt <= 0) {
	if (lines != NULL)
	libzfs_free_str_array(lines, lines_cnt);
	return;
	}

	for (int i = 0; i < lines_cnt; i++)
	if (!is_blank_str(lines[i]))
	printf(" %-14s %s\n", name, lines[i]);

	libzfs_free_str_array(lines, lines_cnt);
	}

	/*
	* Go though the zpool status/iostat -c scripts in the user's path, run their
	* help option (-h), and print out the results.
	*/
	static void
	print_zpool_dir_scripts(char *dirpath)
	{
	DIR *dir;
	struct dirent *ent;
	char fullpath[MAXPATHLEN];
	struct stat dir_stat;

	if ((dir = opendir(dirpath)) != NULL) {
	/* print all the files and directories within directory */
	while ((ent = readdir(dir)) != NULL) {
	if (snprintf(fullpath, sizeof (fullpath), "%s/%s",
	dirpath, ent->d_name) >= sizeof (fullpath)) {
	(void) fprintf(stderr,
	gettext("internal error: "
	"ZPOOL_SCRIPTS_PATH too large.\n"));
	exit(1);
	}

	/* Print the scripts */
	if (stat(fullpath, &dir_stat) == 0)
	if (dir_stat.st_mode & S_IXUSR &&
	S_ISREG(dir_stat.st_mode))
	print_zpool_script_help(ent->d_name,
	fullpath);
	}
	closedir(dir);
	}
	}

	/*
	* Print out help text for all zpool status/iostat -c scripts.
	*/
	static void
	print_zpool_script_list(const char *subcommand)
	{
	char dir, sp, *tmp;

	printf(gettext("Available 'zpool %s -c' commands:\n"), subcommand);

	sp = zpool_get_cmd_search_path();
	if (sp == NULL)
	return;

	for (dir = strtok_r(sp, ":", &tmp);
	dir != NULL;
	dir = strtok_r(NULL, ":", &tmp))
	print_zpool_dir_scripts(dir);

	free(sp);
	}

	/*
	* Set the minimum pool/vdev name column width. The width must be at least 10,
	* but may be as large as the column width - 42 so it still fits on one line.
	* NOTE: 42 is the width of the default capacity/operations/bandwidth output
	*/
	static int
	get_namewidth_iostat(zpool_handle_t zhp, void data)
	{
	iostat_cbdata_t *cb = data;
	int width, available_width;

	/*
	* get_namewidth() returns the maximum width of any name in that column
	* for any pool/vdev/device line that will be output.
	*/
	width = get_namewidth(zhp, cb->cb_namewidth,
	cb->cb_vdevs.cb_name_flags \| VDEV_NAME_TYPE_ID, cb->cb_verbose);

	/*
	* The width we are calculating is the width of the header and also the
	* padding width for names that are less than maximum width. The stats
	* take up 42 characters, so the width available for names is:
	*/
	available_width = get_columns() - 42;

	/*
	* If the maximum width fits on a screen, then great! Make everything
	* line up by justifying all lines to the same width. If that max
	* width is larger than what's available, the name plus stats won't fit
	* on one line, and justifying to that width would cause every line to
	* wrap on the screen. We only want lines with long names to wrap.
	* Limit the padding to what won't wrap.
	*/
	if (width > available_width)
	width = available_width;

	/*
	* And regardless of whatever the screen width is (get_columns can
	* return 0 if the width is not known or less than 42 for a narrow
	* terminal) have the width be a minimum of 10.
	*/
	if (width < 10)
	width = 10;

	/* Save the calculated width */
	cb->cb_namewidth = width;

	return (0);
	}

	/*
	* zpool iostat [[-c [script1,script2,...]] [-lq]\|[-rw]] [-ghHLpPvy] [-n name]
	* [-T d\|u] [[ pool ...]\|[pool vdev ...]\|[vdev ...]]
	* [interval [count]]
	*
	* -c CMD For each vdev, run command CMD
	* -g Display guid for individual vdev name.
	* -L Follow links when resolving vdev path name.
	* -P Display full path for vdev name.
	* -v Display statistics for individual vdevs
	* -h Display help
	* -p Display values in parsable (exact) format.
	* -H Scripted mode. Don't display headers, and separate properties
	* by a single tab.
	* -l Display average latency
	* -q Display queue depths
	* -w Display latency histograms
	* -r Display request size histogram
	* -T Display a timestamp in date(1) or Unix format
	* -n Only print headers once
	*
	* This command can be tricky because we want to be able to deal with pool
	* creation/destruction as well as vdev configuration changes. The bulk of this
	* processing is handled by the pool_list_* routines in zpool_iter.c. We rely
	* on pool_list_update() to detect the addition of new pools. Configuration
	* changes are all handled within libzfs.
	*/
	int
	zpool_do_iostat(int argc, char **argv)
	{
	int c;
	int ret;
	int npools;
	float interval = 0;
	unsigned long count = 0;
	int winheight = 24;
	zpool_list_t *list;
	boolean_t verbose = B_FALSE;
	boolean_t latency = B_FALSE, l_histo = B_FALSE, rq_histo = B_FALSE;
	boolean_t queues = B_FALSE, parsable = B_FALSE, scripted = B_FALSE;
	boolean_t omit_since_boot = B_FALSE;
	boolean_t guid = B_FALSE;
	boolean_t follow_links = B_FALSE;
	boolean_t full_name = B_FALSE;
	boolean_t headers_once = B_FALSE;
	iostat_cbdata_t cb = { 0 };
	char *cmd = NULL;

	/* Used for printing error message */
	const char flag_to_arg[] = {[IOS_LATENCY] = 'l', [IOS_QUEUES] = 'q',
	[IOS_L_HISTO] = 'w', [IOS_RQ_HISTO] = 'r'};

	uint64_t unsupported_flags;

	/* check options */
	while ((c = getopt(argc, argv, "c:gLPT:vyhplqrwnH")) != -1) {
	switch (c) {
	case 'c':
	if (cmd != NULL) {
	fprintf(stderr,
	gettext("Can't set -c flag twice\n"));
	exit(1);
	}

	if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
	!libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
	fprintf(stderr, gettext(
	"Can't run -c, disabled by "
	"ZPOOL_SCRIPTS_ENABLED.\n"));
	exit(1);
	}

	if ((getuid() <= 0 \|\| geteuid() <= 0) &&
	!libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
	fprintf(stderr, gettext(
	"Can't run -c with root privileges "
	"unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
	exit(1);
	}
	cmd = optarg;
	verbose = B_TRUE;
	break;
	case 'g':
	guid = B_TRUE;
	break;
	case 'L':
	follow_links = B_TRUE;
	break;
	case 'P':
	full_name = B_TRUE;
	break;
	case 'T':
	get_timestamp_arg(*optarg);
	break;
	case 'v':
	verbose = B_TRUE;
	break;
	case 'p':
	parsable = B_TRUE;
	break;
	case 'l':
	latency = B_TRUE;
	break;
	case 'q':
	queues = B_TRUE;
	break;
	case 'H':
	scripted = B_TRUE;
	break;
	case 'w':
	l_histo = B_TRUE;
	break;
	case 'r':
	rq_histo = B_TRUE;
	break;
	case 'y':
	omit_since_boot = B_TRUE;
	break;
	case 'n':
	headers_once = B_TRUE;
	break;
	case 'h':
	usage(B_FALSE);
	break;
	case '?':
	if (optopt == 'c') {
	print_zpool_script_list("iostat");
	exit(0);
	} else {
	fprintf(stderr,
	gettext("invalid option '%c'\n"), optopt);
	}
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	cb.cb_literal = parsable;
	cb.cb_scripted = scripted;

	if (guid)
	cb.cb_vdevs.cb_name_flags \|= VDEV_NAME_GUID;
	if (follow_links)
	cb.cb_vdevs.cb_name_flags \|= VDEV_NAME_FOLLOW_LINKS;
	if (full_name)
	cb.cb_vdevs.cb_name_flags \|= VDEV_NAME_PATH;
	cb.cb_iteration = 0;
	cb.cb_namewidth = 0;
	cb.cb_verbose = verbose;

	/* Get our interval and count values (if any) */
	if (guid) {
	get_interval_count_filter_guids(&argc, argv, &interval,
	&count, &cb);
	} else {
	get_interval_count(&argc, argv, &interval, &count);
	}

	if (argc == 0) {
	/* No args, so just print the defaults. */
	} else if (are_all_pools(argc, argv)) {
	/* All the args are pool names */
	} else if (are_vdevs_in_pool(argc, argv, NULL, &cb.cb_vdevs)) {
	/* All the args are vdevs */
	cb.cb_vdevs.cb_names = argv;
	cb.cb_vdevs.cb_names_count = argc;
	argc = 0; /* No pools to process */
	} else if (are_all_pools(1, argv)) {
	/* The first arg is a pool name */
	if (are_vdevs_in_pool(argc - 1, argv + 1, argv[0],
	&cb.cb_vdevs)) {
	/* ...and the rest are vdev names */
	cb.cb_vdevs.cb_names = argv + 1;
	cb.cb_vdevs.cb_names_count = argc - 1;
	argc = 1; /* One pool to process */
	} else {
	fprintf(stderr, gettext("Expected either a list of "));
	fprintf(stderr, gettext("pools, or list of vdevs in"));
	fprintf(stderr, " \"%s\", ", argv[0]);
	fprintf(stderr, gettext("but got:\n"));
	error_list_unresolved_vdevs(argc - 1, argv + 1,
	argv[0], &cb.cb_vdevs);
	fprintf(stderr, "\n");
	usage(B_FALSE);
	return (1);
	}
	} else {
	/*
	* The args don't make sense. The first arg isn't a pool name,
	* nor are all the args vdevs.
	*/
	fprintf(stderr, gettext("Unable to parse pools/vdevs list.\n"));
	fprintf(stderr, "\n");
	return (1);
	}

	if (cb.cb_vdevs.cb_names_count != 0) {
	/*
	* If user specified vdevs, it implies verbose.
	*/
	cb.cb_verbose = B_TRUE;
	}

	/*
	* Construct the list of all interesting pools.
	*/
	ret = 0;
	if ((list = pool_list_get(argc, argv, NULL, ZFS_TYPE_POOL, parsable,
	&ret)) == NULL)
	return (1);

	if (pool_list_count(list) == 0 && argc != 0) {
	pool_list_free(list);
	return (1);
	}

	if (pool_list_count(list) == 0 && interval == 0) {
	pool_list_free(list);
	(void) fprintf(stderr, gettext("no pools available\n"));
	return (1);
	}

	if ((l_histo \|\| rq_histo) && (cmd != NULL \|\| latency \|\| queues)) {
	pool_list_free(list);
	(void) fprintf(stderr,
	gettext("[-r\|-w] isn't allowed with [-c\|-l\|-q]\n"));
	usage(B_FALSE);
	return (1);
	}

	if (l_histo && rq_histo) {
	pool_list_free(list);
	(void) fprintf(stderr,
	gettext("Only one of [-r\|-w] can be passed at a time\n"));
	usage(B_FALSE);
	return (1);
	}

	/*
	* Enter the main iostat loop.
	*/
	cb.cb_list = list;

	if (l_histo) {
	/*
	* Histograms tables look out of place when you try to display
	* them with the other stats, so make a rule that you can only
	* print histograms by themselves.
	*/
	cb.cb_flags = IOS_L_HISTO_M;
	} else if (rq_histo) {
	cb.cb_flags = IOS_RQ_HISTO_M;
	} else {
	cb.cb_flags = IOS_DEFAULT_M;
	if (latency)
	cb.cb_flags \|= IOS_LATENCY_M;
	if (queues)
	cb.cb_flags \|= IOS_QUEUES_M;
	}

	/*
	* See if the module supports all the stats we want to display.
	*/
	unsupported_flags = cb.cb_flags & ~get_stat_flags(list);
	if (unsupported_flags) {
	uint64_t f;
	int idx;
	fprintf(stderr,
	gettext("The loaded zfs module doesn't support:"));

	/* for each bit set in unsupported_flags */
	for (f = unsupported_flags; f; f &= ~(1ULL << idx)) {
	idx = lowbit64(f) - 1;
	fprintf(stderr, " -%c", flag_to_arg[idx]);
	}

	fprintf(stderr, ". Try running a newer module.\n");
	pool_list_free(list);

	return (1);
	}

	for (;;) {
	if ((npools = pool_list_count(list)) == 0)
	(void) fprintf(stderr, gettext("no pools available\n"));
	else {
	/*
	* If this is the first iteration and -y was supplied
	* we skip any printing.
	*/
	boolean_t skip = (omit_since_boot &&
	cb.cb_iteration == 0);

	/*
	* Refresh all statistics. This is done as an
	* explicit step before calculating the maximum name
	* width, so that any * configuration changes are
	* properly accounted for.
	*/
	(void) pool_list_iter(list, B_FALSE, refresh_iostat,
	&cb);

	/*
	* Iterate over all pools to determine the maximum width
	* for the pool / device name column across all pools.
	*/
	cb.cb_namewidth = 0;
	(void) pool_list_iter(list, B_FALSE,
	get_namewidth_iostat, &cb);

	if (timestamp_fmt != NODATE)
	print_timestamp(timestamp_fmt);

	if (cmd != NULL && cb.cb_verbose &&
	!(cb.cb_flags & IOS_ANYHISTO_M)) {
	cb.vcdl = all_pools_for_each_vdev_run(argc,
	argv, cmd, g_zfs, cb.cb_vdevs.cb_names,
	cb.cb_vdevs.cb_names_count,
	cb.cb_vdevs.cb_name_flags);
	} else {
	cb.vcdl = NULL;
	}


	/*
	* Check terminal size so we can print headers
	* even when terminal window has its height
	* changed.
	*/
	winheight = terminal_height();
	/*
	* Are we connected to TTY? If not, headers_once
	* should be true, to avoid breaking scripts.
	*/
	if (winheight < 0)
	headers_once = B_TRUE;

	/*
	* If it's the first time and we're not skipping it,
	* or either skip or verbose mode, print the header.
	*
	* The histogram code explicitly prints its header on
	* every vdev, so skip this for histograms.
	*/
	if (((++cb.cb_iteration == 1 && !skip) \|\|
	(skip != verbose) \|\|
	(!headers_once &&
	(cb.cb_iteration % winheight) == 0)) &&
	(!(cb.cb_flags & IOS_ANYHISTO_M)) &&
	!cb.cb_scripted)
	print_iostat_header(&cb);

	if (skip) {
	(void) fsleep(interval);
	continue;
	}

	pool_list_iter(list, B_FALSE, print_iostat, &cb);

	/*
	* If there's more than one pool, and we're not in
	* verbose mode (which prints a separator for us),
	* then print a separator.
	*
	* In addition, if we're printing specific vdevs then
	* we also want an ending separator.
	*/
	if (((npools > 1 && !verbose &&
	!(cb.cb_flags & IOS_ANYHISTO_M)) \|\|
	(!(cb.cb_flags & IOS_ANYHISTO_M) &&
	cb.cb_vdevs.cb_names_count)) &&
	!cb.cb_scripted) {
	print_iostat_separator(&cb);
	if (cb.vcdl != NULL)
	print_cmd_columns(cb.vcdl, 1);
	printf("\n");
	}

	if (cb.vcdl != NULL)
	free_vdev_cmd_data_list(cb.vcdl);

	}

	/*
	* Flush the output so that redirection to a file isn't buffered
	* indefinitely.
	*/
	(void) fflush(stdout);

	if (interval == 0)
	break;

	if (count != 0 && --count == 0)
	break;

	(void) fsleep(interval);
	}

	pool_list_free(list);

	return (ret);
	}

	typedef struct list_cbdata {
	boolean_t cb_verbose;
	int cb_name_flags;
	int cb_namewidth;
	boolean_t cb_scripted;
	zprop_list_t *cb_proplist;
	boolean_t cb_literal;
	} list_cbdata_t;


	/*
	* Given a list of columns to display, output appropriate headers for each one.
	*/
	static void
	print_header(list_cbdata_t *cb)
	{
	zprop_list_t *pl = cb->cb_proplist;
	char headerbuf[ZPOOL_MAXPROPLEN];
	const char *header;
	boolean_t first = B_TRUE;
	boolean_t right_justify;
	size_t width = 0;

	for (; pl != NULL; pl = pl->pl_next) {
	width = pl->pl_width;
	if (first && cb->cb_verbose) {
	/*
	* Reset the width to accommodate the verbose listing
	* of devices.
	*/
	width = cb->cb_namewidth;
	}

	if (!first)
	(void) fputs(" ", stdout);
	else
	first = B_FALSE;

	right_justify = B_FALSE;
	if (pl->pl_prop != ZPROP_USERPROP) {
	header = zpool_prop_column_name(pl->pl_prop);
	right_justify = zpool_prop_align_right(pl->pl_prop);
	} else {
	int i;

	for (i = 0; pl->pl_user_prop[i] != '\0'; i++)
	headerbuf[i] = toupper(pl->pl_user_prop[i]);
	headerbuf[i] = '\0';
	header = headerbuf;
	}

	if (pl->pl_next == NULL && !right_justify)
	(void) fputs(header, stdout);
	else if (right_justify)
	(void) printf("%*s", (int)width, header);
	else
	(void) printf("%-*s", (int)width, header);
	}

	(void) fputc('\n', stdout);
	}

	/*
	* Given a pool and a list of properties, print out all the properties according
	* to the described layout. Used by zpool_do_list().
	*/
	static void
	print_pool(zpool_handle_t zhp, list_cbdata_t cb)
	{
	zprop_list_t *pl = cb->cb_proplist;
	boolean_t first = B_TRUE;
	char property[ZPOOL_MAXPROPLEN];
	const char *propstr;
	boolean_t right_justify;
	size_t width;

	for (; pl != NULL; pl = pl->pl_next) {

	width = pl->pl_width;
	if (first && cb->cb_verbose) {
	/*
	* Reset the width to accommodate the verbose listing
	* of devices.
	*/
	width = cb->cb_namewidth;
	}

	if (!first) {
	if (cb->cb_scripted)
	(void) fputc('\t', stdout);
	else
	(void) fputs(" ", stdout);
	} else {
	first = B_FALSE;
	}

	right_justify = B_FALSE;
	if (pl->pl_prop != ZPROP_USERPROP) {
	if (zpool_get_prop(zhp, pl->pl_prop, property,
	sizeof (property), NULL, cb->cb_literal) != 0)
	propstr = "-";
	else
	propstr = property;

	right_justify = zpool_prop_align_right(pl->pl_prop);
	} else if ((zpool_prop_feature(pl->pl_user_prop) \|\|
	zpool_prop_unsupported(pl->pl_user_prop)) &&
	zpool_prop_get_feature(zhp, pl->pl_user_prop, property,
	sizeof (property)) == 0) {
	propstr = property;
	} else if (zfs_prop_user(pl->pl_user_prop) &&
	zpool_get_userprop(zhp, pl->pl_user_prop, property,
	sizeof (property), NULL) == 0) {
	propstr = property;
	} else {
	propstr = "-";
	}

	/*
	* If this is being called in scripted mode, or if this is the
	* last column and it is left-justified, don't include a width
	* format specifier.
	*/
	if (cb->cb_scripted \|\| (pl->pl_next == NULL && !right_justify))
	(void) fputs(propstr, stdout);
	else if (right_justify)
	(void) printf("%*s", (int)width, propstr);
	else
	(void) printf("%-*s", (int)width, propstr);
	}

	(void) fputc('\n', stdout);
	}

	static void
	print_one_column(zpool_prop_t prop, uint64_t value, const char *str,
	boolean_t scripted, boolean_t valid, enum zfs_nicenum_format format)
	{
	char propval[64];
	boolean_t fixed;
	size_t width = zprop_width(prop, &fixed, ZFS_TYPE_POOL);

	switch (prop) {
	case ZPOOL_PROP_SIZE:
	case ZPOOL_PROP_EXPANDSZ:
	case ZPOOL_PROP_CHECKPOINT:
	case ZPOOL_PROP_DEDUPRATIO:
	if (value == 0)
	(void) strlcpy(propval, "-", sizeof (propval));
	else
	zfs_nicenum_format(value, propval, sizeof (propval),
	format);
	break;
	case ZPOOL_PROP_FRAGMENTATION:
	if (value == ZFS_FRAG_INVALID) {
	(void) strlcpy(propval, "-", sizeof (propval));
	} else if (format == ZFS_NICENUM_RAW) {
	(void) snprintf(propval, sizeof (propval), "%llu",
	(unsigned long long)value);
	} else {
	(void) snprintf(propval, sizeof (propval), "%llu%%",
	(unsigned long long)value);
	}
	break;
	case ZPOOL_PROP_CAPACITY:
	/* capacity value is in parts-per-10,000 (aka permyriad) */
	if (format == ZFS_NICENUM_RAW)
	(void) snprintf(propval, sizeof (propval), "%llu",
	(unsigned long long)value / 100);
	else
	(void) snprintf(propval, sizeof (propval),
	value < 1000 ? "%1.2f%%" : value < 10000 ?
	"%2.1f%%" : "%3.0f%%", value / 100.0);
	break;
	case ZPOOL_PROP_HEALTH:
	width = 8;
	(void) strlcpy(propval, str, sizeof (propval));
	break;
	default:
	zfs_nicenum_format(value, propval, sizeof (propval), format);
	}

	if (!valid)
	(void) strlcpy(propval, "-", sizeof (propval));

	if (scripted)
	(void) printf("\t%s", propval);
	else
	(void) printf(" %*s", (int)width, propval);
	}

	/*
	* print static default line per vdev
	* not compatible with '-o' <proplist> option
	*/
	static void
	print_list_stats(zpool_handle_t zhp, const char name, nvlist_t *nv,
	list_cbdata_t *cb, int depth, boolean_t isspare)
	{
	nvlist_t **child;
	vdev_stat_t *vs;
	uint_t c, children;
	char *vname;
	boolean_t scripted = cb->cb_scripted;
	uint64_t islog = B_FALSE;
	const char dashes = "%-s - - - - "
	"- - - - -\n";

	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
	(uint64_t **)&vs, &c) == 0);

	if (name != NULL) {
	boolean_t toplevel = (vs->vs_space != 0);
	uint64_t cap;
	enum zfs_nicenum_format format;
	const char *state;

	if (cb->cb_literal)
	format = ZFS_NICENUM_RAW;
	else
	format = ZFS_NICENUM_1024;

	if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
	return;

	if (scripted)
	(void) printf("\t%s", name);
	else if (strlen(name) + depth > cb->cb_namewidth)
	(void) printf("%*s%s", depth, "", name);
	else
	(void) printf("%s%s%s", depth, "", name,
	(int)(cb->cb_namewidth - strlen(name) - depth), "");

	/*
	* Print the properties for the individual vdevs. Some
	* properties are only applicable to toplevel vdevs. The
	* 'toplevel' boolean value is passed to the print_one_column()
	* to indicate that the value is valid.
	*/
	if (VDEV_STAT_VALID(vs_pspace, c) && vs->vs_pspace)
	print_one_column(ZPOOL_PROP_SIZE, vs->vs_pspace, NULL,
	scripted, B_TRUE, format);
	else
	print_one_column(ZPOOL_PROP_SIZE, vs->vs_space, NULL,
	scripted, toplevel, format);
	print_one_column(ZPOOL_PROP_ALLOCATED, vs->vs_alloc, NULL,
	scripted, toplevel, format);
	print_one_column(ZPOOL_PROP_FREE, vs->vs_space - vs->vs_alloc,
	NULL, scripted, toplevel, format);
	print_one_column(ZPOOL_PROP_CHECKPOINT,
	vs->vs_checkpoint_space, NULL, scripted, toplevel, format);
	print_one_column(ZPOOL_PROP_EXPANDSZ, vs->vs_esize, NULL,
	scripted, B_TRUE, format);
	print_one_column(ZPOOL_PROP_FRAGMENTATION,
	vs->vs_fragmentation, NULL, scripted,
	(vs->vs_fragmentation != ZFS_FRAG_INVALID && toplevel),
	format);
	cap = (vs->vs_space == 0) ? 0 :
	(vs->vs_alloc * 10000 / vs->vs_space);
	print_one_column(ZPOOL_PROP_CAPACITY, cap, NULL,
	scripted, toplevel, format);
	print_one_column(ZPOOL_PROP_DEDUPRATIO, 0, NULL,
	scripted, toplevel, format);
	state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
	if (isspare) {
	if (vs->vs_aux == VDEV_AUX_SPARED)
	state = "INUSE";
	else if (vs->vs_state == VDEV_STATE_HEALTHY)
	state = "AVAIL";
	}
	print_one_column(ZPOOL_PROP_HEALTH, 0, state, scripted,
	B_TRUE, format);
	(void) fputc('\n', stdout);
	}

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
	&child, &children) != 0)
	return;

	/* list the normal vdevs first */
	for (c = 0; c < children; c++) {
	uint64_t ishole = B_FALSE;

	if (nvlist_lookup_uint64(child[c],
	ZPOOL_CONFIG_IS_HOLE, &ishole) == 0 && ishole)
	continue;

	if (nvlist_lookup_uint64(child[c],
	ZPOOL_CONFIG_IS_LOG, &islog) == 0 && islog)
	continue;

	if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
	continue;

	vname = zpool_vdev_name(g_zfs, zhp, child[c],
	cb->cb_name_flags \| VDEV_NAME_TYPE_ID);
	print_list_stats(zhp, vname, child[c], cb, depth + 2, B_FALSE);
	free(vname);
	}

	/* list the classes: 'logs', 'dedup', and 'special' */
	for (uint_t n = 0; n < ARRAY_SIZE(class_name); n++) {
	boolean_t printed = B_FALSE;

	for (c = 0; c < children; c++) {
	const char *bias = NULL;
	const char *type = NULL;

	if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
	&islog) == 0 && islog) {
	bias = VDEV_ALLOC_CLASS_LOGS;
	} else {
	(void) nvlist_lookup_string(child[c],
	ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
	(void) nvlist_lookup_string(child[c],
	ZPOOL_CONFIG_TYPE, &type);
	}
	if (bias == NULL \|\| strcmp(bias, class_name[n]) != 0)
	continue;
	if (!islog && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
	continue;

	if (!printed) {
	/* LINTED E_SEC_PRINTF_VAR_FMT */
	(void) printf(dashes, cb->cb_namewidth,
	class_name[n]);
	printed = B_TRUE;
	}
	vname = zpool_vdev_name(g_zfs, zhp, child[c],
	cb->cb_name_flags \| VDEV_NAME_TYPE_ID);
	print_list_stats(zhp, vname, child[c], cb, depth + 2,
	B_FALSE);
	free(vname);
	}
	}

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
	&child, &children) == 0 && children > 0) {
	/* LINTED E_SEC_PRINTF_VAR_FMT */
	(void) printf(dashes, cb->cb_namewidth, "cache");
	for (c = 0; c < children; c++) {
	vname = zpool_vdev_name(g_zfs, zhp, child[c],
	cb->cb_name_flags);
	print_list_stats(zhp, vname, child[c], cb, depth + 2,
	B_FALSE);
	free(vname);
	}
	}

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES, &child,
	&children) == 0 && children > 0) {
	/* LINTED E_SEC_PRINTF_VAR_FMT */
	(void) printf(dashes, cb->cb_namewidth, "spare");
	for (c = 0; c < children; c++) {
	vname = zpool_vdev_name(g_zfs, zhp, child[c],
	cb->cb_name_flags);
	print_list_stats(zhp, vname, child[c], cb, depth + 2,
	B_TRUE);
	free(vname);
	}
	}
	}

	/*
	* Generic callback function to list a pool.
	*/
	static int
	list_callback(zpool_handle_t zhp, void data)
	{
	list_cbdata_t *cbp = data;

	print_pool(zhp, cbp);

	if (cbp->cb_verbose) {
	nvlist_t config, nvroot;

	config = zpool_get_config(zhp, NULL);
	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
	&nvroot) == 0);
	print_list_stats(zhp, NULL, nvroot, cbp, 0, B_FALSE);
	}

	return (0);
	}

	/*
	* Set the minimum pool/vdev name column width. The width must be at least 9,
	* but may be as large as needed.
	*/
	static int
	get_namewidth_list(zpool_handle_t zhp, void data)
	{
	list_cbdata_t *cb = data;
	int width;

	width = get_namewidth(zhp, cb->cb_namewidth,
	cb->cb_name_flags \| VDEV_NAME_TYPE_ID, cb->cb_verbose);

	if (width < 9)
	width = 9;

	cb->cb_namewidth = width;

	return (0);
	}

	/*
	* zpool list [-gHLpP] [-o prop[,prop]*] [-T d\|u] [pool] ... [interval [count]]
	*
	* -g Display guid for individual vdev name.
	* -H Scripted mode. Don't display headers, and separate properties
	* by a single tab.
	* -L Follow links when resolving vdev path name.
	* -o List of properties to display. Defaults to
	* "name,size,allocated,free,expandsize,fragmentation,capacity,"
	* "dedupratio,health,altroot"
	* -p Display values in parsable (exact) format.
	* -P Display full path for vdev name.
	* -T Display a timestamp in date(1) or Unix format
	*
	* List all pools in the system, whether or not they're healthy. Output space
	* statistics for each one, as well as health status summary.
	*/
	int
	zpool_do_list(int argc, char **argv)
	{
	int c;
	int ret = 0;
	list_cbdata_t cb = { 0 };
	static char default_props[] =
	"name,size,allocated,free,checkpoint,expandsize,fragmentation,"
	"capacity,dedupratio,health,altroot";
	char *props = default_props;
	float interval = 0;
	unsigned long count = 0;
	zpool_list_t *list;
	boolean_t first = B_TRUE;
	current_prop_type = ZFS_TYPE_POOL;

	/* check options */
	while ((c = getopt(argc, argv, ":gHLo:pPT:v")) != -1) {
	switch (c) {
	case 'g':
	cb.cb_name_flags \|= VDEV_NAME_GUID;
	break;
	case 'H':
	cb.cb_scripted = B_TRUE;
	break;
	case 'L':
	cb.cb_name_flags \|= VDEV_NAME_FOLLOW_LINKS;
	break;
	case 'o':
	props = optarg;
	break;
	case 'P':
	cb.cb_name_flags \|= VDEV_NAME_PATH;
	break;
	case 'p':
	cb.cb_literal = B_TRUE;
	break;
	case 'T':
	get_timestamp_arg(*optarg);
	break;
	case 'v':
	cb.cb_verbose = B_TRUE;
	cb.cb_namewidth = 8; /* 8 until precalc is avail */
	break;
	case ':':
	(void) fprintf(stderr, gettext("missing argument for "
	"'%c' option\n"), optopt);
	usage(B_FALSE);
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	get_interval_count(&argc, argv, &interval, &count);

	if (zprop_get_list(g_zfs, props, &cb.cb_proplist, ZFS_TYPE_POOL) != 0)
	usage(B_FALSE);

	for (;;) {
	if ((list = pool_list_get(argc, argv, &cb.cb_proplist,
	ZFS_TYPE_POOL, cb.cb_literal, &ret)) == NULL)
	return (1);

	if (pool_list_count(list) == 0)
	break;

	cb.cb_namewidth = 0;
	(void) pool_list_iter(list, B_FALSE, get_namewidth_list, &cb);

	if (timestamp_fmt != NODATE)
	print_timestamp(timestamp_fmt);

	if (!cb.cb_scripted && (first \|\| cb.cb_verbose)) {
	print_header(&cb);
	first = B_FALSE;
	}
	ret = pool_list_iter(list, B_TRUE, list_callback, &cb);

	if (interval == 0)
	break;

	if (count != 0 && --count == 0)
	break;

	pool_list_free(list);
	(void) fsleep(interval);
	}

	if (argc == 0 && !cb.cb_scripted && pool_list_count(list) == 0) {
	(void) printf(gettext("no pools available\n"));
	ret = 0;
	}

	pool_list_free(list);
	zprop_free_list(cb.cb_proplist);
	return (ret);
	}

	static int
	zpool_do_attach_or_replace(int argc, char **argv, int replacing)
	{
	boolean_t force = B_FALSE;
	boolean_t rebuild = B_FALSE;
	boolean_t wait = B_FALSE;
	int c;
	nvlist_t *nvroot;
	char poolname, old_disk, *new_disk;
	zpool_handle_t *zhp;
	nvlist_t *props = NULL;
	char *propval;
	int ret;

	/* check options */
	while ((c = getopt(argc, argv, "fo:sw")) != -1) {
	switch (c) {
	case 'f':
	force = B_TRUE;
	break;
	case 'o':
	if ((propval = strchr(optarg, '=')) == NULL) {
	(void) fprintf(stderr, gettext("missing "
	"'=' for -o option\n"));
	usage(B_FALSE);
	}
	*propval = '\0';
	propval++;

	if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) \|\|
	(add_prop_list(optarg, propval, &props, B_TRUE)))
	usage(B_FALSE);
	break;
	case 's':
	rebuild = B_TRUE;
	break;
	case 'w':
	wait = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* get pool name and check number of arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing pool name argument\n"));
	usage(B_FALSE);
	}

	poolname = argv[0];

	if (argc < 2) {
	(void) fprintf(stderr,
	gettext("missing <device> specification\n"));
	usage(B_FALSE);
	}

	old_disk = argv[1];

	if (argc < 3) {
	if (!replacing) {
	(void) fprintf(stderr,
	gettext("missing <new_device> specification\n"));
	usage(B_FALSE);
	}
	new_disk = old_disk;
	argc -= 1;
	argv += 1;
	} else {
	new_disk = argv[2];
	argc -= 2;
	argv += 2;
	}

	if (argc > 1) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	if ((zhp = zpool_open(g_zfs, poolname)) == NULL) {
	nvlist_free(props);
	return (1);
	}

	if (zpool_get_config(zhp, NULL) == NULL) {
	(void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
	poolname);
	zpool_close(zhp);
	nvlist_free(props);
	return (1);
	}

	/* unless manually specified use "ashift" pool property (if set) */
	if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
	int intval;
	zprop_source_t src;
	char strval[ZPOOL_MAXPROPLEN];

	intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
	if (src != ZPROP_SRC_DEFAULT) {
	(void) sprintf(strval, "%" PRId32, intval);
	verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
	&props, B_TRUE) == 0);
	}
	}

	nvroot = make_root_vdev(zhp, props, force, B_FALSE, replacing, B_FALSE,
	argc, argv);
	if (nvroot == NULL) {
	zpool_close(zhp);
	nvlist_free(props);
	return (1);
	}

	ret = zpool_vdev_attach(zhp, old_disk, new_disk, nvroot, replacing,
	rebuild);

	if (ret == 0 && wait)
	ret = zpool_wait(zhp,
	replacing ? ZPOOL_WAIT_REPLACE : ZPOOL_WAIT_RESILVER);

	nvlist_free(props);
	nvlist_free(nvroot);
	zpool_close(zhp);

	return (ret);
	}

	/*
	* zpool replace [-fsw] [-o property=value] <pool> <device> <new_device>
	*
	* -f Force attach, even if <new_device> appears to be in use.
	* -s Use sequential instead of healing reconstruction for resilver.
	* -o Set property=value.
	* -w Wait for replacing to complete before returning
	*
	* Replace <device> with <new_device>.
	*/
	int
	zpool_do_replace(int argc, char **argv)
	{
	return (zpool_do_attach_or_replace(argc, argv, B_TRUE));
	}

	/*
	* zpool attach [-fsw] [-o property=value] <pool> <device> <new_device>
	*
	* -f Force attach, even if <new_device> appears to be in use.
	* -s Use sequential instead of healing reconstruction for resilver.
	* -o Set property=value.
	* -w Wait for resilvering to complete before returning
	*
	* Attach <new_device> to the mirror containing <device>. If <device> is not
	* part of a mirror, then <device> will be transformed into a mirror of
	* <device> and <new_device>. In either case, <new_device> will begin life
	* with a DTL of [0, now], and will immediately begin to resilver itself.
	*/
	int
	zpool_do_attach(int argc, char **argv)
	{
	return (zpool_do_attach_or_replace(argc, argv, B_FALSE));
	}

	/*
	* zpool detach [-f] <pool> <device>
	*
	* -f Force detach of <device>, even if DTLs argue against it
	* (not supported yet)
	*
	* Detach a device from a mirror. The operation will be refused if <device>
	* is the last device in the mirror, or if the DTLs indicate that this device
	* has the only valid copy of some data.
	*/
	int
	zpool_do_detach(int argc, char **argv)
	{
	int c;
	char poolname, path;
	zpool_handle_t *zhp;
	int ret;

	/* check options */
	while ((c = getopt(argc, argv, "")) != -1) {
	switch (c) {
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* get pool name and check number of arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing pool name argument\n"));
	usage(B_FALSE);
	}

	if (argc < 2) {
	(void) fprintf(stderr,
	gettext("missing <device> specification\n"));
	usage(B_FALSE);
	}

	poolname = argv[0];
	path = argv[1];

	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
	return (1);

	ret = zpool_vdev_detach(zhp, path);

	zpool_close(zhp);

	return (ret);
	}

	/*
	* zpool split [-gLnP] [-o prop=val] ...
	* [-o mntopt] ...
	* [-R altroot] <pool> <newpool> [<device> ...]
	*
	* -g Display guid for individual vdev name.
	* -L Follow links when resolving vdev path name.
	* -n Do not split the pool, but display the resulting layout if
	* it were to be split.
	* -o Set property=value, or set mount options.
	* -P Display full path for vdev name.
	* -R Mount the split-off pool under an alternate root.
	* -l Load encryption keys while importing.
	*
	* Splits the named pool and gives it the new pool name. Devices to be split
	* off may be listed, provided that no more than one device is specified
	* per top-level vdev mirror. The newly split pool is left in an exported
	* state unless -R is specified.
	*
	* Restrictions: the top-level of the pool pool must only be made up of
	* mirrors; all devices in the pool must be healthy; no device may be
	* undergoing a resilvering operation.
	*/
	int
	zpool_do_split(int argc, char **argv)
	{
	char srcpool, newpool, *propval;
	char *mntopts = NULL;
	splitflags_t flags;
	int c, ret = 0;
	int ms_status = 0;
	boolean_t loadkeys = B_FALSE;
	zpool_handle_t *zhp;
	nvlist_t config, props = NULL;

	flags.dryrun = B_FALSE;
	flags.import = B_FALSE;
	flags.name_flags = 0;

	/* check options */
	while ((c = getopt(argc, argv, ":gLR:lno:P")) != -1) {
	switch (c) {
	case 'g':
	flags.name_flags \|= VDEV_NAME_GUID;
	break;
	case 'L':
	flags.name_flags \|= VDEV_NAME_FOLLOW_LINKS;
	break;
	case 'R':
	flags.import = B_TRUE;
	if (add_prop_list(
	zpool_prop_to_name(ZPOOL_PROP_ALTROOT), optarg,
	&props, B_TRUE) != 0) {
	nvlist_free(props);
	usage(B_FALSE);
	}
	break;
	case 'l':
	loadkeys = B_TRUE;
	break;
	case 'n':
	flags.dryrun = B_TRUE;
	break;
	case 'o':
	if ((propval = strchr(optarg, '=')) != NULL) {
	*propval = '\0';
	propval++;
	if (add_prop_list(optarg, propval,
	&props, B_TRUE) != 0) {
	nvlist_free(props);
	usage(B_FALSE);
	}
	} else {
	mntopts = optarg;
	}
	break;
	case 'P':
	flags.name_flags \|= VDEV_NAME_PATH;
	break;
	case ':':
	(void) fprintf(stderr, gettext("missing argument for "
	"'%c' option\n"), optopt);
	usage(B_FALSE);
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	break;
	}
	}

	if (!flags.import && mntopts != NULL) {
	(void) fprintf(stderr, gettext("setting mntopts is only "
	"valid when importing the pool\n"));
	usage(B_FALSE);
	}

	if (!flags.import && loadkeys) {
	(void) fprintf(stderr, gettext("loading keys is only "
	"valid when importing the pool\n"));
	usage(B_FALSE);
	}

	argc -= optind;
	argv += optind;

	if (argc < 1) {
	(void) fprintf(stderr, gettext("Missing pool name\n"));
	usage(B_FALSE);
	}
	if (argc < 2) {
	(void) fprintf(stderr, gettext("Missing new pool name\n"));
	usage(B_FALSE);
	}

	srcpool = argv[0];
	newpool = argv[1];

	argc -= 2;
	argv += 2;

	if ((zhp = zpool_open(g_zfs, srcpool)) == NULL) {
	nvlist_free(props);
	return (1);
	}

	config = split_mirror_vdev(zhp, newpool, props, flags, argc, argv);
	if (config == NULL) {
	ret = 1;
	} else {
	if (flags.dryrun) {
	(void) printf(gettext("would create '%s' with the "
	"following layout:\n\n"), newpool);
	print_vdev_tree(NULL, newpool, config, 0, "",
	flags.name_flags);
	print_vdev_tree(NULL, "dedup", config, 0,
	VDEV_ALLOC_BIAS_DEDUP, 0);
	print_vdev_tree(NULL, "special", config, 0,
	VDEV_ALLOC_BIAS_SPECIAL, 0);
	}
	}

	zpool_close(zhp);

	if (ret != 0 \|\| flags.dryrun \|\| !flags.import) {
	nvlist_free(config);
	nvlist_free(props);
	return (ret);
	}

	/*
	* The split was successful. Now we need to open the new
	* pool and import it.
	*/
	if ((zhp = zpool_open_canfail(g_zfs, newpool)) == NULL) {
	nvlist_free(config);
	nvlist_free(props);
	return (1);
	}

	if (loadkeys) {
	ret = zfs_crypto_attempt_load_keys(g_zfs, newpool);
	if (ret != 0)
	ret = 1;
	}

	if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL) {
	ms_status = zpool_enable_datasets(zhp, mntopts, 0);
	if (ms_status == EZFS_SHAREFAILED) {
	(void) fprintf(stderr, gettext("Split was successful, "
	"datasets are mounted but sharing of some datasets "
	"has failed\n"));
	} else if (ms_status == EZFS_MOUNTFAILED) {
	(void) fprintf(stderr, gettext("Split was successful"
	", but some datasets could not be mounted\n"));
	(void) fprintf(stderr, gettext("Try doing '%s' with a "
	"different altroot\n"), "zpool import");
	}
	}
	zpool_close(zhp);
	nvlist_free(config);
	nvlist_free(props);

	return (ret);
	}

	-#define POWER_OPT 1024

	/*
	* zpool online [--power] <pool> <device> ...
	*
	* --power: Power on the enclosure slot to the drive (if possible)
	*/
	int
	zpool_do_online(int argc, char **argv)
	{
	int c, i;
	char *poolname;
	zpool_handle_t *zhp;
	int ret = 0;
	vdev_state_t newstate;
	int flags = 0;
	boolean_t is_power_on = B_FALSE;
	struct option long_options[] = {
	- {"power", no_argument, NULL, POWER_OPT},
	+ {"power", no_argument, NULL, ZPOOL_OPTION_POWER},
	{0, 0, 0, 0}
	};

	/* check options */
	while ((c = getopt_long(argc, argv, "e", long_options, NULL)) != -1) {
	switch (c) {
	case 'e':
	flags \|= ZFS_ONLINE_EXPAND;
	break;
	- case POWER_OPT:
	+ case ZPOOL_OPTION_POWER:
	is_power_on = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	if (libzfs_envvar_is_set("ZPOOL_AUTO_POWER_ON_SLOT"))
	is_power_on = B_TRUE;

	argc -= optind;
	argv += optind;

	/* get pool name and check number of arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing pool name\n"));
	usage(B_FALSE);
	}
	if (argc < 2) {
	(void) fprintf(stderr, gettext("missing device name\n"));
	usage(B_FALSE);
	}

	poolname = argv[0];

	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
	return (1);

	for (i = 1; i < argc; i++) {
	vdev_state_t oldstate;
	boolean_t avail_spare, l2cache;
	int rc;

	if (is_power_on) {
	rc = zpool_power_on_and_disk_wait(zhp, argv[i]);
	if (rc == ENOTSUP) {
	(void) fprintf(stderr,
	gettext("Power control not supported\n"));
	}
	if (rc != 0)
	return (rc);
	}

	nvlist_t *tgt = zpool_find_vdev(zhp, argv[i], &avail_spare,
	&l2cache, NULL);
	if (tgt == NULL) {
	ret = 1;
	continue;
	}
	uint_t vsc;
	oldstate = ((vdev_stat_t *)fnvlist_lookup_uint64_array(tgt,
	ZPOOL_CONFIG_VDEV_STATS, &vsc))->vs_state;
	if (zpool_vdev_online(zhp, argv[i], flags, &newstate) == 0) {
	if (newstate != VDEV_STATE_HEALTHY) {
	(void) printf(gettext("warning: device '%s' "
	"onlined, but remains in faulted state\n"),
	argv[i]);
	if (newstate == VDEV_STATE_FAULTED)
	(void) printf(gettext("use 'zpool "
	"clear' to restore a faulted "
	"device\n"));
	else
	(void) printf(gettext("use 'zpool "
	"replace' to replace devices "
	"that are no longer present\n"));
	if ((flags & ZFS_ONLINE_EXPAND)) {
	(void) printf(gettext("%s: failed "
	"to expand usable space on "
	"unhealthy device '%s'\n"),
	(oldstate >= VDEV_STATE_DEGRADED ?
	"error" : "warning"), argv[i]);
	if (oldstate >= VDEV_STATE_DEGRADED) {
	ret = 1;
	break;
	}
	}
	}
	} else {
	ret = 1;
	}
	}

	zpool_close(zhp);

	return (ret);
	}

	/*
	* zpool offline [-ft]\|[--power] <pool> <device> ...
	*
	*
	* -f Force the device into a faulted state.
	*
	* -t Only take the device off-line temporarily. The offline/faulted
	* state will not be persistent across reboots.
	*
	* --power Power off the enclosure slot to the drive (if possible)
	*/
	int
	zpool_do_offline(int argc, char **argv)
	{
	int c, i;
	char *poolname;
	zpool_handle_t *zhp;
	int ret = 0;
	boolean_t istmp = B_FALSE;
	boolean_t fault = B_FALSE;
	boolean_t is_power_off = B_FALSE;

	struct option long_options[] = {
	- {"power", no_argument, NULL, POWER_OPT},
	+ {"power", no_argument, NULL, ZPOOL_OPTION_POWER},
	{0, 0, 0, 0}
	};

	/* check options */
	while ((c = getopt_long(argc, argv, "ft", long_options, NULL)) != -1) {
	switch (c) {
	case 'f':
	fault = B_TRUE;
	break;
	case 't':
	istmp = B_TRUE;
	break;
	- case POWER_OPT:
	+ case ZPOOL_OPTION_POWER:
	is_power_off = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	if (is_power_off && fault) {
	(void) fprintf(stderr,
	gettext("-0 and -f cannot be used together\n"));
	usage(B_FALSE);
	return (1);
	}

	if (is_power_off && istmp) {
	(void) fprintf(stderr,
	gettext("-0 and -t cannot be used together\n"));
	usage(B_FALSE);
	return (1);
	}

	argc -= optind;
	argv += optind;

	/* get pool name and check number of arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing pool name\n"));
	usage(B_FALSE);
	}
	if (argc < 2) {
	(void) fprintf(stderr, gettext("missing device name\n"));
	usage(B_FALSE);
	}

	poolname = argv[0];

	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
	return (1);

	for (i = 1; i < argc; i++) {
	uint64_t guid = zpool_vdev_path_to_guid(zhp, argv[i]);
	if (is_power_off) {
	/*
	* Note: we have to power off first, then set REMOVED,
	* or else zpool_vdev_set_removed_state() returns
	* EAGAIN.
	*/
	ret = zpool_power_off(zhp, argv[i]);
	if (ret != 0) {
	(void) fprintf(stderr, "%s %s %d\n",
	gettext("unable to power off slot for"),
	argv[i], ret);
	}
	zpool_vdev_set_removed_state(zhp, guid, VDEV_AUX_NONE);

	} else if (fault) {
	vdev_aux_t aux;
	if (istmp == B_FALSE) {
	/* Force the fault to persist across imports */
	aux = VDEV_AUX_EXTERNAL_PERSIST;
	} else {
	aux = VDEV_AUX_EXTERNAL;
	}

	if (guid == 0 \|\| zpool_vdev_fault(zhp, guid, aux) != 0)
	ret = 1;
	} else {
	if (zpool_vdev_offline(zhp, argv[i], istmp) != 0)
	ret = 1;
	}
	}

	zpool_close(zhp);

	return (ret);
	}

	/*
	* zpool clear [-nF]\|[--power] <pool> [device]
	*
	* Clear all errors associated with a pool or a particular device.
	*/
	int
	zpool_do_clear(int argc, char **argv)
	{
	int c;
	int ret = 0;
	boolean_t dryrun = B_FALSE;
	boolean_t do_rewind = B_FALSE;
	boolean_t xtreme_rewind = B_FALSE;
	boolean_t is_power_on = B_FALSE;
	uint32_t rewind_policy = ZPOOL_NO_REWIND;
	nvlist_t *policy = NULL;
	zpool_handle_t *zhp;
	char pool, device;

	struct option long_options[] = {
	- {"power", no_argument, NULL, POWER_OPT},
	+ {"power", no_argument, NULL, ZPOOL_OPTION_POWER},
	{0, 0, 0, 0}
	};

	/* check options */
	while ((c = getopt_long(argc, argv, "FnX", long_options,
	NULL)) != -1) {
	switch (c) {
	case 'F':
	do_rewind = B_TRUE;
	break;
	case 'n':
	dryrun = B_TRUE;
	break;
	case 'X':
	xtreme_rewind = B_TRUE;
	break;
	- case POWER_OPT:
	+ case ZPOOL_OPTION_POWER:
	is_power_on = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	if (libzfs_envvar_is_set("ZPOOL_AUTO_POWER_ON_SLOT"))
	is_power_on = B_TRUE;

	argc -= optind;
	argv += optind;

	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing pool name\n"));
	usage(B_FALSE);
	}

	if (argc > 2) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	if ((dryrun \|\| xtreme_rewind) && !do_rewind) {
	(void) fprintf(stderr,
	gettext("-n or -X only meaningful with -F\n"));
	usage(B_FALSE);
	}
	if (dryrun)
	rewind_policy = ZPOOL_TRY_REWIND;
	else if (do_rewind)
	rewind_policy = ZPOOL_DO_REWIND;
	if (xtreme_rewind)
	rewind_policy \|= ZPOOL_EXTREME_REWIND;

	/* In future, further rewind policy choices can be passed along here */
	if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 \|\|
	nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
	rewind_policy) != 0) {
	return (1);
	}

	pool = argv[0];
	device = argc == 2 ? argv[1] : NULL;

	if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
	nvlist_free(policy);
	return (1);
	}

	if (is_power_on) {
	if (device == NULL) {
	zpool_power_on_pool_and_wait_for_devices(zhp);
	} else {
	zpool_power_on_and_disk_wait(zhp, device);
	}
	}

	if (zpool_clear(zhp, device, policy) != 0)
	ret = 1;

	zpool_close(zhp);

	nvlist_free(policy);

	return (ret);
	}

	/*
	* zpool reguid <pool>
	*/
	int
	zpool_do_reguid(int argc, char **argv)
	{
	int c;
	char *poolname;
	zpool_handle_t *zhp;
	int ret = 0;

	/* check options */
	while ((c = getopt(argc, argv, "")) != -1) {
	switch (c) {
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* get pool name and check number of arguments */
	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing pool name\n"));
	usage(B_FALSE);
	}

	if (argc > 1) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	poolname = argv[0];
	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
	return (1);

	ret = zpool_reguid(zhp);

	zpool_close(zhp);
	return (ret);
	}


	/*
	* zpool reopen <pool>
	*
	* Reopen the pool so that the kernel can update the sizes of all vdevs.
	*/
	int
	zpool_do_reopen(int argc, char **argv)
	{
	int c;
	int ret = 0;
	boolean_t scrub_restart = B_TRUE;

	/* check options */
	while ((c = getopt(argc, argv, "n")) != -1) {
	switch (c) {
	case 'n':
	scrub_restart = B_FALSE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	/* if argc == 0 we will execute zpool_reopen_one on all pools */
	ret = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
	B_FALSE, zpool_reopen_one, &scrub_restart);

	return (ret);
	}

	typedef struct scrub_cbdata {
	int cb_type;
	pool_scrub_cmd_t cb_scrub_cmd;
	} scrub_cbdata_t;

	static boolean_t
	zpool_has_checkpoint(zpool_handle_t *zhp)
	{
	nvlist_t config, nvroot;

	config = zpool_get_config(zhp, NULL);

	if (config != NULL) {
	pool_checkpoint_stat_t *pcs = NULL;
	uint_t c;

	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
	(void) nvlist_lookup_uint64_array(nvroot,
	ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);

	if (pcs == NULL \|\| pcs->pcs_state == CS_NONE)
	return (B_FALSE);

	assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS \|\|
	pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
	return (B_TRUE);
	}

	return (B_FALSE);
	}

	static int
	scrub_callback(zpool_handle_t zhp, void data)
	{
	scrub_cbdata_t *cb = data;
	int err;

	/*
	* Ignore faulted pools.
	*/
	if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) {
	(void) fprintf(stderr, gettext("cannot scan '%s': pool is "
	"currently unavailable\n"), zpool_get_name(zhp));
	return (1);
	}

	err = zpool_scan(zhp, cb->cb_type, cb->cb_scrub_cmd);

	if (err == 0 && zpool_has_checkpoint(zhp) &&
	cb->cb_type == POOL_SCAN_SCRUB) {
	(void) printf(gettext("warning: will not scrub state that "
	"belongs to the checkpoint of pool '%s'\n"),
	zpool_get_name(zhp));
	}

	return (err != 0);
	}

	static int
	wait_callback(zpool_handle_t zhp, void data)
	{
	zpool_wait_activity_t *act = data;
	return (zpool_wait(zhp, *act));
	}

	/*
	* zpool scrub [-s \| -p] [-w] [-e] <pool> ...
	*
	* -e Only scrub blocks in the error log.
	* -s Stop. Stops any in-progress scrub.
	* -p Pause. Pause in-progress scrub.
	* -w Wait. Blocks until scrub has completed.
	*/
	int
	zpool_do_scrub(int argc, char **argv)
	{
	int c;
	scrub_cbdata_t cb;
	boolean_t wait = B_FALSE;
	int error;

	cb.cb_type = POOL_SCAN_SCRUB;
	cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;

	boolean_t is_error_scrub = B_FALSE;
	boolean_t is_pause = B_FALSE;
	boolean_t is_stop = B_FALSE;

	/* check options */
	while ((c = getopt(argc, argv, "spwe")) != -1) {
	switch (c) {
	case 'e':
	is_error_scrub = B_TRUE;
	break;
	case 's':
	is_stop = B_TRUE;
	break;
	case 'p':
	is_pause = B_TRUE;
	break;
	case 'w':
	wait = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	if (is_pause && is_stop) {
	(void) fprintf(stderr, gettext("invalid option "
	"combination :-s and -p are mutually exclusive\n"));
	usage(B_FALSE);
	} else {
	if (is_error_scrub)
	cb.cb_type = POOL_SCAN_ERRORSCRUB;

	if (is_pause) {
	cb.cb_scrub_cmd = POOL_SCRUB_PAUSE;
	} else if (is_stop) {
	cb.cb_type = POOL_SCAN_NONE;
	} else {
	cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
	}
	}

	if (wait && (cb.cb_type == POOL_SCAN_NONE \|\|
	cb.cb_scrub_cmd == POOL_SCRUB_PAUSE)) {
	(void) fprintf(stderr, gettext("invalid option combination: "
	"-w cannot be used with -p or -s\n"));
	usage(B_FALSE);
	}

	argc -= optind;
	argv += optind;

	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing pool name argument\n"));
	usage(B_FALSE);
	}

	error = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
	B_FALSE, scrub_callback, &cb);

	if (wait && !error) {
	zpool_wait_activity_t act = ZPOOL_WAIT_SCRUB;
	error = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
	B_FALSE, wait_callback, &act);
	}

	return (error);
	}

	/*
	* zpool resilver <pool> ...
	*
	* Restarts any in-progress resilver
	*/
	int
	zpool_do_resilver(int argc, char **argv)
	{
	int c;
	scrub_cbdata_t cb;

	cb.cb_type = POOL_SCAN_RESILVER;
	cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;

	/* check options */
	while ((c = getopt(argc, argv, "")) != -1) {
	switch (c) {
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing pool name argument\n"));
	usage(B_FALSE);
	}

	return (for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
	B_FALSE, scrub_callback, &cb));
	}

	/*
	* zpool trim [-d] [-r <rate>] [-c \| -s] <pool> [<device> ...]
	*
	* -c Cancel. Ends any in-progress trim.
	* -d Secure trim. Requires kernel and device support.
	* -r <rate> Sets the TRIM rate in bytes (per second). Supports
	* adding a multiplier suffix such as 'k' or 'm'.
	* -s Suspend. TRIM can then be restarted with no flags.
	* -w Wait. Blocks until trimming has completed.
	*/
	int
	zpool_do_trim(int argc, char **argv)
	{
	struct option long_options[] = {
	{"cancel", no_argument, NULL, 'c'},
	{"secure", no_argument, NULL, 'd'},
	{"rate", required_argument, NULL, 'r'},
	{"suspend", no_argument, NULL, 's'},
	{"wait", no_argument, NULL, 'w'},
	{0, 0, 0, 0}
	};

	pool_trim_func_t cmd_type = POOL_TRIM_START;
	uint64_t rate = 0;
	boolean_t secure = B_FALSE;
	boolean_t wait = B_FALSE;

	int c;
	while ((c = getopt_long(argc, argv, "cdr:sw", long_options, NULL))
	!= -1) {
	switch (c) {
	case 'c':
	if (cmd_type != POOL_TRIM_START &&
	cmd_type != POOL_TRIM_CANCEL) {
	(void) fprintf(stderr, gettext("-c cannot be "
	"combined with other options\n"));
	usage(B_FALSE);
	}
	cmd_type = POOL_TRIM_CANCEL;
	break;
	case 'd':
	if (cmd_type != POOL_TRIM_START) {
	(void) fprintf(stderr, gettext("-d cannot be "
	"combined with the -c or -s options\n"));
	usage(B_FALSE);
	}
	secure = B_TRUE;
	break;
	case 'r':
	if (cmd_type != POOL_TRIM_START) {
	(void) fprintf(stderr, gettext("-r cannot be "
	"combined with the -c or -s options\n"));
	usage(B_FALSE);
	}
	if (zfs_nicestrtonum(g_zfs, optarg, &rate) == -1) {
	(void) fprintf(stderr, "%s: %s\n",
	gettext("invalid value for rate"),
	libzfs_error_description(g_zfs));
	usage(B_FALSE);
	}
	break;
	case 's':
	if (cmd_type != POOL_TRIM_START &&
	cmd_type != POOL_TRIM_SUSPEND) {
	(void) fprintf(stderr, gettext("-s cannot be "
	"combined with other options\n"));
	usage(B_FALSE);
	}
	cmd_type = POOL_TRIM_SUSPEND;
	break;
	case 'w':
	wait = B_TRUE;
	break;
	case '?':
	if (optopt != 0) {
	(void) fprintf(stderr,
	gettext("invalid option '%c'\n"), optopt);
	} else {
	(void) fprintf(stderr,
	gettext("invalid option '%s'\n"),
	argv[optind - 1]);
	}
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing pool name argument\n"));
	usage(B_FALSE);
	return (-1);
	}

	if (wait && (cmd_type != POOL_TRIM_START)) {
	(void) fprintf(stderr, gettext("-w cannot be used with -c or "
	"-s\n"));
	usage(B_FALSE);
	}

	char *poolname = argv[0];
	zpool_handle_t *zhp = zpool_open(g_zfs, poolname);
	if (zhp == NULL)
	return (-1);

	trimflags_t trim_flags = {
	.secure = secure,
	.rate = rate,
	.wait = wait,
	};

	nvlist_t *vdevs = fnvlist_alloc();
	if (argc == 1) {
	/* no individual leaf vdevs specified, so add them all */
	nvlist_t *config = zpool_get_config(zhp, NULL);
	nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
	ZPOOL_CONFIG_VDEV_TREE);
	zpool_collect_leaves(zhp, nvroot, vdevs);
	trim_flags.fullpool = B_TRUE;
	} else {
	trim_flags.fullpool = B_FALSE;
	for (int i = 1; i < argc; i++) {
	fnvlist_add_boolean(vdevs, argv[i]);
	}
	}

	int error = zpool_trim(zhp, cmd_type, vdevs, &trim_flags);

	fnvlist_free(vdevs);
	zpool_close(zhp);

	return (error);
	}

	/*
	* Converts a total number of seconds to a human readable string broken
	* down in to days/hours/minutes/seconds.
	*/
	static void
	secs_to_dhms(uint64_t total, char *buf)
	{
	uint64_t days = total / 60 / 60 / 24;
	uint64_t hours = (total / 60 / 60) % 24;
	uint64_t mins = (total / 60) % 60;
	uint64_t secs = (total % 60);

	if (days > 0) {
	(void) sprintf(buf, "%llu days %02llu:%02llu:%02llu",
	(u_longlong_t)days, (u_longlong_t)hours,
	(u_longlong_t)mins, (u_longlong_t)secs);
	} else {
	(void) sprintf(buf, "%02llu:%02llu:%02llu",
	(u_longlong_t)hours, (u_longlong_t)mins,
	(u_longlong_t)secs);
	}
	}

	/*
	* Print out detailed error scrub status.
	*/
	static void
	print_err_scrub_status(pool_scan_stat_t *ps)
	{
	time_t start, end, pause;
	uint64_t total_secs_left;
	uint64_t secs_left, mins_left, hours_left, days_left;
	uint64_t examined, to_be_examined;

	if (ps == NULL \|\| ps->pss_error_scrub_func != POOL_SCAN_ERRORSCRUB) {
	return;
	}

	(void) printf(gettext(" scrub: "));

	start = ps->pss_error_scrub_start;
	end = ps->pss_error_scrub_end;
	pause = ps->pss_pass_error_scrub_pause;
	examined = ps->pss_error_scrub_examined;
	to_be_examined = ps->pss_error_scrub_to_be_examined;

	assert(ps->pss_error_scrub_func == POOL_SCAN_ERRORSCRUB);

	if (ps->pss_error_scrub_state == DSS_FINISHED) {
	total_secs_left = end - start;
	days_left = total_secs_left / 60 / 60 / 24;
	hours_left = (total_secs_left / 60 / 60) % 24;
	mins_left = (total_secs_left / 60) % 60;
	secs_left = (total_secs_left % 60);

	(void) printf(gettext("scrubbed %llu error blocks in %llu days "
	"%02llu:%02llu:%02llu on %s"), (u_longlong_t)examined,
	(u_longlong_t)days_left, (u_longlong_t)hours_left,
	(u_longlong_t)mins_left, (u_longlong_t)secs_left,
	ctime(&end));

	return;
	} else if (ps->pss_error_scrub_state == DSS_CANCELED) {
	(void) printf(gettext("error scrub canceled on %s"),
	ctime(&end));
	return;
	}
	assert(ps->pss_error_scrub_state == DSS_ERRORSCRUBBING);

	/* Error scrub is in progress. */
	if (pause == 0) {
	(void) printf(gettext("error scrub in progress since %s"),
	ctime(&start));
	} else {
	(void) printf(gettext("error scrub paused since %s"),
	ctime(&pause));
	(void) printf(gettext("\terror scrub started on %s"),
	ctime(&start));
	}

	double fraction_done = (double)examined / (to_be_examined + examined);
	(void) printf(gettext("\t%.2f%% done, issued I/O for %llu error"
	" blocks"), 100 * fraction_done, (u_longlong_t)examined);

	(void) printf("\n");
	}

	/*
	* Print out detailed scrub status.
	*/
	static void
	print_scan_scrub_resilver_status(pool_scan_stat_t *ps)
	{
	time_t start, end, pause;
	uint64_t pass_scanned, scanned, pass_issued, issued, total_s, total_i;
	uint64_t elapsed, scan_rate, issue_rate;
	double fraction_done;
	char processed_buf[7], scanned_buf[7], issued_buf[7], total_s_buf[7];
	char total_i_buf[7], srate_buf[7], irate_buf[7], time_buf[32];

	printf(" ");
	printf_color(ANSI_BOLD, gettext("scan:"));
	printf(" ");

	/* If there's never been a scan, there's not much to say. */
	if (ps == NULL \|\| ps->pss_func == POOL_SCAN_NONE \|\|
	ps->pss_func >= POOL_SCAN_FUNCS) {
	(void) printf(gettext("none requested\n"));
	return;
	}

	start = ps->pss_start_time;
	end = ps->pss_end_time;
	pause = ps->pss_pass_scrub_pause;

	zfs_nicebytes(ps->pss_processed, processed_buf, sizeof (processed_buf));

	int is_resilver = ps->pss_func == POOL_SCAN_RESILVER;
	int is_scrub = ps->pss_func == POOL_SCAN_SCRUB;
	assert(is_resilver \|\| is_scrub);

	/* Scan is finished or canceled. */
	if (ps->pss_state == DSS_FINISHED) {
	secs_to_dhms(end - start, time_buf);

	if (is_scrub) {
	(void) printf(gettext("scrub repaired %s "
	"in %s with %llu errors on %s"), processed_buf,
	time_buf, (u_longlong_t)ps->pss_errors,
	ctime(&end));
	} else if (is_resilver) {
	(void) printf(gettext("resilvered %s "
	"in %s with %llu errors on %s"), processed_buf,
	time_buf, (u_longlong_t)ps->pss_errors,
	ctime(&end));
	}
	return;
	} else if (ps->pss_state == DSS_CANCELED) {
	if (is_scrub) {
	(void) printf(gettext("scrub canceled on %s"),
	ctime(&end));
	} else if (is_resilver) {
	(void) printf(gettext("resilver canceled on %s"),
	ctime(&end));
	}
	return;
	}

	assert(ps->pss_state == DSS_SCANNING);

	/* Scan is in progress. Resilvers can't be paused. */
	if (is_scrub) {
	if (pause == 0) {
	(void) printf(gettext("scrub in progress since %s"),
	ctime(&start));
	} else {
	(void) printf(gettext("scrub paused since %s"),
	ctime(&pause));
	(void) printf(gettext("\tscrub started on %s"),
	ctime(&start));
	}
	} else if (is_resilver) {
	(void) printf(gettext("resilver in progress since %s"),
	ctime(&start));
	}

	scanned = ps->pss_examined;
	pass_scanned = ps->pss_pass_exam;
	issued = ps->pss_issued;
	pass_issued = ps->pss_pass_issued;
	total_s = ps->pss_to_examine;
	total_i = ps->pss_to_examine - ps->pss_skipped;

	/* we are only done with a block once we have issued the IO for it */
	fraction_done = (double)issued / total_i;

	/* elapsed time for this pass, rounding up to 1 if it's 0 */
	elapsed = time(NULL) - ps->pss_pass_start;
	elapsed -= ps->pss_pass_scrub_spent_paused;
	elapsed = (elapsed != 0) ? elapsed : 1;

	scan_rate = pass_scanned / elapsed;
	issue_rate = pass_issued / elapsed;

	/* format all of the numbers we will be reporting */
	zfs_nicebytes(scanned, scanned_buf, sizeof (scanned_buf));
	zfs_nicebytes(issued, issued_buf, sizeof (issued_buf));
	zfs_nicebytes(total_s, total_s_buf, sizeof (total_s_buf));
	zfs_nicebytes(total_i, total_i_buf, sizeof (total_i_buf));

	/* do not print estimated time if we have a paused scrub */
	(void) printf(gettext("\t%s / %s scanned"), scanned_buf, total_s_buf);
	if (pause == 0 && scan_rate > 0) {
	zfs_nicebytes(scan_rate, srate_buf, sizeof (srate_buf));
	(void) printf(gettext(" at %s/s"), srate_buf);
	}
	(void) printf(gettext(", %s / %s issued"), issued_buf, total_i_buf);
	if (pause == 0 && issue_rate > 0) {
	zfs_nicebytes(issue_rate, irate_buf, sizeof (irate_buf));
	(void) printf(gettext(" at %s/s"), irate_buf);
	}
	(void) printf(gettext("\n"));

	if (is_resilver) {
	(void) printf(gettext("\t%s resilvered, %.2f%% done"),
	processed_buf, 100 * fraction_done);
	} else if (is_scrub) {
	(void) printf(gettext("\t%s repaired, %.2f%% done"),
	processed_buf, 100 * fraction_done);
	}

	if (pause == 0) {
	/*
	* Only provide an estimate iff:
	* 1) we haven't yet issued all we expected, and
	* 2) the issue rate exceeds 10 MB/s, and
	* 3) it's either:
	* a) a resilver which has started repairs, or
	* b) a scrub which has entered the issue phase.
	*/
	if (total_i >= issued && issue_rate >= 10 * 1024 * 1024 &&
	((is_resilver && ps->pss_processed > 0) \|\|
	(is_scrub && issued > 0))) {
	secs_to_dhms((total_i - issued) / issue_rate, time_buf);
	(void) printf(gettext(", %s to go\n"), time_buf);
	} else {
	(void) printf(gettext(", no estimated "
	"completion time\n"));
	}
	} else {
	(void) printf(gettext("\n"));
	}
	}

	static void
	print_rebuild_status_impl(vdev_rebuild_stat_t vrs, uint_t c, char vdev_name)
	{
	if (vrs == NULL \|\| vrs->vrs_state == VDEV_REBUILD_NONE)
	return;

	printf(" ");
	printf_color(ANSI_BOLD, gettext("scan:"));
	printf(" ");

	uint64_t bytes_scanned = vrs->vrs_bytes_scanned;
	uint64_t bytes_issued = vrs->vrs_bytes_issued;
	uint64_t bytes_rebuilt = vrs->vrs_bytes_rebuilt;
	uint64_t bytes_est_s = vrs->vrs_bytes_est;
	uint64_t bytes_est_i = vrs->vrs_bytes_est;
	if (c > offsetof(vdev_rebuild_stat_t, vrs_pass_bytes_skipped) / 8)
	bytes_est_i -= vrs->vrs_pass_bytes_skipped;
	uint64_t scan_rate = (vrs->vrs_pass_bytes_scanned /
	(vrs->vrs_pass_time_ms + 1)) * 1000;
	uint64_t issue_rate = (vrs->vrs_pass_bytes_issued /
	(vrs->vrs_pass_time_ms + 1)) * 1000;
	double scan_pct = MIN((double)bytes_scanned * 100 /
	(bytes_est_s + 1), 100);

	/* Format all of the numbers we will be reporting */
	char bytes_scanned_buf[7], bytes_issued_buf[7];
	char bytes_rebuilt_buf[7], bytes_est_s_buf[7], bytes_est_i_buf[7];
	char scan_rate_buf[7], issue_rate_buf[7], time_buf[32];
	zfs_nicebytes(bytes_scanned, bytes_scanned_buf,
	sizeof (bytes_scanned_buf));
	zfs_nicebytes(bytes_issued, bytes_issued_buf,
	sizeof (bytes_issued_buf));
	zfs_nicebytes(bytes_rebuilt, bytes_rebuilt_buf,
	sizeof (bytes_rebuilt_buf));
	zfs_nicebytes(bytes_est_s, bytes_est_s_buf, sizeof (bytes_est_s_buf));
	zfs_nicebytes(bytes_est_i, bytes_est_i_buf, sizeof (bytes_est_i_buf));

	time_t start = vrs->vrs_start_time;
	time_t end = vrs->vrs_end_time;

	/* Rebuild is finished or canceled. */
	if (vrs->vrs_state == VDEV_REBUILD_COMPLETE) {
	secs_to_dhms(vrs->vrs_scan_time_ms / 1000, time_buf);
	(void) printf(gettext("resilvered (%s) %s in %s "
	"with %llu errors on %s"), vdev_name, bytes_rebuilt_buf,
	time_buf, (u_longlong_t)vrs->vrs_errors, ctime(&end));
	return;
	} else if (vrs->vrs_state == VDEV_REBUILD_CANCELED) {
	(void) printf(gettext("resilver (%s) canceled on %s"),
	vdev_name, ctime(&end));
	return;
	} else if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
	(void) printf(gettext("resilver (%s) in progress since %s"),
	vdev_name, ctime(&start));
	}

	assert(vrs->vrs_state == VDEV_REBUILD_ACTIVE);

	(void) printf(gettext("\t%s / %s scanned"), bytes_scanned_buf,
	bytes_est_s_buf);
	if (scan_rate > 0) {
	zfs_nicebytes(scan_rate, scan_rate_buf, sizeof (scan_rate_buf));
	(void) printf(gettext(" at %s/s"), scan_rate_buf);
	}
	(void) printf(gettext(", %s / %s issued"), bytes_issued_buf,
	bytes_est_i_buf);
	if (issue_rate > 0) {
	zfs_nicebytes(issue_rate, issue_rate_buf,
	sizeof (issue_rate_buf));
	(void) printf(gettext(" at %s/s"), issue_rate_buf);
	}
	(void) printf(gettext("\n"));

	(void) printf(gettext("\t%s resilvered, %.2f%% done"),
	bytes_rebuilt_buf, scan_pct);

	if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
	if (bytes_est_s >= bytes_scanned &&
	scan_rate >= 10 * 1024 * 1024) {
	secs_to_dhms((bytes_est_s - bytes_scanned) / scan_rate,
	time_buf);
	(void) printf(gettext(", %s to go\n"), time_buf);
	} else {
	(void) printf(gettext(", no estimated "
	"completion time\n"));
	}
	} else {
	(void) printf(gettext("\n"));
	}
	}

	/*
	* Print rebuild status for top-level vdevs.
	*/
	static void
	print_rebuild_status(zpool_handle_t zhp, nvlist_t nvroot)
	{
	nvlist_t **child;
	uint_t children;

	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
	&child, &children) != 0)
	children = 0;

	for (uint_t c = 0; c < children; c++) {
	vdev_rebuild_stat_t *vrs;
	uint_t i;

	if (nvlist_lookup_uint64_array(child[c],
	ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i) == 0) {
	char *name = zpool_vdev_name(g_zfs, zhp,
	child[c], VDEV_NAME_TYPE_ID);
	print_rebuild_status_impl(vrs, i, name);
	free(name);
	}
	}
	}

	/*
	* As we don't scrub checkpointed blocks, we want to warn the user that we
	* skipped scanning some blocks if a checkpoint exists or existed at any
	* time during the scan. If a sequential instead of healing reconstruction
	* was performed then the blocks were reconstructed. However, their checksums
	* have not been verified so we still print the warning.
	*/
	static void
	print_checkpoint_scan_warning(pool_scan_stat_t ps, pool_checkpoint_stat_t pcs)
	{
	if (ps == NULL \|\| pcs == NULL)
	return;

	if (pcs->pcs_state == CS_NONE \|\|
	pcs->pcs_state == CS_CHECKPOINT_DISCARDING)
	return;

	assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS);

	if (ps->pss_state == DSS_NONE)
	return;

	if ((ps->pss_state == DSS_FINISHED \|\| ps->pss_state == DSS_CANCELED) &&
	ps->pss_end_time < pcs->pcs_start_time)
	return;

	if (ps->pss_state == DSS_FINISHED \|\| ps->pss_state == DSS_CANCELED) {
	(void) printf(gettext(" scan warning: skipped blocks "
	"that are only referenced by the checkpoint.\n"));
	} else {
	assert(ps->pss_state == DSS_SCANNING);
	(void) printf(gettext(" scan warning: skipping blocks "
	"that are only referenced by the checkpoint.\n"));
	}
	}

	/*
	* Returns B_TRUE if there is an active rebuild in progress. Otherwise,
	* B_FALSE is returned and 'rebuild_end_time' is set to the end time for
	* the last completed (or cancelled) rebuild.
	*/
	static boolean_t
	check_rebuilding(nvlist_t nvroot, uint64_t rebuild_end_time)
	{
	nvlist_t **child;
	uint_t children;
	boolean_t rebuilding = B_FALSE;
	uint64_t end_time = 0;

	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
	&child, &children) != 0)
	children = 0;

	for (uint_t c = 0; c < children; c++) {
	vdev_rebuild_stat_t *vrs;
	uint_t i;

	if (nvlist_lookup_uint64_array(child[c],
	ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i) == 0) {

	if (vrs->vrs_end_time > end_time)
	end_time = vrs->vrs_end_time;

	if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
	rebuilding = B_TRUE;
	end_time = 0;
	break;
	}
	}
	}

	if (rebuild_end_time != NULL)
	*rebuild_end_time = end_time;

	return (rebuilding);
	}

	/*
	* Print the scan status.
	*/
	static void
	print_scan_status(zpool_handle_t zhp, nvlist_t nvroot)
	{
	uint64_t rebuild_end_time = 0, resilver_end_time = 0;
	boolean_t have_resilver = B_FALSE, have_scrub = B_FALSE;
	boolean_t have_errorscrub = B_FALSE;
	boolean_t active_resilver = B_FALSE;
	pool_checkpoint_stat_t *pcs = NULL;
	pool_scan_stat_t *ps = NULL;
	uint_t c;
	time_t scrub_start = 0, errorscrub_start = 0;

	if (nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_SCAN_STATS,
	(uint64_t **)&ps, &c) == 0) {
	if (ps->pss_func == POOL_SCAN_RESILVER) {
	resilver_end_time = ps->pss_end_time;
	active_resilver = (ps->pss_state == DSS_SCANNING);
	}

	have_resilver = (ps->pss_func == POOL_SCAN_RESILVER);
	have_scrub = (ps->pss_func == POOL_SCAN_SCRUB);
	scrub_start = ps->pss_start_time;
	if (c > offsetof(pool_scan_stat_t,
	pss_pass_error_scrub_pause) / 8) {
	have_errorscrub = (ps->pss_error_scrub_func ==
	POOL_SCAN_ERRORSCRUB);
	errorscrub_start = ps->pss_error_scrub_start;
	}
	}

	boolean_t active_rebuild = check_rebuilding(nvroot, &rebuild_end_time);
	boolean_t have_rebuild = (active_rebuild \|\| (rebuild_end_time > 0));

	/* Always print the scrub status when available. */
	if (have_scrub && scrub_start > errorscrub_start)
	print_scan_scrub_resilver_status(ps);
	else if (have_errorscrub && errorscrub_start >= scrub_start)
	print_err_scrub_status(ps);

	/*
	* When there is an active resilver or rebuild print its status.
	* Otherwise print the status of the last resilver or rebuild.
	*/
	if (active_resilver \|\| (!active_rebuild && have_resilver &&
	resilver_end_time && resilver_end_time > rebuild_end_time)) {
	print_scan_scrub_resilver_status(ps);
	} else if (active_rebuild \|\| (!active_resilver && have_rebuild &&
	rebuild_end_time && rebuild_end_time > resilver_end_time)) {
	print_rebuild_status(zhp, nvroot);
	}

	(void) nvlist_lookup_uint64_array(nvroot,
	ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
	print_checkpoint_scan_warning(ps, pcs);
	}

	/*
	* Print out detailed removal status.
	*/
	static void
	print_removal_status(zpool_handle_t zhp, pool_removal_stat_t prs)
	{
	char copied_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
	time_t start, end;
	nvlist_t config, nvroot;
	nvlist_t **child;
	uint_t children;
	char *vdev_name;

	if (prs == NULL \|\| prs->prs_state == DSS_NONE)
	return;

	/*
	* Determine name of vdev.
	*/
	config = zpool_get_config(zhp, NULL);
	nvroot = fnvlist_lookup_nvlist(config,
	ZPOOL_CONFIG_VDEV_TREE);
	verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
	&child, &children) == 0);
	assert(prs->prs_removing_vdev < children);
	vdev_name = zpool_vdev_name(g_zfs, zhp,
	child[prs->prs_removing_vdev], B_TRUE);

	printf_color(ANSI_BOLD, gettext("remove: "));

	start = prs->prs_start_time;
	end = prs->prs_end_time;
	zfs_nicenum(prs->prs_copied, copied_buf, sizeof (copied_buf));

	/*
	* Removal is finished or canceled.
	*/
	if (prs->prs_state == DSS_FINISHED) {
	uint64_t minutes_taken = (end - start) / 60;

	(void) printf(gettext("Removal of vdev %llu copied %s "
	"in %lluh%um, completed on %s"),
	(longlong_t)prs->prs_removing_vdev,
	copied_buf,
	(u_longlong_t)(minutes_taken / 60),
	(uint_t)(minutes_taken % 60),
	ctime((time_t *)&end));
	} else if (prs->prs_state == DSS_CANCELED) {
	(void) printf(gettext("Removal of %s canceled on %s"),
	vdev_name, ctime(&end));
	} else {
	uint64_t copied, total, elapsed, mins_left, hours_left;
	double fraction_done;
	uint_t rate;

	assert(prs->prs_state == DSS_SCANNING);

	/*
	* Removal is in progress.
	*/
	(void) printf(gettext(
	"Evacuation of %s in progress since %s"),
	vdev_name, ctime(&start));

	copied = prs->prs_copied > 0 ? prs->prs_copied : 1;
	total = prs->prs_to_copy;
	fraction_done = (double)copied / total;

	/* elapsed time for this pass */
	elapsed = time(NULL) - prs->prs_start_time;
	elapsed = elapsed > 0 ? elapsed : 1;
	rate = copied / elapsed;
	rate = rate > 0 ? rate : 1;
	mins_left = ((total - copied) / rate) / 60;
	hours_left = mins_left / 60;

	zfs_nicenum(copied, examined_buf, sizeof (examined_buf));
	zfs_nicenum(total, total_buf, sizeof (total_buf));
	zfs_nicenum(rate, rate_buf, sizeof (rate_buf));

	/*
	* do not print estimated time if hours_left is more than
	* 30 days
	*/
	(void) printf(gettext(
	"\t%s copied out of %s at %s/s, %.2f%% done"),
	examined_buf, total_buf, rate_buf, 100 * fraction_done);
	if (hours_left < (30 * 24)) {
	(void) printf(gettext(", %lluh%um to go\n"),
	(u_longlong_t)hours_left, (uint_t)(mins_left % 60));
	} else {
	(void) printf(gettext(
	", (copy is slow, no estimated time)\n"));
	}
	}
	free(vdev_name);

	if (prs->prs_mapping_memory > 0) {
	char mem_buf[7];
	zfs_nicenum(prs->prs_mapping_memory, mem_buf, sizeof (mem_buf));
	(void) printf(gettext(
	"\t%s memory used for removed device mappings\n"),
	mem_buf);
	}
	}

	static void
	print_checkpoint_status(pool_checkpoint_stat_t *pcs)
	{
	time_t start;
	char space_buf[7];

	if (pcs == NULL \|\| pcs->pcs_state == CS_NONE)
	return;

	(void) printf(gettext("checkpoint: "));

	start = pcs->pcs_start_time;
	zfs_nicenum(pcs->pcs_space, space_buf, sizeof (space_buf));

	if (pcs->pcs_state == CS_CHECKPOINT_EXISTS) {
	char *date = ctime(&start);

	/*
	* ctime() adds a newline at the end of the generated
	* string, thus the weird format specifier and the
	* strlen() call used to chop it off from the output.
	*/
	(void) printf(gettext("created %.*s, consumes %s\n"),
	(int)(strlen(date) - 1), date, space_buf);
	return;
	}

	assert(pcs->pcs_state == CS_CHECKPOINT_DISCARDING);

	(void) printf(gettext("discarding, %s remaining.\n"),
	space_buf);
	}

	static void
	print_error_log(zpool_handle_t *zhp)
	{
	nvlist_t *nverrlist = NULL;
	nvpair_t *elem;
	char *pathname;
	size_t len = MAXPATHLEN * 2;

	if (zpool_get_errlog(zhp, &nverrlist) != 0)
	return;

	(void) printf("errors: Permanent errors have been "
	"detected in the following files:\n\n");

	pathname = safe_malloc(len);
	elem = NULL;
	while ((elem = nvlist_next_nvpair(nverrlist, elem)) != NULL) {
	nvlist_t *nv;
	uint64_t dsobj, obj;

	verify(nvpair_value_nvlist(elem, &nv) == 0);
	verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_DATASET,
	&dsobj) == 0);
	verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_OBJECT,
	&obj) == 0);
	zpool_obj_to_path(zhp, dsobj, obj, pathname, len);
	(void) printf("%7s %s\n", "", pathname);
	}
	free(pathname);
	nvlist_free(nverrlist);
	}

	static void
	print_spares(zpool_handle_t zhp, status_cbdata_t cb, nvlist_t **spares,
	uint_t nspares)
	{
	uint_t i;
	char *name;

	if (nspares == 0)
	return;

	(void) printf(gettext("\tspares\n"));

	for (i = 0; i < nspares; i++) {
	name = zpool_vdev_name(g_zfs, zhp, spares[i],
	cb->cb_name_flags);
	print_status_config(zhp, cb, name, spares[i], 2, B_TRUE, NULL);
	free(name);
	}
	}

	static void
	print_l2cache(zpool_handle_t zhp, status_cbdata_t cb, nvlist_t **l2cache,
	uint_t nl2cache)
	{
	uint_t i;
	char *name;

	if (nl2cache == 0)
	return;

	(void) printf(gettext("\tcache\n"));

	for (i = 0; i < nl2cache; i++) {
	name = zpool_vdev_name(g_zfs, zhp, l2cache[i],
	cb->cb_name_flags);
	print_status_config(zhp, cb, name, l2cache[i], 2,
	B_FALSE, NULL);
	free(name);
	}
	}

	static void
	print_dedup_stats(nvlist_t *config)
	{
	ddt_histogram_t *ddh;
	ddt_stat_t *dds;
	ddt_object_t *ddo;
	uint_t c;
	char dspace[6], mspace[6];

	/*
	* If the pool was faulted then we may not have been able to
	* obtain the config. Otherwise, if we have anything in the dedup
	* table continue processing the stats.
	*/
	if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_OBJ_STATS,
	(uint64_t **)&ddo, &c) != 0)
	return;

	(void) printf("\n");
	(void) printf(gettext(" dedup: "));
	if (ddo->ddo_count == 0) {
	(void) printf(gettext("no DDT entries\n"));
	return;
	}

	zfs_nicebytes(ddo->ddo_dspace, dspace, sizeof (dspace));
	zfs_nicebytes(ddo->ddo_mspace, mspace, sizeof (mspace));
	(void) printf("DDT entries %llu, size %s on disk, %s in core\n",
	(u_longlong_t)ddo->ddo_count,
	dspace,
	mspace);

	verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_STATS,
	(uint64_t **)&dds, &c) == 0);
	verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_HISTOGRAM,
	(uint64_t **)&ddh, &c) == 0);
	zpool_dump_ddt(dds, ddh);
	}

	/*
	* Display a summary of pool status. Displays a summary such as:
	*
	* pool: tank
	* status: DEGRADED
	* reason: One or more devices ...
	* see: https://openzfs.github.io/openzfs-docs/msg/ZFS-xxxx-01
	* config:
	* mirror DEGRADED
	* c1t0d0 OK
	* c2t0d0 UNAVAIL
	*
	* When given the '-v' option, we print out the complete config. If the '-e'
	* option is specified, then we print out error rate information as well.
	*/
	static int
	status_callback(zpool_handle_t zhp, void data)
	{
	status_cbdata_t *cbp = data;
	nvlist_t config, nvroot;
	const char *msgid;
	zpool_status_t reason;
	zpool_errata_t errata;
	const char *health;
	uint_t c;
	vdev_stat_t *vs;

	config = zpool_get_config(zhp, NULL);
	reason = zpool_get_status(zhp, &msgid, &errata);

	cbp->cb_count++;

	/*
	* If we were given 'zpool status -x', only report those pools with
	* problems.
	*/
	if (cbp->cb_explain &&
	(reason == ZPOOL_STATUS_OK \|\|
	reason == ZPOOL_STATUS_VERSION_OLDER \|\|
	reason == ZPOOL_STATUS_FEAT_DISABLED \|\|
	reason == ZPOOL_STATUS_COMPATIBILITY_ERR \|\|
	reason == ZPOOL_STATUS_INCOMPATIBLE_FEAT)) {
	if (!cbp->cb_allpools) {
	(void) printf(gettext("pool '%s' is healthy\n"),
	zpool_get_name(zhp));
	if (cbp->cb_first)
	cbp->cb_first = B_FALSE;
	}
	return (0);
	}

	if (cbp->cb_first)
	cbp->cb_first = B_FALSE;
	else
	(void) printf("\n");

	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
	verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
	(uint64_t **)&vs, &c) == 0);

	health = zpool_get_state_str(zhp);

	printf(" ");
	printf_color(ANSI_BOLD, gettext("pool:"));
	printf(" %s\n", zpool_get_name(zhp));
	fputc(' ', stdout);
	printf_color(ANSI_BOLD, gettext("state: "));

	printf_color(health_str_to_color(health), "%s", health);

	fputc('\n', stdout);

	switch (reason) {
	case ZPOOL_STATUS_MISSING_DEV_R:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices could "
	"not be opened. Sufficient replicas exist for\n\tthe pool "
	"to continue functioning in a degraded state.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Attach the missing device "
	"and online it using 'zpool online'.\n"));
	break;

	case ZPOOL_STATUS_MISSING_DEV_NR:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices could "
	"not be opened. There are insufficient\n\treplicas for the"
	" pool to continue functioning.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Attach the missing device "
	"and online it using 'zpool online'.\n"));
	break;

	case ZPOOL_STATUS_CORRUPT_LABEL_R:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices could "
	"not be used because the label is missing or\n\tinvalid. "
	"Sufficient replicas exist for the pool to continue\n\t"
	"functioning in a degraded state.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Replace the device using "
	"'zpool replace'.\n"));
	break;

	case ZPOOL_STATUS_CORRUPT_LABEL_NR:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices could "
	"not be used because the label is missing \n\tor invalid. "
	"There are insufficient replicas for the pool to "
	"continue\n\tfunctioning.\n"));
	zpool_explain_recover(zpool_get_handle(zhp),
	zpool_get_name(zhp), reason, config);
	break;

	case ZPOOL_STATUS_FAILING_DEV:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices has "
	"experienced an unrecoverable error. An\n\tattempt was "
	"made to correct the error. Applications are "
	"unaffected.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Determine if the "
	"device needs to be replaced, and clear the errors\n\tusing"
	" 'zpool clear' or replace the device with 'zpool "
	"replace'.\n"));
	break;

	case ZPOOL_STATUS_OFFLINE_DEV:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices has "
	"been taken offline by the administrator.\n\tSufficient "
	"replicas exist for the pool to continue functioning in "
	"a\n\tdegraded state.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Online the device "
	"using 'zpool online' or replace the device with\n\t'zpool "
	"replace'.\n"));
	break;

	case ZPOOL_STATUS_REMOVED_DEV:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices has "
	"been removed by the administrator.\n\tSufficient "
	"replicas exist for the pool to continue functioning in "
	"a\n\tdegraded state.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Online the device "
	"using zpool online' or replace the device with\n\t'zpool "
	"replace'.\n"));
	break;

	case ZPOOL_STATUS_RESILVERING:
	case ZPOOL_STATUS_REBUILDING:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices is "
	"currently being resilvered. The pool will\n\tcontinue "
	"to function, possibly in a degraded state.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Wait for the resilver to "
	"complete.\n"));
	break;

	case ZPOOL_STATUS_REBUILD_SCRUB:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices have "
	"been sequentially resilvered, scrubbing\n\tthe pool "
	"is recommended.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Use 'zpool scrub' to "
	"verify all data checksums.\n"));
	break;

	case ZPOOL_STATUS_CORRUPT_DATA:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices has "
	"experienced an error resulting in data\n\tcorruption. "
	"Applications may be affected.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Restore the file in question"
	" if possible. Otherwise restore the\n\tentire pool from "
	"backup.\n"));
	break;

	case ZPOOL_STATUS_CORRUPT_POOL:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("The pool metadata is "
	"corrupted and the pool cannot be opened.\n"));
	zpool_explain_recover(zpool_get_handle(zhp),
	zpool_get_name(zhp), reason, config);
	break;

	case ZPOOL_STATUS_VERSION_OLDER:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
	"a legacy on-disk format. The pool can\n\tstill be used, "
	"but some features are unavailable.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Upgrade the pool using "
	"'zpool upgrade'. Once this is done, the\n\tpool will no "
	"longer be accessible on software that does not support\n\t"
	"feature flags.\n"));
	break;

	case ZPOOL_STATUS_VERSION_NEWER:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("The pool has been upgraded "
	"to a newer, incompatible on-disk version.\n\tThe pool "
	"cannot be accessed on this system.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Access the pool from a "
	"system running more recent software, or\n\trestore the "
	"pool from backup.\n"));
	break;

	case ZPOOL_STATUS_FEAT_DISABLED:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("Some supported and "
	"requested features are not enabled on the pool.\n\t"
	"The pool can still be used, but some features are "
	"unavailable.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Enable all features using "
	"'zpool upgrade'. Once this is done,\n\tthe pool may no "
	"longer be accessible by software that does not support\n\t"
	"the features. See zpool-features(7) for details.\n"));
	break;

	case ZPOOL_STATUS_COMPATIBILITY_ERR:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("This pool has a "
	"compatibility list specified, but it could not be\n\t"
	"read/parsed at this time. The pool can still be used, "
	"but this\n\tshould be investigated.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Check the value of the "
	"'compatibility' property against the\n\t"
	"appropriate file in " ZPOOL_SYSCONF_COMPAT_D " or "
	ZPOOL_DATA_COMPAT_D ".\n"));
	break;

	case ZPOOL_STATUS_INCOMPATIBLE_FEAT:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more features "
	"are enabled on the pool despite not being\n\t"
	"requested by the 'compatibility' property.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Consider setting "
	"'compatibility' to an appropriate value, or\n\t"
	"adding needed features to the relevant file in\n\t"
	ZPOOL_SYSCONF_COMPAT_D " or " ZPOOL_DATA_COMPAT_D ".\n"));
	break;

	case ZPOOL_STATUS_UNSUP_FEAT_READ:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("The pool cannot be accessed "
	"on this system because it uses the\n\tfollowing feature(s)"
	" not supported on this system:\n"));
	zpool_print_unsup_feat(config);
	(void) printf("\n");
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Access the pool from a "
	"system that supports the required feature(s),\n\tor "
	"restore the pool from backup.\n"));
	break;

	case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("The pool can only be "
	"accessed in read-only mode on this system. It\n\tcannot be"
	" accessed in read-write mode because it uses the "
	"following\n\tfeature(s) not supported on this system:\n"));
	zpool_print_unsup_feat(config);
	(void) printf("\n");
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("The pool cannot be accessed "
	"in read-write mode. Import the pool with\n"
	"\t\"-o readonly=on\", access the pool from a system that "
	"supports the\n\trequired feature(s), or restore the "
	"pool from backup.\n"));
	break;

	case ZPOOL_STATUS_FAULTED_DEV_R:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices are "
	"faulted in response to persistent errors.\n\tSufficient "
	"replicas exist for the pool to continue functioning "
	"in a\n\tdegraded state.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Replace the faulted device, "
	"or use 'zpool clear' to mark the device\n\trepaired.\n"));
	break;

	case ZPOOL_STATUS_FAULTED_DEV_NR:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices are "
	"faulted in response to persistent errors. There are "
	"insufficient replicas for the pool to\n\tcontinue "
	"functioning.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Destroy and re-create the "
	"pool from a backup source. Manually marking the device\n"
	"\trepaired using 'zpool clear' may allow some data "
	"to be recovered.\n"));
	break;

	case ZPOOL_STATUS_IO_FAILURE_MMP:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("The pool is suspended "
	"because multihost writes failed or were delayed;\n\t"
	"another system could import the pool undetected.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Make sure the pool's devices"
	" are connected, then reboot your system and\n\timport the "
	- "pool.\n"));
	+ "pool or run 'zpool clear' to resume the pool.\n"));
	break;

	case ZPOOL_STATUS_IO_FAILURE_WAIT:
	case ZPOOL_STATUS_IO_FAILURE_CONTINUE:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("One or more devices are "
	"faulted in response to IO failures.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Make sure the affected "
	"devices are connected, then run 'zpool clear'.\n"));
	break;

	case ZPOOL_STATUS_BAD_LOG:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("An intent log record "
	"could not be read.\n"
	"\tWaiting for administrator intervention to fix the "
	"faulted pool.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Either restore the affected "
	"device(s) and run 'zpool online',\n"
	"\tor ignore the intent log records by running "
	"'zpool clear'.\n"));
	break;

	case ZPOOL_STATUS_NON_NATIVE_ASHIFT:
	(void) printf(gettext("status: One or more devices are "
	"configured to use a non-native block size.\n"
	"\tExpect reduced performance.\n"));
	(void) printf(gettext("action: Replace affected devices with "
	"devices that support the\n\tconfigured block size, or "
	"migrate data to a properly configured\n\tpool.\n"));
	break;

	case ZPOOL_STATUS_HOSTID_MISMATCH:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("Mismatch between pool hostid"
	" and system hostid on imported pool.\n\tThis pool was "
	"previously imported into a system with a different "
	"hostid,\n\tand then was verbatim imported into this "
	"system.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("Export this pool on all "
	"systems on which it is imported.\n"
	"\tThen import it to correct the mismatch.\n"));
	break;

	case ZPOOL_STATUS_ERRATA:
	printf_color(ANSI_BOLD, gettext("status: "));
	printf_color(ANSI_YELLOW, gettext("Errata #%d detected.\n"),
	errata);

	switch (errata) {
	case ZPOOL_ERRATA_NONE:
	break;

	case ZPOOL_ERRATA_ZOL_2094_SCRUB:
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("To correct the issue"
	" run 'zpool scrub'.\n"));
	break;

	case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
	(void) printf(gettext("\tExisting encrypted datasets "
	"contain an on-disk incompatibility\n\twhich "
	"needs to be corrected.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("To correct the issue"
	" backup existing encrypted datasets to new\n\t"
	"encrypted datasets and destroy the old ones. "
	"'zfs mount -o ro' can\n\tbe used to temporarily "
	"mount existing encrypted datasets readonly.\n"));
	break;

	case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
	(void) printf(gettext("\tExisting encrypted snapshots "
	"and bookmarks contain an on-disk\n\tincompat"
	"ibility. This may cause on-disk corruption if "
	"they are used\n\twith 'zfs recv'.\n"));
	printf_color(ANSI_BOLD, gettext("action: "));
	printf_color(ANSI_YELLOW, gettext("To correct the"
	"issue, enable the bookmark_v2 feature. No "
	"additional\n\taction is needed if there are no "
	"encrypted snapshots or bookmarks.\n\tIf preserving"
	"the encrypted snapshots and bookmarks is required,"
	" use\n\ta non-raw send to backup and restore them."
	" Alternately, they may be\n\tremoved to resolve "
	"the incompatibility.\n"));
	break;

	default:
	/*
	* All errata which allow the pool to be imported
	* must contain an action message.
	*/
	assert(0);
	}
	break;

	default:
	/*
	* The remaining errors can't actually be generated, yet.
	*/
	assert(reason == ZPOOL_STATUS_OK);
	}

	if (msgid != NULL) {
	printf(" ");
	printf_color(ANSI_BOLD, gettext("see:"));
	printf(gettext(
	" https://openzfs.github.io/openzfs-docs/msg/%s\n"),
	msgid);
	}

	if (config != NULL) {
	uint64_t nerr;
	nvlist_t spares, l2cache;
	uint_t nspares, nl2cache;
	pool_checkpoint_stat_t *pcs = NULL;
	pool_removal_stat_t *prs = NULL;

	print_scan_status(zhp, nvroot);

	(void) nvlist_lookup_uint64_array(nvroot,
	ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
	print_removal_status(zhp, prs);

	(void) nvlist_lookup_uint64_array(nvroot,
	ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
	print_checkpoint_status(pcs);

	cbp->cb_namewidth = max_width(zhp, nvroot, 0, 0,
	cbp->cb_name_flags \| VDEV_NAME_TYPE_ID);
	if (cbp->cb_namewidth < 10)
	cbp->cb_namewidth = 10;

	color_start(ANSI_BOLD);
	(void) printf(gettext("config:\n\n"));
	(void) printf(gettext("\t%-*s %-8s %5s %5s %5s"),
	cbp->cb_namewidth, "NAME", "STATE", "READ", "WRITE",
	"CKSUM");
	color_end();

	if (cbp->cb_print_slow_ios) {
	printf_color(ANSI_BOLD, " %5s", gettext("SLOW"));
	}

	if (cbp->cb_print_power) {
	printf_color(ANSI_BOLD, " %5s", gettext("POWER"));
	}

	if (cbp->vcdl != NULL)
	print_cmd_columns(cbp->vcdl, 0);

	printf("\n");

	print_status_config(zhp, cbp, zpool_get_name(zhp), nvroot, 0,
	B_FALSE, NULL);

	print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_BIAS_DEDUP);
	print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_BIAS_SPECIAL);
	print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_CLASS_LOGS);

	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
	&l2cache, &nl2cache) == 0)
	print_l2cache(zhp, cbp, l2cache, nl2cache);

	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
	&spares, &nspares) == 0)
	print_spares(zhp, cbp, spares, nspares);

	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRCOUNT,
	&nerr) == 0) {
	(void) printf("\n");
	if (nerr == 0) {
	(void) printf(gettext(
	"errors: No known data errors\n"));
	} else if (!cbp->cb_verbose) {
	color_start(ANSI_RED);
	(void) printf(gettext("errors: %llu data "
	"errors, use '-v' for a list\n"),
	(u_longlong_t)nerr);
	color_end();
	} else {
	print_error_log(zhp);
	}
	}

	if (cbp->cb_dedup_stats)
	print_dedup_stats(config);
	} else {
	(void) printf(gettext("config: The configuration cannot be "
	"determined.\n"));
	}

	return (0);
	}

	/*
	- * zpool status [-c [script1,script2,...]] [-igLpPstvx] [--power] [-T d\|u] ...
	+ * zpool status [-c [script1,script2,...]] [-DegiLpPstvx] [--power] [-T d\|u] ...
	* [pool] [interval [count]]
	*
	* -c CMD For each vdev, run command CMD
	+ * -D Display dedup status (undocumented)
	* -e Display only unhealthy vdevs
	- * -i Display vdev initialization status.
	* -g Display guid for individual vdev name.
	+ * -i Display vdev initialization status.
	* -L Follow links when resolving vdev path name.
	* -p Display values in parsable (exact) format.
	* -P Display full path for vdev name.
	* -s Display slow IOs column.
	- * -v Display complete error logs
	- * -x Display only pools with potential problems
	- * -D Display dedup status (undocumented)
	* -t Display vdev TRIM status.
	* -T Display a timestamp in date(1) or Unix format
	+ * -v Display complete error logs
	+ * -x Display only pools with potential problems
	* --power Display vdev enclosure slot power status
	*
	* Describes the health status of all pools or some subset.
	*/
	int
	zpool_do_status(int argc, char **argv)
	{
	int c;
	int ret;
	float interval = 0;
	unsigned long count = 0;
	status_cbdata_t cb = { 0 };
	char *cmd = NULL;

	struct option long_options[] = {
	- {"power", no_argument, NULL, POWER_OPT},
	+ {"power", no_argument, NULL, ZPOOL_OPTION_POWER},
	{0, 0, 0, 0}
	};

	/* check options */
	- while ((c = getopt_long(argc, argv, "c:eigLpPsvxDtT:", long_options,
	+ while ((c = getopt_long(argc, argv, "c:DegiLpPstT:vx", long_options,
	NULL)) != -1) {
	switch (c) {
	case 'c':
	if (cmd != NULL) {
	fprintf(stderr,
	gettext("Can't set -c flag twice\n"));
	exit(1);
	}

	if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
	!libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
	fprintf(stderr, gettext(
	"Can't run -c, disabled by "
	"ZPOOL_SCRIPTS_ENABLED.\n"));
	exit(1);
	}

	if ((getuid() <= 0 \|\| geteuid() <= 0) &&
	!libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
	fprintf(stderr, gettext(
	"Can't run -c with root privileges "
	"unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
	exit(1);
	}
	cmd = optarg;
	break;
	+ case 'D':
	+ cb.cb_dedup_stats = B_TRUE;
	+ break;
	case 'e':
	cb.cb_print_unhealthy = B_TRUE;
	break;
	- case 'i':
	- cb.cb_print_vdev_init = B_TRUE;
	- break;
	case 'g':
	cb.cb_name_flags \|= VDEV_NAME_GUID;
	break;
	+ case 'i':
	+ cb.cb_print_vdev_init = B_TRUE;
	+ break;
	case 'L':
	cb.cb_name_flags \|= VDEV_NAME_FOLLOW_LINKS;
	break;
	case 'p':
	cb.cb_literal = B_TRUE;
	break;
	case 'P':
	cb.cb_name_flags \|= VDEV_NAME_PATH;
	break;
	case 's':
	cb.cb_print_slow_ios = B_TRUE;
	break;
	- case 'v':
	- cb.cb_verbose = B_TRUE;
	- break;
	- case 'x':
	- cb.cb_explain = B_TRUE;
	- break;
	- case 'D':
	- cb.cb_dedup_stats = B_TRUE;
	- break;
	case 't':
	cb.cb_print_vdev_trim = B_TRUE;
	break;
	case 'T':
	get_timestamp_arg(*optarg);
	break;
	- case POWER_OPT:
	+ case 'v':
	+ cb.cb_verbose = B_TRUE;
	+ break;
	+ case 'x':
	+ cb.cb_explain = B_TRUE;
	+ break;
	+ case ZPOOL_OPTION_POWER:
	cb.cb_print_power = B_TRUE;
	break;
	case '?':
	if (optopt == 'c') {
	print_zpool_script_list("status");
	exit(0);
	} else {
	fprintf(stderr,
	gettext("invalid option '%c'\n"), optopt);
	}
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	get_interval_count(&argc, argv, &interval, &count);

	if (argc == 0)
	cb.cb_allpools = B_TRUE;

	cb.cb_first = B_TRUE;
	cb.cb_print_status = B_TRUE;

	for (;;) {
	if (timestamp_fmt != NODATE)
	print_timestamp(timestamp_fmt);

	if (cmd != NULL)
	cb.vcdl = all_pools_for_each_vdev_run(argc, argv, cmd,
	NULL, NULL, 0, 0);

	ret = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
	cb.cb_literal, status_callback, &cb);

	if (cb.vcdl != NULL)
	free_vdev_cmd_data_list(cb.vcdl);
	-
	if (argc == 0 && cb.cb_count == 0)
	(void) fprintf(stderr, gettext("no pools available\n"));
	else if (cb.cb_explain && cb.cb_first && cb.cb_allpools)
	(void) printf(gettext("all pools are healthy\n"));

	if (ret != 0)
	return (ret);

	if (interval == 0)
	break;

	if (count != 0 && --count == 0)
	break;

	(void) fsleep(interval);
	}

	return (0);
	}

	typedef struct upgrade_cbdata {
	int cb_first;
	int cb_argc;
	uint64_t cb_version;
	char **cb_argv;
	} upgrade_cbdata_t;

	static int
	check_unsupp_fs(zfs_handle_t zhp, void unsupp_fs)
	{
	int zfs_version = (int)zfs_prop_get_int(zhp, ZFS_PROP_VERSION);
	int count = (int )unsupp_fs;

	if (zfs_version > ZPL_VERSION) {
	(void) printf(gettext("%s (v%d) is not supported by this "
	"implementation of ZFS.\n"),
	zfs_get_name(zhp), zfs_version);
	(*count)++;
	}

	zfs_iter_filesystems_v2(zhp, 0, check_unsupp_fs, unsupp_fs);

	zfs_close(zhp);

	return (0);
	}

	static int
	upgrade_version(zpool_handle_t *zhp, uint64_t version)
	{
	int ret;
	nvlist_t *config;
	uint64_t oldversion;
	int unsupp_fs = 0;

	config = zpool_get_config(zhp, NULL);
	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
	&oldversion) == 0);

	char compat[ZFS_MAXPROPLEN];
	if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY, compat,
	ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
	compat[0] = '\0';

	assert(SPA_VERSION_IS_SUPPORTED(oldversion));
	assert(oldversion < version);

	ret = zfs_iter_root(zpool_get_handle(zhp), check_unsupp_fs, &unsupp_fs);
	if (ret != 0)
	return (ret);

	if (unsupp_fs) {
	(void) fprintf(stderr, gettext("Upgrade not performed due "
	"to %d unsupported filesystems (max v%d).\n"),
	unsupp_fs, (int)ZPL_VERSION);
	return (1);
	}

	if (strcmp(compat, ZPOOL_COMPAT_LEGACY) == 0) {
	(void) fprintf(stderr, gettext("Upgrade not performed because "
	"'compatibility' property set to '"
	ZPOOL_COMPAT_LEGACY "'.\n"));
	return (1);
	}

	ret = zpool_upgrade(zhp, version);
	if (ret != 0)
	return (ret);

	if (version >= SPA_VERSION_FEATURES) {
	(void) printf(gettext("Successfully upgraded "
	"'%s' from version %llu to feature flags.\n"),
	zpool_get_name(zhp), (u_longlong_t)oldversion);
	} else {
	(void) printf(gettext("Successfully upgraded "
	"'%s' from version %llu to version %llu.\n"),
	zpool_get_name(zhp), (u_longlong_t)oldversion,
	(u_longlong_t)version);
	}

	return (0);
	}

	static int
	upgrade_enable_all(zpool_handle_t zhp, int countp)
	{
	int i, ret, count;
	boolean_t firstff = B_TRUE;
	nvlist_t *enabled = zpool_get_features(zhp);

	char compat[ZFS_MAXPROPLEN];
	if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY, compat,
	ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
	compat[0] = '\0';

	boolean_t requested_features[SPA_FEATURES];
	if (zpool_do_load_compat(compat, requested_features) !=
	ZPOOL_COMPATIBILITY_OK)
	return (-1);

	count = 0;
	for (i = 0; i < SPA_FEATURES; i++) {
	const char *fname = spa_feature_table[i].fi_uname;
	const char *fguid = spa_feature_table[i].fi_guid;

	if (!spa_feature_table[i].fi_zfs_mod_supported)
	continue;

	if (!nvlist_exists(enabled, fguid) && requested_features[i]) {
	char *propname;
	verify(-1 != asprintf(&propname, "feature@%s", fname));
	ret = zpool_set_prop(zhp, propname,
	ZFS_FEATURE_ENABLED);
	if (ret != 0) {
	free(propname);
	return (ret);
	}
	count++;

	if (firstff) {
	(void) printf(gettext("Enabled the "
	"following features on '%s':\n"),
	zpool_get_name(zhp));
	firstff = B_FALSE;
	}
	(void) printf(gettext(" %s\n"), fname);
	free(propname);
	}
	}

	if (countp != NULL)
	*countp = count;
	return (0);
	}

	static int
	upgrade_cb(zpool_handle_t zhp, void arg)
	{
	upgrade_cbdata_t *cbp = arg;
	nvlist_t *config;
	uint64_t version;
	boolean_t modified_pool = B_FALSE;
	int ret;

	config = zpool_get_config(zhp, NULL);
	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
	&version) == 0);

	assert(SPA_VERSION_IS_SUPPORTED(version));

	if (version < cbp->cb_version) {
	cbp->cb_first = B_FALSE;
	ret = upgrade_version(zhp, cbp->cb_version);
	if (ret != 0)
	return (ret);
	modified_pool = B_TRUE;

	/*
	* If they did "zpool upgrade -a", then we could
	* be doing ioctls to different pools. We need
	* to log this history once to each pool, and bypass
	* the normal history logging that happens in main().
	*/
	(void) zpool_log_history(g_zfs, history_str);
	log_history = B_FALSE;
	}

	if (cbp->cb_version >= SPA_VERSION_FEATURES) {
	int count;
	ret = upgrade_enable_all(zhp, &count);
	if (ret != 0)
	return (ret);

	if (count > 0) {
	cbp->cb_first = B_FALSE;
	modified_pool = B_TRUE;
	}
	}

	if (modified_pool) {
	(void) printf("\n");
	(void) after_zpool_upgrade(zhp);
	}

	return (0);
	}

	static int
	upgrade_list_older_cb(zpool_handle_t zhp, void arg)
	{
	upgrade_cbdata_t *cbp = arg;
	nvlist_t *config;
	uint64_t version;

	config = zpool_get_config(zhp, NULL);
	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
	&version) == 0);

	assert(SPA_VERSION_IS_SUPPORTED(version));

	if (version < SPA_VERSION_FEATURES) {
	if (cbp->cb_first) {
	(void) printf(gettext("The following pools are "
	"formatted with legacy version numbers and can\n"
	"be upgraded to use feature flags. After "
	"being upgraded, these pools\nwill no "
	"longer be accessible by software that does not "
	"support feature\nflags.\n\n"
	"Note that setting a pool's 'compatibility' "
	"feature to '" ZPOOL_COMPAT_LEGACY "' will\n"
	"inhibit upgrades.\n\n"));
	(void) printf(gettext("VER POOL\n"));
	(void) printf(gettext("--- ------------\n"));
	cbp->cb_first = B_FALSE;
	}

	(void) printf("%2llu %s\n", (u_longlong_t)version,
	zpool_get_name(zhp));
	}

	return (0);
	}

	static int
	upgrade_list_disabled_cb(zpool_handle_t zhp, void arg)
	{
	upgrade_cbdata_t *cbp = arg;
	nvlist_t *config;
	uint64_t version;

	config = zpool_get_config(zhp, NULL);
	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
	&version) == 0);

	if (version >= SPA_VERSION_FEATURES) {
	int i;
	boolean_t poolfirst = B_TRUE;
	nvlist_t *enabled = zpool_get_features(zhp);

	for (i = 0; i < SPA_FEATURES; i++) {
	const char *fguid = spa_feature_table[i].fi_guid;
	const char *fname = spa_feature_table[i].fi_uname;

	if (!spa_feature_table[i].fi_zfs_mod_supported)
	continue;

	if (!nvlist_exists(enabled, fguid)) {
	if (cbp->cb_first) {
	(void) printf(gettext("\nSome "
	"supported features are not "
	"enabled on the following pools. "
	"Once a\nfeature is enabled the "
	"pool may become incompatible with "
	"software\nthat does not support "
	"the feature. See "
	"zpool-features(7) for "
	"details.\n\n"
	"Note that the pool "
	"'compatibility' feature can be "
	"used to inhibit\nfeature "
	"upgrades.\n\n"));
	(void) printf(gettext("POOL "
	"FEATURE\n"));
	(void) printf(gettext("------"
	"---------\n"));
	cbp->cb_first = B_FALSE;
	}

	if (poolfirst) {
	(void) printf(gettext("%s\n"),
	zpool_get_name(zhp));
	poolfirst = B_FALSE;
	}

	(void) printf(gettext(" %s\n"), fname);
	}
	/*
	* If they did "zpool upgrade -a", then we could
	* be doing ioctls to different pools. We need
	* to log this history once to each pool, and bypass
	* the normal history logging that happens in main().
	*/
	(void) zpool_log_history(g_zfs, history_str);
	log_history = B_FALSE;
	}
	}

	return (0);
	}

	static int
	upgrade_one(zpool_handle_t zhp, void data)
	{
	boolean_t modified_pool = B_FALSE;
	upgrade_cbdata_t *cbp = data;
	uint64_t cur_version;
	int ret;

	if (strcmp("log", zpool_get_name(zhp)) == 0) {
	(void) fprintf(stderr, gettext("'log' is now a reserved word\n"
	"Pool 'log' must be renamed using export and import"
	" to upgrade.\n"));
	return (1);
	}

	cur_version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
	if (cur_version > cbp->cb_version) {
	(void) printf(gettext("Pool '%s' is already formatted "
	"using more current version '%llu'.\n\n"),
	zpool_get_name(zhp), (u_longlong_t)cur_version);
	return (0);
	}

	if (cbp->cb_version != SPA_VERSION && cur_version == cbp->cb_version) {
	(void) printf(gettext("Pool '%s' is already formatted "
	"using version %llu.\n\n"), zpool_get_name(zhp),
	(u_longlong_t)cbp->cb_version);
	return (0);
	}

	if (cur_version != cbp->cb_version) {
	modified_pool = B_TRUE;
	ret = upgrade_version(zhp, cbp->cb_version);
	if (ret != 0)
	return (ret);
	}

	if (cbp->cb_version >= SPA_VERSION_FEATURES) {
	int count = 0;
	ret = upgrade_enable_all(zhp, &count);
	if (ret != 0)
	return (ret);

	if (count != 0) {
	modified_pool = B_TRUE;
	} else if (cur_version == SPA_VERSION) {
	(void) printf(gettext("Pool '%s' already has all "
	"supported and requested features enabled.\n"),
	zpool_get_name(zhp));
	}
	}

	if (modified_pool) {
	(void) printf("\n");
	(void) after_zpool_upgrade(zhp);
	}

	return (0);
	}

	/*
	* zpool upgrade
	* zpool upgrade -v
	* zpool upgrade [-V version] <-a \| pool ...>
	*
	* With no arguments, display downrev'd ZFS pool available for upgrade.
	* Individual pools can be upgraded by specifying the pool, and '-a' will
	* upgrade all pools.
	*/
	int
	zpool_do_upgrade(int argc, char **argv)
	{
	int c;
	upgrade_cbdata_t cb = { 0 };
	int ret = 0;
	boolean_t showversions = B_FALSE;
	boolean_t upgradeall = B_FALSE;
	char *end;


	/* check options */
	while ((c = getopt(argc, argv, ":avV:")) != -1) {
	switch (c) {
	case 'a':
	upgradeall = B_TRUE;
	break;
	case 'v':
	showversions = B_TRUE;
	break;
	case 'V':
	cb.cb_version = strtoll(optarg, &end, 10);
	if (*end != '\0' \|\|
	!SPA_VERSION_IS_SUPPORTED(cb.cb_version)) {
	(void) fprintf(stderr,
	gettext("invalid version '%s'\n"), optarg);
	usage(B_FALSE);
	}
	break;
	case ':':
	(void) fprintf(stderr, gettext("missing argument for "
	"'%c' option\n"), optopt);
	usage(B_FALSE);
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	cb.cb_argc = argc;
	cb.cb_argv = argv;
	argc -= optind;
	argv += optind;

	if (cb.cb_version == 0) {
	cb.cb_version = SPA_VERSION;
	} else if (!upgradeall && argc == 0) {
	(void) fprintf(stderr, gettext("-V option is "
	"incompatible with other arguments\n"));
	usage(B_FALSE);
	}

	if (showversions) {
	if (upgradeall \|\| argc != 0) {
	(void) fprintf(stderr, gettext("-v option is "
	"incompatible with other arguments\n"));
	usage(B_FALSE);
	}
	} else if (upgradeall) {
	if (argc != 0) {
	(void) fprintf(stderr, gettext("-a option should not "
	"be used along with a pool name\n"));
	usage(B_FALSE);
	}
	}

	(void) printf("%s", gettext("This system supports ZFS pool feature "
	"flags.\n\n"));
	if (showversions) {
	int i;

	(void) printf(gettext("The following features are "
	"supported:\n\n"));
	(void) printf(gettext("FEAT DESCRIPTION\n"));
	(void) printf("----------------------------------------------"
	"---------------\n");
	for (i = 0; i < SPA_FEATURES; i++) {
	zfeature_info_t *fi = &spa_feature_table[i];
	if (!fi->fi_zfs_mod_supported)
	continue;
	const char *ro =
	(fi->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
	" (read-only compatible)" : "";

	(void) printf("%-37s%s\n", fi->fi_uname, ro);
	(void) printf(" %s\n", fi->fi_desc);
	}
	(void) printf("\n");

	(void) printf(gettext("The following legacy versions are also "
	"supported:\n\n"));
	(void) printf(gettext("VER DESCRIPTION\n"));
	(void) printf("--- -----------------------------------------"
	"---------------\n");
	(void) printf(gettext(" 1 Initial ZFS version\n"));
	(void) printf(gettext(" 2 Ditto blocks "
	"(replicated metadata)\n"));
	(void) printf(gettext(" 3 Hot spares and double parity "
	"RAID-Z\n"));
	(void) printf(gettext(" 4 zpool history\n"));
	(void) printf(gettext(" 5 Compression using the gzip "
	"algorithm\n"));
	(void) printf(gettext(" 6 bootfs pool property\n"));
	(void) printf(gettext(" 7 Separate intent log devices\n"));
	(void) printf(gettext(" 8 Delegated administration\n"));
	(void) printf(gettext(" 9 refquota and refreservation "
	"properties\n"));
	(void) printf(gettext(" 10 Cache devices\n"));
	(void) printf(gettext(" 11 Improved scrub performance\n"));
	(void) printf(gettext(" 12 Snapshot properties\n"));
	(void) printf(gettext(" 13 snapused property\n"));
	(void) printf(gettext(" 14 passthrough-x aclinherit\n"));
	(void) printf(gettext(" 15 user/group space accounting\n"));
	(void) printf(gettext(" 16 stmf property support\n"));
	(void) printf(gettext(" 17 Triple-parity RAID-Z\n"));
	(void) printf(gettext(" 18 Snapshot user holds\n"));
	(void) printf(gettext(" 19 Log device removal\n"));
	(void) printf(gettext(" 20 Compression using zle "
	"(zero-length encoding)\n"));
	(void) printf(gettext(" 21 Deduplication\n"));
	(void) printf(gettext(" 22 Received properties\n"));
	(void) printf(gettext(" 23 Slim ZIL\n"));
	(void) printf(gettext(" 24 System attributes\n"));
	(void) printf(gettext(" 25 Improved scrub stats\n"));
	(void) printf(gettext(" 26 Improved snapshot deletion "
	"performance\n"));
	(void) printf(gettext(" 27 Improved snapshot creation "
	"performance\n"));
	(void) printf(gettext(" 28 Multiple vdev replacements\n"));
	(void) printf(gettext("\nFor more information on a particular "
	"version, including supported releases,\n"));
	(void) printf(gettext("see the ZFS Administration Guide.\n\n"));
	} else if (argc == 0 && upgradeall) {
	cb.cb_first = B_TRUE;
	ret = zpool_iter(g_zfs, upgrade_cb, &cb);
	if (ret == 0 && cb.cb_first) {
	if (cb.cb_version == SPA_VERSION) {
	(void) printf(gettext("All pools are already "
	"formatted using feature flags.\n\n"));
	(void) printf(gettext("Every feature flags "
	"pool already has all supported and "
	"requested features enabled.\n"));
	} else {
	(void) printf(gettext("All pools are already "
	"formatted with version %llu or higher.\n"),
	(u_longlong_t)cb.cb_version);
	}
	}
	} else if (argc == 0) {
	cb.cb_first = B_TRUE;
	ret = zpool_iter(g_zfs, upgrade_list_older_cb, &cb);
	assert(ret == 0);

	if (cb.cb_first) {
	(void) printf(gettext("All pools are formatted "
	"using feature flags.\n\n"));
	} else {
	(void) printf(gettext("\nUse 'zpool upgrade -v' "
	"for a list of available legacy versions.\n"));
	}

	cb.cb_first = B_TRUE;
	ret = zpool_iter(g_zfs, upgrade_list_disabled_cb, &cb);
	assert(ret == 0);

	if (cb.cb_first) {
	(void) printf(gettext("Every feature flags pool has "
	"all supported and requested features enabled.\n"));
	} else {
	(void) printf(gettext("\n"));
	}
	} else {
	ret = for_each_pool(argc, argv, B_FALSE, NULL, ZFS_TYPE_POOL,
	B_FALSE, upgrade_one, &cb);
	}

	return (ret);
	}

	typedef struct hist_cbdata {
	boolean_t first;
	boolean_t longfmt;
	boolean_t internal;
	} hist_cbdata_t;

	static void
	print_history_records(nvlist_t nvhis, hist_cbdata_t cb)
	{
	nvlist_t **records;
	uint_t numrecords;
	int i;

	verify(nvlist_lookup_nvlist_array(nvhis, ZPOOL_HIST_RECORD,
	&records, &numrecords) == 0);
	for (i = 0; i < numrecords; i++) {
	nvlist_t *rec = records[i];
	char tbuf[64] = "";

	if (nvlist_exists(rec, ZPOOL_HIST_TIME)) {
	time_t tsec;
	struct tm t;

	tsec = fnvlist_lookup_uint64(records[i],
	ZPOOL_HIST_TIME);
	(void) localtime_r(&tsec, &t);
	(void) strftime(tbuf, sizeof (tbuf), "%F.%T", &t);
	}

	if (nvlist_exists(rec, ZPOOL_HIST_ELAPSED_NS)) {
	uint64_t elapsed_ns = fnvlist_lookup_int64(records[i],
	ZPOOL_HIST_ELAPSED_NS);
	(void) snprintf(tbuf + strlen(tbuf),
	sizeof (tbuf) - strlen(tbuf),
	" (%lldms)", (long long)elapsed_ns / 1000 / 1000);
	}

	if (nvlist_exists(rec, ZPOOL_HIST_CMD)) {
	(void) printf("%s %s", tbuf,
	fnvlist_lookup_string(rec, ZPOOL_HIST_CMD));
	} else if (nvlist_exists(rec, ZPOOL_HIST_INT_EVENT)) {
	int ievent =
	fnvlist_lookup_uint64(rec, ZPOOL_HIST_INT_EVENT);
	if (!cb->internal)
	continue;
	if (ievent >= ZFS_NUM_LEGACY_HISTORY_EVENTS) {
	(void) printf("%s unrecognized record:\n",
	tbuf);
	dump_nvlist(rec, 4);
	continue;
	}
	(void) printf("%s [internal %s txg:%lld] %s", tbuf,
	zfs_history_event_names[ievent],
	(longlong_t)fnvlist_lookup_uint64(
	rec, ZPOOL_HIST_TXG),
	fnvlist_lookup_string(rec, ZPOOL_HIST_INT_STR));
	} else if (nvlist_exists(rec, ZPOOL_HIST_INT_NAME)) {
	if (!cb->internal)
	continue;
	(void) printf("%s [txg:%lld] %s", tbuf,
	(longlong_t)fnvlist_lookup_uint64(
	rec, ZPOOL_HIST_TXG),
	fnvlist_lookup_string(rec, ZPOOL_HIST_INT_NAME));
	if (nvlist_exists(rec, ZPOOL_HIST_DSNAME)) {
	(void) printf(" %s (%llu)",
	fnvlist_lookup_string(rec,
	ZPOOL_HIST_DSNAME),
	(u_longlong_t)fnvlist_lookup_uint64(rec,
	ZPOOL_HIST_DSID));
	}
	(void) printf(" %s", fnvlist_lookup_string(rec,
	ZPOOL_HIST_INT_STR));
	} else if (nvlist_exists(rec, ZPOOL_HIST_IOCTL)) {
	if (!cb->internal)
	continue;
	(void) printf("%s ioctl %s\n", tbuf,
	fnvlist_lookup_string(rec, ZPOOL_HIST_IOCTL));
	if (nvlist_exists(rec, ZPOOL_HIST_INPUT_NVL)) {
	(void) printf(" input:\n");
	dump_nvlist(fnvlist_lookup_nvlist(rec,
	ZPOOL_HIST_INPUT_NVL), 8);
	}
	if (nvlist_exists(rec, ZPOOL_HIST_OUTPUT_NVL)) {
	(void) printf(" output:\n");
	dump_nvlist(fnvlist_lookup_nvlist(rec,
	ZPOOL_HIST_OUTPUT_NVL), 8);
	}
	if (nvlist_exists(rec, ZPOOL_HIST_OUTPUT_SIZE)) {
	(void) printf(" output nvlist omitted; "
	"original size: %lldKB\n",
	(longlong_t)fnvlist_lookup_int64(rec,
	ZPOOL_HIST_OUTPUT_SIZE) / 1024);
	}
	if (nvlist_exists(rec, ZPOOL_HIST_ERRNO)) {
	(void) printf(" errno: %lld\n",
	(longlong_t)fnvlist_lookup_int64(rec,
	ZPOOL_HIST_ERRNO));
	}
	} else {
	if (!cb->internal)
	continue;
	(void) printf("%s unrecognized record:\n", tbuf);
	dump_nvlist(rec, 4);
	}

	if (!cb->longfmt) {
	(void) printf("\n");
	continue;
	}
	(void) printf(" [");
	if (nvlist_exists(rec, ZPOOL_HIST_WHO)) {
	uid_t who = fnvlist_lookup_uint64(rec, ZPOOL_HIST_WHO);
	struct passwd *pwd = getpwuid(who);
	(void) printf("user %d ", (int)who);
	if (pwd != NULL)
	(void) printf("(%s) ", pwd->pw_name);
	}
	if (nvlist_exists(rec, ZPOOL_HIST_HOST)) {
	(void) printf("on %s",
	fnvlist_lookup_string(rec, ZPOOL_HIST_HOST));
	}
	if (nvlist_exists(rec, ZPOOL_HIST_ZONE)) {
	(void) printf(":%s",
	fnvlist_lookup_string(rec, ZPOOL_HIST_ZONE));
	}

	(void) printf("]");
	(void) printf("\n");
	}
	}

	/*
	* Print out the command history for a specific pool.
	*/
	static int
	get_history_one(zpool_handle_t zhp, void data)
	{
	nvlist_t *nvhis;
	int ret;
	hist_cbdata_t cb = (hist_cbdata_t )data;
	uint64_t off = 0;
	boolean_t eof = B_FALSE;

	cb->first = B_FALSE;

	(void) printf(gettext("History for '%s':\n"), zpool_get_name(zhp));

	while (!eof) {
	if ((ret = zpool_get_history(zhp, &nvhis, &off, &eof)) != 0)
	return (ret);

	print_history_records(nvhis, cb);
	nvlist_free(nvhis);
	}
	(void) printf("\n");

	return (ret);
	}

	/*
	* zpool history <pool>
	*
	* Displays the history of commands that modified pools.
	*/
	int
	zpool_do_history(int argc, char **argv)
	{
	hist_cbdata_t cbdata = { 0 };
	int ret;
	int c;

	cbdata.first = B_TRUE;
	/* check options */
	while ((c = getopt(argc, argv, "li")) != -1) {
	switch (c) {
	case 'l':
	cbdata.longfmt = B_TRUE;
	break;
	case 'i':
	cbdata.internal = B_TRUE;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}
	argc -= optind;
	argv += optind;

	ret = for_each_pool(argc, argv, B_FALSE, NULL, ZFS_TYPE_POOL,
	B_FALSE, get_history_one, &cbdata);

	if (argc == 0 && cbdata.first == B_TRUE) {
	(void) fprintf(stderr, gettext("no pools available\n"));
	return (0);
	}

	return (ret);
	}

	typedef struct ev_opts {
	int verbose;
	int scripted;
	int follow;
	int clear;
	char poolname[ZFS_MAX_DATASET_NAME_LEN];
	} ev_opts_t;

	static void
	zpool_do_events_short(nvlist_t nvl, ev_opts_t opts)
	{
	char ctime_str[26], str[32];
	const char *ptr;
	int64_t *tv;
	uint_t n;

	verify(nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tv, &n) == 0);
	memset(str, ' ', 32);
	(void) ctime_r((const time_t *)&tv[0], ctime_str);
	(void) memcpy(str, ctime_str+4, 6); /* 'Jun 30' */
	(void) memcpy(str+7, ctime_str+20, 4); /* '1993' */
	(void) memcpy(str+12, ctime_str+11, 8); /* '21:49:08' */
	(void) sprintf(str+20, ".%09lld", (longlong_t)tv[1]); /* '.123456789' */
	if (opts->scripted)
	(void) printf(gettext("%s\t"), str);
	else
	(void) printf(gettext("%s "), str);

	verify(nvlist_lookup_string(nvl, FM_CLASS, &ptr) == 0);
	(void) printf(gettext("%s\n"), ptr);
	}

	static void
	zpool_do_events_nvprint(nvlist_t *nvl, int depth)
	{
	nvpair_t *nvp;

	for (nvp = nvlist_next_nvpair(nvl, NULL);
	nvp != NULL; nvp = nvlist_next_nvpair(nvl, nvp)) {

	data_type_t type = nvpair_type(nvp);
	const char *name = nvpair_name(nvp);

	boolean_t b;
	uint8_t i8;
	uint16_t i16;
	uint32_t i32;
	uint64_t i64;
	const char *str;
	nvlist_t *cnv;

	printf(gettext("%*s%s = "), depth, "", name);

	switch (type) {
	case DATA_TYPE_BOOLEAN:
	printf(gettext("%s"), "1");
	break;

	case DATA_TYPE_BOOLEAN_VALUE:
	(void) nvpair_value_boolean_value(nvp, &b);
	printf(gettext("%s"), b ? "1" : "0");
	break;

	case DATA_TYPE_BYTE:
	(void) nvpair_value_byte(nvp, &i8);
	printf(gettext("0x%x"), i8);
	break;

	case DATA_TYPE_INT8:
	(void) nvpair_value_int8(nvp, (void *)&i8);
	printf(gettext("0x%x"), i8);
	break;

	case DATA_TYPE_UINT8:
	(void) nvpair_value_uint8(nvp, &i8);
	printf(gettext("0x%x"), i8);
	break;

	case DATA_TYPE_INT16:
	(void) nvpair_value_int16(nvp, (void *)&i16);
	printf(gettext("0x%x"), i16);
	break;

	case DATA_TYPE_UINT16:
	(void) nvpair_value_uint16(nvp, &i16);
	printf(gettext("0x%x"), i16);
	break;

	case DATA_TYPE_INT32:
	(void) nvpair_value_int32(nvp, (void *)&i32);
	printf(gettext("0x%x"), i32);
	break;

	case DATA_TYPE_UINT32:
	(void) nvpair_value_uint32(nvp, &i32);
	printf(gettext("0x%x"), i32);
	break;

	case DATA_TYPE_INT64:
	(void) nvpair_value_int64(nvp, (void *)&i64);
	printf(gettext("0x%llx"), (u_longlong_t)i64);
	break;

	case DATA_TYPE_UINT64:
	(void) nvpair_value_uint64(nvp, &i64);
	/*
	* translate vdev state values to readable
	* strings to aide zpool events consumers
	*/
	if (strcmp(name,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE) == 0 \|\|
	strcmp(name,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_LASTSTATE) == 0) {
	printf(gettext("\"%s\" (0x%llx)"),
	zpool_state_to_name(i64, VDEV_AUX_NONE),
	(u_longlong_t)i64);
	} else {
	printf(gettext("0x%llx"), (u_longlong_t)i64);
	}
	break;

	case DATA_TYPE_HRTIME:
	(void) nvpair_value_hrtime(nvp, (void *)&i64);
	printf(gettext("0x%llx"), (u_longlong_t)i64);
	break;

	case DATA_TYPE_STRING:
	(void) nvpair_value_string(nvp, &str);
	printf(gettext("\"%s\""), str ? str : "<NULL>");
	break;

	case DATA_TYPE_NVLIST:
	printf(gettext("(embedded nvlist)\n"));
	(void) nvpair_value_nvlist(nvp, &cnv);
	zpool_do_events_nvprint(cnv, depth + 8);
	printf(gettext("%*s(end %s)"), depth, "", name);
	break;

	case DATA_TYPE_NVLIST_ARRAY: {
	nvlist_t **val;
	uint_t i, nelem;

	(void) nvpair_value_nvlist_array(nvp, &val, &nelem);
	printf(gettext("(%d embedded nvlists)\n"), nelem);
	for (i = 0; i < nelem; i++) {
	printf(gettext("%*s%s[%d] = %s\n"),
	depth, "", name, i, "(embedded nvlist)");
	zpool_do_events_nvprint(val[i], depth + 8);
	printf(gettext("%*s(end %s[%i])\n"),
	depth, "", name, i);
	}
	printf(gettext("%*s(end %s)\n"), depth, "", name);
	}
	break;

	case DATA_TYPE_INT8_ARRAY: {
	int8_t *val;
	uint_t i, nelem;

	(void) nvpair_value_int8_array(nvp, &val, &nelem);
	for (i = 0; i < nelem; i++)
	printf(gettext("0x%x "), val[i]);

	break;
	}

	case DATA_TYPE_UINT8_ARRAY: {
	uint8_t *val;
	uint_t i, nelem;

	(void) nvpair_value_uint8_array(nvp, &val, &nelem);
	for (i = 0; i < nelem; i++)
	printf(gettext("0x%x "), val[i]);

	break;
	}

	case DATA_TYPE_INT16_ARRAY: {
	int16_t *val;
	uint_t i, nelem;

	(void) nvpair_value_int16_array(nvp, &val, &nelem);
	for (i = 0; i < nelem; i++)
	printf(gettext("0x%x "), val[i]);

	break;
	}

	case DATA_TYPE_UINT16_ARRAY: {
	uint16_t *val;
	uint_t i, nelem;

	(void) nvpair_value_uint16_array(nvp, &val, &nelem);
	for (i = 0; i < nelem; i++)
	printf(gettext("0x%x "), val[i]);

	break;
	}

	case DATA_TYPE_INT32_ARRAY: {
	int32_t *val;
	uint_t i, nelem;

	(void) nvpair_value_int32_array(nvp, &val, &nelem);
	for (i = 0; i < nelem; i++)
	printf(gettext("0x%x "), val[i]);

	break;
	}

	case DATA_TYPE_UINT32_ARRAY: {
	uint32_t *val;
	uint_t i, nelem;

	(void) nvpair_value_uint32_array(nvp, &val, &nelem);
	for (i = 0; i < nelem; i++)
	printf(gettext("0x%x "), val[i]);

	break;
	}

	case DATA_TYPE_INT64_ARRAY: {
	int64_t *val;
	uint_t i, nelem;

	(void) nvpair_value_int64_array(nvp, &val, &nelem);
	for (i = 0; i < nelem; i++)
	printf(gettext("0x%llx "),
	(u_longlong_t)val[i]);

	break;
	}

	case DATA_TYPE_UINT64_ARRAY: {
	uint64_t *val;
	uint_t i, nelem;

	(void) nvpair_value_uint64_array(nvp, &val, &nelem);
	for (i = 0; i < nelem; i++)
	printf(gettext("0x%llx "),
	(u_longlong_t)val[i]);

	break;
	}

	case DATA_TYPE_STRING_ARRAY: {
	const char **str;
	uint_t i, nelem;

	(void) nvpair_value_string_array(nvp, &str, &nelem);
	for (i = 0; i < nelem; i++)
	printf(gettext("\"%s\" "),
	str[i] ? str[i] : "<NULL>");

	break;
	}

	case DATA_TYPE_BOOLEAN_ARRAY:
	case DATA_TYPE_BYTE_ARRAY:
	case DATA_TYPE_DOUBLE:
	case DATA_TYPE_DONTCARE:
	case DATA_TYPE_UNKNOWN:
	printf(gettext("<unknown>"));
	break;
	}

	printf(gettext("\n"));
	}
	}

	static int
	zpool_do_events_next(ev_opts_t *opts)
	{
	nvlist_t *nvl;
	int zevent_fd, ret, dropped;
	const char *pool;

	zevent_fd = open(ZFS_DEV, O_RDWR);
	VERIFY(zevent_fd >= 0);

	if (!opts->scripted)
	(void) printf(gettext("%-30s %s\n"), "TIME", "CLASS");

	while (1) {
	ret = zpool_events_next(g_zfs, &nvl, &dropped,
	(opts->follow ? ZEVENT_NONE : ZEVENT_NONBLOCK), zevent_fd);
	if (ret \|\| nvl == NULL)
	break;

	if (dropped > 0)
	(void) printf(gettext("dropped %d events\n"), dropped);

	if (strlen(opts->poolname) > 0 &&
	nvlist_lookup_string(nvl, FM_FMRI_ZFS_POOL, &pool) == 0 &&
	strcmp(opts->poolname, pool) != 0)
	continue;

	zpool_do_events_short(nvl, opts);

	if (opts->verbose) {
	zpool_do_events_nvprint(nvl, 8);
	printf(gettext("\n"));
	}
	(void) fflush(stdout);

	nvlist_free(nvl);
	}

	VERIFY(0 == close(zevent_fd));

	return (ret);
	}

	static int
	zpool_do_events_clear(void)
	{
	int count, ret;

	ret = zpool_events_clear(g_zfs, &count);
	if (!ret)
	(void) printf(gettext("cleared %d events\n"), count);

	return (ret);
	}

	/*
	* zpool events [-vHf [pool] \| -c]
	*
	* Displays events logs by ZFS.
	*/
	int
	zpool_do_events(int argc, char **argv)
	{
	ev_opts_t opts = { 0 };
	int ret;
	int c;

	/* check options */
	while ((c = getopt(argc, argv, "vHfc")) != -1) {
	switch (c) {
	case 'v':
	opts.verbose = 1;
	break;
	case 'H':
	opts.scripted = 1;
	break;
	case 'f':
	opts.follow = 1;
	break;
	case 'c':
	opts.clear = 1;
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}
	argc -= optind;
	argv += optind;

	if (argc > 1) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	} else if (argc == 1) {
	(void) strlcpy(opts.poolname, argv[0], sizeof (opts.poolname));
	if (!zfs_name_valid(opts.poolname, ZFS_TYPE_POOL)) {
	(void) fprintf(stderr,
	gettext("invalid pool name '%s'\n"), opts.poolname);
	usage(B_FALSE);
	}
	}

	if ((argc == 1 \|\| opts.verbose \|\| opts.scripted \|\| opts.follow) &&
	opts.clear) {
	(void) fprintf(stderr,
	gettext("invalid options combined with -c\n"));
	usage(B_FALSE);
	}

	if (opts.clear)
	ret = zpool_do_events_clear();
	else
	ret = zpool_do_events_next(&opts);

	return (ret);
	}

	static int
	get_callback_vdev(zpool_handle_t zhp, char vdevname, void *data)
	{
	zprop_get_cbdata_t cbp = (zprop_get_cbdata_t )data;
	char value[ZFS_MAXPROPLEN];
	zprop_source_t srctype;

	for (zprop_list_t *pl = cbp->cb_proplist; pl != NULL;
	pl = pl->pl_next) {
	char *prop_name;
	/*
	* If the first property is pool name, it is a special
	* placeholder that we can skip. This will also skip
	* over the name property when 'all' is specified.
	*/
	if (pl->pl_prop == ZPOOL_PROP_NAME &&
	pl == cbp->cb_proplist)
	continue;

	if (pl->pl_prop == ZPROP_INVAL) {
	prop_name = pl->pl_user_prop;
	} else {
	prop_name = (char *)vdev_prop_to_name(pl->pl_prop);
	}
	if (zpool_get_vdev_prop(zhp, vdevname, pl->pl_prop,
	prop_name, value, sizeof (value), &srctype,
	cbp->cb_literal) == 0) {
	zprop_print_one_property(vdevname, cbp, prop_name,
	value, srctype, NULL, NULL);
	}
	}

	return (0);
	}

	static int
	get_callback_vdev_cb(void zhp_data, nvlist_t nv, void *data)
	{
	zpool_handle_t *zhp = zhp_data;
	zprop_get_cbdata_t cbp = (zprop_get_cbdata_t )data;
	char *vdevname;
	const char *type;
	int ret;

	/*
	* zpool_vdev_name() transforms the root vdev name (i.e., root-0) to the
	* pool name for display purposes, which is not desired. Fallback to
	* zpool_vdev_name() when not dealing with the root vdev.
	*/
	type = fnvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE);
	if (zhp != NULL && strcmp(type, "root") == 0)
	vdevname = strdup("root-0");
	else
	vdevname = zpool_vdev_name(g_zfs, zhp, nv,
	cbp->cb_vdevs.cb_name_flags);

	(void) vdev_expand_proplist(zhp, vdevname, &cbp->cb_proplist);

	ret = get_callback_vdev(zhp, vdevname, data);

	free(vdevname);

	return (ret);
	}

	static int
	get_callback(zpool_handle_t zhp, void data)
	{
	zprop_get_cbdata_t cbp = (zprop_get_cbdata_t )data;
	char value[ZFS_MAXPROPLEN];
	zprop_source_t srctype;
	zprop_list_t *pl;
	int vid;

	if (cbp->cb_type == ZFS_TYPE_VDEV) {
	if (strcmp(cbp->cb_vdevs.cb_names[0], "all-vdevs") == 0) {
	for_each_vdev(zhp, get_callback_vdev_cb, data);
	} else {
	/* Adjust column widths for vdev properties */
	for (vid = 0; vid < cbp->cb_vdevs.cb_names_count;
	vid++) {
	vdev_expand_proplist(zhp,
	cbp->cb_vdevs.cb_names[vid],
	&cbp->cb_proplist);
	}
	/* Display the properties */
	for (vid = 0; vid < cbp->cb_vdevs.cb_names_count;
	vid++) {
	get_callback_vdev(zhp,
	cbp->cb_vdevs.cb_names[vid], data);
	}
	}
	} else {
	assert(cbp->cb_type == ZFS_TYPE_POOL);
	for (pl = cbp->cb_proplist; pl != NULL; pl = pl->pl_next) {
	/*
	* Skip the special fake placeholder. This will also
	* skip over the name property when 'all' is specified.
	*/
	if (pl->pl_prop == ZPOOL_PROP_NAME &&
	pl == cbp->cb_proplist)
	continue;

	if (pl->pl_prop == ZPROP_INVAL &&
	zfs_prop_user(pl->pl_user_prop)) {
	srctype = ZPROP_SRC_LOCAL;

	if (zpool_get_userprop(zhp, pl->pl_user_prop,
	value, sizeof (value), &srctype) != 0)
	continue;

	zprop_print_one_property(zpool_get_name(zhp),
	cbp, pl->pl_user_prop, value, srctype,
	NULL, NULL);
	} else if (pl->pl_prop == ZPROP_INVAL &&
	(zpool_prop_feature(pl->pl_user_prop) \|\|
	zpool_prop_unsupported(pl->pl_user_prop))) {
	srctype = ZPROP_SRC_LOCAL;

	if (zpool_prop_get_feature(zhp,
	pl->pl_user_prop, value,
	sizeof (value)) == 0) {
	zprop_print_one_property(
	zpool_get_name(zhp), cbp,
	pl->pl_user_prop, value, srctype,
	NULL, NULL);
	}
	} else {
	if (zpool_get_prop(zhp, pl->pl_prop, value,
	sizeof (value), &srctype,
	cbp->cb_literal) != 0)
	continue;

	zprop_print_one_property(zpool_get_name(zhp),
	cbp, zpool_prop_to_name(pl->pl_prop),
	value, srctype, NULL, NULL);
	}
	}
	}

	return (0);
	}

	/*
	* zpool get [-Hp] [-o "all" \| field[,...]] <"all" \| property[,...]> <pool> ...
	*
	* -H Scripted mode. Don't display headers, and separate properties
	* by a single tab.
	* -o List of columns to display. Defaults to
	* "name,property,value,source".
	* -p Display values in parsable (exact) format.
	*
	* Get properties of pools in the system. Output space statistics
	* for each one as well as other attributes.
	*/
	int
	zpool_do_get(int argc, char **argv)
	{
	zprop_get_cbdata_t cb = { 0 };
	zprop_list_t fake_name = { 0 };
	int ret;
	int c, i;
	char *propstr = NULL;
	char *vdev = NULL;

	cb.cb_first = B_TRUE;

	/*
	* Set up default columns and sources.
	*/
	cb.cb_sources = ZPROP_SRC_ALL;
	cb.cb_columns[0] = GET_COL_NAME;
	cb.cb_columns[1] = GET_COL_PROPERTY;
	cb.cb_columns[2] = GET_COL_VALUE;
	cb.cb_columns[3] = GET_COL_SOURCE;
	cb.cb_type = ZFS_TYPE_POOL;
	cb.cb_vdevs.cb_name_flags \|= VDEV_NAME_TYPE_ID;
	current_prop_type = cb.cb_type;

	/* check options */
	while ((c = getopt(argc, argv, ":Hpo:")) != -1) {
	switch (c) {
	case 'p':
	cb.cb_literal = B_TRUE;
	break;
	case 'H':
	cb.cb_scripted = B_TRUE;
	break;
	case 'o':
	memset(&cb.cb_columns, 0, sizeof (cb.cb_columns));
	i = 0;

	for (char *tok; (tok = strsep(&optarg, ",")); ) {
	static const char *const col_opts[] =
	{ "name", "property", "value", "source",
	"all" };
	static const zfs_get_column_t col_cols[] =
	{ GET_COL_NAME, GET_COL_PROPERTY, GET_COL_VALUE,
	GET_COL_SOURCE };

	if (i == ZFS_GET_NCOLS - 1) {
	(void) fprintf(stderr, gettext("too "
	"many fields given to -o "
	"option\n"));
	usage(B_FALSE);
	}

	for (c = 0; c < ARRAY_SIZE(col_opts); ++c)
	if (strcmp(tok, col_opts[c]) == 0)
	goto found;

	(void) fprintf(stderr,
	gettext("invalid column name '%s'\n"), tok);
	usage(B_FALSE);

	found:
	if (c >= 4) {
	if (i > 0) {
	(void) fprintf(stderr,
	gettext("\"all\" conflicts "
	"with specific fields "
	"given to -o option\n"));
	usage(B_FALSE);
	}

	memcpy(cb.cb_columns, col_cols,
	sizeof (col_cols));
	i = ZFS_GET_NCOLS - 1;
	} else
	cb.cb_columns[i++] = col_cols[c];
	}
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing property "
	"argument\n"));
	usage(B_FALSE);
	}

	/* Properties list is needed later by zprop_get_list() */
	propstr = argv[0];

	argc--;
	argv++;

	if (argc == 0) {
	/* No args, so just print the defaults. */
	} else if (are_all_pools(argc, argv)) {
	/* All the args are pool names */
	} else if (are_all_pools(1, argv)) {
	/* The first arg is a pool name */
	if ((argc == 2 && strcmp(argv[1], "all-vdevs") == 0) \|\|
	(argc == 2 && strcmp(argv[1], "root") == 0) \|\|
	are_vdevs_in_pool(argc - 1, argv + 1, argv[0],
	&cb.cb_vdevs)) {

	if (strcmp(argv[1], "root") == 0)
	vdev = strdup("root-0");
	else
	vdev = strdup(argv[1]);

	/* ... and the rest are vdev names */
	cb.cb_vdevs.cb_names = &vdev;
	cb.cb_vdevs.cb_names_count = argc - 1;
	cb.cb_type = ZFS_TYPE_VDEV;
	argc = 1; /* One pool to process */
	} else {
	fprintf(stderr, gettext("Expected a list of vdevs in"
	" \"%s\", but got:\n"), argv[0]);
	error_list_unresolved_vdevs(argc - 1, argv + 1,
	argv[0], &cb.cb_vdevs);
	fprintf(stderr, "\n");
	usage(B_FALSE);
	return (1);
	}
	} else {
	/*
	- * The first arg isn't a pool name,
	+ * The first arg isn't the name of a valid pool.
	*/
	- fprintf(stderr, gettext("missing pool name.\n"));
	- fprintf(stderr, "\n");
	- usage(B_FALSE);
	+ fprintf(stderr, gettext("Cannot get properties of %s: "
	+ "no such pool available.\n"), argv[0]);
	return (1);
	}

	if (zprop_get_list(g_zfs, propstr, &cb.cb_proplist,
	cb.cb_type) != 0) {
	/* Use correct list of valid properties (pool or vdev) */
	current_prop_type = cb.cb_type;
	usage(B_FALSE);
	}

	if (cb.cb_proplist != NULL) {
	fake_name.pl_prop = ZPOOL_PROP_NAME;
	fake_name.pl_width = strlen(gettext("NAME"));
	fake_name.pl_next = cb.cb_proplist;
	cb.cb_proplist = &fake_name;
	}

	ret = for_each_pool(argc, argv, B_TRUE, &cb.cb_proplist, cb.cb_type,
	cb.cb_literal, get_callback, &cb);

	if (cb.cb_proplist == &fake_name)
	zprop_free_list(fake_name.pl_next);
	else
	zprop_free_list(cb.cb_proplist);

	if (vdev != NULL)
	free(vdev);

	return (ret);
	}

	typedef struct set_cbdata {
	char *cb_propname;
	char *cb_value;
	zfs_type_t cb_type;
	vdev_cbdata_t cb_vdevs;
	boolean_t cb_any_successful;
	} set_cbdata_t;

	static int
	set_pool_callback(zpool_handle_t zhp, set_cbdata_t cb)
	{
	int error;

	/* Check if we have out-of-bounds features */
	if (strcmp(cb->cb_propname, ZPOOL_CONFIG_COMPATIBILITY) == 0) {
	boolean_t features[SPA_FEATURES];
	if (zpool_do_load_compat(cb->cb_value, features) !=
	ZPOOL_COMPATIBILITY_OK)
	return (-1);

	nvlist_t *enabled = zpool_get_features(zhp);
	spa_feature_t i;
	for (i = 0; i < SPA_FEATURES; i++) {
	const char *fguid = spa_feature_table[i].fi_guid;
	if (nvlist_exists(enabled, fguid) && !features[i])
	break;
	}
	if (i < SPA_FEATURES)
	(void) fprintf(stderr, gettext("Warning: one or "
	"more features already enabled on pool '%s'\n"
	"are not present in this compatibility set.\n"),
	zpool_get_name(zhp));
	}

	/* if we're setting a feature, check it's in compatibility set */
	if (zpool_prop_feature(cb->cb_propname) &&
	strcmp(cb->cb_value, ZFS_FEATURE_ENABLED) == 0) {
	char *fname = strchr(cb->cb_propname, '@') + 1;
	spa_feature_t f;

	if (zfeature_lookup_name(fname, &f) == 0) {
	char compat[ZFS_MAXPROPLEN];
	if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY,
	compat, ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
	compat[0] = '\0';

	boolean_t features[SPA_FEATURES];
	if (zpool_do_load_compat(compat, features) !=
	ZPOOL_COMPATIBILITY_OK) {
	(void) fprintf(stderr, gettext("Error: "
	"cannot enable feature '%s' on pool '%s'\n"
	"because the pool's 'compatibility' "
	"property cannot be parsed.\n"),
	fname, zpool_get_name(zhp));
	return (-1);
	}

	if (!features[f]) {
	(void) fprintf(stderr, gettext("Error: "
	"cannot enable feature '%s' on pool '%s'\n"
	"as it is not specified in this pool's "
	"current compatibility set.\n"
	"Consider setting 'compatibility' to a "
	"less restrictive set, or to 'off'.\n"),
	fname, zpool_get_name(zhp));
	return (-1);
	}
	}
	}

	error = zpool_set_prop(zhp, cb->cb_propname, cb->cb_value);

	return (error);
	}

	static int
	set_callback(zpool_handle_t zhp, void data)
	{
	int error;
	set_cbdata_t cb = (set_cbdata_t )data;

	if (cb->cb_type == ZFS_TYPE_VDEV) {
	error = zpool_set_vdev_prop(zhp, *cb->cb_vdevs.cb_names,
	cb->cb_propname, cb->cb_value);
	} else {
	assert(cb->cb_type == ZFS_TYPE_POOL);
	error = set_pool_callback(zhp, cb);
	}

	cb->cb_any_successful = !error;
	return (error);
	}

	int
	zpool_do_set(int argc, char **argv)
	{
	set_cbdata_t cb = { 0 };
	int error;
	char *vdev = NULL;

	current_prop_type = ZFS_TYPE_POOL;
	if (argc > 1 && argv[1][0] == '-') {
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	argv[1][1]);
	usage(B_FALSE);
	}

	if (argc < 2) {
	(void) fprintf(stderr, gettext("missing property=value "
	"argument\n"));
	usage(B_FALSE);
	}

	if (argc < 3) {
	(void) fprintf(stderr, gettext("missing pool name\n"));
	usage(B_FALSE);
	}

	if (argc > 4) {
	(void) fprintf(stderr, gettext("too many pool names\n"));
	usage(B_FALSE);
	}

	cb.cb_propname = argv[1];
	cb.cb_type = ZFS_TYPE_POOL;
	cb.cb_vdevs.cb_name_flags \|= VDEV_NAME_TYPE_ID;
	cb.cb_value = strchr(cb.cb_propname, '=');
	if (cb.cb_value == NULL) {
	(void) fprintf(stderr, gettext("missing value in "
	"property=value argument\n"));
	usage(B_FALSE);
	}

	*(cb.cb_value) = '\0';
	cb.cb_value++;
	argc -= 2;
	argv += 2;

	/* argv[0] is pool name */
	if (!is_pool(argv[0])) {
	(void) fprintf(stderr,
	gettext("cannot open '%s': is not a pool\n"), argv[0]);
	return (EINVAL);
	}

	/* argv[1], when supplied, is vdev name */
	if (argc == 2) {

	if (strcmp(argv[1], "root") == 0)
	vdev = strdup("root-0");
	else
	vdev = strdup(argv[1]);

	if (!are_vdevs_in_pool(1, &vdev, argv[0], &cb.cb_vdevs)) {
	(void) fprintf(stderr, gettext(
	"cannot find '%s' in '%s': device not in pool\n"),
	vdev, argv[0]);
	free(vdev);
	return (EINVAL);
	}
	cb.cb_vdevs.cb_names = &vdev;
	cb.cb_vdevs.cb_names_count = 1;
	cb.cb_type = ZFS_TYPE_VDEV;
	}

	error = for_each_pool(1, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
	B_FALSE, set_callback, &cb);

	if (vdev != NULL)
	free(vdev);

	return (error);
	}

	/* Add up the total number of bytes left to initialize/trim across all vdevs */
	static uint64_t
	vdev_activity_remaining(nvlist_t *nv, zpool_wait_activity_t activity)
	{
	uint64_t bytes_remaining;
	nvlist_t **child;
	uint_t c, children;
	vdev_stat_t *vs;

	assert(activity == ZPOOL_WAIT_INITIALIZE \|\|
	activity == ZPOOL_WAIT_TRIM);

	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
	(uint64_t **)&vs, &c) == 0);

	if (activity == ZPOOL_WAIT_INITIALIZE &&
	vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE)
	bytes_remaining = vs->vs_initialize_bytes_est -
	vs->vs_initialize_bytes_done;
	else if (activity == ZPOOL_WAIT_TRIM &&
	vs->vs_trim_state == VDEV_TRIM_ACTIVE)
	bytes_remaining = vs->vs_trim_bytes_est -
	vs->vs_trim_bytes_done;
	else
	bytes_remaining = 0;

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
	&child, &children) != 0)
	children = 0;

	for (c = 0; c < children; c++)
	bytes_remaining += vdev_activity_remaining(child[c], activity);

	return (bytes_remaining);
	}

	/* Add up the total number of bytes left to rebuild across top-level vdevs */
	static uint64_t
	vdev_activity_top_remaining(nvlist_t *nv)
	{
	uint64_t bytes_remaining = 0;
	nvlist_t **child;
	uint_t children;
	int error;

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
	&child, &children) != 0)
	children = 0;

	for (uint_t c = 0; c < children; c++) {
	vdev_rebuild_stat_t *vrs;
	uint_t i;

	error = nvlist_lookup_uint64_array(child[c],
	ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i);
	if (error == 0) {
	if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
	bytes_remaining += (vrs->vrs_bytes_est -
	vrs->vrs_bytes_rebuilt);
	}
	}
	}

	return (bytes_remaining);
	}

	/* Whether any vdevs are 'spare' or 'replacing' vdevs */
	static boolean_t
	vdev_any_spare_replacing(nvlist_t *nv)
	{
	nvlist_t **child;
	uint_t c, children;
	const char *vdev_type;

	(void) nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &vdev_type);

	if (strcmp(vdev_type, VDEV_TYPE_REPLACING) == 0 \|\|
	strcmp(vdev_type, VDEV_TYPE_SPARE) == 0 \|\|
	strcmp(vdev_type, VDEV_TYPE_DRAID_SPARE) == 0) {
	return (B_TRUE);
	}

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
	&child, &children) != 0)
	children = 0;

	for (c = 0; c < children; c++) {
	if (vdev_any_spare_replacing(child[c]))
	return (B_TRUE);
	}

	return (B_FALSE);
	}

	typedef struct wait_data {
	char *wd_poolname;
	boolean_t wd_scripted;
	boolean_t wd_exact;
	boolean_t wd_headers_once;
	boolean_t wd_should_exit;
	/* Which activities to wait for */
	boolean_t wd_enabled[ZPOOL_WAIT_NUM_ACTIVITIES];
	float wd_interval;
	pthread_cond_t wd_cv;
	pthread_mutex_t wd_mutex;
	} wait_data_t;

	/*
	* Print to stdout a single line, containing one column for each activity that
	* we are waiting for specifying how many bytes of work are left for that
	* activity.
	*/
	static void
	print_wait_status_row(wait_data_t wd, zpool_handle_t zhp, int row)
	{
	nvlist_t config, nvroot;
	uint_t c;
	int i;
	pool_checkpoint_stat_t *pcs = NULL;
	pool_scan_stat_t *pss = NULL;
	pool_removal_stat_t *prs = NULL;
	const char *const headers[] = {"DISCARD", "FREE", "INITIALIZE",
	"REPLACE", "REMOVE", "RESILVER", "SCRUB", "TRIM"};
	int col_widths[ZPOOL_WAIT_NUM_ACTIVITIES];

	/* Calculate the width of each column */
	for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
	/*
	* Make sure we have enough space in the col for pretty-printed
	* numbers and for the column header, and then leave a couple
	* spaces between cols for readability.
	*/
	col_widths[i] = MAX(strlen(headers[i]), 6) + 2;
	}

	if (timestamp_fmt != NODATE)
	print_timestamp(timestamp_fmt);

	/* Print header if appropriate */
	int term_height = terminal_height();
	boolean_t reprint_header = (!wd->wd_headers_once && term_height > 0 &&
	row % (term_height-1) == 0);
	if (!wd->wd_scripted && (row == 0 \|\| reprint_header)) {
	for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
	if (wd->wd_enabled[i])
	(void) printf("%*s", col_widths[i], headers[i]);
	}
	(void) fputc('\n', stdout);
	}

	/* Bytes of work remaining in each activity */
	int64_t bytes_rem[ZPOOL_WAIT_NUM_ACTIVITIES] = {0};

	bytes_rem[ZPOOL_WAIT_FREE] =
	zpool_get_prop_int(zhp, ZPOOL_PROP_FREEING, NULL);

	config = zpool_get_config(zhp, NULL);
	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);

	(void) nvlist_lookup_uint64_array(nvroot,
	ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
	if (pcs != NULL && pcs->pcs_state == CS_CHECKPOINT_DISCARDING)
	bytes_rem[ZPOOL_WAIT_CKPT_DISCARD] = pcs->pcs_space;

	(void) nvlist_lookup_uint64_array(nvroot,
	ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
	if (prs != NULL && prs->prs_state == DSS_SCANNING)
	bytes_rem[ZPOOL_WAIT_REMOVE] = prs->prs_to_copy -
	prs->prs_copied;

	(void) nvlist_lookup_uint64_array(nvroot,
	ZPOOL_CONFIG_SCAN_STATS, (uint64_t **)&pss, &c);
	if (pss != NULL && pss->pss_state == DSS_SCANNING &&
	pss->pss_pass_scrub_pause == 0) {
	int64_t rem = pss->pss_to_examine - pss->pss_issued;
	if (pss->pss_func == POOL_SCAN_SCRUB)
	bytes_rem[ZPOOL_WAIT_SCRUB] = rem;
	else
	bytes_rem[ZPOOL_WAIT_RESILVER] = rem;
	} else if (check_rebuilding(nvroot, NULL)) {
	bytes_rem[ZPOOL_WAIT_RESILVER] =
	vdev_activity_top_remaining(nvroot);
	}

	bytes_rem[ZPOOL_WAIT_INITIALIZE] =
	vdev_activity_remaining(nvroot, ZPOOL_WAIT_INITIALIZE);
	bytes_rem[ZPOOL_WAIT_TRIM] =
	vdev_activity_remaining(nvroot, ZPOOL_WAIT_TRIM);

	/*
	* A replace finishes after resilvering finishes, so the amount of work
	* left for a replace is the same as for resilvering.
	*
	* It isn't quite correct to say that if we have any 'spare' or
	* 'replacing' vdevs and a resilver is happening, then a replace is in
	* progress, like we do here. When a hot spare is used, the faulted vdev
	* is not removed after the hot spare is resilvered, so parent 'spare'
	* vdev is not removed either. So we could have a 'spare' vdev, but be
	* resilvering for a different reason. However, we use it as a heuristic
	* because we don't have access to the DTLs, which could tell us whether
	* or not we have really finished resilvering a hot spare.
	*/
	if (vdev_any_spare_replacing(nvroot))
	bytes_rem[ZPOOL_WAIT_REPLACE] = bytes_rem[ZPOOL_WAIT_RESILVER];

	for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
	char buf[64];
	if (!wd->wd_enabled[i])
	continue;

	if (wd->wd_exact)
	(void) snprintf(buf, sizeof (buf), "%" PRIi64,
	bytes_rem[i]);
	else
	zfs_nicenum(bytes_rem[i], buf, sizeof (buf));

	if (wd->wd_scripted)
	(void) printf(i == 0 ? "%s" : "\t%s", buf);
	else
	(void) printf(" %*s", col_widths[i] - 1, buf);
	}
	(void) printf("\n");
	(void) fflush(stdout);
	}

	static void *
	wait_status_thread(void *arg)
	{
	wait_data_t wd = (wait_data_t )arg;
	zpool_handle_t *zhp;

	if ((zhp = zpool_open(g_zfs, wd->wd_poolname)) == NULL)
	return (void *)(1);

	for (int row = 0; ; row++) {
	boolean_t missing;
	struct timespec timeout;
	int ret = 0;
	(void) clock_gettime(CLOCK_REALTIME, &timeout);

	if (zpool_refresh_stats(zhp, &missing) != 0 \|\| missing \|\|
	zpool_props_refresh(zhp) != 0) {
	zpool_close(zhp);
	return (void *)(uintptr_t)(missing ? 0 : 1);
	}

	print_wait_status_row(wd, zhp, row);

	timeout.tv_sec += floor(wd->wd_interval);
	long nanos = timeout.tv_nsec +
	(wd->wd_interval - floor(wd->wd_interval)) * NANOSEC;
	if (nanos >= NANOSEC) {
	timeout.tv_sec++;
	timeout.tv_nsec = nanos - NANOSEC;
	} else {
	timeout.tv_nsec = nanos;
	}
	pthread_mutex_lock(&wd->wd_mutex);
	if (!wd->wd_should_exit)
	ret = pthread_cond_timedwait(&wd->wd_cv, &wd->wd_mutex,
	&timeout);
	pthread_mutex_unlock(&wd->wd_mutex);
	if (ret == 0) {
	break; /* signaled by main thread */
	} else if (ret != ETIMEDOUT) {
	(void) fprintf(stderr, gettext("pthread_cond_timedwait "
	"failed: %s\n"), strerror(ret));
	zpool_close(zhp);
	return (void *)(uintptr_t)(1);
	}
	}

	zpool_close(zhp);
	return (void *)(0);
	}

	int
	zpool_do_wait(int argc, char **argv)
	{
	boolean_t verbose = B_FALSE;
	int c, i;
	unsigned long count;
	pthread_t status_thr;
	int error = 0;
	zpool_handle_t *zhp;

	wait_data_t wd;
	wd.wd_scripted = B_FALSE;
	wd.wd_exact = B_FALSE;
	wd.wd_headers_once = B_FALSE;
	wd.wd_should_exit = B_FALSE;

	pthread_mutex_init(&wd.wd_mutex, NULL);
	pthread_cond_init(&wd.wd_cv, NULL);

	/* By default, wait for all types of activity. */
	for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++)
	wd.wd_enabled[i] = B_TRUE;

	while ((c = getopt(argc, argv, "HpT:t:")) != -1) {
	switch (c) {
	case 'H':
	wd.wd_scripted = B_TRUE;
	break;
	case 'n':
	wd.wd_headers_once = B_TRUE;
	break;
	case 'p':
	wd.wd_exact = B_TRUE;
	break;
	case 'T':
	get_timestamp_arg(*optarg);
	break;
	case 't':
	/* Reset activities array */
	memset(&wd.wd_enabled, 0, sizeof (wd.wd_enabled));

	for (char *tok; (tok = strsep(&optarg, ",")); ) {
	static const char *const col_opts[] = {
	"discard", "free", "initialize", "replace",
	"remove", "resilver", "scrub", "trim" };

	for (i = 0; i < ARRAY_SIZE(col_opts); ++i)
	if (strcmp(tok, col_opts[i]) == 0) {
	wd.wd_enabled[i] = B_TRUE;
	goto found;
	}

	(void) fprintf(stderr,
	gettext("invalid activity '%s'\n"), tok);
	usage(B_FALSE);
	found:;
	}
	break;
	case '?':
	(void) fprintf(stderr, gettext("invalid option '%c'\n"),
	optopt);
	usage(B_FALSE);
	}
	}

	argc -= optind;
	argv += optind;

	get_interval_count(&argc, argv, &wd.wd_interval, &count);
	if (count != 0) {
	/* This subcmd only accepts an interval, not a count */
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	if (wd.wd_interval != 0)
	verbose = B_TRUE;

	if (argc < 1) {
	(void) fprintf(stderr, gettext("missing 'pool' argument\n"));
	usage(B_FALSE);
	}
	if (argc > 1) {
	(void) fprintf(stderr, gettext("too many arguments\n"));
	usage(B_FALSE);
	}

	wd.wd_poolname = argv[0];

	if ((zhp = zpool_open(g_zfs, wd.wd_poolname)) == NULL)
	return (1);

	if (verbose) {
	/*
	* We use a separate thread for printing status updates because
	* the main thread will call lzc_wait(), which blocks as long
	* as an activity is in progress, which can be a long time.
	*/
	if (pthread_create(&status_thr, NULL, wait_status_thread, &wd)
	!= 0) {
	(void) fprintf(stderr, gettext("failed to create status"
	"thread: %s\n"), strerror(errno));
	zpool_close(zhp);
	return (1);
	}
	}

	/*
	* Loop over all activities that we are supposed to wait for until none
	* of them are in progress. Note that this means we can end up waiting
	* for more activities to complete than just those that were in progress
	* when we began waiting; if an activity we are interested in begins
	* while we are waiting for another activity, we will wait for both to
	* complete before exiting.
	*/
	for (;;) {
	boolean_t missing = B_FALSE;
	boolean_t any_waited = B_FALSE;

	for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
	boolean_t waited;

	if (!wd.wd_enabled[i])
	continue;

	error = zpool_wait_status(zhp, i, &missing, &waited);
	if (error != 0 \|\| missing)
	break;

	any_waited = (any_waited \|\| waited);
	}

	if (error != 0 \|\| missing \|\| !any_waited)
	break;
	}

	zpool_close(zhp);

	if (verbose) {
	uintptr_t status;
	pthread_mutex_lock(&wd.wd_mutex);
	wd.wd_should_exit = B_TRUE;
	pthread_cond_signal(&wd.wd_cv);
	pthread_mutex_unlock(&wd.wd_mutex);
	(void) pthread_join(status_thr, (void *)&status);
	if (status != 0)
	error = status;
	}

	pthread_mutex_destroy(&wd.wd_mutex);
	pthread_cond_destroy(&wd.wd_cv);
	return (error);
	}

	static int
	find_command_idx(const char command, int idx)
	{
	for (int i = 0; i < NCOMMAND; ++i) {
	if (command_table[i].name == NULL)
	continue;

	if (strcmp(command, command_table[i].name) == 0) {
	*idx = i;
	return (0);
	}
	}
	return (1);
	}

	/*
	* Display version message
	*/
	static int
	zpool_do_version(int argc, char **argv)
	{
	(void) argc, (void) argv;
	return (zfs_version_print() != 0);
	}

	/* Display documentation */
	static int
	zpool_do_help(int argc, char **argv)
	{
	char page[MAXNAMELEN];
	if (argc < 3 \|\| strcmp(argv[2], "zpool") == 0)
	strcpy(page, "zpool");
	else if (strcmp(argv[2], "concepts") == 0 \|\|
	strcmp(argv[2], "props") == 0)
	snprintf(page, sizeof (page), "zpool%s", argv[2]);
	else
	snprintf(page, sizeof (page), "zpool-%s", argv[2]);

	execlp("man", "man", page, NULL);

	fprintf(stderr, "couldn't run man program: %s", strerror(errno));
	return (-1);
	}

	/*
	* Do zpool_load_compat() and print error message on failure
	*/
	static zpool_compat_status_t
	zpool_do_load_compat(const char compat, boolean_t list)
	{
	char report[1024];

	zpool_compat_status_t ret;

	ret = zpool_load_compat(compat, list, report, 1024);
	switch (ret) {

	case ZPOOL_COMPATIBILITY_OK:
	break;

	case ZPOOL_COMPATIBILITY_NOFILES:
	case ZPOOL_COMPATIBILITY_BADFILE:
	case ZPOOL_COMPATIBILITY_BADTOKEN:
	(void) fprintf(stderr, "Error: %s\n", report);
	break;

	case ZPOOL_COMPATIBILITY_WARNTOKEN:
	(void) fprintf(stderr, "Warning: %s\n", report);
	ret = ZPOOL_COMPATIBILITY_OK;
	break;
	}
	return (ret);
	}

	int
	main(int argc, char **argv)
	{
	int ret = 0;
	int i = 0;
	char *cmdname;
	char **newargv;

	(void) setlocale(LC_ALL, "");
	(void) setlocale(LC_NUMERIC, "C");
	(void) textdomain(TEXT_DOMAIN);
	srand(time(NULL));

	opterr = 0;

	/*
	* Make sure the user has specified some command.
	*/
	if (argc < 2) {
	(void) fprintf(stderr, gettext("missing command\n"));
	usage(B_FALSE);
	}

	cmdname = argv[1];

	/*
	* Special case '-?'
	*/
	if ((strcmp(cmdname, "-?") == 0) \|\| strcmp(cmdname, "--help") == 0)
	usage(B_TRUE);

	/*
	* Special case '-V\|--version'
	*/
	if ((strcmp(cmdname, "-V") == 0) \|\| (strcmp(cmdname, "--version") == 0))
	return (zpool_do_version(argc, argv));

	/*
	* Special case 'help'
	*/
	if (strcmp(cmdname, "help") == 0)
	return (zpool_do_help(argc, argv));

	if ((g_zfs = libzfs_init()) == NULL) {
	(void) fprintf(stderr, "%s\n", libzfs_error_init(errno));
	return (1);
	}

	libzfs_print_on_error(g_zfs, B_TRUE);

	zfs_save_arguments(argc, argv, history_str, sizeof (history_str));

	/*
	* Many commands modify input strings for string parsing reasons.
	* We create a copy to protect the original argv.
	*/
	newargv = safe_malloc((argc + 1) * sizeof (newargv[0]));
	for (i = 0; i < argc; i++)
	newargv[i] = strdup(argv[i]);
	newargv[argc] = NULL;

	/*
	* Run the appropriate command.
	*/
	if (find_command_idx(cmdname, &i) == 0) {
	current_command = &command_table[i];
	ret = command_table[i].func(argc - 1, newargv + 1);
	} else if (strchr(cmdname, '=')) {
	verify(find_command_idx("set", &i) == 0);
	current_command = &command_table[i];
	ret = command_table[i].func(argc, newargv);
	} else if (strcmp(cmdname, "freeze") == 0 && argc == 3) {
	/*
	* 'freeze' is a vile debugging abomination, so we treat
	* it as such.
	*/
	zfs_cmd_t zc = {"\0"};

	(void) strlcpy(zc.zc_name, argv[2], sizeof (zc.zc_name));
	ret = zfs_ioctl(g_zfs, ZFS_IOC_POOL_FREEZE, &zc);
	if (ret != 0) {
	(void) fprintf(stderr,
	gettext("failed to freeze pool: %d\n"), errno);
	ret = 1;
	}

	log_history = 0;
	} else {
	(void) fprintf(stderr, gettext("unrecognized "
	"command '%s'\n"), cmdname);
	usage(B_FALSE);
	ret = 1;
	}

	for (i = 0; i < argc; i++)
	free(newargv[i]);
	free(newargv);

	if (ret == 0 && log_history)
	(void) zpool_log_history(g_zfs, history_str);

	libzfs_fini(g_zfs);

	/*
	* The 'ZFS_ABORT' environment variable causes us to dump core on exit
	* for the purposes of running ::findleaks.
	*/
	if (getenv("ZFS_ABORT") != NULL) {
	(void) printf("dumping core by request\n");
	abort();
	}

	return (ret);
	}
	diff --git a/sys/contrib/openzfs/cmd/ztest.c b/sys/contrib/openzfs/cmd/ztest.c
	index 8cfbdfe1c2e2..34744d12b592 100644
	--- a/sys/contrib/openzfs/cmd/ztest.c
	+++ b/sys/contrib/openzfs/cmd/ztest.c
	@@ -1,8326 +1,8326 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	- * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
	+ * Copyright (c) 2011, 2024 by Delphix. All rights reserved.
	* Copyright 2011 Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2013 Steven Hartland. All rights reserved.
	* Copyright (c) 2014 Integros [integros.com]
	* Copyright 2017 Joyent, Inc.
	* Copyright (c) 2017, Intel Corporation.
	*/

	/*
	* The objective of this program is to provide a DMU/ZAP/SPA stress test
	* that runs entirely in userland, is easy to use, and easy to extend.
	*
	* The overall design of the ztest program is as follows:
	*
	* (1) For each major functional area (e.g. adding vdevs to a pool,
	* creating and destroying datasets, reading and writing objects, etc)
	* we have a simple routine to test that functionality. These
	* individual routines do not have to do anything "stressful".
	*
	* (2) We turn these simple functionality tests into a stress test by
	* running them all in parallel, with as many threads as desired,
	* and spread across as many datasets, objects, and vdevs as desired.
	*
	* (3) While all this is happening, we inject faults into the pool to
	* verify that self-healing data really works.
	*
	* (4) Every time we open a dataset, we change its checksum and compression
	* functions. Thus even individual objects vary from block to block
	* in which checksum they use and whether they're compressed.
	*
	* (5) To verify that we never lose on-disk consistency after a crash,
	* we run the entire test in a child of the main process.
	* At random times, the child self-immolates with a SIGKILL.
	* This is the software equivalent of pulling the power cord.
	* The parent then runs the test again, using the existing
	* storage pool, as many times as desired. If backwards compatibility
	* testing is enabled ztest will sometimes run the "older" version
	* of ztest after a SIGKILL.
	*
	* (6) To verify that we don't have future leaks or temporal incursions,
	* many of the functional tests record the transaction group number
	* as part of their data. When reading old data, they verify that
	* the transaction group number is less than the current, open txg.
	* If you add a new test, please do this if applicable.
	*
	* (7) Threads are created with a reduced stack size, for sanity checking.
	* Therefore, it's important not to allocate huge buffers on the stack.
	*
	* When run with no arguments, ztest runs for about five minutes and
	* produces no output if successful. To get a little bit of information,
	* specify -V. To get more information, specify -VV, and so on.
	*
	* To turn this into an overnight stress test, use -T to specify run time.
	*
	* You can ask more vdevs [-v], datasets [-d], or threads [-t]
	* to increase the pool capacity, fanout, and overall stress level.
	*
	* Use the -k option to set the desired frequency of kills.
	*
	* When ztest invokes itself it passes all relevant information through a
	* temporary file which is mmap-ed in the child process. This allows shared
	* memory to survive the exec syscall. The ztest_shared_hdr_t struct is always
	* stored at offset 0 of this file and contains information on the size and
	* number of shared structures in the file. The information stored in this file
	* must remain backwards compatible with older versions of ztest so that
	* ztest can invoke them during backwards compatibility testing (-B).
	*/

	#include <sys/zfs_context.h>
	#include <sys/spa.h>
	#include <sys/dmu.h>
	#include <sys/txg.h>
	#include <sys/dbuf.h>
	#include <sys/zap.h>
	#include <sys/dmu_objset.h>
	#include <sys/poll.h>
	#include <sys/stat.h>
	#include <sys/time.h>
	#include <sys/wait.h>
	#include <sys/mman.h>
	#include <sys/resource.h>
	#include <sys/zio.h>
	#include <sys/zil.h>
	#include <sys/zil_impl.h>
	#include <sys/vdev_draid.h>
	#include <sys/vdev_impl.h>
	#include <sys/vdev_file.h>
	#include <sys/vdev_initialize.h>
	#include <sys/vdev_raidz.h>
	#include <sys/vdev_trim.h>
	#include <sys/spa_impl.h>
	#include <sys/metaslab_impl.h>
	#include <sys/dsl_prop.h>
	#include <sys/dsl_dataset.h>
	#include <sys/dsl_destroy.h>
	#include <sys/dsl_scan.h>
	#include <sys/zio_checksum.h>
	#include <sys/zfs_refcount.h>
	#include <sys/zfeature.h>
	#include <sys/dsl_userhold.h>
	#include <sys/abd.h>
	#include <sys/blake3.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <unistd.h>
	#include <getopt.h>
	#include <signal.h>
	#include <umem.h>
	#include <ctype.h>
	#include <math.h>
	#include <sys/fs/zfs.h>
	#include <zfs_fletcher.h>
	#include <libnvpair.h>
	#include <libzutil.h>
	#include <sys/crypto/icp.h>
	#include <sys/zfs_impl.h>
	#if (__GLIBC__ && !__UCLIBC__)
	#include <execinfo.h> /* for backtrace() */
	#endif

	static int ztest_fd_data = -1;
	static int ztest_fd_rand = -1;

	typedef struct ztest_shared_hdr {
	uint64_t zh_hdr_size;
	uint64_t zh_opts_size;
	uint64_t zh_size;
	uint64_t zh_stats_size;
	uint64_t zh_stats_count;
	uint64_t zh_ds_size;
	uint64_t zh_ds_count;
	} ztest_shared_hdr_t;

	static ztest_shared_hdr_t *ztest_shared_hdr;

	enum ztest_class_state {
	ZTEST_VDEV_CLASS_OFF,
	ZTEST_VDEV_CLASS_ON,
	ZTEST_VDEV_CLASS_RND
	};

	#define ZO_GVARS_MAX_ARGLEN ((size_t)64)
	#define ZO_GVARS_MAX_COUNT ((size_t)10)

	typedef struct ztest_shared_opts {
	char zo_pool[ZFS_MAX_DATASET_NAME_LEN];
	char zo_dir[ZFS_MAX_DATASET_NAME_LEN];
	char zo_alt_ztest[MAXNAMELEN];
	char zo_alt_libpath[MAXNAMELEN];
	uint64_t zo_vdevs;
	uint64_t zo_vdevtime;
	size_t zo_vdev_size;
	int zo_ashift;
	int zo_mirrors;
	int zo_raid_children;
	int zo_raid_parity;
	char zo_raid_type[8];
	int zo_draid_data;
	int zo_draid_spares;
	int zo_datasets;
	int zo_threads;
	uint64_t zo_passtime;
	uint64_t zo_killrate;
	int zo_verbose;
	int zo_init;
	uint64_t zo_time;
	uint64_t zo_maxloops;
	uint64_t zo_metaslab_force_ganging;
	int zo_mmp_test;
	int zo_special_vdevs;
	int zo_dump_dbgmsg;
	int zo_gvars_count;
	char zo_gvars[ZO_GVARS_MAX_COUNT][ZO_GVARS_MAX_ARGLEN];
	} ztest_shared_opts_t;

	/* Default values for command line options. */
	#define DEFAULT_POOL "ztest"
	#define DEFAULT_VDEV_DIR "/tmp"
	#define DEFAULT_VDEV_COUNT 5
	#define DEFAULT_VDEV_SIZE (SPA_MINDEVSIZE * 4) /* 256m default size */
	#define DEFAULT_VDEV_SIZE_STR "256M"
	#define DEFAULT_ASHIFT SPA_MINBLOCKSHIFT
	#define DEFAULT_MIRRORS 2
	#define DEFAULT_RAID_CHILDREN 4
	#define DEFAULT_RAID_PARITY 1
	#define DEFAULT_DRAID_DATA 4
	#define DEFAULT_DRAID_SPARES 1
	#define DEFAULT_DATASETS_COUNT 7
	#define DEFAULT_THREADS 23
	#define DEFAULT_RUN_TIME 300 /* 300 seconds */
	#define DEFAULT_RUN_TIME_STR "300 sec"
	#define DEFAULT_PASS_TIME 60 /* 60 seconds */
	#define DEFAULT_PASS_TIME_STR "60 sec"
	#define DEFAULT_KILL_RATE 70 /* 70% kill rate */
	#define DEFAULT_KILLRATE_STR "70%"
	#define DEFAULT_INITS 1
	#define DEFAULT_MAX_LOOPS 50 /* 5 minutes */
	#define DEFAULT_FORCE_GANGING (64 << 10)
	#define DEFAULT_FORCE_GANGING_STR "64K"

	/* Simplifying assumption: -1 is not a valid default. */
	#define NO_DEFAULT -1

	static const ztest_shared_opts_t ztest_opts_defaults = {
	.zo_pool = DEFAULT_POOL,
	.zo_dir = DEFAULT_VDEV_DIR,
	.zo_alt_ztest = { '\0' },
	.zo_alt_libpath = { '\0' },
	.zo_vdevs = DEFAULT_VDEV_COUNT,
	.zo_ashift = DEFAULT_ASHIFT,
	.zo_mirrors = DEFAULT_MIRRORS,
	.zo_raid_children = DEFAULT_RAID_CHILDREN,
	.zo_raid_parity = DEFAULT_RAID_PARITY,
	.zo_raid_type = VDEV_TYPE_RAIDZ,
	.zo_vdev_size = DEFAULT_VDEV_SIZE,
	.zo_draid_data = DEFAULT_DRAID_DATA, /* data drives */
	.zo_draid_spares = DEFAULT_DRAID_SPARES, /* distributed spares */
	.zo_datasets = DEFAULT_DATASETS_COUNT,
	.zo_threads = DEFAULT_THREADS,
	.zo_passtime = DEFAULT_PASS_TIME,
	.zo_killrate = DEFAULT_KILL_RATE,
	.zo_verbose = 0,
	.zo_mmp_test = 0,
	.zo_init = DEFAULT_INITS,
	.zo_time = DEFAULT_RUN_TIME,
	.zo_maxloops = DEFAULT_MAX_LOOPS, /* max loops during spa_freeze() */
	.zo_metaslab_force_ganging = DEFAULT_FORCE_GANGING,
	.zo_special_vdevs = ZTEST_VDEV_CLASS_RND,
	.zo_gvars_count = 0,
	};

	extern uint64_t metaslab_force_ganging;
	extern uint64_t metaslab_df_alloc_threshold;
	extern uint64_t zfs_deadman_synctime_ms;
	extern uint_t metaslab_preload_limit;
	extern int zfs_compressed_arc_enabled;
	extern int zfs_abd_scatter_enabled;
	extern uint_t dmu_object_alloc_chunk_shift;
	extern boolean_t zfs_force_some_double_word_sm_entries;
	extern unsigned long zio_decompress_fail_fraction;
	extern unsigned long zfs_reconstruct_indirect_damage_fraction;


	static ztest_shared_opts_t *ztest_shared_opts;
	static ztest_shared_opts_t ztest_opts;
	static const char *const ztest_wkeydata = "abcdefghijklmnopqrstuvwxyz012345";

	typedef struct ztest_shared_ds {
	uint64_t zd_seq;
	} ztest_shared_ds_t;

	static ztest_shared_ds_t *ztest_shared_ds;
	#define ZTEST_GET_SHARED_DS(d) (&ztest_shared_ds[d])

	#define BT_MAGIC 0x123456789abcdefULL
	#define MAXFAULTS(zs) \
	(MAX((zs)->zs_mirrors, 1) * (ztest_opts.zo_raid_parity + 1) - 1)

	enum ztest_io_type {
	ZTEST_IO_WRITE_TAG,
	ZTEST_IO_WRITE_PATTERN,
	ZTEST_IO_WRITE_ZEROES,
	ZTEST_IO_TRUNCATE,
	ZTEST_IO_SETATTR,
	ZTEST_IO_REWRITE,
	ZTEST_IO_TYPES
	};

	typedef struct ztest_block_tag {
	uint64_t bt_magic;
	uint64_t bt_objset;
	uint64_t bt_object;
	uint64_t bt_dnodesize;
	uint64_t bt_offset;
	uint64_t bt_gen;
	uint64_t bt_txg;
	uint64_t bt_crtxg;
	} ztest_block_tag_t;

	typedef struct bufwad {
	uint64_t bw_index;
	uint64_t bw_txg;
	uint64_t bw_data;
	} bufwad_t;

	/*
	* It would be better to use a rangelock_t per object. Unfortunately
	* the rangelock_t is not a drop-in replacement for rl_t, because we
	* still need to map from object ID to rangelock_t.
	*/
	typedef enum {
	RL_READER,
	RL_WRITER,
	RL_APPEND
	} rl_type_t;

	typedef struct rll {
	void *rll_writer;
	int rll_readers;
	kmutex_t rll_lock;
	kcondvar_t rll_cv;
	} rll_t;

	typedef struct rl {
	uint64_t rl_object;
	uint64_t rl_offset;
	uint64_t rl_size;
	rll_t *rl_lock;
	} rl_t;

	#define ZTEST_RANGE_LOCKS 64
	#define ZTEST_OBJECT_LOCKS 64

	/*
	* Object descriptor. Used as a template for object lookup/create/remove.
	*/
	typedef struct ztest_od {
	uint64_t od_dir;
	uint64_t od_object;
	dmu_object_type_t od_type;
	dmu_object_type_t od_crtype;
	uint64_t od_blocksize;
	uint64_t od_crblocksize;
	uint64_t od_crdnodesize;
	uint64_t od_gen;
	uint64_t od_crgen;
	char od_name[ZFS_MAX_DATASET_NAME_LEN];
	} ztest_od_t;

	/*
	* Per-dataset state.
	*/
	typedef struct ztest_ds {
	ztest_shared_ds_t *zd_shared;
	objset_t *zd_os;
	pthread_rwlock_t zd_zilog_lock;
	zilog_t *zd_zilog;
	ztest_od_t zd_od; / debugging aid */
	char zd_name[ZFS_MAX_DATASET_NAME_LEN];
	kmutex_t zd_dirobj_lock;
	rll_t zd_object_lock[ZTEST_OBJECT_LOCKS];
	rll_t zd_range_lock[ZTEST_RANGE_LOCKS];
	} ztest_ds_t;

	/*
	* Per-iteration state.
	*/
	typedef void ztest_func_t(ztest_ds_t *zd, uint64_t id);

	typedef struct ztest_info {
	ztest_func_t zi_func; / test function */
	uint64_t zi_iters; /* iterations per execution */
	uint64_t zi_interval; / execute every <interval> seconds */
	const char zi_funcname; / name of test function */
	} ztest_info_t;

	typedef struct ztest_shared_callstate {
	uint64_t zc_count; /* per-pass count */
	uint64_t zc_time; /* per-pass time */
	uint64_t zc_next; /* next time to call this function */
	} ztest_shared_callstate_t;

	static ztest_shared_callstate_t *ztest_shared_callstate;
	#define ZTEST_GET_SHARED_CALLSTATE(c) (&ztest_shared_callstate[c])

	ztest_func_t ztest_dmu_read_write;
	ztest_func_t ztest_dmu_write_parallel;
	ztest_func_t ztest_dmu_object_alloc_free;
	ztest_func_t ztest_dmu_object_next_chunk;
	ztest_func_t ztest_dmu_commit_callbacks;
	ztest_func_t ztest_zap;
	ztest_func_t ztest_zap_parallel;
	ztest_func_t ztest_zil_commit;
	ztest_func_t ztest_zil_remount;
	ztest_func_t ztest_dmu_read_write_zcopy;
	ztest_func_t ztest_dmu_objset_create_destroy;
	ztest_func_t ztest_dmu_prealloc;
	ztest_func_t ztest_fzap;
	ztest_func_t ztest_dmu_snapshot_create_destroy;
	ztest_func_t ztest_dsl_prop_get_set;
	ztest_func_t ztest_spa_prop_get_set;
	ztest_func_t ztest_spa_create_destroy;
	ztest_func_t ztest_fault_inject;
	ztest_func_t ztest_dmu_snapshot_hold;
	ztest_func_t ztest_mmp_enable_disable;
	ztest_func_t ztest_scrub;
	ztest_func_t ztest_dsl_dataset_promote_busy;
	ztest_func_t ztest_vdev_attach_detach;
	ztest_func_t ztest_vdev_LUN_growth;
	ztest_func_t ztest_vdev_add_remove;
	ztest_func_t ztest_vdev_class_add;
	ztest_func_t ztest_vdev_aux_add_remove;
	ztest_func_t ztest_split_pool;
	ztest_func_t ztest_reguid;
	ztest_func_t ztest_spa_upgrade;
	ztest_func_t ztest_device_removal;
	ztest_func_t ztest_spa_checkpoint_create_discard;
	ztest_func_t ztest_initialize;
	ztest_func_t ztest_trim;
	ztest_func_t ztest_blake3;
	ztest_func_t ztest_fletcher;
	ztest_func_t ztest_fletcher_incr;
	ztest_func_t ztest_verify_dnode_bt;

	static uint64_t zopt_always = 0ULL * NANOSEC; /* all the time */
	static uint64_t zopt_incessant = 1ULL * NANOSEC / 10; /* every 1/10 second */
	static uint64_t zopt_often = 1ULL * NANOSEC; /* every second */
	static uint64_t zopt_sometimes = 10ULL * NANOSEC; /* every 10 seconds */
	static uint64_t zopt_rarely = 60ULL * NANOSEC; /* every 60 seconds */

	#define ZTI_INIT(func, iters, interval) \
	{ .zi_func = (func), \
	.zi_iters = (iters), \
	.zi_interval = (interval), \
	.zi_funcname = # func }

	static ztest_info_t ztest_info[] = {
	ZTI_INIT(ztest_dmu_read_write, 1, &zopt_always),
	ZTI_INIT(ztest_dmu_write_parallel, 10, &zopt_always),
	ZTI_INIT(ztest_dmu_object_alloc_free, 1, &zopt_always),
	ZTI_INIT(ztest_dmu_object_next_chunk, 1, &zopt_sometimes),
	ZTI_INIT(ztest_dmu_commit_callbacks, 1, &zopt_always),
	ZTI_INIT(ztest_zap, 30, &zopt_always),
	ZTI_INIT(ztest_zap_parallel, 100, &zopt_always),
	ZTI_INIT(ztest_split_pool, 1, &zopt_sometimes),
	ZTI_INIT(ztest_zil_commit, 1, &zopt_incessant),
	ZTI_INIT(ztest_zil_remount, 1, &zopt_sometimes),
	ZTI_INIT(ztest_dmu_read_write_zcopy, 1, &zopt_often),
	ZTI_INIT(ztest_dmu_objset_create_destroy, 1, &zopt_often),
	ZTI_INIT(ztest_dsl_prop_get_set, 1, &zopt_often),
	ZTI_INIT(ztest_spa_prop_get_set, 1, &zopt_sometimes),
	#if 0
	ZTI_INIT(ztest_dmu_prealloc, 1, &zopt_sometimes),
	#endif
	ZTI_INIT(ztest_fzap, 1, &zopt_sometimes),
	ZTI_INIT(ztest_dmu_snapshot_create_destroy, 1, &zopt_sometimes),
	ZTI_INIT(ztest_spa_create_destroy, 1, &zopt_sometimes),
	ZTI_INIT(ztest_fault_inject, 1, &zopt_sometimes),
	ZTI_INIT(ztest_dmu_snapshot_hold, 1, &zopt_sometimes),
	ZTI_INIT(ztest_mmp_enable_disable, 1, &zopt_sometimes),
	ZTI_INIT(ztest_reguid, 1, &zopt_rarely),
	ZTI_INIT(ztest_scrub, 1, &zopt_rarely),
	ZTI_INIT(ztest_spa_upgrade, 1, &zopt_rarely),
	ZTI_INIT(ztest_dsl_dataset_promote_busy, 1, &zopt_rarely),
	ZTI_INIT(ztest_vdev_attach_detach, 1, &zopt_sometimes),
	ZTI_INIT(ztest_vdev_LUN_growth, 1, &zopt_rarely),
	ZTI_INIT(ztest_vdev_add_remove, 1, &ztest_opts.zo_vdevtime),
	ZTI_INIT(ztest_vdev_class_add, 1, &ztest_opts.zo_vdevtime),
	ZTI_INIT(ztest_vdev_aux_add_remove, 1, &ztest_opts.zo_vdevtime),
	ZTI_INIT(ztest_device_removal, 1, &zopt_sometimes),
	ZTI_INIT(ztest_spa_checkpoint_create_discard, 1, &zopt_rarely),
	ZTI_INIT(ztest_initialize, 1, &zopt_sometimes),
	ZTI_INIT(ztest_trim, 1, &zopt_sometimes),
	ZTI_INIT(ztest_blake3, 1, &zopt_rarely),
	ZTI_INIT(ztest_fletcher, 1, &zopt_rarely),
	ZTI_INIT(ztest_fletcher_incr, 1, &zopt_rarely),
	ZTI_INIT(ztest_verify_dnode_bt, 1, &zopt_sometimes),
	};

	#define ZTEST_FUNCS (sizeof (ztest_info) / sizeof (ztest_info_t))

	/*
	* The following struct is used to hold a list of uncalled commit callbacks.
	* The callbacks are ordered by txg number.
	*/
	typedef struct ztest_cb_list {
	kmutex_t zcl_callbacks_lock;
	list_t zcl_callbacks;
	} ztest_cb_list_t;

	/*
	* Stuff we need to share writably between parent and child.
	*/
	typedef struct ztest_shared {
	boolean_t zs_do_init;
	hrtime_t zs_proc_start;
	hrtime_t zs_proc_stop;
	hrtime_t zs_thread_start;
	hrtime_t zs_thread_stop;
	hrtime_t zs_thread_kill;
	uint64_t zs_enospc_count;
	uint64_t zs_vdev_next_leaf;
	uint64_t zs_vdev_aux;
	uint64_t zs_alloc;
	uint64_t zs_space;
	uint64_t zs_splits;
	uint64_t zs_mirrors;
	uint64_t zs_metaslab_sz;
	uint64_t zs_metaslab_df_alloc_threshold;
	uint64_t zs_guid;
	} ztest_shared_t;

	#define ID_PARALLEL -1ULL

	static char ztest_dev_template[] = "%s/%s.%llua";
	static char ztest_aux_template[] = "%s/%s.%s.%llu";
	static ztest_shared_t *ztest_shared;

	static spa_t *ztest_spa = NULL;
	static ztest_ds_t *ztest_ds;

	static kmutex_t ztest_vdev_lock;
	static boolean_t ztest_device_removal_active = B_FALSE;
	static boolean_t ztest_pool_scrubbed = B_FALSE;
	static kmutex_t ztest_checkpoint_lock;

	/*
	* The ztest_name_lock protects the pool and dataset namespace used by
	* the individual tests. To modify the namespace, consumers must grab
	* this lock as writer. Grabbing the lock as reader will ensure that the
	* namespace does not change while the lock is held.
	*/
	static pthread_rwlock_t ztest_name_lock;

	static boolean_t ztest_dump_core = B_TRUE;
	static boolean_t ztest_exiting;

	/* Global commit callback list */
	static ztest_cb_list_t zcl;
	/* Commit cb delay */
	static uint64_t zc_min_txg_delay = UINT64_MAX;
	static int zc_cb_counter = 0;

	/*
	* Minimum number of commit callbacks that need to be registered for us to check
	* whether the minimum txg delay is acceptable.
	*/
	#define ZTEST_COMMIT_CB_MIN_REG 100

	/*
	* If a number of txgs equal to this threshold have been created after a commit
	* callback has been registered but not called, then we assume there is an
	* implementation bug.
	*/
	#define ZTEST_COMMIT_CB_THRESH (TXG_CONCURRENT_STATES + 1000)

	enum ztest_object {
	ZTEST_META_DNODE = 0,
	ZTEST_DIROBJ,
	ZTEST_OBJECTS
	};

	static __attribute__((noreturn)) void usage(boolean_t requested);
	static int ztest_scrub_impl(spa_t *spa);

	/*
	* These libumem hooks provide a reasonable set of defaults for the allocator's
	* debugging facilities.
	*/
	const char *
	_umem_debug_init(void)
	{
	return ("default,verbose"); /* $UMEM_DEBUG setting */
	}

	const char *
	_umem_logging_init(void)
	{
	return ("fail,contents"); /* $UMEM_LOGGING setting */
	}

	static void
	dump_debug_buffer(void)
	{
	ssize_t ret __attribute__((unused));

	if (!ztest_opts.zo_dump_dbgmsg)
	return;

	/*
	* We use write() instead of printf() so that this function
	* is safe to call from a signal handler.
	*/
	ret = write(STDOUT_FILENO, "\n", 1);
	zfs_dbgmsg_print("ztest");
	}

	#define BACKTRACE_SZ 100

	static void sig_handler(int signo)
	{
	struct sigaction action;
	#if (__GLIBC__ && !__UCLIBC__) /* backtrace() is a GNU extension */
	int nptrs;
	void *buffer[BACKTRACE_SZ];

	nptrs = backtrace(buffer, BACKTRACE_SZ);
	backtrace_symbols_fd(buffer, nptrs, STDERR_FILENO);
	#endif
	dump_debug_buffer();

	/*
	* Restore default action and re-raise signal so SIGSEGV and
	* SIGABRT can trigger a core dump.
	*/
	action.sa_handler = SIG_DFL;
	sigemptyset(&action.sa_mask);
	action.sa_flags = 0;
	(void) sigaction(signo, &action, NULL);
	raise(signo);
	}

	#define FATAL_MSG_SZ 1024

	static const char *fatal_msg;

	static __attribute__((format(printf, 2, 3))) __attribute__((noreturn)) void
	fatal(int do_perror, const char *message, ...)
	{
	va_list args;
	int save_errno = errno;
	char *buf;

	(void) fflush(stdout);
	buf = umem_alloc(FATAL_MSG_SZ, UMEM_NOFAIL);
	if (buf == NULL)
	goto out;

	va_start(args, message);
	(void) sprintf(buf, "ztest: ");
	/* LINTED */
	(void) vsprintf(buf + strlen(buf), message, args);
	va_end(args);
	if (do_perror) {
	(void) snprintf(buf + strlen(buf), FATAL_MSG_SZ - strlen(buf),
	": %s", strerror(save_errno));
	}
	(void) fprintf(stderr, "%s\n", buf);
	fatal_msg = buf; /* to ease debugging */

	out:
	if (ztest_dump_core)
	abort();
	else
	dump_debug_buffer();

	exit(3);
	}

	static int
	str2shift(const char *buf)
	{
	const char *ends = "BKMGTPEZ";
	int i;

	if (buf[0] == '\0')
	return (0);
	for (i = 0; i < strlen(ends); i++) {
	if (toupper(buf[0]) == ends[i])
	break;
	}
	if (i == strlen(ends)) {
	(void) fprintf(stderr, "ztest: invalid bytes suffix: %s\n",
	buf);
	usage(B_FALSE);
	}
	if (buf[1] == '\0' \|\| (toupper(buf[1]) == 'B' && buf[2] == '\0')) {
	return (10*i);
	}
	(void) fprintf(stderr, "ztest: invalid bytes suffix: %s\n", buf);
	usage(B_FALSE);
	}

	static uint64_t
	nicenumtoull(const char *buf)
	{
	char *end;
	uint64_t val;

	val = strtoull(buf, &end, 0);
	if (end == buf) {
	(void) fprintf(stderr, "ztest: bad numeric value: %s\n", buf);
	usage(B_FALSE);
	} else if (end[0] == '.') {
	double fval = strtod(buf, &end);
	fval *= pow(2, str2shift(end));
	/*
	* UINT64_MAX is not exactly representable as a double.
	* The closest representation is UINT64_MAX + 1, so we
	* use a >= comparison instead of > for the bounds check.
	*/
	if (fval >= (double)UINT64_MAX) {
	(void) fprintf(stderr, "ztest: value too large: %s\n",
	buf);
	usage(B_FALSE);
	}
	val = (uint64_t)fval;
	} else {
	int shift = str2shift(end);
	if (shift >= 64 \|\| (val << shift) >> shift != val) {
	(void) fprintf(stderr, "ztest: value too large: %s\n",
	buf);
	usage(B_FALSE);
	}
	val <<= shift;
	}
	return (val);
	}

	typedef struct ztest_option {
	const char short_opt;
	const char *long_opt;
	const char *long_opt_param;
	const char *comment;
	unsigned int default_int;
	const char *default_str;
	} ztest_option_t;

	/*
	* The following option_table is used for generating the usage info as well as
	* the long and short option information for calling getopt_long().
	*/
	static ztest_option_t option_table[] = {
	{ 'v', "vdevs", "INTEGER", "Number of vdevs", DEFAULT_VDEV_COUNT,
	NULL},
	{ 's', "vdev-size", "INTEGER", "Size of each vdev",
	NO_DEFAULT, DEFAULT_VDEV_SIZE_STR},
	{ 'a', "alignment-shift", "INTEGER",
	"Alignment shift; use 0 for random", DEFAULT_ASHIFT, NULL},
	{ 'm', "mirror-copies", "INTEGER", "Number of mirror copies",
	DEFAULT_MIRRORS, NULL},
	{ 'r', "raid-disks", "INTEGER", "Number of raidz/draid disks",
	DEFAULT_RAID_CHILDREN, NULL},
	{ 'R', "raid-parity", "INTEGER", "Raid parity",
	DEFAULT_RAID_PARITY, NULL},
	{ 'K', "raid-kind", "raidz\|draid\|random", "Raid kind",
	NO_DEFAULT, "random"},
	{ 'D', "draid-data", "INTEGER", "Number of draid data drives",
	DEFAULT_DRAID_DATA, NULL},
	{ 'S', "draid-spares", "INTEGER", "Number of draid spares",
	DEFAULT_DRAID_SPARES, NULL},
	{ 'd', "datasets", "INTEGER", "Number of datasets",
	DEFAULT_DATASETS_COUNT, NULL},
	{ 't', "threads", "INTEGER", "Number of ztest threads",
	DEFAULT_THREADS, NULL},
	{ 'g', "gang-block-threshold", "INTEGER",
	"Metaslab gang block threshold",
	NO_DEFAULT, DEFAULT_FORCE_GANGING_STR},
	{ 'i', "init-count", "INTEGER", "Number of times to initialize pool",
	DEFAULT_INITS, NULL},
	{ 'k', "kill-percentage", "INTEGER", "Kill percentage",
	NO_DEFAULT, DEFAULT_KILLRATE_STR},
	{ 'p', "pool-name", "STRING", "Pool name",
	NO_DEFAULT, DEFAULT_POOL},
	{ 'f', "vdev-file-directory", "PATH", "File directory for vdev files",
	NO_DEFAULT, DEFAULT_VDEV_DIR},
	{ 'M', "multi-host", NULL,
	"Multi-host; simulate pool imported on remote host",
	NO_DEFAULT, NULL},
	{ 'E', "use-existing-pool", NULL,
	"Use existing pool instead of creating new one", NO_DEFAULT, NULL},
	{ 'T', "run-time", "INTEGER", "Total run time",
	NO_DEFAULT, DEFAULT_RUN_TIME_STR},
	{ 'P', "pass-time", "INTEGER", "Time per pass",
	NO_DEFAULT, DEFAULT_PASS_TIME_STR},
	{ 'F', "freeze-loops", "INTEGER", "Max loops in spa_freeze()",
	DEFAULT_MAX_LOOPS, NULL},
	{ 'B', "alt-ztest", "PATH", "Alternate ztest path",
	NO_DEFAULT, NULL},
	{ 'C', "vdev-class-state", "on\|off\|random", "vdev class state",
	NO_DEFAULT, "random"},
	{ 'o', "option", "\"OPTION=INTEGER\"",
	"Set global variable to an unsigned 32-bit integer value",
	NO_DEFAULT, NULL},
	{ 'G', "dump-debug-msg", NULL,
	"Dump zfs_dbgmsg buffer before exiting due to an error",
	NO_DEFAULT, NULL},
	{ 'V', "verbose", NULL,
	"Verbose (use multiple times for ever more verbosity)",
	NO_DEFAULT, NULL},
	{ 'h', "help", NULL, "Show this help",
	NO_DEFAULT, NULL},
	{0, 0, 0, 0, 0, 0}
	};

	static struct option *long_opts = NULL;
	static char *short_opts = NULL;

	static void
	init_options(void)
	{
	ASSERT3P(long_opts, ==, NULL);
	ASSERT3P(short_opts, ==, NULL);

	int count = sizeof (option_table) / sizeof (option_table[0]);
	long_opts = umem_alloc(sizeof (struct option) * count, UMEM_NOFAIL);

	short_opts = umem_alloc(sizeof (char) * 2 * count, UMEM_NOFAIL);
	int short_opt_index = 0;

	for (int i = 0; i < count; i++) {
	long_opts[i].val = option_table[i].short_opt;
	long_opts[i].name = option_table[i].long_opt;
	long_opts[i].has_arg = option_table[i].long_opt_param != NULL
	? required_argument : no_argument;
	long_opts[i].flag = NULL;
	short_opts[short_opt_index++] = option_table[i].short_opt;
	if (option_table[i].long_opt_param != NULL) {
	short_opts[short_opt_index++] = ':';
	}
	}
	}

	static void
	fini_options(void)
	{
	int count = sizeof (option_table) / sizeof (option_table[0]);

	umem_free(long_opts, sizeof (struct option) * count);
	umem_free(short_opts, sizeof (char) * 2 * count);

	long_opts = NULL;
	short_opts = NULL;
	}

	static __attribute__((noreturn)) void
	usage(boolean_t requested)
	{
	char option[80];
	FILE *fp = requested ? stdout : stderr;

	(void) fprintf(fp, "Usage: %s [OPTIONS...]\n", DEFAULT_POOL);
	for (int i = 0; option_table[i].short_opt != 0; i++) {
	if (option_table[i].long_opt_param != NULL) {
	(void) sprintf(option, " -%c --%s=%s",
	option_table[i].short_opt,
	option_table[i].long_opt,
	option_table[i].long_opt_param);
	} else {
	(void) sprintf(option, " -%c --%s",
	option_table[i].short_opt,
	option_table[i].long_opt);
	}
	(void) fprintf(fp, " %-40s%s", option,
	option_table[i].comment);

	if (option_table[i].long_opt_param != NULL) {
	if (option_table[i].default_str != NULL) {
	(void) fprintf(fp, " (default: %s)",
	option_table[i].default_str);
	} else if (option_table[i].default_int != NO_DEFAULT) {
	(void) fprintf(fp, " (default: %u)",
	option_table[i].default_int);
	}
	}
	(void) fprintf(fp, "\n");
	}
	exit(requested ? 0 : 1);
	}

	static uint64_t
	ztest_random(uint64_t range)
	{
	uint64_t r;

	ASSERT3S(ztest_fd_rand, >=, 0);

	if (range == 0)
	return (0);

	if (read(ztest_fd_rand, &r, sizeof (r)) != sizeof (r))
	fatal(B_TRUE, "short read from /dev/urandom");

	return (r % range);
	}

	static void
	ztest_parse_name_value(const char input, ztest_shared_opts_t zo)
	{
	char name[32];
	char *value;
	int state = ZTEST_VDEV_CLASS_RND;

	(void) strlcpy(name, input, sizeof (name));

	value = strchr(name, '=');
	if (value == NULL) {
	(void) fprintf(stderr, "missing value in property=value "
	"'-C' argument (%s)\n", input);
	usage(B_FALSE);
	}
	*(value) = '\0';
	value++;

	if (strcmp(value, "on") == 0) {
	state = ZTEST_VDEV_CLASS_ON;
	} else if (strcmp(value, "off") == 0) {
	state = ZTEST_VDEV_CLASS_OFF;
	} else if (strcmp(value, "random") == 0) {
	state = ZTEST_VDEV_CLASS_RND;
	} else {
	(void) fprintf(stderr, "invalid property value '%s'\n", value);
	usage(B_FALSE);
	}

	if (strcmp(name, "special") == 0) {
	zo->zo_special_vdevs = state;
	} else {
	(void) fprintf(stderr, "invalid property name '%s'\n", name);
	usage(B_FALSE);
	}
	if (zo->zo_verbose >= 3)
	(void) printf("%s vdev state is '%s'\n", name, value);
	}

	static void
	process_options(int argc, char **argv)
	{
	char *path;
	ztest_shared_opts_t *zo = &ztest_opts;

	int opt;
	uint64_t value;
	const char *raid_kind = "random";

	memcpy(zo, &ztest_opts_defaults, sizeof (*zo));

	init_options();

	while ((opt = getopt_long(argc, argv, short_opts, long_opts,
	NULL)) != EOF) {
	value = 0;
	switch (opt) {
	case 'v':
	case 's':
	case 'a':
	case 'm':
	case 'r':
	case 'R':
	case 'D':
	case 'S':
	case 'd':
	case 't':
	case 'g':
	case 'i':
	case 'k':
	case 'T':
	case 'P':
	case 'F':
	value = nicenumtoull(optarg);
	}
	switch (opt) {
	case 'v':
	zo->zo_vdevs = value;
	break;
	case 's':
	zo->zo_vdev_size = MAX(SPA_MINDEVSIZE, value);
	break;
	case 'a':
	zo->zo_ashift = value;
	break;
	case 'm':
	zo->zo_mirrors = value;
	break;
	case 'r':
	zo->zo_raid_children = MAX(1, value);
	break;
	case 'R':
	zo->zo_raid_parity = MIN(MAX(value, 1), 3);
	break;
	case 'K':
	raid_kind = optarg;
	break;
	case 'D':
	zo->zo_draid_data = MAX(1, value);
	break;
	case 'S':
	zo->zo_draid_spares = MAX(1, value);
	break;
	case 'd':
	zo->zo_datasets = MAX(1, value);
	break;
	case 't':
	zo->zo_threads = MAX(1, value);
	break;
	case 'g':
	zo->zo_metaslab_force_ganging =
	MAX(SPA_MINBLOCKSIZE << 1, value);
	break;
	case 'i':
	zo->zo_init = value;
	break;
	case 'k':
	zo->zo_killrate = value;
	break;
	case 'p':
	(void) strlcpy(zo->zo_pool, optarg,
	sizeof (zo->zo_pool));
	break;
	case 'f':
	path = realpath(optarg, NULL);
	if (path == NULL) {
	(void) fprintf(stderr, "error: %s: %s\n",
	optarg, strerror(errno));
	usage(B_FALSE);
	} else {
	(void) strlcpy(zo->zo_dir, path,
	sizeof (zo->zo_dir));
	free(path);
	}
	break;
	case 'M':
	zo->zo_mmp_test = 1;
	break;
	case 'V':
	zo->zo_verbose++;
	break;
	case 'E':
	zo->zo_init = 0;
	break;
	case 'T':
	zo->zo_time = value;
	break;
	case 'P':
	zo->zo_passtime = MAX(1, value);
	break;
	case 'F':
	zo->zo_maxloops = MAX(1, value);
	break;
	case 'B':
	(void) strlcpy(zo->zo_alt_ztest, optarg,
	sizeof (zo->zo_alt_ztest));
	break;
	case 'C':
	ztest_parse_name_value(optarg, zo);
	break;
	case 'o':
	if (zo->zo_gvars_count >= ZO_GVARS_MAX_COUNT) {
	(void) fprintf(stderr,
	"max global var count (%zu) exceeded\n",
	ZO_GVARS_MAX_COUNT);
	usage(B_FALSE);
	}
	char *v = zo->zo_gvars[zo->zo_gvars_count];
	if (strlcpy(v, optarg, ZO_GVARS_MAX_ARGLEN) >=
	ZO_GVARS_MAX_ARGLEN) {
	(void) fprintf(stderr,
	"global var option '%s' is too long\n",
	optarg);
	usage(B_FALSE);
	}
	zo->zo_gvars_count++;
	break;
	case 'G':
	zo->zo_dump_dbgmsg = 1;
	break;
	case 'h':
	usage(B_TRUE);
	break;
	case '?':
	default:
	usage(B_FALSE);
	break;
	}
	}

	fini_options();

	/* When raid choice is 'random' add a draid pool 50% of the time */
	if (strcmp(raid_kind, "random") == 0) {
	raid_kind = (ztest_random(2) == 0) ? "draid" : "raidz";

	if (ztest_opts.zo_verbose >= 3)
	(void) printf("choosing RAID type '%s'\n", raid_kind);
	}

	if (strcmp(raid_kind, "draid") == 0) {
	uint64_t min_devsize;

	/* With fewer disk use 256M, otherwise 128M is OK */
	min_devsize = (ztest_opts.zo_raid_children < 16) ?
	(256ULL << 20) : (128ULL << 20);

	/* No top-level mirrors with dRAID for now */
	zo->zo_mirrors = 0;

	/* Use more appropriate defaults for dRAID */
	if (zo->zo_vdevs == ztest_opts_defaults.zo_vdevs)
	zo->zo_vdevs = 1;
	if (zo->zo_raid_children ==
	ztest_opts_defaults.zo_raid_children)
	zo->zo_raid_children = 16;
	if (zo->zo_ashift < 12)
	zo->zo_ashift = 12;
	if (zo->zo_vdev_size < min_devsize)
	zo->zo_vdev_size = min_devsize;

	if (zo->zo_draid_data + zo->zo_raid_parity >
	zo->zo_raid_children - zo->zo_draid_spares) {
	(void) fprintf(stderr, "error: too few draid "
	"children (%d) for stripe width (%d)\n",
	zo->zo_raid_children,
	zo->zo_draid_data + zo->zo_raid_parity);
	usage(B_FALSE);
	}

	(void) strlcpy(zo->zo_raid_type, VDEV_TYPE_DRAID,
	sizeof (zo->zo_raid_type));

	} else /* using raidz */ {
	ASSERT0(strcmp(raid_kind, "raidz"));

	zo->zo_raid_parity = MIN(zo->zo_raid_parity,
	zo->zo_raid_children - 1);
	}

	zo->zo_vdevtime =
	(zo->zo_vdevs > 0 ? zo->zo_time * NANOSEC / zo->zo_vdevs :
	UINT64_MAX >> 2);

	if (*zo->zo_alt_ztest) {
	const char *invalid_what = "ztest";
	char *val = zo->zo_alt_ztest;
	if (0 != access(val, X_OK) \|\|
	(strrchr(val, '/') == NULL && (errno == EINVAL)))
	goto invalid;

	int dirlen = strrchr(val, '/') - val;
	strlcpy(zo->zo_alt_libpath, val,
	MIN(sizeof (zo->zo_alt_libpath), dirlen + 1));
	invalid_what = "library path", val = zo->zo_alt_libpath;
	if (strrchr(val, '/') == NULL && (errno == EINVAL))
	goto invalid;
	*strrchr(val, '/') = '\0';
	strlcat(val, "/lib", sizeof (zo->zo_alt_libpath));

	if (0 != access(zo->zo_alt_libpath, X_OK))
	goto invalid;
	return;

	invalid:
	ztest_dump_core = B_FALSE;
	fatal(B_TRUE, "invalid alternate %s %s", invalid_what, val);
	}
	}

	static void
	ztest_kill(ztest_shared_t *zs)
	{
	zs->zs_alloc = metaslab_class_get_alloc(spa_normal_class(ztest_spa));
	zs->zs_space = metaslab_class_get_space(spa_normal_class(ztest_spa));

	/*
	* Before we kill ourselves, make sure that the config is updated.
	* See comment above spa_write_cachefile().
	*/
	mutex_enter(&spa_namespace_lock);
	spa_write_cachefile(ztest_spa, B_FALSE, B_FALSE, B_FALSE);
	mutex_exit(&spa_namespace_lock);

	(void) raise(SIGKILL);
	}

	static void
	ztest_record_enospc(const char *s)
	{
	(void) s;
	ztest_shared->zs_enospc_count++;
	}

	static uint64_t
	ztest_get_ashift(void)
	{
	if (ztest_opts.zo_ashift == 0)
	return (SPA_MINBLOCKSHIFT + ztest_random(5));
	return (ztest_opts.zo_ashift);
	}

	static boolean_t
	ztest_is_draid_spare(const char *name)
	{
	uint64_t spare_id = 0, parity = 0, vdev_id = 0;

	if (sscanf(name, VDEV_TYPE_DRAID "%"PRIu64"-%"PRIu64"-%"PRIu64"",
	&parity, &vdev_id, &spare_id) == 3) {
	return (B_TRUE);
	}

	return (B_FALSE);
	}

	static nvlist_t *
	make_vdev_file(const char path, const char aux, const char *pool,
	size_t size, uint64_t ashift)
	{
	char *pathbuf = NULL;
	uint64_t vdev;
	nvlist_t *file;
	boolean_t draid_spare = B_FALSE;


	if (ashift == 0)
	ashift = ztest_get_ashift();

	if (path == NULL) {
	pathbuf = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
	path = pathbuf;

	if (aux != NULL) {
	vdev = ztest_shared->zs_vdev_aux;
	(void) snprintf(pathbuf, MAXPATHLEN,
	ztest_aux_template, ztest_opts.zo_dir,
	pool == NULL ? ztest_opts.zo_pool : pool,
	aux, vdev);
	} else {
	vdev = ztest_shared->zs_vdev_next_leaf++;
	(void) snprintf(pathbuf, MAXPATHLEN,
	ztest_dev_template, ztest_opts.zo_dir,
	pool == NULL ? ztest_opts.zo_pool : pool, vdev);
	}
	} else {
	draid_spare = ztest_is_draid_spare(path);
	}

	if (size != 0 && !draid_spare) {
	int fd = open(path, O_RDWR \| O_CREAT \| O_TRUNC, 0666);
	if (fd == -1)
	fatal(B_TRUE, "can't open %s", path);
	if (ftruncate(fd, size) != 0)
	fatal(B_TRUE, "can't ftruncate %s", path);
	(void) close(fd);
	}

	file = fnvlist_alloc();
	fnvlist_add_string(file, ZPOOL_CONFIG_TYPE,
	draid_spare ? VDEV_TYPE_DRAID_SPARE : VDEV_TYPE_FILE);
	fnvlist_add_string(file, ZPOOL_CONFIG_PATH, path);
	fnvlist_add_uint64(file, ZPOOL_CONFIG_ASHIFT, ashift);
	umem_free(pathbuf, MAXPATHLEN);

	return (file);
	}

	static nvlist_t *
	make_vdev_raid(const char path, const char aux, const char *pool, size_t size,
	uint64_t ashift, int r)
	{
	nvlist_t raid, *child;
	int c;

	if (r < 2)
	return (make_vdev_file(path, aux, pool, size, ashift));
	child = umem_alloc(r * sizeof (nvlist_t *), UMEM_NOFAIL);

	for (c = 0; c < r; c++)
	child[c] = make_vdev_file(path, aux, pool, size, ashift);

	raid = fnvlist_alloc();
	fnvlist_add_string(raid, ZPOOL_CONFIG_TYPE,
	ztest_opts.zo_raid_type);
	fnvlist_add_uint64(raid, ZPOOL_CONFIG_NPARITY,
	ztest_opts.zo_raid_parity);
	fnvlist_add_nvlist_array(raid, ZPOOL_CONFIG_CHILDREN,
	(const nvlist_t **)child, r);

	if (strcmp(ztest_opts.zo_raid_type, VDEV_TYPE_DRAID) == 0) {
	uint64_t ndata = ztest_opts.zo_draid_data;
	uint64_t nparity = ztest_opts.zo_raid_parity;
	uint64_t nspares = ztest_opts.zo_draid_spares;
	uint64_t children = ztest_opts.zo_raid_children;
	uint64_t ngroups = 1;

	/*
	* Calculate the minimum number of groups required to fill a
	* slice. This is the LCM of the stripe width (data + parity)
	* and the number of data drives (children - spares).
	*/
	while (ngroups * (ndata + nparity) % (children - nspares) != 0)
	ngroups++;

	/* Store the basic dRAID configuration. */
	fnvlist_add_uint64(raid, ZPOOL_CONFIG_DRAID_NDATA, ndata);
	fnvlist_add_uint64(raid, ZPOOL_CONFIG_DRAID_NSPARES, nspares);
	fnvlist_add_uint64(raid, ZPOOL_CONFIG_DRAID_NGROUPS, ngroups);
	}

	for (c = 0; c < r; c++)
	fnvlist_free(child[c]);

	umem_free(child, r * sizeof (nvlist_t *));

	return (raid);
	}

	static nvlist_t *
	make_vdev_mirror(const char path, const char aux, const char *pool,
	size_t size, uint64_t ashift, int r, int m)
	{
	nvlist_t mirror, *child;
	int c;

	if (m < 1)
	return (make_vdev_raid(path, aux, pool, size, ashift, r));

	child = umem_alloc(m * sizeof (nvlist_t *), UMEM_NOFAIL);

	for (c = 0; c < m; c++)
	child[c] = make_vdev_raid(path, aux, pool, size, ashift, r);

	mirror = fnvlist_alloc();
	fnvlist_add_string(mirror, ZPOOL_CONFIG_TYPE, VDEV_TYPE_MIRROR);
	fnvlist_add_nvlist_array(mirror, ZPOOL_CONFIG_CHILDREN,
	(const nvlist_t **)child, m);

	for (c = 0; c < m; c++)
	fnvlist_free(child[c]);

	umem_free(child, m * sizeof (nvlist_t *));

	return (mirror);
	}

	static nvlist_t *
	make_vdev_root(const char path, const char aux, const char *pool, size_t size,
	uint64_t ashift, const char *class, int r, int m, int t)
	{
	nvlist_t root, *child;
	int c;
	boolean_t log;

	ASSERT3S(t, >, 0);

	log = (class != NULL && strcmp(class, "log") == 0);

	child = umem_alloc(t * sizeof (nvlist_t *), UMEM_NOFAIL);

	for (c = 0; c < t; c++) {
	child[c] = make_vdev_mirror(path, aux, pool, size, ashift,
	r, m);
	fnvlist_add_uint64(child[c], ZPOOL_CONFIG_IS_LOG, log);

	if (class != NULL && class[0] != '\0') {
	ASSERT(m > 1 \|\| log); /* expecting a mirror */
	fnvlist_add_string(child[c],
	ZPOOL_CONFIG_ALLOCATION_BIAS, class);
	}
	}

	root = fnvlist_alloc();
	fnvlist_add_string(root, ZPOOL_CONFIG_TYPE, VDEV_TYPE_ROOT);
	fnvlist_add_nvlist_array(root, aux ? aux : ZPOOL_CONFIG_CHILDREN,
	(const nvlist_t **)child, t);

	for (c = 0; c < t; c++)
	fnvlist_free(child[c]);

	umem_free(child, t * sizeof (nvlist_t *));

	return (root);
	}

	/*
	* Find a random spa version. Returns back a random spa version in the
	* range [initial_version, SPA_VERSION_FEATURES].
	*/
	static uint64_t
	ztest_random_spa_version(uint64_t initial_version)
	{
	uint64_t version = initial_version;

	if (version <= SPA_VERSION_BEFORE_FEATURES) {
	version = version +
	ztest_random(SPA_VERSION_BEFORE_FEATURES - version + 1);
	}

	if (version > SPA_VERSION_BEFORE_FEATURES)
	version = SPA_VERSION_FEATURES;

	ASSERT(SPA_VERSION_IS_SUPPORTED(version));
	return (version);
	}

	static int
	ztest_random_blocksize(void)
	{
	ASSERT3U(ztest_spa->spa_max_ashift, !=, 0);

	/*
	* Choose a block size >= the ashift.
	* If the SPA supports new MAXBLOCKSIZE, test up to 1MB blocks.
	*/
	int maxbs = SPA_OLD_MAXBLOCKSHIFT;
	if (spa_maxblocksize(ztest_spa) == SPA_MAXBLOCKSIZE)
	maxbs = 20;
	uint64_t block_shift =
	ztest_random(maxbs - ztest_spa->spa_max_ashift + 1);
	return (1 << (SPA_MINBLOCKSHIFT + block_shift));
	}

	static int
	ztest_random_dnodesize(void)
	{
	int slots;
	int max_slots = spa_maxdnodesize(ztest_spa) >> DNODE_SHIFT;

	if (max_slots == DNODE_MIN_SLOTS)
	return (DNODE_MIN_SIZE);

	/*
	* Weight the random distribution more heavily toward smaller
	* dnode sizes since that is more likely to reflect real-world
	* usage.
	*/
	ASSERT3U(max_slots, >, 4);
	switch (ztest_random(10)) {
	case 0:
	slots = 5 + ztest_random(max_slots - 4);
	break;
	case 1 ... 4:
	slots = 2 + ztest_random(3);
	break;
	default:
	slots = 1;
	break;
	}

	return (slots << DNODE_SHIFT);
	}

	static int
	ztest_random_ibshift(void)
	{
	return (DN_MIN_INDBLKSHIFT +
	ztest_random(DN_MAX_INDBLKSHIFT - DN_MIN_INDBLKSHIFT + 1));
	}

	static uint64_t
	ztest_random_vdev_top(spa_t *spa, boolean_t log_ok)
	{
	uint64_t top;
	vdev_t *rvd = spa->spa_root_vdev;
	vdev_t *tvd;

	ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0);

	do {
	top = ztest_random(rvd->vdev_children);
	tvd = rvd->vdev_child[top];
	} while (!vdev_is_concrete(tvd) \|\| (tvd->vdev_islog && !log_ok) \|\|
	tvd->vdev_mg == NULL \|\| tvd->vdev_mg->mg_class == NULL);

	return (top);
	}

	static uint64_t
	ztest_random_dsl_prop(zfs_prop_t prop)
	{
	uint64_t value;

	do {
	value = zfs_prop_random_value(prop, ztest_random(-1ULL));
	} while (prop == ZFS_PROP_CHECKSUM && value == ZIO_CHECKSUM_OFF);

	return (value);
	}

	static int
	ztest_dsl_prop_set_uint64(char *osname, zfs_prop_t prop, uint64_t value,
	boolean_t inherit)
	{
	const char *propname = zfs_prop_to_name(prop);
	const char *valname;
	char *setpoint;
	uint64_t curval;
	int error;

	error = dsl_prop_set_int(osname, propname,
	(inherit ? ZPROP_SRC_NONE : ZPROP_SRC_LOCAL), value);

	if (error == ENOSPC) {
	ztest_record_enospc(FTAG);
	return (error);
	}
	ASSERT0(error);

	setpoint = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
	VERIFY0(dsl_prop_get_integer(osname, propname, &curval, setpoint));

	if (ztest_opts.zo_verbose >= 6) {
	int err;

	err = zfs_prop_index_to_string(prop, curval, &valname);
	if (err)
	(void) printf("%s %s = %llu at '%s'\n", osname,
	propname, (unsigned long long)curval, setpoint);
	else
	(void) printf("%s %s = %s at '%s'\n",
	osname, propname, valname, setpoint);
	}
	umem_free(setpoint, MAXPATHLEN);

	return (error);
	}

	static int
	ztest_spa_prop_set_uint64(zpool_prop_t prop, uint64_t value)
	{
	spa_t *spa = ztest_spa;
	nvlist_t *props = NULL;
	int error;

	props = fnvlist_alloc();
	fnvlist_add_uint64(props, zpool_prop_to_name(prop), value);

	error = spa_prop_set(spa, props);

	fnvlist_free(props);

	if (error == ENOSPC) {
	ztest_record_enospc(FTAG);
	return (error);
	}
	ASSERT0(error);

	return (error);
	}

	static int
	ztest_dmu_objset_own(const char *name, dmu_objset_type_t type,
	boolean_t readonly, boolean_t decrypt, const void tag, objset_t *osp)
	{
	int err;
	char *cp = NULL;
	char ddname[ZFS_MAX_DATASET_NAME_LEN];

	strlcpy(ddname, name, sizeof (ddname));
	cp = strchr(ddname, '@');
	if (cp != NULL)
	*cp = '\0';

	err = dmu_objset_own(name, type, readonly, decrypt, tag, osp);
	while (decrypt && err == EACCES) {
	dsl_crypto_params_t *dcp;
	nvlist_t *crypto_args = fnvlist_alloc();

	fnvlist_add_uint8_array(crypto_args, "wkeydata",
	(uint8_t *)ztest_wkeydata, WRAPPING_KEY_LEN);
	VERIFY0(dsl_crypto_params_create_nvlist(DCP_CMD_NONE, NULL,
	crypto_args, &dcp));
	err = spa_keystore_load_wkey(ddname, dcp, B_FALSE);
	/*
	* Note: if there was an error loading, the wkey was not
	* consumed, and needs to be freed.
	*/
	dsl_crypto_params_free(dcp, (err != 0));
	fnvlist_free(crypto_args);

	if (err == EINVAL) {
	/*
	* We couldn't load a key for this dataset so try
	* the parent. This loop will eventually hit the
	* encryption root since ztest only makes clones
	* as children of their origin datasets.
	*/
	cp = strrchr(ddname, '/');
	if (cp == NULL)
	return (err);

	*cp = '\0';
	err = EACCES;
	continue;
	} else if (err != 0) {
	break;
	}

	err = dmu_objset_own(name, type, readonly, decrypt, tag, osp);
	break;
	}

	return (err);
	}

	static void
	ztest_rll_init(rll_t *rll)
	{
	rll->rll_writer = NULL;
	rll->rll_readers = 0;
	mutex_init(&rll->rll_lock, NULL, MUTEX_DEFAULT, NULL);
	cv_init(&rll->rll_cv, NULL, CV_DEFAULT, NULL);
	}

	static void
	ztest_rll_destroy(rll_t *rll)
	{
	ASSERT3P(rll->rll_writer, ==, NULL);
	ASSERT0(rll->rll_readers);
	mutex_destroy(&rll->rll_lock);
	cv_destroy(&rll->rll_cv);
	}

	static void
	ztest_rll_lock(rll_t *rll, rl_type_t type)
	{
	mutex_enter(&rll->rll_lock);

	if (type == RL_READER) {
	while (rll->rll_writer != NULL)
	(void) cv_wait(&rll->rll_cv, &rll->rll_lock);
	rll->rll_readers++;
	} else {
	while (rll->rll_writer != NULL \|\| rll->rll_readers)
	(void) cv_wait(&rll->rll_cv, &rll->rll_lock);
	rll->rll_writer = curthread;
	}

	mutex_exit(&rll->rll_lock);
	}

	static void
	ztest_rll_unlock(rll_t *rll)
	{
	mutex_enter(&rll->rll_lock);

	if (rll->rll_writer) {
	ASSERT0(rll->rll_readers);
	rll->rll_writer = NULL;
	} else {
	ASSERT3S(rll->rll_readers, >, 0);
	ASSERT3P(rll->rll_writer, ==, NULL);
	rll->rll_readers--;
	}

	if (rll->rll_writer == NULL && rll->rll_readers == 0)
	cv_broadcast(&rll->rll_cv);

	mutex_exit(&rll->rll_lock);
	}

	static void
	ztest_object_lock(ztest_ds_t *zd, uint64_t object, rl_type_t type)
	{
	rll_t *rll = &zd->zd_object_lock[object & (ZTEST_OBJECT_LOCKS - 1)];

	ztest_rll_lock(rll, type);
	}

	static void
	ztest_object_unlock(ztest_ds_t *zd, uint64_t object)
	{
	rll_t *rll = &zd->zd_object_lock[object & (ZTEST_OBJECT_LOCKS - 1)];

	ztest_rll_unlock(rll);
	}

	static rl_t *
	ztest_range_lock(ztest_ds_t *zd, uint64_t object, uint64_t offset,
	uint64_t size, rl_type_t type)
	{
	uint64_t hash = object ^ (offset % (ZTEST_RANGE_LOCKS + 1));
	rll_t *rll = &zd->zd_range_lock[hash & (ZTEST_RANGE_LOCKS - 1)];
	rl_t *rl;

	rl = umem_alloc(sizeof (*rl), UMEM_NOFAIL);
	rl->rl_object = object;
	rl->rl_offset = offset;
	rl->rl_size = size;
	rl->rl_lock = rll;

	ztest_rll_lock(rll, type);

	return (rl);
	}

	static void
	ztest_range_unlock(rl_t *rl)
	{
	rll_t *rll = rl->rl_lock;

	ztest_rll_unlock(rll);

	umem_free(rl, sizeof (*rl));
	}

	static void
	ztest_zd_init(ztest_ds_t zd, ztest_shared_ds_t szd, objset_t *os)
	{
	zd->zd_os = os;
	zd->zd_zilog = dmu_objset_zil(os);
	zd->zd_shared = szd;
	dmu_objset_name(os, zd->zd_name);
	int l;

	if (zd->zd_shared != NULL)
	zd->zd_shared->zd_seq = 0;

	VERIFY0(pthread_rwlock_init(&zd->zd_zilog_lock, NULL));
	mutex_init(&zd->zd_dirobj_lock, NULL, MUTEX_DEFAULT, NULL);

	for (l = 0; l < ZTEST_OBJECT_LOCKS; l++)
	ztest_rll_init(&zd->zd_object_lock[l]);

	for (l = 0; l < ZTEST_RANGE_LOCKS; l++)
	ztest_rll_init(&zd->zd_range_lock[l]);
	}

	static void
	ztest_zd_fini(ztest_ds_t *zd)
	{
	int l;

	mutex_destroy(&zd->zd_dirobj_lock);
	(void) pthread_rwlock_destroy(&zd->zd_zilog_lock);

	for (l = 0; l < ZTEST_OBJECT_LOCKS; l++)
	ztest_rll_destroy(&zd->zd_object_lock[l]);

	for (l = 0; l < ZTEST_RANGE_LOCKS; l++)
	ztest_rll_destroy(&zd->zd_range_lock[l]);
	}

	#define TXG_MIGHTWAIT (ztest_random(10) == 0 ? TXG_NOWAIT : TXG_WAIT)

	static uint64_t
	ztest_tx_assign(dmu_tx_t tx, uint64_t txg_how, const char tag)
	{
	uint64_t txg;
	int error;

	/*
	* Attempt to assign tx to some transaction group.
	*/
	error = dmu_tx_assign(tx, txg_how);
	if (error) {
	if (error == ERESTART) {
	ASSERT3U(txg_how, ==, TXG_NOWAIT);
	dmu_tx_wait(tx);
	} else {
	ASSERT3U(error, ==, ENOSPC);
	ztest_record_enospc(tag);
	}
	dmu_tx_abort(tx);
	return (0);
	}
	txg = dmu_tx_get_txg(tx);
	ASSERT3U(txg, !=, 0);
	return (txg);
	}

	static void
	ztest_bt_generate(ztest_block_tag_t bt, objset_t os, uint64_t object,
	uint64_t dnodesize, uint64_t offset, uint64_t gen, uint64_t txg,
	uint64_t crtxg)
	{
	bt->bt_magic = BT_MAGIC;
	bt->bt_objset = dmu_objset_id(os);
	bt->bt_object = object;
	bt->bt_dnodesize = dnodesize;
	bt->bt_offset = offset;
	bt->bt_gen = gen;
	bt->bt_txg = txg;
	bt->bt_crtxg = crtxg;
	}

	static void
	ztest_bt_verify(ztest_block_tag_t bt, objset_t os, uint64_t object,
	uint64_t dnodesize, uint64_t offset, uint64_t gen, uint64_t txg,
	uint64_t crtxg)
	{
	ASSERT3U(bt->bt_magic, ==, BT_MAGIC);
	ASSERT3U(bt->bt_objset, ==, dmu_objset_id(os));
	ASSERT3U(bt->bt_object, ==, object);
	ASSERT3U(bt->bt_dnodesize, ==, dnodesize);
	ASSERT3U(bt->bt_offset, ==, offset);
	ASSERT3U(bt->bt_gen, <=, gen);
	ASSERT3U(bt->bt_txg, <=, txg);
	ASSERT3U(bt->bt_crtxg, ==, crtxg);
	}

	static ztest_block_tag_t *
	ztest_bt_bonus(dmu_buf_t *db)
	{
	dmu_object_info_t doi;
	ztest_block_tag_t *bt;

	dmu_object_info_from_db(db, &doi);
	ASSERT3U(doi.doi_bonus_size, <=, db->db_size);
	ASSERT3U(doi.doi_bonus_size, >=, sizeof (*bt));
	bt = (void )((char )db->db_data + doi.doi_bonus_size - sizeof (*bt));

	return (bt);
	}

	/*
	* Generate a token to fill up unused bonus buffer space. Try to make
	* it unique to the object, generation, and offset to verify that data
	* is not getting overwritten by data from other dnodes.
	*/
	#define ZTEST_BONUS_FILL_TOKEN(obj, ds, gen, offset) \
	(((ds) << 48) \| ((gen) << 32) \| ((obj) << 8) \| (offset))

	/*
	* Fill up the unused bonus buffer region before the block tag with a
	* verifiable pattern. Filling the whole bonus area with non-zero data
	* helps ensure that all dnode traversal code properly skips the
	* interior regions of large dnodes.
	*/
	static void
	ztest_fill_unused_bonus(dmu_buf_t db, void end, uint64_t obj,
	objset_t *os, uint64_t gen)
	{
	uint64_t *bonusp;

	ASSERT(IS_P2ALIGNED((char )end - (char )db->db_data, 8));

	for (bonusp = db->db_data; bonusp < (uint64_t *)end; bonusp++) {
	uint64_t token = ZTEST_BONUS_FILL_TOKEN(obj, dmu_objset_id(os),
	gen, bonusp - (uint64_t *)db->db_data);
	*bonusp = token;
	}
	}

	/*
	* Verify that the unused area of a bonus buffer is filled with the
	* expected tokens.
	*/
	static void
	ztest_verify_unused_bonus(dmu_buf_t db, void end, uint64_t obj,
	objset_t *os, uint64_t gen)
	{
	uint64_t *bonusp;

	for (bonusp = db->db_data; bonusp < (uint64_t *)end; bonusp++) {
	uint64_t token = ZTEST_BONUS_FILL_TOKEN(obj, dmu_objset_id(os),
	gen, bonusp - (uint64_t *)db->db_data);
	VERIFY3U(*bonusp, ==, token);
	}
	}

	/*
	* ZIL logging ops
	*/

	#define lrz_type lr_mode
	#define lrz_blocksize lr_uid
	#define lrz_ibshift lr_gid
	#define lrz_bonustype lr_rdev
	#define lrz_dnodesize lr_crtime[1]

	static void
	ztest_log_create(ztest_ds_t zd, dmu_tx_t tx, lr_create_t *lr)
	{
	char name = (void )(lr + 1); /* name follows lr */
	size_t namesize = strlen(name) + 1;
	itx_t *itx;

	if (zil_replaying(zd->zd_zilog, tx))
	return;

	itx = zil_itx_create(TX_CREATE, sizeof (*lr) + namesize);
	memcpy(&itx->itx_lr + 1, &lr->lr_common + 1,
	sizeof (*lr) + namesize - sizeof (lr_t));

	zil_itx_assign(zd->zd_zilog, itx, tx);
	}

	static void
	ztest_log_remove(ztest_ds_t zd, dmu_tx_t tx, lr_remove_t *lr, uint64_t object)
	{
	char name = (void )(lr + 1); /* name follows lr */
	size_t namesize = strlen(name) + 1;
	itx_t *itx;

	if (zil_replaying(zd->zd_zilog, tx))
	return;

	itx = zil_itx_create(TX_REMOVE, sizeof (*lr) + namesize);
	memcpy(&itx->itx_lr + 1, &lr->lr_common + 1,
	sizeof (*lr) + namesize - sizeof (lr_t));

	itx->itx_oid = object;
	zil_itx_assign(zd->zd_zilog, itx, tx);
	}

	static void
	ztest_log_write(ztest_ds_t zd, dmu_tx_t tx, lr_write_t *lr)
	{
	itx_t *itx;
	itx_wr_state_t write_state = ztest_random(WR_NUM_STATES);

	if (zil_replaying(zd->zd_zilog, tx))
	return;

	if (lr->lr_length > zil_max_log_data(zd->zd_zilog, sizeof (lr_write_t)))
	write_state = WR_INDIRECT;

	itx = zil_itx_create(TX_WRITE,
	sizeof (*lr) + (write_state == WR_COPIED ? lr->lr_length : 0));

	if (write_state == WR_COPIED &&
	dmu_read(zd->zd_os, lr->lr_foid, lr->lr_offset, lr->lr_length,
	((lr_write_t *)&itx->itx_lr) + 1, DMU_READ_NO_PREFETCH) != 0) {
	zil_itx_destroy(itx);
	itx = zil_itx_create(TX_WRITE, sizeof (*lr));
	write_state = WR_NEED_COPY;
	}
	itx->itx_private = zd;
	itx->itx_wr_state = write_state;
	itx->itx_sync = (ztest_random(8) == 0);

	memcpy(&itx->itx_lr + 1, &lr->lr_common + 1,
	sizeof (*lr) - sizeof (lr_t));

	zil_itx_assign(zd->zd_zilog, itx, tx);
	}

	static void
	ztest_log_truncate(ztest_ds_t zd, dmu_tx_t tx, lr_truncate_t *lr)
	{
	itx_t *itx;

	if (zil_replaying(zd->zd_zilog, tx))
	return;

	itx = zil_itx_create(TX_TRUNCATE, sizeof (*lr));
	memcpy(&itx->itx_lr + 1, &lr->lr_common + 1,
	sizeof (*lr) - sizeof (lr_t));

	itx->itx_sync = B_FALSE;
	zil_itx_assign(zd->zd_zilog, itx, tx);
	}

	static void
	ztest_log_setattr(ztest_ds_t zd, dmu_tx_t tx, lr_setattr_t *lr)
	{
	itx_t *itx;

	if (zil_replaying(zd->zd_zilog, tx))
	return;

	itx = zil_itx_create(TX_SETATTR, sizeof (*lr));
	memcpy(&itx->itx_lr + 1, &lr->lr_common + 1,
	sizeof (*lr) - sizeof (lr_t));

	itx->itx_sync = B_FALSE;
	zil_itx_assign(zd->zd_zilog, itx, tx);
	}

	/*
	* ZIL replay ops
	*/
	static int
	ztest_replay_create(void arg1, void arg2, boolean_t byteswap)
	{
	ztest_ds_t *zd = arg1;
	lr_create_t *lr = arg2;
	char name = (void )(lr + 1); /* name follows lr */
	objset_t *os = zd->zd_os;
	ztest_block_tag_t *bbt;
	dmu_buf_t *db;
	dmu_tx_t *tx;
	uint64_t txg;
	int error = 0;
	int bonuslen;

	if (byteswap)
	byteswap_uint64_array(lr, sizeof (*lr));

	ASSERT3U(lr->lr_doid, ==, ZTEST_DIROBJ);
	ASSERT3S(name[0], !=, '\0');

	tx = dmu_tx_create(os);

	dmu_tx_hold_zap(tx, lr->lr_doid, B_TRUE, name);

	if (lr->lrz_type == DMU_OT_ZAP_OTHER) {
	dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, B_TRUE, NULL);
	} else {
	dmu_tx_hold_bonus(tx, DMU_NEW_OBJECT);
	}

	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
	if (txg == 0)
	return (ENOSPC);

	ASSERT3U(dmu_objset_zil(os)->zl_replay, ==, !!lr->lr_foid);
	bonuslen = DN_BONUS_SIZE(lr->lrz_dnodesize);

	if (lr->lrz_type == DMU_OT_ZAP_OTHER) {
	if (lr->lr_foid == 0) {
	lr->lr_foid = zap_create_dnsize(os,
	lr->lrz_type, lr->lrz_bonustype,
	bonuslen, lr->lrz_dnodesize, tx);
	} else {
	error = zap_create_claim_dnsize(os, lr->lr_foid,
	lr->lrz_type, lr->lrz_bonustype,
	bonuslen, lr->lrz_dnodesize, tx);
	}
	} else {
	if (lr->lr_foid == 0) {
	lr->lr_foid = dmu_object_alloc_dnsize(os,
	lr->lrz_type, 0, lr->lrz_bonustype,
	bonuslen, lr->lrz_dnodesize, tx);
	} else {
	error = dmu_object_claim_dnsize(os, lr->lr_foid,
	lr->lrz_type, 0, lr->lrz_bonustype,
	bonuslen, lr->lrz_dnodesize, tx);
	}
	}

	if (error) {
	ASSERT3U(error, ==, EEXIST);
	ASSERT(zd->zd_zilog->zl_replay);
	dmu_tx_commit(tx);
	return (error);
	}

	ASSERT3U(lr->lr_foid, !=, 0);

	if (lr->lrz_type != DMU_OT_ZAP_OTHER)
	VERIFY0(dmu_object_set_blocksize(os, lr->lr_foid,
	lr->lrz_blocksize, lr->lrz_ibshift, tx));

	VERIFY0(dmu_bonus_hold(os, lr->lr_foid, FTAG, &db));
	bbt = ztest_bt_bonus(db);
	dmu_buf_will_dirty(db, tx);
	ztest_bt_generate(bbt, os, lr->lr_foid, lr->lrz_dnodesize, -1ULL,
	lr->lr_gen, txg, txg);
	ztest_fill_unused_bonus(db, bbt, lr->lr_foid, os, lr->lr_gen);
	dmu_buf_rele(db, FTAG);

	VERIFY0(zap_add(os, lr->lr_doid, name, sizeof (uint64_t), 1,
	&lr->lr_foid, tx));

	(void) ztest_log_create(zd, tx, lr);

	dmu_tx_commit(tx);

	return (0);
	}

	static int
	ztest_replay_remove(void arg1, void arg2, boolean_t byteswap)
	{
	ztest_ds_t *zd = arg1;
	lr_remove_t *lr = arg2;
	char name = (void )(lr + 1); /* name follows lr */
	objset_t *os = zd->zd_os;
	dmu_object_info_t doi;
	dmu_tx_t *tx;
	uint64_t object, txg;

	if (byteswap)
	byteswap_uint64_array(lr, sizeof (*lr));

	ASSERT3U(lr->lr_doid, ==, ZTEST_DIROBJ);
	ASSERT3S(name[0], !=, '\0');

	VERIFY0(
	zap_lookup(os, lr->lr_doid, name, sizeof (object), 1, &object));
	ASSERT3U(object, !=, 0);

	ztest_object_lock(zd, object, RL_WRITER);

	VERIFY0(dmu_object_info(os, object, &doi));

	tx = dmu_tx_create(os);

	dmu_tx_hold_zap(tx, lr->lr_doid, B_FALSE, name);
	dmu_tx_hold_free(tx, object, 0, DMU_OBJECT_END);

	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
	if (txg == 0) {
	ztest_object_unlock(zd, object);
	return (ENOSPC);
	}

	if (doi.doi_type == DMU_OT_ZAP_OTHER) {
	VERIFY0(zap_destroy(os, object, tx));
	} else {
	VERIFY0(dmu_object_free(os, object, tx));
	}

	VERIFY0(zap_remove(os, lr->lr_doid, name, tx));

	(void) ztest_log_remove(zd, tx, lr, object);

	dmu_tx_commit(tx);

	ztest_object_unlock(zd, object);

	return (0);
	}

	static int
	ztest_replay_write(void arg1, void arg2, boolean_t byteswap)
	{
	ztest_ds_t *zd = arg1;
	lr_write_t *lr = arg2;
	objset_t *os = zd->zd_os;
	void data = lr + 1; / data follows lr */
	uint64_t offset, length;
	ztest_block_tag_t *bt = data;
	ztest_block_tag_t *bbt;
	uint64_t gen, txg, lrtxg, crtxg;
	dmu_object_info_t doi;
	dmu_tx_t *tx;
	dmu_buf_t *db;
	arc_buf_t *abuf = NULL;
	rl_t *rl;

	if (byteswap)
	byteswap_uint64_array(lr, sizeof (*lr));

	offset = lr->lr_offset;
	length = lr->lr_length;

	/* If it's a dmu_sync() block, write the whole block */
	if (lr->lr_common.lrc_reclen == sizeof (lr_write_t)) {
	uint64_t blocksize = BP_GET_LSIZE(&lr->lr_blkptr);
	if (length < blocksize) {
	offset -= offset % blocksize;
	length = blocksize;
	}
	}

	if (bt->bt_magic == BSWAP_64(BT_MAGIC))
	byteswap_uint64_array(bt, sizeof (*bt));

	if (bt->bt_magic != BT_MAGIC)
	bt = NULL;

	ztest_object_lock(zd, lr->lr_foid, RL_READER);
	rl = ztest_range_lock(zd, lr->lr_foid, offset, length, RL_WRITER);

	VERIFY0(dmu_bonus_hold(os, lr->lr_foid, FTAG, &db));

	dmu_object_info_from_db(db, &doi);

	bbt = ztest_bt_bonus(db);
	ASSERT3U(bbt->bt_magic, ==, BT_MAGIC);
	gen = bbt->bt_gen;
	crtxg = bbt->bt_crtxg;
	lrtxg = lr->lr_common.lrc_txg;

	tx = dmu_tx_create(os);

	dmu_tx_hold_write(tx, lr->lr_foid, offset, length);

	if (ztest_random(8) == 0 && length == doi.doi_data_block_size &&
	P2PHASE(offset, length) == 0)
	abuf = dmu_request_arcbuf(db, length);

	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
	if (txg == 0) {
	if (abuf != NULL)
	dmu_return_arcbuf(abuf);
	dmu_buf_rele(db, FTAG);
	ztest_range_unlock(rl);
	ztest_object_unlock(zd, lr->lr_foid);
	return (ENOSPC);
	}

	if (bt != NULL) {
	/*
	* Usually, verify the old data before writing new data --
	* but not always, because we also want to verify correct
	* behavior when the data was not recently read into cache.
	*/
	ASSERT(doi.doi_data_block_size);
	ASSERT0(offset % doi.doi_data_block_size);
	if (ztest_random(4) != 0) {
	int prefetch = ztest_random(2) ?
	DMU_READ_PREFETCH : DMU_READ_NO_PREFETCH;
	ztest_block_tag_t rbt;

	VERIFY(dmu_read(os, lr->lr_foid, offset,
	sizeof (rbt), &rbt, prefetch) == 0);
	if (rbt.bt_magic == BT_MAGIC) {
	ztest_bt_verify(&rbt, os, lr->lr_foid, 0,
	offset, gen, txg, crtxg);
	}
	}

	/*
	* Writes can appear to be newer than the bonus buffer because
	* the ztest_get_data() callback does a dmu_read() of the
	* open-context data, which may be different than the data
	* as it was when the write was generated.
	*/
	if (zd->zd_zilog->zl_replay) {
	ztest_bt_verify(bt, os, lr->lr_foid, 0, offset,
	MAX(gen, bt->bt_gen), MAX(txg, lrtxg),
	bt->bt_crtxg);
	}

	/*
	* Set the bt's gen/txg to the bonus buffer's gen/txg
	* so that all of the usual ASSERTs will work.
	*/
	ztest_bt_generate(bt, os, lr->lr_foid, 0, offset, gen, txg,
	crtxg);
	}

	if (abuf == NULL) {
	dmu_write(os, lr->lr_foid, offset, length, data, tx);
	} else {
	memcpy(abuf->b_data, data, length);
	VERIFY0(dmu_assign_arcbuf_by_dbuf(db, offset, abuf, tx));
	}

	(void) ztest_log_write(zd, tx, lr);

	dmu_buf_rele(db, FTAG);

	dmu_tx_commit(tx);

	ztest_range_unlock(rl);
	ztest_object_unlock(zd, lr->lr_foid);

	return (0);
	}

	static int
	ztest_replay_truncate(void arg1, void arg2, boolean_t byteswap)
	{
	ztest_ds_t *zd = arg1;
	lr_truncate_t *lr = arg2;
	objset_t *os = zd->zd_os;
	dmu_tx_t *tx;
	uint64_t txg;
	rl_t *rl;

	if (byteswap)
	byteswap_uint64_array(lr, sizeof (*lr));

	ztest_object_lock(zd, lr->lr_foid, RL_READER);
	rl = ztest_range_lock(zd, lr->lr_foid, lr->lr_offset, lr->lr_length,
	RL_WRITER);

	tx = dmu_tx_create(os);

	dmu_tx_hold_free(tx, lr->lr_foid, lr->lr_offset, lr->lr_length);

	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
	if (txg == 0) {
	ztest_range_unlock(rl);
	ztest_object_unlock(zd, lr->lr_foid);
	return (ENOSPC);
	}

	VERIFY0(dmu_free_range(os, lr->lr_foid, lr->lr_offset,
	lr->lr_length, tx));

	(void) ztest_log_truncate(zd, tx, lr);

	dmu_tx_commit(tx);

	ztest_range_unlock(rl);
	ztest_object_unlock(zd, lr->lr_foid);

	return (0);
	}

	static int
	ztest_replay_setattr(void arg1, void arg2, boolean_t byteswap)
	{
	ztest_ds_t *zd = arg1;
	lr_setattr_t *lr = arg2;
	objset_t *os = zd->zd_os;
	dmu_tx_t *tx;
	dmu_buf_t *db;
	ztest_block_tag_t *bbt;
	uint64_t txg, lrtxg, crtxg, dnodesize;

	if (byteswap)
	byteswap_uint64_array(lr, sizeof (*lr));

	ztest_object_lock(zd, lr->lr_foid, RL_WRITER);

	VERIFY0(dmu_bonus_hold(os, lr->lr_foid, FTAG, &db));

	tx = dmu_tx_create(os);
	dmu_tx_hold_bonus(tx, lr->lr_foid);

	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
	if (txg == 0) {
	dmu_buf_rele(db, FTAG);
	ztest_object_unlock(zd, lr->lr_foid);
	return (ENOSPC);
	}

	bbt = ztest_bt_bonus(db);
	ASSERT3U(bbt->bt_magic, ==, BT_MAGIC);
	crtxg = bbt->bt_crtxg;
	lrtxg = lr->lr_common.lrc_txg;
	dnodesize = bbt->bt_dnodesize;

	if (zd->zd_zilog->zl_replay) {
	ASSERT3U(lr->lr_size, !=, 0);
	ASSERT3U(lr->lr_mode, !=, 0);
	ASSERT3U(lrtxg, !=, 0);
	} else {
	/*
	* Randomly change the size and increment the generation.
	*/
	lr->lr_size = (ztest_random(db->db_size / sizeof (bbt)) + 1)
	sizeof (*bbt);
	lr->lr_mode = bbt->bt_gen + 1;
	ASSERT0(lrtxg);
	}

	/*
	* Verify that the current bonus buffer is not newer than our txg.
	*/
	ztest_bt_verify(bbt, os, lr->lr_foid, dnodesize, -1ULL, lr->lr_mode,
	MAX(txg, lrtxg), crtxg);

	dmu_buf_will_dirty(db, tx);

	ASSERT3U(lr->lr_size, >=, sizeof (*bbt));
	ASSERT3U(lr->lr_size, <=, db->db_size);
	VERIFY0(dmu_set_bonus(db, lr->lr_size, tx));
	bbt = ztest_bt_bonus(db);

	ztest_bt_generate(bbt, os, lr->lr_foid, dnodesize, -1ULL, lr->lr_mode,
	txg, crtxg);
	ztest_fill_unused_bonus(db, bbt, lr->lr_foid, os, bbt->bt_gen);
	dmu_buf_rele(db, FTAG);

	(void) ztest_log_setattr(zd, tx, lr);

	dmu_tx_commit(tx);

	ztest_object_unlock(zd, lr->lr_foid);

	return (0);
	}

	static zil_replay_func_t *ztest_replay_vector[TX_MAX_TYPE] = {
	NULL, /* 0 no such transaction type */
	ztest_replay_create, /* TX_CREATE */
	NULL, /* TX_MKDIR */
	NULL, /* TX_MKXATTR */
	NULL, /* TX_SYMLINK */
	ztest_replay_remove, /* TX_REMOVE */
	NULL, /* TX_RMDIR */
	NULL, /* TX_LINK */
	NULL, /* TX_RENAME */
	ztest_replay_write, /* TX_WRITE */
	ztest_replay_truncate, /* TX_TRUNCATE */
	ztest_replay_setattr, /* TX_SETATTR */
	NULL, /* TX_ACL */
	NULL, /* TX_CREATE_ACL */
	NULL, /* TX_CREATE_ATTR */
	NULL, /* TX_CREATE_ACL_ATTR */
	NULL, /* TX_MKDIR_ACL */
	NULL, /* TX_MKDIR_ATTR */
	NULL, /* TX_MKDIR_ACL_ATTR */
	NULL, /* TX_WRITE2 */
	NULL, /* TX_SETSAXATTR */
	NULL, /* TX_RENAME_EXCHANGE */
	NULL, /* TX_RENAME_WHITEOUT */
	};

	/*
	* ZIL get_data callbacks
	*/

	static void
	ztest_get_done(zgd_t *zgd, int error)
	{
	(void) error;
	ztest_ds_t *zd = zgd->zgd_private;
	uint64_t object = ((rl_t *)zgd->zgd_lr)->rl_object;

	if (zgd->zgd_db)
	dmu_buf_rele(zgd->zgd_db, zgd);

	ztest_range_unlock((rl_t *)zgd->zgd_lr);
	ztest_object_unlock(zd, object);

	umem_free(zgd, sizeof (*zgd));
	}

	static int
	ztest_get_data(void arg, uint64_t arg2, lr_write_t lr, char *buf,
	struct lwb lwb, zio_t zio)
	{
	(void) arg2;
	ztest_ds_t *zd = arg;
	objset_t *os = zd->zd_os;
	uint64_t object = lr->lr_foid;
	uint64_t offset = lr->lr_offset;
	uint64_t size = lr->lr_length;
	uint64_t txg = lr->lr_common.lrc_txg;
	uint64_t crtxg;
	dmu_object_info_t doi;
	dmu_buf_t *db;
	zgd_t *zgd;
	int error;

	ASSERT3P(lwb, !=, NULL);
	ASSERT3U(size, !=, 0);

	ztest_object_lock(zd, object, RL_READER);
	error = dmu_bonus_hold(os, object, FTAG, &db);
	if (error) {
	ztest_object_unlock(zd, object);
	return (error);
	}

	crtxg = ztest_bt_bonus(db)->bt_crtxg;

	if (crtxg == 0 \|\| crtxg > txg) {
	dmu_buf_rele(db, FTAG);
	ztest_object_unlock(zd, object);
	return (ENOENT);
	}

	dmu_object_info_from_db(db, &doi);
	dmu_buf_rele(db, FTAG);
	db = NULL;

	zgd = umem_zalloc(sizeof (*zgd), UMEM_NOFAIL);
	zgd->zgd_lwb = lwb;
	zgd->zgd_private = zd;

	if (buf != NULL) { /* immediate write */
	zgd->zgd_lr = (struct zfs_locked_range *)ztest_range_lock(zd,
	object, offset, size, RL_READER);

	error = dmu_read(os, object, offset, size, buf,
	DMU_READ_NO_PREFETCH);
	ASSERT0(error);
	} else {
	ASSERT3P(zio, !=, NULL);
	size = doi.doi_data_block_size;
	if (ISP2(size)) {
	offset = P2ALIGN(offset, size);
	} else {
	ASSERT3U(offset, <, size);
	offset = 0;
	}

	zgd->zgd_lr = (struct zfs_locked_range *)ztest_range_lock(zd,
	object, offset, size, RL_READER);

	error = dmu_buf_hold_noread(os, object, offset, zgd, &db);

	if (error == 0) {
	blkptr_t *bp = &lr->lr_blkptr;

	zgd->zgd_db = db;
	zgd->zgd_bp = bp;

	ASSERT3U(db->db_offset, ==, offset);
	ASSERT3U(db->db_size, ==, size);

	error = dmu_sync(zio, lr->lr_common.lrc_txg,
	ztest_get_done, zgd);

	if (error == 0)
	return (0);
	}
	}

	ztest_get_done(zgd, error);

	return (error);
	}

	static void *
	ztest_lr_alloc(size_t lrsize, char *name)
	{
	char *lr;
	size_t namesize = name ? strlen(name) + 1 : 0;

	lr = umem_zalloc(lrsize + namesize, UMEM_NOFAIL);

	if (name)
	memcpy(lr + lrsize, name, namesize);

	return (lr);
	}

	static void
	ztest_lr_free(void lr, size_t lrsize, char name)
	{
	size_t namesize = name ? strlen(name) + 1 : 0;

	umem_free(lr, lrsize + namesize);
	}

	/*
	* Lookup a bunch of objects. Returns the number of objects not found.
	*/
	static int
	ztest_lookup(ztest_ds_t zd, ztest_od_t od, int count)
	{
	int missing = 0;
	int error;
	int i;

	ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));

	for (i = 0; i < count; i++, od++) {
	od->od_object = 0;
	error = zap_lookup(zd->zd_os, od->od_dir, od->od_name,
	sizeof (uint64_t), 1, &od->od_object);
	if (error) {
	ASSERT3S(error, ==, ENOENT);
	ASSERT0(od->od_object);
	missing++;
	} else {
	dmu_buf_t *db;
	ztest_block_tag_t *bbt;
	dmu_object_info_t doi;

	ASSERT3U(od->od_object, !=, 0);
	ASSERT0(missing); /* there should be no gaps */

	ztest_object_lock(zd, od->od_object, RL_READER);
	VERIFY0(dmu_bonus_hold(zd->zd_os, od->od_object,
	FTAG, &db));
	dmu_object_info_from_db(db, &doi);
	bbt = ztest_bt_bonus(db);
	ASSERT3U(bbt->bt_magic, ==, BT_MAGIC);
	od->od_type = doi.doi_type;
	od->od_blocksize = doi.doi_data_block_size;
	od->od_gen = bbt->bt_gen;
	dmu_buf_rele(db, FTAG);
	ztest_object_unlock(zd, od->od_object);
	}
	}

	return (missing);
	}

	static int
	ztest_create(ztest_ds_t zd, ztest_od_t od, int count)
	{
	int missing = 0;
	int i;

	ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));

	for (i = 0; i < count; i++, od++) {
	if (missing) {
	od->od_object = 0;
	missing++;
	continue;
	}

	lr_create_t lr = ztest_lr_alloc(sizeof (lr), od->od_name);

	lr->lr_doid = od->od_dir;
	lr->lr_foid = 0; /* 0 to allocate, > 0 to claim */
	lr->lrz_type = od->od_crtype;
	lr->lrz_blocksize = od->od_crblocksize;
	lr->lrz_ibshift = ztest_random_ibshift();
	lr->lrz_bonustype = DMU_OT_UINT64_OTHER;
	lr->lrz_dnodesize = od->od_crdnodesize;
	lr->lr_gen = od->od_crgen;
	lr->lr_crtime[0] = time(NULL);

	if (ztest_replay_create(zd, lr, B_FALSE) != 0) {
	ASSERT0(missing);
	od->od_object = 0;
	missing++;
	} else {
	od->od_object = lr->lr_foid;
	od->od_type = od->od_crtype;
	od->od_blocksize = od->od_crblocksize;
	od->od_gen = od->od_crgen;
	ASSERT3U(od->od_object, !=, 0);
	}

	ztest_lr_free(lr, sizeof (*lr), od->od_name);
	}

	return (missing);
	}

	static int
	ztest_remove(ztest_ds_t zd, ztest_od_t od, int count)
	{
	int missing = 0;
	int error;
	int i;

	ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));

	od += count - 1;

	for (i = count - 1; i >= 0; i--, od--) {
	if (missing) {
	missing++;
	continue;
	}

	/*
	* No object was found.
	*/
	if (od->od_object == 0)
	continue;

	lr_remove_t lr = ztest_lr_alloc(sizeof (lr), od->od_name);

	lr->lr_doid = od->od_dir;

	if ((error = ztest_replay_remove(zd, lr, B_FALSE)) != 0) {
	ASSERT3U(error, ==, ENOSPC);
	missing++;
	} else {
	od->od_object = 0;
	}
	ztest_lr_free(lr, sizeof (*lr), od->od_name);
	}

	return (missing);
	}

	static int
	ztest_write(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size,
	void *data)
	{
	lr_write_t *lr;
	int error;

	lr = ztest_lr_alloc(sizeof (*lr) + size, NULL);

	lr->lr_foid = object;
	lr->lr_offset = offset;
	lr->lr_length = size;
	lr->lr_blkoff = 0;
	BP_ZERO(&lr->lr_blkptr);

	memcpy(lr + 1, data, size);

	error = ztest_replay_write(zd, lr, B_FALSE);

	ztest_lr_free(lr, sizeof (*lr) + size, NULL);

	return (error);
	}

	static int
	ztest_truncate(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size)
	{
	lr_truncate_t *lr;
	int error;

	lr = ztest_lr_alloc(sizeof (*lr), NULL);

	lr->lr_foid = object;
	lr->lr_offset = offset;
	lr->lr_length = size;

	error = ztest_replay_truncate(zd, lr, B_FALSE);

	ztest_lr_free(lr, sizeof (*lr), NULL);

	return (error);
	}

	static int
	ztest_setattr(ztest_ds_t *zd, uint64_t object)
	{
	lr_setattr_t *lr;
	int error;

	lr = ztest_lr_alloc(sizeof (*lr), NULL);

	lr->lr_foid = object;
	lr->lr_size = 0;
	lr->lr_mode = 0;

	error = ztest_replay_setattr(zd, lr, B_FALSE);

	ztest_lr_free(lr, sizeof (*lr), NULL);

	return (error);
	}

	static void
	ztest_prealloc(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size)
	{
	objset_t *os = zd->zd_os;
	dmu_tx_t *tx;
	uint64_t txg;
	rl_t *rl;

	txg_wait_synced(dmu_objset_pool(os), 0);

	ztest_object_lock(zd, object, RL_READER);
	rl = ztest_range_lock(zd, object, offset, size, RL_WRITER);

	tx = dmu_tx_create(os);

	dmu_tx_hold_write(tx, object, offset, size);

	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);

	if (txg != 0) {
	dmu_prealloc(os, object, offset, size, tx);
	dmu_tx_commit(tx);
	txg_wait_synced(dmu_objset_pool(os), txg);
	} else {
	(void) dmu_free_long_range(os, object, offset, size);
	}

	ztest_range_unlock(rl);
	ztest_object_unlock(zd, object);
	}

	static void
	ztest_io(ztest_ds_t *zd, uint64_t object, uint64_t offset)
	{
	int err;
	ztest_block_tag_t wbt;
	dmu_object_info_t doi;
	enum ztest_io_type io_type;
	uint64_t blocksize;
	void *data;

	VERIFY0(dmu_object_info(zd->zd_os, object, &doi));
	blocksize = doi.doi_data_block_size;
	data = umem_alloc(blocksize, UMEM_NOFAIL);

	/*
	* Pick an i/o type at random, biased toward writing block tags.
	*/
	io_type = ztest_random(ZTEST_IO_TYPES);
	if (ztest_random(2) == 0)
	io_type = ZTEST_IO_WRITE_TAG;

	(void) pthread_rwlock_rdlock(&zd->zd_zilog_lock);

	switch (io_type) {

	case ZTEST_IO_WRITE_TAG:
	ztest_bt_generate(&wbt, zd->zd_os, object, doi.doi_dnodesize,
	offset, 0, 0, 0);
	(void) ztest_write(zd, object, offset, sizeof (wbt), &wbt);
	break;

	case ZTEST_IO_WRITE_PATTERN:
	(void) memset(data, 'a' + (object + offset) % 5, blocksize);
	if (ztest_random(2) == 0) {
	/*
	* Induce fletcher2 collisions to ensure that
	* zio_ddt_collision() detects and resolves them
	* when using fletcher2-verify for deduplication.
	*/
	((uint64_t *)data)[0] ^= 1ULL << 63;
	((uint64_t *)data)[4] ^= 1ULL << 63;
	}
	(void) ztest_write(zd, object, offset, blocksize, data);
	break;

	case ZTEST_IO_WRITE_ZEROES:
	memset(data, 0, blocksize);
	(void) ztest_write(zd, object, offset, blocksize, data);
	break;

	case ZTEST_IO_TRUNCATE:
	(void) ztest_truncate(zd, object, offset, blocksize);
	break;

	case ZTEST_IO_SETATTR:
	(void) ztest_setattr(zd, object);
	break;
	default:
	break;

	case ZTEST_IO_REWRITE:
	(void) pthread_rwlock_rdlock(&ztest_name_lock);
	err = ztest_dsl_prop_set_uint64(zd->zd_name,
	ZFS_PROP_CHECKSUM, spa_dedup_checksum(ztest_spa),
	B_FALSE);
	ASSERT(err == 0 \|\| err == ENOSPC);
	err = ztest_dsl_prop_set_uint64(zd->zd_name,
	ZFS_PROP_COMPRESSION,
	ztest_random_dsl_prop(ZFS_PROP_COMPRESSION),
	B_FALSE);
	ASSERT(err == 0 \|\| err == ENOSPC);
	(void) pthread_rwlock_unlock(&ztest_name_lock);

	VERIFY0(dmu_read(zd->zd_os, object, offset, blocksize, data,
	DMU_READ_NO_PREFETCH));

	(void) ztest_write(zd, object, offset, blocksize, data);
	break;
	}

	(void) pthread_rwlock_unlock(&zd->zd_zilog_lock);

	umem_free(data, blocksize);
	}

	/*
	* Initialize an object description template.
	*/
	static void
	ztest_od_init(ztest_od_t od, uint64_t id, const char tag, uint64_t index,
	dmu_object_type_t type, uint64_t blocksize, uint64_t dnodesize,
	uint64_t gen)
	{
	od->od_dir = ZTEST_DIROBJ;
	od->od_object = 0;

	od->od_crtype = type;
	od->od_crblocksize = blocksize ? blocksize : ztest_random_blocksize();
	od->od_crdnodesize = dnodesize ? dnodesize : ztest_random_dnodesize();
	od->od_crgen = gen;

	od->od_type = DMU_OT_NONE;
	od->od_blocksize = 0;
	od->od_gen = 0;

	(void) snprintf(od->od_name, sizeof (od->od_name),
	"%s(%"PRId64")[%"PRIu64"]",
	tag, id, index);
	}

	/*
	* Lookup or create the objects for a test using the od template.
	* If the objects do not all exist, or if 'remove' is specified,
	* remove any existing objects and create new ones. Otherwise,
	* use the existing objects.
	*/
	static int
	ztest_object_init(ztest_ds_t zd, ztest_od_t od, size_t size, boolean_t remove)
	{
	int count = size / sizeof (*od);
	int rv = 0;

	mutex_enter(&zd->zd_dirobj_lock);
	if ((ztest_lookup(zd, od, count) != 0 \|\| remove) &&
	(ztest_remove(zd, od, count) != 0 \|\|
	ztest_create(zd, od, count) != 0))
	rv = -1;
	zd->zd_od = od;
	mutex_exit(&zd->zd_dirobj_lock);

	return (rv);
	}

	void
	ztest_zil_commit(ztest_ds_t *zd, uint64_t id)
	{
	(void) id;
	zilog_t *zilog = zd->zd_zilog;

	(void) pthread_rwlock_rdlock(&zd->zd_zilog_lock);

	zil_commit(zilog, ztest_random(ZTEST_OBJECTS));

	/*
	* Remember the committed values in zd, which is in parent/child
	* shared memory. If we die, the next iteration of ztest_run()
	* will verify that the log really does contain this record.
	*/
	mutex_enter(&zilog->zl_lock);
	ASSERT3P(zd->zd_shared, !=, NULL);
	ASSERT3U(zd->zd_shared->zd_seq, <=, zilog->zl_commit_lr_seq);
	zd->zd_shared->zd_seq = zilog->zl_commit_lr_seq;
	mutex_exit(&zilog->zl_lock);

	(void) pthread_rwlock_unlock(&zd->zd_zilog_lock);
	}

	/*
	* This function is designed to simulate the operations that occur during a
	* mount/unmount operation. We hold the dataset across these operations in an
	* attempt to expose any implicit assumptions about ZIL management.
	*/
	void
	ztest_zil_remount(ztest_ds_t *zd, uint64_t id)
	{
	(void) id;
	objset_t *os = zd->zd_os;

	/*
	* We hold the ztest_vdev_lock so we don't cause problems with
	* other threads that wish to remove a log device, such as
	* ztest_device_removal().
	*/
	mutex_enter(&ztest_vdev_lock);

	/*
	* We grab the zd_dirobj_lock to ensure that no other thread is
	* updating the zil (i.e. adding in-memory log records) and the
	* zd_zilog_lock to block any I/O.
	*/
	mutex_enter(&zd->zd_dirobj_lock);
	(void) pthread_rwlock_wrlock(&zd->zd_zilog_lock);

	/* zfsvfs_teardown() */
	zil_close(zd->zd_zilog);

	/* zfsvfs_setup() */
	VERIFY3P(zil_open(os, ztest_get_data, NULL), ==, zd->zd_zilog);
	zil_replay(os, zd, ztest_replay_vector);

	(void) pthread_rwlock_unlock(&zd->zd_zilog_lock);
	mutex_exit(&zd->zd_dirobj_lock);
	mutex_exit(&ztest_vdev_lock);
	}

	/*
	* Verify that we can't destroy an active pool, create an existing pool,
	* or create a pool with a bad vdev spec.
	*/
	void
	ztest_spa_create_destroy(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	ztest_shared_opts_t *zo = &ztest_opts;
	spa_t *spa;
	nvlist_t *nvroot;

	if (zo->zo_mmp_test)
	return;

	/*
	* Attempt to create using a bad file.
	*/
	nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, NULL, 0, 0, 1);
	VERIFY3U(ENOENT, ==,
	spa_create("ztest_bad_file", nvroot, NULL, NULL, NULL));
	fnvlist_free(nvroot);

	/*
	* Attempt to create using a bad mirror.
	*/
	nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, NULL, 0, 2, 1);
	VERIFY3U(ENOENT, ==,
	spa_create("ztest_bad_mirror", nvroot, NULL, NULL, NULL));
	fnvlist_free(nvroot);

	/*
	* Attempt to create an existing pool. It shouldn't matter
	* what's in the nvroot; we should fail with EEXIST.
	*/
	(void) pthread_rwlock_rdlock(&ztest_name_lock);
	nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, NULL, 0, 0, 1);
	VERIFY3U(EEXIST, ==,
	spa_create(zo->zo_pool, nvroot, NULL, NULL, NULL));
	fnvlist_free(nvroot);

	/*
	* We open a reference to the spa and then we try to export it
	* expecting one of the following errors:
	*
	* EBUSY
	* Because of the reference we just opened.
	*
	* ZFS_ERR_EXPORT_IN_PROGRESS
	* For the case that there is another ztest thread doing
	* an export concurrently.
	*/
	VERIFY0(spa_open(zo->zo_pool, &spa, FTAG));
	int error = spa_destroy(zo->zo_pool);
	if (error != EBUSY && error != ZFS_ERR_EXPORT_IN_PROGRESS) {
	fatal(B_FALSE, "spa_destroy(%s) returned unexpected value %d",
	spa->spa_name, error);
	}
	spa_close(spa, FTAG);

	(void) pthread_rwlock_unlock(&ztest_name_lock);
	}

	/*
	* Start and then stop the MMP threads to ensure the startup and shutdown code
	* works properly. Actual protection and property-related code tested via ZTS.
	*/
	void
	ztest_mmp_enable_disable(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	ztest_shared_opts_t *zo = &ztest_opts;
	spa_t *spa = ztest_spa;

	if (zo->zo_mmp_test)
	return;

	/*
	* Since enabling MMP involves setting a property, it could not be done
	* while the pool is suspended.
	*/
	if (spa_suspended(spa))
	return;

	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
	mutex_enter(&spa->spa_props_lock);

	zfs_multihost_fail_intervals = 0;

	if (!spa_multihost(spa)) {
	spa->spa_multihost = B_TRUE;
	mmp_thread_start(spa);
	}

	mutex_exit(&spa->spa_props_lock);
	spa_config_exit(spa, SCL_CONFIG, FTAG);

	txg_wait_synced(spa_get_dsl(spa), 0);
	mmp_signal_all_threads();
	txg_wait_synced(spa_get_dsl(spa), 0);

	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
	mutex_enter(&spa->spa_props_lock);

	if (spa_multihost(spa)) {
	mmp_thread_stop(spa);
	spa->spa_multihost = B_FALSE;
	}

	mutex_exit(&spa->spa_props_lock);
	spa_config_exit(spa, SCL_CONFIG, FTAG);
	}

	void
	ztest_spa_upgrade(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	spa_t *spa;
	uint64_t initial_version = SPA_VERSION_INITIAL;
	uint64_t version, newversion;
	nvlist_t nvroot, props;
	char *name;

	if (ztest_opts.zo_mmp_test)
	return;

	/* dRAID added after feature flags, skip upgrade test. */
	if (strcmp(ztest_opts.zo_raid_type, VDEV_TYPE_DRAID) == 0)
	return;

	mutex_enter(&ztest_vdev_lock);
	name = kmem_asprintf("%s_upgrade", ztest_opts.zo_pool);

	/*
	* Clean up from previous runs.
	*/
	(void) spa_destroy(name);

	nvroot = make_vdev_root(NULL, NULL, name, ztest_opts.zo_vdev_size, 0,
	NULL, ztest_opts.zo_raid_children, ztest_opts.zo_mirrors, 1);

	/*
	* If we're configuring a RAIDZ device then make sure that the
	* initial version is capable of supporting that feature.
	*/
	switch (ztest_opts.zo_raid_parity) {
	case 0:
	case 1:
	initial_version = SPA_VERSION_INITIAL;
	break;
	case 2:
	initial_version = SPA_VERSION_RAIDZ2;
	break;
	case 3:
	initial_version = SPA_VERSION_RAIDZ3;
	break;
	}

	/*
	* Create a pool with a spa version that can be upgraded. Pick
	* a value between initial_version and SPA_VERSION_BEFORE_FEATURES.
	*/
	do {
	version = ztest_random_spa_version(initial_version);
	} while (version > SPA_VERSION_BEFORE_FEATURES);

	props = fnvlist_alloc();
	fnvlist_add_uint64(props,
	zpool_prop_to_name(ZPOOL_PROP_VERSION), version);
	VERIFY0(spa_create(name, nvroot, props, NULL, NULL));
	fnvlist_free(nvroot);
	fnvlist_free(props);

	VERIFY0(spa_open(name, &spa, FTAG));
	VERIFY3U(spa_version(spa), ==, version);
	newversion = ztest_random_spa_version(version + 1);

	if (ztest_opts.zo_verbose >= 4) {
	(void) printf("upgrading spa version from "
	"%"PRIu64" to %"PRIu64"\n",
	version, newversion);
	}

	spa_upgrade(spa, newversion);
	VERIFY3U(spa_version(spa), >, version);
	VERIFY3U(spa_version(spa), ==, fnvlist_lookup_uint64(spa->spa_config,
	zpool_prop_to_name(ZPOOL_PROP_VERSION)));
	spa_close(spa, FTAG);

	kmem_strfree(name);
	mutex_exit(&ztest_vdev_lock);
	}

	static void
	ztest_spa_checkpoint(spa_t *spa)
	{
	ASSERT(MUTEX_HELD(&ztest_checkpoint_lock));

	int error = spa_checkpoint(spa->spa_name);

	switch (error) {
	case 0:
	case ZFS_ERR_DEVRM_IN_PROGRESS:
	case ZFS_ERR_DISCARDING_CHECKPOINT:
	case ZFS_ERR_CHECKPOINT_EXISTS:
	break;
	case ENOSPC:
	ztest_record_enospc(FTAG);
	break;
	default:
	fatal(B_FALSE, "spa_checkpoint(%s) = %d", spa->spa_name, error);
	}
	}

	static void
	ztest_spa_discard_checkpoint(spa_t *spa)
	{
	ASSERT(MUTEX_HELD(&ztest_checkpoint_lock));

	int error = spa_checkpoint_discard(spa->spa_name);

	switch (error) {
	case 0:
	case ZFS_ERR_DISCARDING_CHECKPOINT:
	case ZFS_ERR_NO_CHECKPOINT:
	break;
	default:
	fatal(B_FALSE, "spa_discard_checkpoint(%s) = %d",
	spa->spa_name, error);
	}

	}

	void
	ztest_spa_checkpoint_create_discard(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	spa_t *spa = ztest_spa;

	mutex_enter(&ztest_checkpoint_lock);
	if (ztest_random(2) == 0) {
	ztest_spa_checkpoint(spa);
	} else {
	ztest_spa_discard_checkpoint(spa);
	}
	mutex_exit(&ztest_checkpoint_lock);
	}


	static vdev_t *
	vdev_lookup_by_path(vdev_t vd, const char path)
	{
	vdev_t *mvd;
	int c;

	if (vd->vdev_path != NULL && strcmp(path, vd->vdev_path) == 0)
	return (vd);

	for (c = 0; c < vd->vdev_children; c++)
	if ((mvd = vdev_lookup_by_path(vd->vdev_child[c], path)) !=
	NULL)
	return (mvd);

	return (NULL);
	}

	static int
	spa_num_top_vdevs(spa_t *spa)
	{
	vdev_t *rvd = spa->spa_root_vdev;
	ASSERT3U(spa_config_held(spa, SCL_VDEV, RW_READER), ==, SCL_VDEV);
	return (rvd->vdev_children);
	}

	/*
	* Verify that vdev_add() works as expected.
	*/
	void
	ztest_vdev_add_remove(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	ztest_shared_t *zs = ztest_shared;
	spa_t *spa = ztest_spa;
	uint64_t leaves;
	uint64_t guid;
	nvlist_t *nvroot;
	int error;

	if (ztest_opts.zo_mmp_test)
	return;

	mutex_enter(&ztest_vdev_lock);
	leaves = MAX(zs->zs_mirrors + zs->zs_splits, 1) *
	ztest_opts.zo_raid_children;

	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);

	ztest_shared->zs_vdev_next_leaf = spa_num_top_vdevs(spa) * leaves;

	/*
	* If we have slogs then remove them 1/4 of the time.
	*/
	if (spa_has_slogs(spa) && ztest_random(4) == 0) {
	metaslab_group_t *mg;

	/*
	* find the first real slog in log allocation class
	*/
	mg = spa_log_class(spa)->mc_allocator[0].mca_rotor;
	while (!mg->mg_vd->vdev_islog)
	mg = mg->mg_next;

	guid = mg->mg_vd->vdev_guid;

	spa_config_exit(spa, SCL_VDEV, FTAG);

	/*
	* We have to grab the zs_name_lock as writer to
	* prevent a race between removing a slog (dmu_objset_find)
	* and destroying a dataset. Removing the slog will
	* grab a reference on the dataset which may cause
	* dsl_destroy_head() to fail with EBUSY thus
	* leaving the dataset in an inconsistent state.
	*/
	pthread_rwlock_wrlock(&ztest_name_lock);
	error = spa_vdev_remove(spa, guid, B_FALSE);
	pthread_rwlock_unlock(&ztest_name_lock);

	switch (error) {
	case 0:
	case EEXIST: /* Generic zil_reset() error */
	case EBUSY: /* Replay required */
	case EACCES: /* Crypto key not loaded */
	case ZFS_ERR_CHECKPOINT_EXISTS:
	case ZFS_ERR_DISCARDING_CHECKPOINT:
	break;
	default:
	fatal(B_FALSE, "spa_vdev_remove() = %d", error);
	}
	} else {
	spa_config_exit(spa, SCL_VDEV, FTAG);

	/*
	* Make 1/4 of the devices be log devices
	*/
	nvroot = make_vdev_root(NULL, NULL, NULL,
	ztest_opts.zo_vdev_size, 0, (ztest_random(4) == 0) ?
	"log" : NULL, ztest_opts.zo_raid_children, zs->zs_mirrors,
	1);

	- error = spa_vdev_add(spa, nvroot);
	+ error = spa_vdev_add(spa, nvroot, B_FALSE);
	fnvlist_free(nvroot);

	switch (error) {
	case 0:
	break;
	case ENOSPC:
	ztest_record_enospc("spa_vdev_add");
	break;
	default:
	fatal(B_FALSE, "spa_vdev_add() = %d", error);
	}
	}

	mutex_exit(&ztest_vdev_lock);
	}

	void
	ztest_vdev_class_add(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	ztest_shared_t *zs = ztest_shared;
	spa_t *spa = ztest_spa;
	uint64_t leaves;
	nvlist_t *nvroot;
	const char *class = (ztest_random(2) == 0) ?
	VDEV_ALLOC_BIAS_SPECIAL : VDEV_ALLOC_BIAS_DEDUP;
	int error;

	/*
	* By default add a special vdev 50% of the time
	*/
	if ((ztest_opts.zo_special_vdevs == ZTEST_VDEV_CLASS_OFF) \|\|
	(ztest_opts.zo_special_vdevs == ZTEST_VDEV_CLASS_RND &&
	ztest_random(2) == 0)) {
	return;
	}

	mutex_enter(&ztest_vdev_lock);

	/* Only test with mirrors */
	if (zs->zs_mirrors < 2) {
	mutex_exit(&ztest_vdev_lock);
	return;
	}

	/* requires feature@allocation_classes */
	if (!spa_feature_is_enabled(spa, SPA_FEATURE_ALLOCATION_CLASSES)) {
	mutex_exit(&ztest_vdev_lock);
	return;
	}

	leaves = MAX(zs->zs_mirrors + zs->zs_splits, 1) *
	ztest_opts.zo_raid_children;

	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
	ztest_shared->zs_vdev_next_leaf = spa_num_top_vdevs(spa) * leaves;
	spa_config_exit(spa, SCL_VDEV, FTAG);

	nvroot = make_vdev_root(NULL, NULL, NULL, ztest_opts.zo_vdev_size, 0,
	class, ztest_opts.zo_raid_children, zs->zs_mirrors, 1);

	- error = spa_vdev_add(spa, nvroot);
	+ error = spa_vdev_add(spa, nvroot, B_FALSE);
	fnvlist_free(nvroot);

	if (error == ENOSPC)
	ztest_record_enospc("spa_vdev_add");
	else if (error != 0)
	fatal(B_FALSE, "spa_vdev_add() = %d", error);

	/*
	* 50% of the time allow small blocks in the special class
	*/
	if (error == 0 &&
	spa_special_class(spa)->mc_groups == 1 && ztest_random(2) == 0) {
	if (ztest_opts.zo_verbose >= 3)
	(void) printf("Enabling special VDEV small blocks\n");
	error = ztest_dsl_prop_set_uint64(zd->zd_name,
	ZFS_PROP_SPECIAL_SMALL_BLOCKS, 32768, B_FALSE);
	ASSERT(error == 0 \|\| error == ENOSPC);
	}

	mutex_exit(&ztest_vdev_lock);

	if (ztest_opts.zo_verbose >= 3) {
	metaslab_class_t *mc;

	if (strcmp(class, VDEV_ALLOC_BIAS_SPECIAL) == 0)
	mc = spa_special_class(spa);
	else
	mc = spa_dedup_class(spa);
	(void) printf("Added a %s mirrored vdev (of %d)\n",
	class, (int)mc->mc_groups);
	}
	}

	/*
	* Verify that adding/removing aux devices (l2arc, hot spare) works as expected.
	*/
	void
	ztest_vdev_aux_add_remove(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	ztest_shared_t *zs = ztest_shared;
	spa_t *spa = ztest_spa;
	vdev_t *rvd = spa->spa_root_vdev;
	spa_aux_vdev_t *sav;
	const char *aux;
	char *path;
	uint64_t guid = 0;
	int error, ignore_err = 0;

	if (ztest_opts.zo_mmp_test)
	return;

	path = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);

	if (ztest_random(2) == 0) {
	sav = &spa->spa_spares;
	aux = ZPOOL_CONFIG_SPARES;
	} else {
	sav = &spa->spa_l2cache;
	aux = ZPOOL_CONFIG_L2CACHE;
	}

	mutex_enter(&ztest_vdev_lock);

	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);

	if (sav->sav_count != 0 && ztest_random(4) == 0) {
	/*
	* Pick a random device to remove.
	*/
	vdev_t *svd = sav->sav_vdevs[ztest_random(sav->sav_count)];

	/* dRAID spares cannot be removed; try anyways to see ENOTSUP */
	if (strstr(svd->vdev_path, VDEV_TYPE_DRAID) != NULL)
	ignore_err = ENOTSUP;

	guid = svd->vdev_guid;
	} else {
	/*
	* Find an unused device we can add.
	*/
	zs->zs_vdev_aux = 0;
	for (;;) {
	int c;
	(void) snprintf(path, MAXPATHLEN, ztest_aux_template,
	ztest_opts.zo_dir, ztest_opts.zo_pool, aux,
	zs->zs_vdev_aux);
	for (c = 0; c < sav->sav_count; c++)
	if (strcmp(sav->sav_vdevs[c]->vdev_path,
	path) == 0)
	break;
	if (c == sav->sav_count &&
	vdev_lookup_by_path(rvd, path) == NULL)
	break;
	zs->zs_vdev_aux++;
	}
	}

	spa_config_exit(spa, SCL_VDEV, FTAG);

	if (guid == 0) {
	/*
	* Add a new device.
	*/
	nvlist_t *nvroot = make_vdev_root(NULL, aux, NULL,
	(ztest_opts.zo_vdev_size * 5) / 4, 0, NULL, 0, 0, 1);
	- error = spa_vdev_add(spa, nvroot);
	+ error = spa_vdev_add(spa, nvroot, B_FALSE);

	switch (error) {
	case 0:
	break;
	default:
	fatal(B_FALSE, "spa_vdev_add(%p) = %d", nvroot, error);
	}
	fnvlist_free(nvroot);
	} else {
	/*
	* Remove an existing device. Sometimes, dirty its
	* vdev state first to make sure we handle removal
	* of devices that have pending state changes.
	*/
	if (ztest_random(2) == 0)
	(void) vdev_online(spa, guid, 0, NULL);

	error = spa_vdev_remove(spa, guid, B_FALSE);

	switch (error) {
	case 0:
	case EBUSY:
	case ZFS_ERR_CHECKPOINT_EXISTS:
	case ZFS_ERR_DISCARDING_CHECKPOINT:
	break;
	default:
	if (error != ignore_err)
	fatal(B_FALSE,
	"spa_vdev_remove(%"PRIu64") = %d",
	guid, error);
	}
	}

	mutex_exit(&ztest_vdev_lock);

	umem_free(path, MAXPATHLEN);
	}

	/*
	* split a pool if it has mirror tlvdevs
	*/
	void
	ztest_split_pool(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	ztest_shared_t *zs = ztest_shared;
	spa_t *spa = ztest_spa;
	vdev_t *rvd = spa->spa_root_vdev;
	nvlist_t tree, child, config, split, *schild;
	uint_t c, children, schildren = 0, lastlogid = 0;
	int error = 0;

	if (ztest_opts.zo_mmp_test)
	return;

	mutex_enter(&ztest_vdev_lock);

	/* ensure we have a usable config; mirrors of raidz aren't supported */
	if (zs->zs_mirrors < 3 \|\| ztest_opts.zo_raid_children > 1) {
	mutex_exit(&ztest_vdev_lock);
	return;
	}

	/* clean up the old pool, if any */
	(void) spa_destroy("splitp");

	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);

	/* generate a config from the existing config */
	mutex_enter(&spa->spa_props_lock);
	tree = fnvlist_lookup_nvlist(spa->spa_config, ZPOOL_CONFIG_VDEV_TREE);
	mutex_exit(&spa->spa_props_lock);

	VERIFY0(nvlist_lookup_nvlist_array(tree, ZPOOL_CONFIG_CHILDREN,
	&child, &children));

	schild = umem_alloc(rvd->vdev_children * sizeof (nvlist_t *),
	UMEM_NOFAIL);
	for (c = 0; c < children; c++) {
	vdev_t *tvd = rvd->vdev_child[c];
	nvlist_t **mchild;
	uint_t mchildren;

	if (tvd->vdev_islog \|\| tvd->vdev_ops == &vdev_hole_ops) {
	schild[schildren] = fnvlist_alloc();
	fnvlist_add_string(schild[schildren],
	ZPOOL_CONFIG_TYPE, VDEV_TYPE_HOLE);
	fnvlist_add_uint64(schild[schildren],
	ZPOOL_CONFIG_IS_HOLE, 1);
	if (lastlogid == 0)
	lastlogid = schildren;
	++schildren;
	continue;
	}
	lastlogid = 0;
	VERIFY0(nvlist_lookup_nvlist_array(child[c],
	ZPOOL_CONFIG_CHILDREN, &mchild, &mchildren));
	schild[schildren++] = fnvlist_dup(mchild[0]);
	}

	/* OK, create a config that can be used to split */
	split = fnvlist_alloc();
	fnvlist_add_string(split, ZPOOL_CONFIG_TYPE, VDEV_TYPE_ROOT);
	fnvlist_add_nvlist_array(split, ZPOOL_CONFIG_CHILDREN,
	(const nvlist_t **)schild, lastlogid != 0 ? lastlogid : schildren);

	config = fnvlist_alloc();
	fnvlist_add_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, split);

	for (c = 0; c < schildren; c++)
	fnvlist_free(schild[c]);
	umem_free(schild, rvd->vdev_children * sizeof (nvlist_t *));
	fnvlist_free(split);

	spa_config_exit(spa, SCL_VDEV, FTAG);

	(void) pthread_rwlock_wrlock(&ztest_name_lock);
	error = spa_vdev_split_mirror(spa, "splitp", config, NULL, B_FALSE);
	(void) pthread_rwlock_unlock(&ztest_name_lock);

	fnvlist_free(config);

	if (error == 0) {
	(void) printf("successful split - results:\n");
	mutex_enter(&spa_namespace_lock);
	show_pool_stats(spa);
	show_pool_stats(spa_lookup("splitp"));
	mutex_exit(&spa_namespace_lock);
	++zs->zs_splits;
	--zs->zs_mirrors;
	}
	mutex_exit(&ztest_vdev_lock);
	}

	/*
	* Verify that we can attach and detach devices.
	*/
	void
	ztest_vdev_attach_detach(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	ztest_shared_t *zs = ztest_shared;
	spa_t *spa = ztest_spa;
	spa_aux_vdev_t *sav = &spa->spa_spares;
	vdev_t *rvd = spa->spa_root_vdev;
	vdev_t oldvd, newvd, *pvd;
	nvlist_t *root;
	uint64_t leaves;
	uint64_t leaf, top;
	uint64_t ashift = ztest_get_ashift();
	uint64_t oldguid, pguid;
	uint64_t oldsize, newsize;
	char oldpath, newpath;
	int replacing;
	int oldvd_has_siblings = B_FALSE;
	int newvd_is_spare = B_FALSE;
	int newvd_is_dspare = B_FALSE;
	int oldvd_is_log;
	int oldvd_is_special;
	int error, expected_error;

	if (ztest_opts.zo_mmp_test)
	return;

	oldpath = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
	newpath = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);

	mutex_enter(&ztest_vdev_lock);
	leaves = MAX(zs->zs_mirrors, 1) * ztest_opts.zo_raid_children;

	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);

	/*
	* If a vdev is in the process of being removed, its removal may
	* finish while we are in progress, leading to an unexpected error
	* value. Don't bother trying to attach while we are in the middle
	* of removal.
	*/
	if (ztest_device_removal_active) {
	spa_config_exit(spa, SCL_ALL, FTAG);
	goto out;
	}

	/*
	* Decide whether to do an attach or a replace.
	*/
	replacing = ztest_random(2);

	/*
	* Pick a random top-level vdev.
	*/
	top = ztest_random_vdev_top(spa, B_TRUE);

	/*
	* Pick a random leaf within it.
	*/
	leaf = ztest_random(leaves);

	/*
	* Locate this vdev.
	*/
	oldvd = rvd->vdev_child[top];

	/* pick a child from the mirror */
	if (zs->zs_mirrors >= 1) {
	ASSERT3P(oldvd->vdev_ops, ==, &vdev_mirror_ops);
	ASSERT3U(oldvd->vdev_children, >=, zs->zs_mirrors);
	oldvd = oldvd->vdev_child[leaf / ztest_opts.zo_raid_children];
	}

	/* pick a child out of the raidz group */
	if (ztest_opts.zo_raid_children > 1) {
	if (strcmp(oldvd->vdev_ops->vdev_op_type, "raidz") == 0)
	ASSERT3P(oldvd->vdev_ops, ==, &vdev_raidz_ops);
	else
	ASSERT3P(oldvd->vdev_ops, ==, &vdev_draid_ops);
	ASSERT3U(oldvd->vdev_children, ==, ztest_opts.zo_raid_children);
	oldvd = oldvd->vdev_child[leaf % ztest_opts.zo_raid_children];
	}

	/*
	* If we're already doing an attach or replace, oldvd may be a
	* mirror vdev -- in which case, pick a random child.
	*/
	while (oldvd->vdev_children != 0) {
	oldvd_has_siblings = B_TRUE;
	ASSERT3U(oldvd->vdev_children, >=, 2);
	oldvd = oldvd->vdev_child[ztest_random(oldvd->vdev_children)];
	}

	oldguid = oldvd->vdev_guid;
	oldsize = vdev_get_min_asize(oldvd);
	oldvd_is_log = oldvd->vdev_top->vdev_islog;
	oldvd_is_special =
	oldvd->vdev_top->vdev_alloc_bias == VDEV_BIAS_SPECIAL \|\|
	oldvd->vdev_top->vdev_alloc_bias == VDEV_BIAS_DEDUP;
	(void) strlcpy(oldpath, oldvd->vdev_path, MAXPATHLEN);
	pvd = oldvd->vdev_parent;
	pguid = pvd->vdev_guid;

	/*
	* If oldvd has siblings, then half of the time, detach it. Prior
	* to the detach the pool is scrubbed in order to prevent creating
	* unrepairable blocks as a result of the data corruption injection.
	*/
	if (oldvd_has_siblings && ztest_random(2) == 0) {
	spa_config_exit(spa, SCL_ALL, FTAG);

	error = ztest_scrub_impl(spa);
	if (error)
	goto out;

	error = spa_vdev_detach(spa, oldguid, pguid, B_FALSE);
	if (error != 0 && error != ENODEV && error != EBUSY &&
	error != ENOTSUP && error != ZFS_ERR_CHECKPOINT_EXISTS &&
	error != ZFS_ERR_DISCARDING_CHECKPOINT)
	fatal(B_FALSE, "detach (%s) returned %d",
	oldpath, error);
	goto out;
	}

	/*
	* For the new vdev, choose with equal probability between the two
	* standard paths (ending in either 'a' or 'b') or a random hot spare.
	*/
	if (sav->sav_count != 0 && ztest_random(3) == 0) {
	newvd = sav->sav_vdevs[ztest_random(sav->sav_count)];
	newvd_is_spare = B_TRUE;

	if (newvd->vdev_ops == &vdev_draid_spare_ops)
	newvd_is_dspare = B_TRUE;

	(void) strlcpy(newpath, newvd->vdev_path, MAXPATHLEN);
	} else {
	(void) snprintf(newpath, MAXPATHLEN, ztest_dev_template,
	ztest_opts.zo_dir, ztest_opts.zo_pool,
	top * leaves + leaf);
	if (ztest_random(2) == 0)
	newpath[strlen(newpath) - 1] = 'b';
	newvd = vdev_lookup_by_path(rvd, newpath);
	}

	if (newvd) {
	/*
	* Reopen to ensure the vdev's asize field isn't stale.
	*/
	vdev_reopen(newvd);
	newsize = vdev_get_min_asize(newvd);
	} else {
	/*
	* Make newsize a little bigger or smaller than oldsize.
	* If it's smaller, the attach should fail.
	* If it's larger, and we're doing a replace,
	* we should get dynamic LUN growth when we're done.
	*/
	newsize = 10 * oldsize / (9 + ztest_random(3));
	}

	/*
	* If pvd is not a mirror or root, the attach should fail with ENOTSUP,
	* unless it's a replace; in that case any non-replacing parent is OK.
	*
	* If newvd is already part of the pool, it should fail with EBUSY.
	*
	* If newvd is too small, it should fail with EOVERFLOW.
	*
	* If newvd is a distributed spare and it's being attached to a
	* dRAID which is not its parent it should fail with EINVAL.
	*/
	if (pvd->vdev_ops != &vdev_mirror_ops &&
	pvd->vdev_ops != &vdev_root_ops && (!replacing \|\|
	pvd->vdev_ops == &vdev_replacing_ops \|\|
	pvd->vdev_ops == &vdev_spare_ops))
	expected_error = ENOTSUP;
	else if (newvd_is_spare &&
	(!replacing \|\| oldvd_is_log \|\| oldvd_is_special))
	expected_error = ENOTSUP;
	else if (newvd == oldvd)
	expected_error = replacing ? 0 : EBUSY;
	else if (vdev_lookup_by_path(rvd, newpath) != NULL)
	expected_error = EBUSY;
	else if (!newvd_is_dspare && newsize < oldsize)
	expected_error = EOVERFLOW;
	else if (ashift > oldvd->vdev_top->vdev_ashift)
	expected_error = EDOM;
	else if (newvd_is_dspare && pvd != vdev_draid_spare_get_parent(newvd))
	expected_error = EINVAL;
	else
	expected_error = 0;

	spa_config_exit(spa, SCL_ALL, FTAG);

	/*
	* Build the nvlist describing newpath.
	*/
	root = make_vdev_root(newpath, NULL, NULL, newvd == NULL ? newsize : 0,
	ashift, NULL, 0, 0, 1);

	/*
	* When supported select either a healing or sequential resilver.
	*/
	boolean_t rebuilding = B_FALSE;
	if (pvd->vdev_ops == &vdev_mirror_ops \|\|
	pvd->vdev_ops == &vdev_root_ops) {
	rebuilding = !!ztest_random(2);
	}

	error = spa_vdev_attach(spa, oldguid, root, replacing, rebuilding);

	fnvlist_free(root);

	/*
	* If our parent was the replacing vdev, but the replace completed,
	* then instead of failing with ENOTSUP we may either succeed,
	* fail with ENODEV, or fail with EOVERFLOW.
	*/
	if (expected_error == ENOTSUP &&
	(error == 0 \|\| error == ENODEV \|\| error == EOVERFLOW))
	expected_error = error;

	/*
	* If someone grew the LUN, the replacement may be too small.
	*/
	if (error == EOVERFLOW \|\| error == EBUSY)
	expected_error = error;

	if (error == ZFS_ERR_CHECKPOINT_EXISTS \|\|
	error == ZFS_ERR_DISCARDING_CHECKPOINT \|\|
	error == ZFS_ERR_RESILVER_IN_PROGRESS \|\|
	error == ZFS_ERR_REBUILD_IN_PROGRESS)
	expected_error = error;

	if (error != expected_error && expected_error != EBUSY) {
	fatal(B_FALSE, "attach (%s %"PRIu64", %s %"PRIu64", %d) "
	"returned %d, expected %d",
	oldpath, oldsize, newpath,
	newsize, replacing, error, expected_error);
	}
	out:
	mutex_exit(&ztest_vdev_lock);

	umem_free(oldpath, MAXPATHLEN);
	umem_free(newpath, MAXPATHLEN);
	}

	void
	ztest_device_removal(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	spa_t *spa = ztest_spa;
	vdev_t *vd;
	uint64_t guid;
	int error;

	mutex_enter(&ztest_vdev_lock);

	if (ztest_device_removal_active) {
	mutex_exit(&ztest_vdev_lock);
	return;
	}

	/*
	* Remove a random top-level vdev and wait for removal to finish.
	*/
	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
	vd = vdev_lookup_top(spa, ztest_random_vdev_top(spa, B_FALSE));
	guid = vd->vdev_guid;
	spa_config_exit(spa, SCL_VDEV, FTAG);

	error = spa_vdev_remove(spa, guid, B_FALSE);
	if (error == 0) {
	ztest_device_removal_active = B_TRUE;
	mutex_exit(&ztest_vdev_lock);

	/*
	* spa->spa_vdev_removal is created in a sync task that
	* is initiated via dsl_sync_task_nowait(). Since the
	* task may not run before spa_vdev_remove() returns, we
	* must wait at least 1 txg to ensure that the removal
	* struct has been created.
	*/
	txg_wait_synced(spa_get_dsl(spa), 0);

	while (spa->spa_removing_phys.sr_state == DSS_SCANNING)
	txg_wait_synced(spa_get_dsl(spa), 0);
	} else {
	mutex_exit(&ztest_vdev_lock);
	return;
	}

	/*
	* The pool needs to be scrubbed after completing device removal.
	* Failure to do so may result in checksum errors due to the
	* strategy employed by ztest_fault_inject() when selecting which
	* offset are redundant and can be damaged.
	*/
	error = spa_scan(spa, POOL_SCAN_SCRUB);
	if (error == 0) {
	while (dsl_scan_scrubbing(spa_get_dsl(spa)))
	txg_wait_synced(spa_get_dsl(spa), 0);
	}

	mutex_enter(&ztest_vdev_lock);
	ztest_device_removal_active = B_FALSE;
	mutex_exit(&ztest_vdev_lock);
	}

	/*
	* Callback function which expands the physical size of the vdev.
	*/
	static vdev_t *
	grow_vdev(vdev_t vd, void arg)
	{
	spa_t *spa __maybe_unused = vd->vdev_spa;
	size_t *newsize = arg;
	size_t fsize;
	int fd;

	ASSERT3S(spa_config_held(spa, SCL_STATE, RW_READER), ==, SCL_STATE);
	ASSERT(vd->vdev_ops->vdev_op_leaf);

	if ((fd = open(vd->vdev_path, O_RDWR)) == -1)
	return (vd);

	fsize = lseek(fd, 0, SEEK_END);
	VERIFY0(ftruncate(fd, *newsize));

	if (ztest_opts.zo_verbose >= 6) {
	(void) printf("%s grew from %lu to %lu bytes\n",
	vd->vdev_path, (ulong_t)fsize, (ulong_t)*newsize);
	}
	(void) close(fd);
	return (NULL);
	}

	/*
	* Callback function which expands a given vdev by calling vdev_online().
	*/
	static vdev_t *
	online_vdev(vdev_t vd, void arg)
	{
	(void) arg;
	spa_t *spa = vd->vdev_spa;
	vdev_t *tvd = vd->vdev_top;
	uint64_t guid = vd->vdev_guid;
	uint64_t generation = spa->spa_config_generation + 1;
	vdev_state_t newstate = VDEV_STATE_UNKNOWN;
	int error;

	ASSERT3S(spa_config_held(spa, SCL_STATE, RW_READER), ==, SCL_STATE);
	ASSERT(vd->vdev_ops->vdev_op_leaf);

	/* Calling vdev_online will initialize the new metaslabs */
	spa_config_exit(spa, SCL_STATE, spa);
	error = vdev_online(spa, guid, ZFS_ONLINE_EXPAND, &newstate);
	spa_config_enter(spa, SCL_STATE, spa, RW_READER);

	/*
	* If vdev_online returned an error or the underlying vdev_open
	* failed then we abort the expand. The only way to know that
	* vdev_open fails is by checking the returned newstate.
	*/
	if (error \|\| newstate != VDEV_STATE_HEALTHY) {
	if (ztest_opts.zo_verbose >= 5) {
	(void) printf("Unable to expand vdev, state %u, "
	"error %d\n", newstate, error);
	}
	return (vd);
	}
	ASSERT3U(newstate, ==, VDEV_STATE_HEALTHY);

	/*
	* Since we dropped the lock we need to ensure that we're
	* still talking to the original vdev. It's possible this
	* vdev may have been detached/replaced while we were
	* trying to online it.
	*/
	if (generation != spa->spa_config_generation) {
	if (ztest_opts.zo_verbose >= 5) {
	(void) printf("vdev configuration has changed, "
	"guid %"PRIu64", state %"PRIu64", "
	"expected gen %"PRIu64", got gen %"PRIu64"\n",
	guid,
	tvd->vdev_state,
	generation,
	spa->spa_config_generation);
	}
	return (vd);
	}
	return (NULL);
	}

	/*
	* Traverse the vdev tree calling the supplied function.
	* We continue to walk the tree until we either have walked all
	* children or we receive a non-NULL return from the callback.
	* If a NULL callback is passed, then we just return back the first
	* leaf vdev we encounter.
	*/
	static vdev_t *
	vdev_walk_tree(vdev_t vd, vdev_t (func)(vdev_t , void ), void arg)
	{
	uint_t c;

	if (vd->vdev_ops->vdev_op_leaf) {
	if (func == NULL)
	return (vd);
	else
	return (func(vd, arg));
	}

	for (c = 0; c < vd->vdev_children; c++) {
	vdev_t *cvd = vd->vdev_child[c];
	if ((cvd = vdev_walk_tree(cvd, func, arg)) != NULL)
	return (cvd);
	}
	return (NULL);
	}

	/*
	* Verify that dynamic LUN growth works as expected.
	*/
	void
	ztest_vdev_LUN_growth(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	spa_t *spa = ztest_spa;
	vdev_t vd, tvd;
	metaslab_class_t *mc;
	metaslab_group_t *mg;
	size_t psize, newsize;
	uint64_t top;
	uint64_t old_class_space, new_class_space, old_ms_count, new_ms_count;

	mutex_enter(&ztest_checkpoint_lock);
	mutex_enter(&ztest_vdev_lock);
	spa_config_enter(spa, SCL_STATE, spa, RW_READER);

	/*
	* If there is a vdev removal in progress, it could complete while
	* we are running, in which case we would not be able to verify
	* that the metaslab_class space increased (because it decreases
	* when the device removal completes).
	*/
	if (ztest_device_removal_active) {
	spa_config_exit(spa, SCL_STATE, spa);
	mutex_exit(&ztest_vdev_lock);
	mutex_exit(&ztest_checkpoint_lock);
	return;
	}

	top = ztest_random_vdev_top(spa, B_TRUE);

	tvd = spa->spa_root_vdev->vdev_child[top];
	mg = tvd->vdev_mg;
	mc = mg->mg_class;
	old_ms_count = tvd->vdev_ms_count;
	old_class_space = metaslab_class_get_space(mc);

	/*
	* Determine the size of the first leaf vdev associated with
	* our top-level device.
	*/
	vd = vdev_walk_tree(tvd, NULL, NULL);
	ASSERT3P(vd, !=, NULL);
	ASSERT(vd->vdev_ops->vdev_op_leaf);

	psize = vd->vdev_psize;

	/*
	* We only try to expand the vdev if it's healthy, less than 4x its
	* original size, and it has a valid psize.
	*/
	if (tvd->vdev_state != VDEV_STATE_HEALTHY \|\|
	psize == 0 \|\| psize >= 4 * ztest_opts.zo_vdev_size) {
	spa_config_exit(spa, SCL_STATE, spa);
	mutex_exit(&ztest_vdev_lock);
	mutex_exit(&ztest_checkpoint_lock);
	return;
	}
	ASSERT3U(psize, >, 0);
	newsize = psize + MAX(psize / 8, SPA_MAXBLOCKSIZE);
	ASSERT3U(newsize, >, psize);

	if (ztest_opts.zo_verbose >= 6) {
	(void) printf("Expanding LUN %s from %lu to %lu\n",
	vd->vdev_path, (ulong_t)psize, (ulong_t)newsize);
	}

	/*
	* Growing the vdev is a two step process:
	* 1). expand the physical size (i.e. relabel)
	* 2). online the vdev to create the new metaslabs
	*/
	if (vdev_walk_tree(tvd, grow_vdev, &newsize) != NULL \|\|
	vdev_walk_tree(tvd, online_vdev, NULL) != NULL \|\|
	tvd->vdev_state != VDEV_STATE_HEALTHY) {
	if (ztest_opts.zo_verbose >= 5) {
	(void) printf("Could not expand LUN because "
	"the vdev configuration changed.\n");
	}
	spa_config_exit(spa, SCL_STATE, spa);
	mutex_exit(&ztest_vdev_lock);
	mutex_exit(&ztest_checkpoint_lock);
	return;
	}

	spa_config_exit(spa, SCL_STATE, spa);

	/*
	* Expanding the LUN will update the config asynchronously,
	* thus we must wait for the async thread to complete any
	* pending tasks before proceeding.
	*/
	for (;;) {
	boolean_t done;
	mutex_enter(&spa->spa_async_lock);
	done = (spa->spa_async_thread == NULL && !spa->spa_async_tasks);
	mutex_exit(&spa->spa_async_lock);
	if (done)
	break;
	txg_wait_synced(spa_get_dsl(spa), 0);
	(void) poll(NULL, 0, 100);
	}

	spa_config_enter(spa, SCL_STATE, spa, RW_READER);

	tvd = spa->spa_root_vdev->vdev_child[top];
	new_ms_count = tvd->vdev_ms_count;
	new_class_space = metaslab_class_get_space(mc);

	if (tvd->vdev_mg != mg \|\| mg->mg_class != mc) {
	if (ztest_opts.zo_verbose >= 5) {
	(void) printf("Could not verify LUN expansion due to "
	"intervening vdev offline or remove.\n");
	}
	spa_config_exit(spa, SCL_STATE, spa);
	mutex_exit(&ztest_vdev_lock);
	mutex_exit(&ztest_checkpoint_lock);
	return;
	}

	/*
	* Make sure we were able to grow the vdev.
	*/
	if (new_ms_count <= old_ms_count) {
	fatal(B_FALSE,
	"LUN expansion failed: ms_count %"PRIu64" < %"PRIu64"\n",
	old_ms_count, new_ms_count);
	}

	/*
	* Make sure we were able to grow the pool.
	*/
	if (new_class_space <= old_class_space) {
	fatal(B_FALSE,
	"LUN expansion failed: class_space %"PRIu64" < %"PRIu64"\n",
	old_class_space, new_class_space);
	}

	if (ztest_opts.zo_verbose >= 5) {
	char oldnumbuf[NN_NUMBUF_SZ], newnumbuf[NN_NUMBUF_SZ];

	nicenum(old_class_space, oldnumbuf, sizeof (oldnumbuf));
	nicenum(new_class_space, newnumbuf, sizeof (newnumbuf));
	(void) printf("%s grew from %s to %s\n",
	spa->spa_name, oldnumbuf, newnumbuf);
	}

	spa_config_exit(spa, SCL_STATE, spa);
	mutex_exit(&ztest_vdev_lock);
	mutex_exit(&ztest_checkpoint_lock);
	}

	/*
	* Verify that dmu_objset_{create,destroy,open,close} work as expected.
	*/
	static void
	ztest_objset_create_cb(objset_t os, void arg, cred_t cr, dmu_tx_t tx)
	{
	(void) arg, (void) cr;

	/*
	* Create the objects common to all ztest datasets.
	*/
	VERIFY0(zap_create_claim(os, ZTEST_DIROBJ,
	DMU_OT_ZAP_OTHER, DMU_OT_NONE, 0, tx));
	}

	static int
	ztest_dataset_create(char *dsname)
	{
	int err;
	uint64_t rand;
	dsl_crypto_params_t *dcp = NULL;

	/*
	* 50% of the time, we create encrypted datasets
	* using a random cipher suite and a hard-coded
	* wrapping key.
	*/
	rand = ztest_random(2);
	if (rand != 0) {
	nvlist_t *crypto_args = fnvlist_alloc();
	nvlist_t *props = fnvlist_alloc();

	/* slight bias towards the default cipher suite */
	rand = ztest_random(ZIO_CRYPT_FUNCTIONS);
	if (rand < ZIO_CRYPT_AES_128_CCM)
	rand = ZIO_CRYPT_ON;

	fnvlist_add_uint64(props,
	zfs_prop_to_name(ZFS_PROP_ENCRYPTION), rand);
	fnvlist_add_uint8_array(crypto_args, "wkeydata",
	(uint8_t *)ztest_wkeydata, WRAPPING_KEY_LEN);

	/*
	* These parameters aren't really used by the kernel. They
	* are simply stored so that userspace knows how to load
	* the wrapping key.
	*/
	fnvlist_add_uint64(props,
	zfs_prop_to_name(ZFS_PROP_KEYFORMAT), ZFS_KEYFORMAT_RAW);
	fnvlist_add_string(props,
	zfs_prop_to_name(ZFS_PROP_KEYLOCATION), "prompt");
	fnvlist_add_uint64(props,
	zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 0ULL);
	fnvlist_add_uint64(props,
	zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 0ULL);

	VERIFY0(dsl_crypto_params_create_nvlist(DCP_CMD_NONE, props,
	crypto_args, &dcp));

	/*
	* Cycle through all available encryption implementations
	* to verify interoperability.
	*/
	VERIFY0(gcm_impl_set("cycle"));
	VERIFY0(aes_impl_set("cycle"));

	fnvlist_free(crypto_args);
	fnvlist_free(props);
	}

	err = dmu_objset_create(dsname, DMU_OST_OTHER, 0, dcp,
	ztest_objset_create_cb, NULL);
	dsl_crypto_params_free(dcp, !!err);

	rand = ztest_random(100);
	if (err \|\| rand < 80)
	return (err);

	if (ztest_opts.zo_verbose >= 5)
	(void) printf("Setting dataset %s to sync always\n", dsname);
	return (ztest_dsl_prop_set_uint64(dsname, ZFS_PROP_SYNC,
	ZFS_SYNC_ALWAYS, B_FALSE));
	}

	static int
	ztest_objset_destroy_cb(const char name, void arg)
	{
	(void) arg;
	objset_t *os;
	dmu_object_info_t doi;
	int error;

	/*
	* Verify that the dataset contains a directory object.
	*/
	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_TRUE,
	B_TRUE, FTAG, &os));
	error = dmu_object_info(os, ZTEST_DIROBJ, &doi);
	if (error != ENOENT) {
	/* We could have crashed in the middle of destroying it */
	ASSERT0(error);
	ASSERT3U(doi.doi_type, ==, DMU_OT_ZAP_OTHER);
	ASSERT3S(doi.doi_physical_blocks_512, >=, 0);
	}
	dmu_objset_disown(os, B_TRUE, FTAG);

	/*
	* Destroy the dataset.
	*/
	if (strchr(name, '@') != NULL) {
	error = dsl_destroy_snapshot(name, B_TRUE);
	if (error != ECHRNG) {
	/*
	* The program was executed, but encountered a runtime
	* error, such as insufficient slop, or a hold on the
	* dataset.
	*/
	ASSERT0(error);
	}
	} else {
	error = dsl_destroy_head(name);
	if (error == ENOSPC) {
	/* There could be checkpoint or insufficient slop */
	ztest_record_enospc(FTAG);
	} else if (error != EBUSY) {
	/* There could be a hold on this dataset */
	ASSERT0(error);
	}
	}
	return (0);
	}

	static boolean_t
	ztest_snapshot_create(char *osname, uint64_t id)
	{
	char snapname[ZFS_MAX_DATASET_NAME_LEN];
	int error;

	(void) snprintf(snapname, sizeof (snapname), "%"PRIu64"", id);

	error = dmu_objset_snapshot_one(osname, snapname);
	if (error == ENOSPC) {
	ztest_record_enospc(FTAG);
	return (B_FALSE);
	}
	if (error != 0 && error != EEXIST && error != ECHRNG) {
	fatal(B_FALSE, "ztest_snapshot_create(%s@%s) = %d", osname,
	snapname, error);
	}
	return (B_TRUE);
	}

	static boolean_t
	ztest_snapshot_destroy(char *osname, uint64_t id)
	{
	char snapname[ZFS_MAX_DATASET_NAME_LEN];
	int error;

	(void) snprintf(snapname, sizeof (snapname), "%s@%"PRIu64"",
	osname, id);

	error = dsl_destroy_snapshot(snapname, B_FALSE);
	if (error != 0 && error != ENOENT && error != ECHRNG)
	fatal(B_FALSE, "ztest_snapshot_destroy(%s) = %d",
	snapname, error);
	return (B_TRUE);
	}

	void
	ztest_dmu_objset_create_destroy(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd;
	ztest_ds_t *zdtmp;
	int iters;
	int error;
	objset_t os, os2;
	char name[ZFS_MAX_DATASET_NAME_LEN];
	zilog_t *zilog;
	int i;

	zdtmp = umem_alloc(sizeof (ztest_ds_t), UMEM_NOFAIL);

	(void) pthread_rwlock_rdlock(&ztest_name_lock);

	(void) snprintf(name, sizeof (name), "%s/temp_%"PRIu64"",
	ztest_opts.zo_pool, id);

	/*
	* If this dataset exists from a previous run, process its replay log
	* half of the time. If we don't replay it, then dsl_destroy_head()
	* (invoked from ztest_objset_destroy_cb()) should just throw it away.
	*/
	if (ztest_random(2) == 0 &&
	ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE,
	B_TRUE, FTAG, &os) == 0) {
	ztest_zd_init(zdtmp, NULL, os);
	zil_replay(os, zdtmp, ztest_replay_vector);
	ztest_zd_fini(zdtmp);
	dmu_objset_disown(os, B_TRUE, FTAG);
	}

	/*
	* There may be an old instance of the dataset we're about to
	* create lying around from a previous run. If so, destroy it
	* and all of its snapshots.
	*/
	(void) dmu_objset_find(name, ztest_objset_destroy_cb, NULL,
	DS_FIND_CHILDREN \| DS_FIND_SNAPSHOTS);

	/*
	* Verify that the destroyed dataset is no longer in the namespace.
	* It may still be present if the destroy above fails with ENOSPC.
	*/
	error = ztest_dmu_objset_own(name, DMU_OST_OTHER, B_TRUE, B_TRUE,
	FTAG, &os);
	if (error == 0) {
	dmu_objset_disown(os, B_TRUE, FTAG);
	ztest_record_enospc(FTAG);
	goto out;
	}
	VERIFY3U(ENOENT, ==, error);

	/*
	* Verify that we can create a new dataset.
	*/
	error = ztest_dataset_create(name);
	if (error) {
	if (error == ENOSPC) {
	ztest_record_enospc(FTAG);
	goto out;
	}
	fatal(B_FALSE, "dmu_objset_create(%s) = %d", name, error);
	}

	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE, B_TRUE,
	FTAG, &os));

	ztest_zd_init(zdtmp, NULL, os);

	/*
	* Open the intent log for it.
	*/
	zilog = zil_open(os, ztest_get_data, NULL);

	/*
	* Put some objects in there, do a little I/O to them,
	* and randomly take a couple of snapshots along the way.
	*/
	iters = ztest_random(5);
	for (i = 0; i < iters; i++) {
	ztest_dmu_object_alloc_free(zdtmp, id);
	if (ztest_random(iters) == 0)
	(void) ztest_snapshot_create(name, i);
	}

	/*
	* Verify that we cannot create an existing dataset.
	*/
	VERIFY3U(EEXIST, ==,
	dmu_objset_create(name, DMU_OST_OTHER, 0, NULL, NULL, NULL));

	/*
	* Verify that we can hold an objset that is also owned.
	*/
	VERIFY0(dmu_objset_hold(name, FTAG, &os2));
	dmu_objset_rele(os2, FTAG);

	/*
	* Verify that we cannot own an objset that is already owned.
	*/
	VERIFY3U(EBUSY, ==, ztest_dmu_objset_own(name, DMU_OST_OTHER,
	B_FALSE, B_TRUE, FTAG, &os2));

	zil_close(zilog);
	dmu_objset_disown(os, B_TRUE, FTAG);
	ztest_zd_fini(zdtmp);
	out:
	(void) pthread_rwlock_unlock(&ztest_name_lock);

	umem_free(zdtmp, sizeof (ztest_ds_t));
	}

	/*
	* Verify that dmu_snapshot_{create,destroy,open,close} work as expected.
	*/
	void
	ztest_dmu_snapshot_create_destroy(ztest_ds_t *zd, uint64_t id)
	{
	(void) pthread_rwlock_rdlock(&ztest_name_lock);
	(void) ztest_snapshot_destroy(zd->zd_name, id);
	(void) ztest_snapshot_create(zd->zd_name, id);
	(void) pthread_rwlock_unlock(&ztest_name_lock);
	}

	/*
	* Cleanup non-standard snapshots and clones.
	*/
	static void
	ztest_dsl_dataset_cleanup(char *osname, uint64_t id)
	{
	char *snap1name;
	char *clone1name;
	char *snap2name;
	char *clone2name;
	char *snap3name;
	int error;

	snap1name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
	clone1name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
	snap2name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
	clone2name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
	snap3name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);

	(void) snprintf(snap1name, ZFS_MAX_DATASET_NAME_LEN, "%s@s1_%"PRIu64"",
	osname, id);
	(void) snprintf(clone1name, ZFS_MAX_DATASET_NAME_LEN, "%s/c1_%"PRIu64"",
	osname, id);
	(void) snprintf(snap2name, ZFS_MAX_DATASET_NAME_LEN, "%s@s2_%"PRIu64"",
	clone1name, id);
	(void) snprintf(clone2name, ZFS_MAX_DATASET_NAME_LEN, "%s/c2_%"PRIu64"",
	osname, id);
	(void) snprintf(snap3name, ZFS_MAX_DATASET_NAME_LEN, "%s@s3_%"PRIu64"",
	clone1name, id);

	error = dsl_destroy_head(clone2name);
	if (error && error != ENOENT)
	fatal(B_FALSE, "dsl_destroy_head(%s) = %d", clone2name, error);
	error = dsl_destroy_snapshot(snap3name, B_FALSE);
	if (error && error != ENOENT)
	fatal(B_FALSE, "dsl_destroy_snapshot(%s) = %d",
	snap3name, error);
	error = dsl_destroy_snapshot(snap2name, B_FALSE);
	if (error && error != ENOENT)
	fatal(B_FALSE, "dsl_destroy_snapshot(%s) = %d",
	snap2name, error);
	error = dsl_destroy_head(clone1name);
	if (error && error != ENOENT)
	fatal(B_FALSE, "dsl_destroy_head(%s) = %d", clone1name, error);
	error = dsl_destroy_snapshot(snap1name, B_FALSE);
	if (error && error != ENOENT)
	fatal(B_FALSE, "dsl_destroy_snapshot(%s) = %d",
	snap1name, error);

	umem_free(snap1name, ZFS_MAX_DATASET_NAME_LEN);
	umem_free(clone1name, ZFS_MAX_DATASET_NAME_LEN);
	umem_free(snap2name, ZFS_MAX_DATASET_NAME_LEN);
	umem_free(clone2name, ZFS_MAX_DATASET_NAME_LEN);
	umem_free(snap3name, ZFS_MAX_DATASET_NAME_LEN);
	}

	/*
	* Verify dsl_dataset_promote handles EBUSY
	*/
	void
	ztest_dsl_dataset_promote_busy(ztest_ds_t *zd, uint64_t id)
	{
	objset_t *os;
	char *snap1name;
	char *clone1name;
	char *snap2name;
	char *clone2name;
	char *snap3name;
	char *osname = zd->zd_name;
	int error;

	snap1name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
	clone1name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
	snap2name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
	clone2name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
	snap3name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);

	(void) pthread_rwlock_rdlock(&ztest_name_lock);

	ztest_dsl_dataset_cleanup(osname, id);

	(void) snprintf(snap1name, ZFS_MAX_DATASET_NAME_LEN, "%s@s1_%"PRIu64"",
	osname, id);
	(void) snprintf(clone1name, ZFS_MAX_DATASET_NAME_LEN, "%s/c1_%"PRIu64"",
	osname, id);
	(void) snprintf(snap2name, ZFS_MAX_DATASET_NAME_LEN, "%s@s2_%"PRIu64"",
	clone1name, id);
	(void) snprintf(clone2name, ZFS_MAX_DATASET_NAME_LEN, "%s/c2_%"PRIu64"",
	osname, id);
	(void) snprintf(snap3name, ZFS_MAX_DATASET_NAME_LEN, "%s@s3_%"PRIu64"",
	clone1name, id);

	error = dmu_objset_snapshot_one(osname, strchr(snap1name, '@') + 1);
	if (error && error != EEXIST) {
	if (error == ENOSPC) {
	ztest_record_enospc(FTAG);
	goto out;
	}
	fatal(B_FALSE, "dmu_take_snapshot(%s) = %d", snap1name, error);
	}

	error = dmu_objset_clone(clone1name, snap1name);
	if (error) {
	if (error == ENOSPC) {
	ztest_record_enospc(FTAG);
	goto out;
	}
	fatal(B_FALSE, "dmu_objset_create(%s) = %d", clone1name, error);
	}

	error = dmu_objset_snapshot_one(clone1name, strchr(snap2name, '@') + 1);
	if (error && error != EEXIST) {
	if (error == ENOSPC) {
	ztest_record_enospc(FTAG);
	goto out;
	}
	fatal(B_FALSE, "dmu_open_snapshot(%s) = %d", snap2name, error);
	}

	error = dmu_objset_snapshot_one(clone1name, strchr(snap3name, '@') + 1);
	if (error && error != EEXIST) {
	if (error == ENOSPC) {
	ztest_record_enospc(FTAG);
	goto out;
	}
	fatal(B_FALSE, "dmu_open_snapshot(%s) = %d", snap3name, error);
	}

	error = dmu_objset_clone(clone2name, snap3name);
	if (error) {
	if (error == ENOSPC) {
	ztest_record_enospc(FTAG);
	goto out;
	}
	fatal(B_FALSE, "dmu_objset_create(%s) = %d", clone2name, error);
	}

	error = ztest_dmu_objset_own(snap2name, DMU_OST_ANY, B_TRUE, B_TRUE,
	FTAG, &os);
	if (error)
	fatal(B_FALSE, "dmu_objset_own(%s) = %d", snap2name, error);
	error = dsl_dataset_promote(clone2name, NULL);
	if (error == ENOSPC) {
	dmu_objset_disown(os, B_TRUE, FTAG);
	ztest_record_enospc(FTAG);
	goto out;
	}
	if (error != EBUSY)
	fatal(B_FALSE, "dsl_dataset_promote(%s), %d, not EBUSY",
	clone2name, error);
	dmu_objset_disown(os, B_TRUE, FTAG);

	out:
	ztest_dsl_dataset_cleanup(osname, id);

	(void) pthread_rwlock_unlock(&ztest_name_lock);

	umem_free(snap1name, ZFS_MAX_DATASET_NAME_LEN);
	umem_free(clone1name, ZFS_MAX_DATASET_NAME_LEN);
	umem_free(snap2name, ZFS_MAX_DATASET_NAME_LEN);
	umem_free(clone2name, ZFS_MAX_DATASET_NAME_LEN);
	umem_free(snap3name, ZFS_MAX_DATASET_NAME_LEN);
	}

	#undef OD_ARRAY_SIZE
	#define OD_ARRAY_SIZE 4

	/*
	* Verify that dmu_object_{alloc,free} work as expected.
	*/
	void
	ztest_dmu_object_alloc_free(ztest_ds_t *zd, uint64_t id)
	{
	ztest_od_t *od;
	int batchsize;
	int size;
	int b;

	size = sizeof (ztest_od_t) * OD_ARRAY_SIZE;
	od = umem_alloc(size, UMEM_NOFAIL);
	batchsize = OD_ARRAY_SIZE;

	for (b = 0; b < batchsize; b++)
	ztest_od_init(od + b, id, FTAG, b, DMU_OT_UINT64_OTHER,
	0, 0, 0);

	/*
	* Destroy the previous batch of objects, create a new batch,
	* and do some I/O on the new objects.
	*/
	if (ztest_object_init(zd, od, size, B_TRUE) != 0) {
	zd->zd_od = NULL;
	umem_free(od, size);
	return;
	}

	while (ztest_random(4 * batchsize) != 0)
	ztest_io(zd, od[ztest_random(batchsize)].od_object,
	ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT);

	umem_free(od, size);
	}

	/*
	* Rewind the global allocator to verify object allocation backfilling.
	*/
	void
	ztest_dmu_object_next_chunk(ztest_ds_t *zd, uint64_t id)
	{
	(void) id;
	objset_t *os = zd->zd_os;
	uint_t dnodes_per_chunk = 1 << dmu_object_alloc_chunk_shift;
	uint64_t object;

	/*
	* Rewind the global allocator randomly back to a lower object number
	* to force backfilling and reclamation of recently freed dnodes.
	*/
	mutex_enter(&os->os_obj_lock);
	object = ztest_random(os->os_obj_next_chunk);
	os->os_obj_next_chunk = P2ALIGN(object, dnodes_per_chunk);
	mutex_exit(&os->os_obj_lock);
	}

	#undef OD_ARRAY_SIZE
	#define OD_ARRAY_SIZE 2

	/*
	* Verify that dmu_{read,write} work as expected.
	*/
	void
	ztest_dmu_read_write(ztest_ds_t *zd, uint64_t id)
	{
	int size;
	ztest_od_t *od;

	objset_t *os = zd->zd_os;
	size = sizeof (ztest_od_t) * OD_ARRAY_SIZE;
	od = umem_alloc(size, UMEM_NOFAIL);
	dmu_tx_t *tx;
	int freeit, error;
	uint64_t i, n, s, txg;
	bufwad_t packbuf, bigbuf, pack, bigH, *bigT;
	uint64_t packobj, packoff, packsize, bigobj, bigoff, bigsize;
	uint64_t chunksize = (1000 + ztest_random(1000)) * sizeof (uint64_t);
	uint64_t regions = 997;
	uint64_t stride = 123456789ULL;
	uint64_t width = 40;
	int free_percent = 5;

	/*
	* This test uses two objects, packobj and bigobj, that are always
	* updated together (i.e. in the same tx) so that their contents are
	* in sync and can be compared. Their contents relate to each other
	* in a simple way: packobj is a dense array of 'bufwad' structures,
	* while bigobj is a sparse array of the same bufwads. Specifically,
	* for any index n, there are three bufwads that should be identical:
	*
	* packobj, at offset n * sizeof (bufwad_t)
	* bigobj, at the head of the nth chunk
	* bigobj, at the tail of the nth chunk
	*
	* The chunk size is arbitrary. It doesn't have to be a power of two,
	* and it doesn't have any relation to the object blocksize.
	* The only requirement is that it can hold at least two bufwads.
	*
	* Normally, we write the bufwad to each of these locations.
	* However, free_percent of the time we instead write zeroes to
	* packobj and perform a dmu_free_range() on bigobj. By comparing
	* bigobj to packobj, we can verify that the DMU is correctly
	* tracking which parts of an object are allocated and free,
	* and that the contents of the allocated blocks are correct.
	*/

	/*
	* Read the directory info. If it's the first time, set things up.
	*/
	ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, chunksize);
	ztest_od_init(od + 1, id, FTAG, 1, DMU_OT_UINT64_OTHER, 0, 0,
	chunksize);

	if (ztest_object_init(zd, od, size, B_FALSE) != 0) {
	umem_free(od, size);
	return;
	}

	bigobj = od[0].od_object;
	packobj = od[1].od_object;
	chunksize = od[0].od_gen;
	ASSERT3U(chunksize, ==, od[1].od_gen);

	/*
	* Prefetch a random chunk of the big object.
	* Our aim here is to get some async reads in flight
	* for blocks that we may free below; the DMU should
	* handle this race correctly.
	*/
	n = ztest_random(regions) * stride + ztest_random(width);
	s = 1 + ztest_random(2 * width - 1);
	dmu_prefetch(os, bigobj, 0, n * chunksize, s * chunksize,
	ZIO_PRIORITY_SYNC_READ);

	/*
	* Pick a random index and compute the offsets into packobj and bigobj.
	*/
	n = ztest_random(regions) * stride + ztest_random(width);
	s = 1 + ztest_random(width - 1);

	packoff = n * sizeof (bufwad_t);
	packsize = s * sizeof (bufwad_t);

	bigoff = n * chunksize;
	bigsize = s * chunksize;

	packbuf = umem_alloc(packsize, UMEM_NOFAIL);
	bigbuf = umem_alloc(bigsize, UMEM_NOFAIL);

	/*
	* free_percent of the time, free a range of bigobj rather than
	* overwriting it.
	*/
	freeit = (ztest_random(100) < free_percent);

	/*
	* Read the current contents of our objects.
	*/
	error = dmu_read(os, packobj, packoff, packsize, packbuf,
	DMU_READ_PREFETCH);
	ASSERT0(error);
	error = dmu_read(os, bigobj, bigoff, bigsize, bigbuf,
	DMU_READ_PREFETCH);
	ASSERT0(error);

	/*
	* Get a tx for the mods to both packobj and bigobj.
	*/
	tx = dmu_tx_create(os);

	dmu_tx_hold_write(tx, packobj, packoff, packsize);

	if (freeit)
	dmu_tx_hold_free(tx, bigobj, bigoff, bigsize);
	else
	dmu_tx_hold_write(tx, bigobj, bigoff, bigsize);

	/* This accounts for setting the checksum/compression. */
	dmu_tx_hold_bonus(tx, bigobj);

	txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
	if (txg == 0) {
	umem_free(packbuf, packsize);
	umem_free(bigbuf, bigsize);
	umem_free(od, size);
	return;
	}

	enum zio_checksum cksum;
	do {
	cksum = (enum zio_checksum)
	ztest_random_dsl_prop(ZFS_PROP_CHECKSUM);
	} while (cksum >= ZIO_CHECKSUM_LEGACY_FUNCTIONS);
	dmu_object_set_checksum(os, bigobj, cksum, tx);

	enum zio_compress comp;
	do {
	comp = (enum zio_compress)
	ztest_random_dsl_prop(ZFS_PROP_COMPRESSION);
	} while (comp >= ZIO_COMPRESS_LEGACY_FUNCTIONS);
	dmu_object_set_compress(os, bigobj, comp, tx);

	/*
	* For each index from n to n + s, verify that the existing bufwad
	* in packobj matches the bufwads at the head and tail of the
	* corresponding chunk in bigobj. Then update all three bufwads
	* with the new values we want to write out.
	*/
	for (i = 0; i < s; i++) {
	/* LINTED */
	pack = (bufwad_t )((char )packbuf + i * sizeof (bufwad_t));
	/* LINTED */
	bigH = (bufwad_t )((char )bigbuf + i * chunksize);
	/* LINTED */
	bigT = (bufwad_t )((char )bigH + chunksize) - 1;

	ASSERT3U((uintptr_t)bigH - (uintptr_t)bigbuf, <, bigsize);
	ASSERT3U((uintptr_t)bigT - (uintptr_t)bigbuf, <, bigsize);

	if (pack->bw_txg > txg)
	fatal(B_FALSE,
	"future leak: got %"PRIx64", open txg is %"PRIx64"",
	pack->bw_txg, txg);

	if (pack->bw_data != 0 && pack->bw_index != n + i)
	fatal(B_FALSE, "wrong index: "
	"got %"PRIx64", wanted %"PRIx64"+%"PRIx64"",
	pack->bw_index, n, i);

	if (memcmp(pack, bigH, sizeof (bufwad_t)) != 0)
	fatal(B_FALSE, "pack/bigH mismatch in %p/%p",
	pack, bigH);

	if (memcmp(pack, bigT, sizeof (bufwad_t)) != 0)
	fatal(B_FALSE, "pack/bigT mismatch in %p/%p",
	pack, bigT);

	if (freeit) {
	memset(pack, 0, sizeof (bufwad_t));
	} else {
	pack->bw_index = n + i;
	pack->bw_txg = txg;
	pack->bw_data = 1 + ztest_random(-2ULL);
	}
	bigH = pack;
	bigT = pack;
	}

	/*
	* We've verified all the old bufwads, and made new ones.
	* Now write them out.
	*/
	dmu_write(os, packobj, packoff, packsize, packbuf, tx);

	if (freeit) {
	if (ztest_opts.zo_verbose >= 7) {
	(void) printf("freeing offset %"PRIx64" size %"PRIx64""
	" txg %"PRIx64"\n",
	bigoff, bigsize, txg);
	}
	VERIFY0(dmu_free_range(os, bigobj, bigoff, bigsize, tx));
	} else {
	if (ztest_opts.zo_verbose >= 7) {
	(void) printf("writing offset %"PRIx64" size %"PRIx64""
	" txg %"PRIx64"\n",
	bigoff, bigsize, txg);
	}
	dmu_write(os, bigobj, bigoff, bigsize, bigbuf, tx);
	}

	dmu_tx_commit(tx);

	/*
	* Sanity check the stuff we just wrote.
	*/
	{
	void *packcheck = umem_alloc(packsize, UMEM_NOFAIL);
	void *bigcheck = umem_alloc(bigsize, UMEM_NOFAIL);

	VERIFY0(dmu_read(os, packobj, packoff,
	packsize, packcheck, DMU_READ_PREFETCH));
	VERIFY0(dmu_read(os, bigobj, bigoff,
	bigsize, bigcheck, DMU_READ_PREFETCH));

	ASSERT0(memcmp(packbuf, packcheck, packsize));
	ASSERT0(memcmp(bigbuf, bigcheck, bigsize));

	umem_free(packcheck, packsize);
	umem_free(bigcheck, bigsize);
	}

	umem_free(packbuf, packsize);
	umem_free(bigbuf, bigsize);
	umem_free(od, size);
	}

	static void
	compare_and_update_pbbufs(uint64_t s, bufwad_t packbuf, bufwad_t bigbuf,
	uint64_t bigsize, uint64_t n, uint64_t chunksize, uint64_t txg)
	{
	uint64_t i;
	bufwad_t *pack;
	bufwad_t *bigH;
	bufwad_t *bigT;

	/*
	* For each index from n to n + s, verify that the existing bufwad
	* in packobj matches the bufwads at the head and tail of the
	* corresponding chunk in bigobj. Then update all three bufwads
	* with the new values we want to write out.
	*/
	for (i = 0; i < s; i++) {
	/* LINTED */
	pack = (bufwad_t )((char )packbuf + i * sizeof (bufwad_t));
	/* LINTED */
	bigH = (bufwad_t )((char )bigbuf + i * chunksize);
	/* LINTED */
	bigT = (bufwad_t )((char )bigH + chunksize) - 1;

	ASSERT3U((uintptr_t)bigH - (uintptr_t)bigbuf, <, bigsize);
	ASSERT3U((uintptr_t)bigT - (uintptr_t)bigbuf, <, bigsize);

	if (pack->bw_txg > txg)
	fatal(B_FALSE,
	"future leak: got %"PRIx64", open txg is %"PRIx64"",
	pack->bw_txg, txg);

	if (pack->bw_data != 0 && pack->bw_index != n + i)
	fatal(B_FALSE, "wrong index: "
	"got %"PRIx64", wanted %"PRIx64"+%"PRIx64"",
	pack->bw_index, n, i);

	if (memcmp(pack, bigH, sizeof (bufwad_t)) != 0)
	fatal(B_FALSE, "pack/bigH mismatch in %p/%p",
	pack, bigH);

	if (memcmp(pack, bigT, sizeof (bufwad_t)) != 0)
	fatal(B_FALSE, "pack/bigT mismatch in %p/%p",
	pack, bigT);

	pack->bw_index = n + i;
	pack->bw_txg = txg;
	pack->bw_data = 1 + ztest_random(-2ULL);

	bigH = pack;
	bigT = pack;
	}
	}

	#undef OD_ARRAY_SIZE
	#define OD_ARRAY_SIZE 2

	void
	ztest_dmu_read_write_zcopy(ztest_ds_t *zd, uint64_t id)
	{
	objset_t *os = zd->zd_os;
	ztest_od_t *od;
	dmu_tx_t *tx;
	uint64_t i;
	int error;
	int size;
	uint64_t n, s, txg;
	bufwad_t packbuf, bigbuf;
	uint64_t packobj, packoff, packsize, bigobj, bigoff, bigsize;
	uint64_t blocksize = ztest_random_blocksize();
	uint64_t chunksize = blocksize;
	uint64_t regions = 997;
	uint64_t stride = 123456789ULL;
	uint64_t width = 9;
	dmu_buf_t *bonus_db;
	arc_buf_t **bigbuf_arcbufs;
	dmu_object_info_t doi;

	size = sizeof (ztest_od_t) * OD_ARRAY_SIZE;
	od = umem_alloc(size, UMEM_NOFAIL);

	/*
	* This test uses two objects, packobj and bigobj, that are always
	* updated together (i.e. in the same tx) so that their contents are
	* in sync and can be compared. Their contents relate to each other
	* in a simple way: packobj is a dense array of 'bufwad' structures,
	* while bigobj is a sparse array of the same bufwads. Specifically,
	* for any index n, there are three bufwads that should be identical:
	*
	* packobj, at offset n * sizeof (bufwad_t)
	* bigobj, at the head of the nth chunk
	* bigobj, at the tail of the nth chunk
	*
	* The chunk size is set equal to bigobj block size so that
	* dmu_assign_arcbuf_by_dbuf() can be tested for object updates.
	*/

	/*
	* Read the directory info. If it's the first time, set things up.
	*/
	ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, blocksize, 0, 0);
	ztest_od_init(od + 1, id, FTAG, 1, DMU_OT_UINT64_OTHER, 0, 0,
	chunksize);


	if (ztest_object_init(zd, od, size, B_FALSE) != 0) {
	umem_free(od, size);
	return;
	}

	bigobj = od[0].od_object;
	packobj = od[1].od_object;
	blocksize = od[0].od_blocksize;
	chunksize = blocksize;
	ASSERT3U(chunksize, ==, od[1].od_gen);

	VERIFY0(dmu_object_info(os, bigobj, &doi));
	VERIFY(ISP2(doi.doi_data_block_size));
	VERIFY3U(chunksize, ==, doi.doi_data_block_size);
	VERIFY3U(chunksize, >=, 2 * sizeof (bufwad_t));

	/*
	* Pick a random index and compute the offsets into packobj and bigobj.
	*/
	n = ztest_random(regions) * stride + ztest_random(width);
	s = 1 + ztest_random(width - 1);

	packoff = n * sizeof (bufwad_t);
	packsize = s * sizeof (bufwad_t);

	bigoff = n * chunksize;
	bigsize = s * chunksize;

	packbuf = umem_zalloc(packsize, UMEM_NOFAIL);
	bigbuf = umem_zalloc(bigsize, UMEM_NOFAIL);

	VERIFY0(dmu_bonus_hold(os, bigobj, FTAG, &bonus_db));

	bigbuf_arcbufs = umem_zalloc(2 * s * sizeof (arc_buf_t *), UMEM_NOFAIL);

	/*
	* Iteration 0 test zcopy for DB_UNCACHED dbufs.
	* Iteration 1 test zcopy to already referenced dbufs.
	* Iteration 2 test zcopy to dirty dbuf in the same txg.
	* Iteration 3 test zcopy to dbuf dirty in previous txg.
	* Iteration 4 test zcopy when dbuf is no longer dirty.
	* Iteration 5 test zcopy when it can't be done.
	* Iteration 6 one more zcopy write.
	*/
	for (i = 0; i < 7; i++) {
	uint64_t j;
	uint64_t off;

	/*
	* In iteration 5 (i == 5) use arcbufs
	* that don't match bigobj blksz to test
	* dmu_assign_arcbuf_by_dbuf() when it can't directly
	* assign an arcbuf to a dbuf.
	*/
	for (j = 0; j < s; j++) {
	if (i != 5 \|\| chunksize < (SPA_MINBLOCKSIZE * 2)) {
	bigbuf_arcbufs[j] =
	dmu_request_arcbuf(bonus_db, chunksize);
	} else {
	bigbuf_arcbufs[2 * j] =
	dmu_request_arcbuf(bonus_db, chunksize / 2);
	bigbuf_arcbufs[2 * j + 1] =
	dmu_request_arcbuf(bonus_db, chunksize / 2);
	}
	}

	/*
	* Get a tx for the mods to both packobj and bigobj.
	*/
	tx = dmu_tx_create(os);

	dmu_tx_hold_write(tx, packobj, packoff, packsize);
	dmu_tx_hold_write(tx, bigobj, bigoff, bigsize);

	txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
	if (txg == 0) {
	umem_free(packbuf, packsize);
	umem_free(bigbuf, bigsize);
	for (j = 0; j < s; j++) {
	if (i != 5 \|\|
	chunksize < (SPA_MINBLOCKSIZE * 2)) {
	dmu_return_arcbuf(bigbuf_arcbufs[j]);
	} else {
	dmu_return_arcbuf(
	bigbuf_arcbufs[2 * j]);
	dmu_return_arcbuf(
	bigbuf_arcbufs[2 * j + 1]);
	}
	}
	umem_free(bigbuf_arcbufs, 2 * s * sizeof (arc_buf_t *));
	umem_free(od, size);
	dmu_buf_rele(bonus_db, FTAG);
	return;
	}

	/*
	* 50% of the time don't read objects in the 1st iteration to
	* test dmu_assign_arcbuf_by_dbuf() for the case when there are
	* no existing dbufs for the specified offsets.
	*/
	if (i != 0 \|\| ztest_random(2) != 0) {
	error = dmu_read(os, packobj, packoff,
	packsize, packbuf, DMU_READ_PREFETCH);
	ASSERT0(error);
	error = dmu_read(os, bigobj, bigoff, bigsize,
	bigbuf, DMU_READ_PREFETCH);
	ASSERT0(error);
	}
	compare_and_update_pbbufs(s, packbuf, bigbuf, bigsize,
	n, chunksize, txg);

	/*
	* We've verified all the old bufwads, and made new ones.
	* Now write them out.
	*/
	dmu_write(os, packobj, packoff, packsize, packbuf, tx);
	if (ztest_opts.zo_verbose >= 7) {
	(void) printf("writing offset %"PRIx64" size %"PRIx64""
	" txg %"PRIx64"\n",
	bigoff, bigsize, txg);
	}
	for (off = bigoff, j = 0; j < s; j++, off += chunksize) {
	dmu_buf_t *dbt;
	if (i != 5 \|\| chunksize < (SPA_MINBLOCKSIZE * 2)) {
	memcpy(bigbuf_arcbufs[j]->b_data,
	(caddr_t)bigbuf + (off - bigoff),
	chunksize);
	} else {
	memcpy(bigbuf_arcbufs[2 * j]->b_data,
	(caddr_t)bigbuf + (off - bigoff),
	chunksize / 2);
	memcpy(bigbuf_arcbufs[2 * j + 1]->b_data,
	(caddr_t)bigbuf + (off - bigoff) +
	chunksize / 2,
	chunksize / 2);
	}

	if (i == 1) {
	VERIFY(dmu_buf_hold(os, bigobj, off,
	FTAG, &dbt, DMU_READ_NO_PREFETCH) == 0);
	}
	if (i != 5 \|\| chunksize < (SPA_MINBLOCKSIZE * 2)) {
	VERIFY0(dmu_assign_arcbuf_by_dbuf(bonus_db,
	off, bigbuf_arcbufs[j], tx));
	} else {
	VERIFY0(dmu_assign_arcbuf_by_dbuf(bonus_db,
	off, bigbuf_arcbufs[2 * j], tx));
	VERIFY0(dmu_assign_arcbuf_by_dbuf(bonus_db,
	off + chunksize / 2,
	bigbuf_arcbufs[2 * j + 1], tx));
	}
	if (i == 1) {
	dmu_buf_rele(dbt, FTAG);
	}
	}
	dmu_tx_commit(tx);

	/*
	* Sanity check the stuff we just wrote.
	*/
	{
	void *packcheck = umem_alloc(packsize, UMEM_NOFAIL);
	void *bigcheck = umem_alloc(bigsize, UMEM_NOFAIL);

	VERIFY0(dmu_read(os, packobj, packoff,
	packsize, packcheck, DMU_READ_PREFETCH));
	VERIFY0(dmu_read(os, bigobj, bigoff,
	bigsize, bigcheck, DMU_READ_PREFETCH));

	ASSERT0(memcmp(packbuf, packcheck, packsize));
	ASSERT0(memcmp(bigbuf, bigcheck, bigsize));

	umem_free(packcheck, packsize);
	umem_free(bigcheck, bigsize);
	}
	if (i == 2) {
	txg_wait_open(dmu_objset_pool(os), 0, B_TRUE);
	} else if (i == 3) {
	txg_wait_synced(dmu_objset_pool(os), 0);
	}
	}

	dmu_buf_rele(bonus_db, FTAG);
	umem_free(packbuf, packsize);
	umem_free(bigbuf, bigsize);
	umem_free(bigbuf_arcbufs, 2 * s * sizeof (arc_buf_t *));
	umem_free(od, size);
	}

	void
	ztest_dmu_write_parallel(ztest_ds_t *zd, uint64_t id)
	{
	(void) id;
	ztest_od_t *od;

	od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
	uint64_t offset = (1ULL << (ztest_random(20) + 43)) +
	(ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT);

	/*
	* Have multiple threads write to large offsets in an object
	* to verify that parallel writes to an object -- even to the
	* same blocks within the object -- doesn't cause any trouble.
	*/
	ztest_od_init(od, ID_PARALLEL, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, 0);

	if (ztest_object_init(zd, od, sizeof (ztest_od_t), B_FALSE) != 0)
	return;

	while (ztest_random(10) != 0)
	ztest_io(zd, od->od_object, offset);

	umem_free(od, sizeof (ztest_od_t));
	}

	void
	ztest_dmu_prealloc(ztest_ds_t *zd, uint64_t id)
	{
	ztest_od_t *od;
	uint64_t offset = (1ULL << (ztest_random(4) + SPA_MAXBLOCKSHIFT)) +
	(ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT);
	uint64_t count = ztest_random(20) + 1;
	uint64_t blocksize = ztest_random_blocksize();
	void *data;

	od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);

	ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, blocksize, 0, 0);

	if (ztest_object_init(zd, od, sizeof (ztest_od_t),
	!ztest_random(2)) != 0) {
	umem_free(od, sizeof (ztest_od_t));
	return;
	}

	if (ztest_truncate(zd, od->od_object, offset, count * blocksize) != 0) {
	umem_free(od, sizeof (ztest_od_t));
	return;
	}

	ztest_prealloc(zd, od->od_object, offset, count * blocksize);

	data = umem_zalloc(blocksize, UMEM_NOFAIL);

	while (ztest_random(count) != 0) {
	uint64_t randoff = offset + (ztest_random(count) * blocksize);
	if (ztest_write(zd, od->od_object, randoff, blocksize,
	data) != 0)
	break;
	while (ztest_random(4) != 0)
	ztest_io(zd, od->od_object, randoff);
	}

	umem_free(data, blocksize);
	umem_free(od, sizeof (ztest_od_t));
	}

	/*
	* Verify that zap_{create,destroy,add,remove,update} work as expected.
	*/
	#define ZTEST_ZAP_MIN_INTS 1
	#define ZTEST_ZAP_MAX_INTS 4
	#define ZTEST_ZAP_MAX_PROPS 1000

	void
	ztest_zap(ztest_ds_t *zd, uint64_t id)
	{
	objset_t *os = zd->zd_os;
	ztest_od_t *od;
	uint64_t object;
	uint64_t txg, last_txg;
	uint64_t value[ZTEST_ZAP_MAX_INTS];
	uint64_t zl_ints, zl_intsize, prop;
	int i, ints;
	dmu_tx_t *tx;
	char propname[100], txgname[100];
	int error;
	const char *const hc[2] = { "s.acl.h", ".s.open.h.hyLZlg" };

	od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
	ztest_od_init(od, id, FTAG, 0, DMU_OT_ZAP_OTHER, 0, 0, 0);

	if (ztest_object_init(zd, od, sizeof (ztest_od_t),
	!ztest_random(2)) != 0)
	goto out;

	object = od->od_object;

	/*
	* Generate a known hash collision, and verify that
	* we can lookup and remove both entries.
	*/
	tx = dmu_tx_create(os);
	dmu_tx_hold_zap(tx, object, B_TRUE, NULL);
	txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
	if (txg == 0)
	goto out;
	for (i = 0; i < 2; i++) {
	value[i] = i;
	VERIFY0(zap_add(os, object, hc[i], sizeof (uint64_t),
	1, &value[i], tx));
	}
	for (i = 0; i < 2; i++) {
	VERIFY3U(EEXIST, ==, zap_add(os, object, hc[i],
	sizeof (uint64_t), 1, &value[i], tx));
	VERIFY0(
	zap_length(os, object, hc[i], &zl_intsize, &zl_ints));
	ASSERT3U(zl_intsize, ==, sizeof (uint64_t));
	ASSERT3U(zl_ints, ==, 1);
	}
	for (i = 0; i < 2; i++) {
	VERIFY0(zap_remove(os, object, hc[i], tx));
	}
	dmu_tx_commit(tx);

	/*
	* Generate a bunch of random entries.
	*/
	ints = MAX(ZTEST_ZAP_MIN_INTS, object % ZTEST_ZAP_MAX_INTS);

	prop = ztest_random(ZTEST_ZAP_MAX_PROPS);
	(void) sprintf(propname, "prop_%"PRIu64"", prop);
	(void) sprintf(txgname, "txg_%"PRIu64"", prop);
	memset(value, 0, sizeof (value));
	last_txg = 0;

	/*
	* If these zap entries already exist, validate their contents.
	*/
	error = zap_length(os, object, txgname, &zl_intsize, &zl_ints);
	if (error == 0) {
	ASSERT3U(zl_intsize, ==, sizeof (uint64_t));
	ASSERT3U(zl_ints, ==, 1);

	VERIFY0(zap_lookup(os, object, txgname, zl_intsize,
	zl_ints, &last_txg));

	VERIFY0(zap_length(os, object, propname, &zl_intsize,
	&zl_ints));

	ASSERT3U(zl_intsize, ==, sizeof (uint64_t));
	ASSERT3U(zl_ints, ==, ints);

	VERIFY0(zap_lookup(os, object, propname, zl_intsize,
	zl_ints, value));

	for (i = 0; i < ints; i++) {
	ASSERT3U(value[i], ==, last_txg + object + i);
	}
	} else {
	ASSERT3U(error, ==, ENOENT);
	}

	/*
	* Atomically update two entries in our zap object.
	* The first is named txg_%llu, and contains the txg
	* in which the property was last updated. The second
	* is named prop_%llu, and the nth element of its value
	* should be txg + object + n.
	*/
	tx = dmu_tx_create(os);
	dmu_tx_hold_zap(tx, object, B_TRUE, NULL);
	txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
	if (txg == 0)
	goto out;

	if (last_txg > txg)
	fatal(B_FALSE, "zap future leak: old %"PRIu64" new %"PRIu64"",
	last_txg, txg);

	for (i = 0; i < ints; i++)
	value[i] = txg + object + i;

	VERIFY0(zap_update(os, object, txgname, sizeof (uint64_t),
	1, &txg, tx));
	VERIFY0(zap_update(os, object, propname, sizeof (uint64_t),
	ints, value, tx));

	dmu_tx_commit(tx);

	/*
	* Remove a random pair of entries.
	*/
	prop = ztest_random(ZTEST_ZAP_MAX_PROPS);
	(void) sprintf(propname, "prop_%"PRIu64"", prop);
	(void) sprintf(txgname, "txg_%"PRIu64"", prop);

	error = zap_length(os, object, txgname, &zl_intsize, &zl_ints);

	if (error == ENOENT)
	goto out;

	ASSERT0(error);

	tx = dmu_tx_create(os);
	dmu_tx_hold_zap(tx, object, B_TRUE, NULL);
	txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
	if (txg == 0)
	goto out;
	VERIFY0(zap_remove(os, object, txgname, tx));
	VERIFY0(zap_remove(os, object, propname, tx));
	dmu_tx_commit(tx);
	out:
	umem_free(od, sizeof (ztest_od_t));
	}

	/*
	* Test case to test the upgrading of a microzap to fatzap.
	*/
	void
	ztest_fzap(ztest_ds_t *zd, uint64_t id)
	{
	objset_t *os = zd->zd_os;
	ztest_od_t *od;
	uint64_t object, txg, value;

	od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
	ztest_od_init(od, id, FTAG, 0, DMU_OT_ZAP_OTHER, 0, 0, 0);

	if (ztest_object_init(zd, od, sizeof (ztest_od_t),
	!ztest_random(2)) != 0)
	goto out;
	object = od->od_object;

	/*
	* Add entries to this ZAP and make sure it spills over
	* and gets upgraded to a fatzap. Also, since we are adding
	* 2050 entries we should see ptrtbl growth and leaf-block split.
	*/
	for (value = 0; value < 2050; value++) {
	char name[ZFS_MAX_DATASET_NAME_LEN];
	dmu_tx_t *tx;
	int error;

	(void) snprintf(name, sizeof (name), "fzap-%"PRIu64"-%"PRIu64"",
	id, value);

	tx = dmu_tx_create(os);
	dmu_tx_hold_zap(tx, object, B_TRUE, name);
	txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
	if (txg == 0)
	goto out;
	error = zap_add(os, object, name, sizeof (uint64_t), 1,
	&value, tx);
	ASSERT(error == 0 \|\| error == EEXIST);
	dmu_tx_commit(tx);
	}
	out:
	umem_free(od, sizeof (ztest_od_t));
	}

	void
	ztest_zap_parallel(ztest_ds_t *zd, uint64_t id)
	{
	(void) id;
	objset_t *os = zd->zd_os;
	ztest_od_t *od;
	uint64_t txg, object, count, wsize, wc, zl_wsize, zl_wc;
	dmu_tx_t *tx;
	int i, namelen, error;
	int micro = ztest_random(2);
	char name[20], string_value[20];
	void *data;

	od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
	ztest_od_init(od, ID_PARALLEL, FTAG, micro, DMU_OT_ZAP_OTHER, 0, 0, 0);

	if (ztest_object_init(zd, od, sizeof (ztest_od_t), B_FALSE) != 0) {
	umem_free(od, sizeof (ztest_od_t));
	return;
	}

	object = od->od_object;

	/*
	* Generate a random name of the form 'xxx.....' where each
	* x is a random printable character and the dots are dots.
	* There are 94 such characters, and the name length goes from
	* 6 to 20, so there are 94^3 * 15 = 12,458,760 possible names.
	*/
	namelen = ztest_random(sizeof (name) - 5) + 5 + 1;

	for (i = 0; i < 3; i++)
	name[i] = '!' + ztest_random('~' - '!' + 1);
	for (; i < namelen - 1; i++)
	name[i] = '.';
	name[i] = '\0';

	if ((namelen & 1) \|\| micro) {
	wsize = sizeof (txg);
	wc = 1;
	data = &txg;
	} else {
	wsize = 1;
	wc = namelen;
	data = string_value;
	}

	count = -1ULL;
	VERIFY0(zap_count(os, object, &count));
	ASSERT3S(count, !=, -1ULL);

	/*
	* Select an operation: length, lookup, add, update, remove.
	*/
	i = ztest_random(5);

	if (i >= 2) {
	tx = dmu_tx_create(os);
	dmu_tx_hold_zap(tx, object, B_TRUE, NULL);
	txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
	if (txg == 0) {
	umem_free(od, sizeof (ztest_od_t));
	return;
	}
	memcpy(string_value, name, namelen);
	} else {
	tx = NULL;
	txg = 0;
	memset(string_value, 0, namelen);
	}

	switch (i) {

	case 0:
	error = zap_length(os, object, name, &zl_wsize, &zl_wc);
	if (error == 0) {
	ASSERT3U(wsize, ==, zl_wsize);
	ASSERT3U(wc, ==, zl_wc);
	} else {
	ASSERT3U(error, ==, ENOENT);
	}
	break;

	case 1:
	error = zap_lookup(os, object, name, wsize, wc, data);
	if (error == 0) {
	if (data == string_value &&
	memcmp(name, data, namelen) != 0)
	fatal(B_FALSE, "name '%s' != val '%s' len %d",
	name, (char *)data, namelen);
	} else {
	ASSERT3U(error, ==, ENOENT);
	}
	break;

	case 2:
	error = zap_add(os, object, name, wsize, wc, data, tx);
	ASSERT(error == 0 \|\| error == EEXIST);
	break;

	case 3:
	VERIFY0(zap_update(os, object, name, wsize, wc, data, tx));
	break;

	case 4:
	error = zap_remove(os, object, name, tx);
	ASSERT(error == 0 \|\| error == ENOENT);
	break;
	}

	if (tx != NULL)
	dmu_tx_commit(tx);

	umem_free(od, sizeof (ztest_od_t));
	}

	/*
	* Commit callback data.
	*/
	typedef struct ztest_cb_data {
	list_node_t zcd_node;
	uint64_t zcd_txg;
	int zcd_expected_err;
	boolean_t zcd_added;
	boolean_t zcd_called;
	spa_t *zcd_spa;
	} ztest_cb_data_t;

	/* This is the actual commit callback function */
	static void
	ztest_commit_callback(void *arg, int error)
	{
	ztest_cb_data_t *data = arg;
	uint64_t synced_txg;

	VERIFY3P(data, !=, NULL);
	VERIFY3S(data->zcd_expected_err, ==, error);
	VERIFY(!data->zcd_called);

	synced_txg = spa_last_synced_txg(data->zcd_spa);
	if (data->zcd_txg > synced_txg)
	fatal(B_FALSE,
	"commit callback of txg %"PRIu64" called prematurely, "
	"last synced txg = %"PRIu64"\n",
	data->zcd_txg, synced_txg);

	data->zcd_called = B_TRUE;

	if (error == ECANCELED) {
	ASSERT0(data->zcd_txg);
	ASSERT(!data->zcd_added);

	/*
	* The private callback data should be destroyed here, but
	* since we are going to check the zcd_called field after
	* dmu_tx_abort(), we will destroy it there.
	*/
	return;
	}

	ASSERT(data->zcd_added);
	ASSERT3U(data->zcd_txg, !=, 0);

	(void) mutex_enter(&zcl.zcl_callbacks_lock);

	/* See if this cb was called more quickly */
	if ((synced_txg - data->zcd_txg) < zc_min_txg_delay)
	zc_min_txg_delay = synced_txg - data->zcd_txg;

	/* Remove our callback from the list */
	list_remove(&zcl.zcl_callbacks, data);

	(void) mutex_exit(&zcl.zcl_callbacks_lock);

	umem_free(data, sizeof (ztest_cb_data_t));
	}

	/* Allocate and initialize callback data structure */
	static ztest_cb_data_t *
	ztest_create_cb_data(objset_t *os, uint64_t txg)
	{
	ztest_cb_data_t *cb_data;

	cb_data = umem_zalloc(sizeof (ztest_cb_data_t), UMEM_NOFAIL);

	cb_data->zcd_txg = txg;
	cb_data->zcd_spa = dmu_objset_spa(os);
	list_link_init(&cb_data->zcd_node);

	return (cb_data);
	}

	/*
	* Commit callback test.
	*/
	void
	ztest_dmu_commit_callbacks(ztest_ds_t *zd, uint64_t id)
	{
	objset_t *os = zd->zd_os;
	ztest_od_t *od;
	dmu_tx_t *tx;
	ztest_cb_data_t cb_data[3], tmp_cb;
	uint64_t old_txg, txg;
	int i, error = 0;

	od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
	ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, 0);

	if (ztest_object_init(zd, od, sizeof (ztest_od_t), B_FALSE) != 0) {
	umem_free(od, sizeof (ztest_od_t));
	return;
	}

	tx = dmu_tx_create(os);

	cb_data[0] = ztest_create_cb_data(os, 0);
	dmu_tx_callback_register(tx, ztest_commit_callback, cb_data[0]);

	dmu_tx_hold_write(tx, od->od_object, 0, sizeof (uint64_t));

	/* Every once in a while, abort the transaction on purpose */
	if (ztest_random(100) == 0)
	error = -1;

	if (!error)
	error = dmu_tx_assign(tx, TXG_NOWAIT);

	txg = error ? 0 : dmu_tx_get_txg(tx);

	cb_data[0]->zcd_txg = txg;
	cb_data[1] = ztest_create_cb_data(os, txg);
	dmu_tx_callback_register(tx, ztest_commit_callback, cb_data[1]);

	if (error) {
	/*
	* It's not a strict requirement to call the registered
	* callbacks from inside dmu_tx_abort(), but that's what
	* it's supposed to happen in the current implementation
	* so we will check for that.
	*/
	for (i = 0; i < 2; i++) {
	cb_data[i]->zcd_expected_err = ECANCELED;
	VERIFY(!cb_data[i]->zcd_called);
	}

	dmu_tx_abort(tx);

	for (i = 0; i < 2; i++) {
	VERIFY(cb_data[i]->zcd_called);
	umem_free(cb_data[i], sizeof (ztest_cb_data_t));
	}

	umem_free(od, sizeof (ztest_od_t));
	return;
	}

	cb_data[2] = ztest_create_cb_data(os, txg);
	dmu_tx_callback_register(tx, ztest_commit_callback, cb_data[2]);

	/*
	* Read existing data to make sure there isn't a future leak.
	*/
	VERIFY0(dmu_read(os, od->od_object, 0, sizeof (uint64_t),
	&old_txg, DMU_READ_PREFETCH));

	if (old_txg > txg)
	fatal(B_FALSE,
	"future leak: got %"PRIu64", open txg is %"PRIu64"",
	old_txg, txg);

	dmu_write(os, od->od_object, 0, sizeof (uint64_t), &txg, tx);

	(void) mutex_enter(&zcl.zcl_callbacks_lock);

	/*
	* Since commit callbacks don't have any ordering requirement and since
	* it is theoretically possible for a commit callback to be called
	* after an arbitrary amount of time has elapsed since its txg has been
	* synced, it is difficult to reliably determine whether a commit
	* callback hasn't been called due to high load or due to a flawed
	* implementation.
	*
	* In practice, we will assume that if after a certain number of txgs a
	* commit callback hasn't been called, then most likely there's an
	* implementation bug..
	*/
	tmp_cb = list_head(&zcl.zcl_callbacks);
	if (tmp_cb != NULL &&
	tmp_cb->zcd_txg + ZTEST_COMMIT_CB_THRESH < txg) {
	fatal(B_FALSE,
	"Commit callback threshold exceeded, "
	"oldest txg: %"PRIu64", open txg: %"PRIu64"\n",
	tmp_cb->zcd_txg, txg);
	}

	/*
	* Let's find the place to insert our callbacks.
	*
	* Even though the list is ordered by txg, it is possible for the
	* insertion point to not be the end because our txg may already be
	* quiescing at this point and other callbacks in the open txg
	* (from other objsets) may have sneaked in.
	*/
	tmp_cb = list_tail(&zcl.zcl_callbacks);
	while (tmp_cb != NULL && tmp_cb->zcd_txg > txg)
	tmp_cb = list_prev(&zcl.zcl_callbacks, tmp_cb);

	/* Add the 3 callbacks to the list */
	for (i = 0; i < 3; i++) {
	if (tmp_cb == NULL)
	list_insert_head(&zcl.zcl_callbacks, cb_data[i]);
	else
	list_insert_after(&zcl.zcl_callbacks, tmp_cb,
	cb_data[i]);

	cb_data[i]->zcd_added = B_TRUE;
	VERIFY(!cb_data[i]->zcd_called);

	tmp_cb = cb_data[i];
	}

	zc_cb_counter += 3;

	(void) mutex_exit(&zcl.zcl_callbacks_lock);

	dmu_tx_commit(tx);

	umem_free(od, sizeof (ztest_od_t));
	}

	/*
	* Visit each object in the dataset. Verify that its properties
	* are consistent what was stored in the block tag when it was created,
	* and that its unused bonus buffer space has not been overwritten.
	*/
	void
	ztest_verify_dnode_bt(ztest_ds_t *zd, uint64_t id)
	{
	(void) id;
	objset_t *os = zd->zd_os;
	uint64_t obj;
	int err = 0;

	for (obj = 0; err == 0; err = dmu_object_next(os, &obj, FALSE, 0)) {
	ztest_block_tag_t *bt = NULL;
	dmu_object_info_t doi;
	dmu_buf_t *db;

	ztest_object_lock(zd, obj, RL_READER);
	if (dmu_bonus_hold(os, obj, FTAG, &db) != 0) {
	ztest_object_unlock(zd, obj);
	continue;
	}

	dmu_object_info_from_db(db, &doi);
	if (doi.doi_bonus_size >= sizeof (*bt))
	bt = ztest_bt_bonus(db);

	if (bt && bt->bt_magic == BT_MAGIC) {
	ztest_bt_verify(bt, os, obj, doi.doi_dnodesize,
	bt->bt_offset, bt->bt_gen, bt->bt_txg,
	bt->bt_crtxg);
	ztest_verify_unused_bonus(db, bt, obj, os, bt->bt_gen);
	}

	dmu_buf_rele(db, FTAG);
	ztest_object_unlock(zd, obj);
	}
	}

	void
	ztest_dsl_prop_get_set(ztest_ds_t *zd, uint64_t id)
	{
	(void) id;
	zfs_prop_t proplist[] = {
	ZFS_PROP_CHECKSUM,
	ZFS_PROP_COMPRESSION,
	ZFS_PROP_COPIES,
	ZFS_PROP_DEDUP
	};

	(void) pthread_rwlock_rdlock(&ztest_name_lock);

	for (int p = 0; p < sizeof (proplist) / sizeof (proplist[0]); p++) {
	int error = ztest_dsl_prop_set_uint64(zd->zd_name, proplist[p],
	ztest_random_dsl_prop(proplist[p]), (int)ztest_random(2));
	ASSERT(error == 0 \|\| error == ENOSPC);
	}

	int error = ztest_dsl_prop_set_uint64(zd->zd_name, ZFS_PROP_RECORDSIZE,
	ztest_random_blocksize(), (int)ztest_random(2));
	ASSERT(error == 0 \|\| error == ENOSPC);

	(void) pthread_rwlock_unlock(&ztest_name_lock);
	}

	void
	ztest_spa_prop_get_set(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	nvlist_t *props = NULL;

	(void) pthread_rwlock_rdlock(&ztest_name_lock);

	(void) ztest_spa_prop_set_uint64(ZPOOL_PROP_AUTOTRIM, ztest_random(2));

	VERIFY0(spa_prop_get(ztest_spa, &props));

	if (ztest_opts.zo_verbose >= 6)
	dump_nvlist(props, 4);

	fnvlist_free(props);

	(void) pthread_rwlock_unlock(&ztest_name_lock);
	}

	static int
	user_release_one(const char snapname, const char holdname)
	{
	nvlist_t snaps, holds;
	int error;

	snaps = fnvlist_alloc();
	holds = fnvlist_alloc();
	fnvlist_add_boolean(holds, holdname);
	fnvlist_add_nvlist(snaps, snapname, holds);
	fnvlist_free(holds);
	error = dsl_dataset_user_release(snaps, NULL);
	fnvlist_free(snaps);
	return (error);
	}

	/*
	* Test snapshot hold/release and deferred destroy.
	*/
	void
	ztest_dmu_snapshot_hold(ztest_ds_t *zd, uint64_t id)
	{
	int error;
	objset_t *os = zd->zd_os;
	objset_t *origin;
	char snapname[100];
	char fullname[100];
	char clonename[100];
	char tag[100];
	char osname[ZFS_MAX_DATASET_NAME_LEN];
	nvlist_t *holds;

	(void) pthread_rwlock_rdlock(&ztest_name_lock);

	dmu_objset_name(os, osname);

	(void) snprintf(snapname, sizeof (snapname), "sh1_%"PRIu64"", id);
	(void) snprintf(fullname, sizeof (fullname), "%s@%s", osname, snapname);
	(void) snprintf(clonename, sizeof (clonename), "%s/ch1_%"PRIu64"",
	osname, id);
	(void) snprintf(tag, sizeof (tag), "tag_%"PRIu64"", id);

	/*
	* Clean up from any previous run.
	*/
	error = dsl_destroy_head(clonename);
	if (error != ENOENT)
	ASSERT0(error);
	error = user_release_one(fullname, tag);
	if (error != ESRCH && error != ENOENT)
	ASSERT0(error);
	error = dsl_destroy_snapshot(fullname, B_FALSE);
	if (error != ENOENT)
	ASSERT0(error);

	/*
	* Create snapshot, clone it, mark snap for deferred destroy,
	* destroy clone, verify snap was also destroyed.
	*/
	error = dmu_objset_snapshot_one(osname, snapname);
	if (error) {
	if (error == ENOSPC) {
	ztest_record_enospc("dmu_objset_snapshot");
	goto out;
	}
	fatal(B_FALSE, "dmu_objset_snapshot(%s) = %d", fullname, error);
	}

	error = dmu_objset_clone(clonename, fullname);
	if (error) {
	if (error == ENOSPC) {
	ztest_record_enospc("dmu_objset_clone");
	goto out;
	}
	fatal(B_FALSE, "dmu_objset_clone(%s) = %d", clonename, error);
	}

	error = dsl_destroy_snapshot(fullname, B_TRUE);
	if (error) {
	fatal(B_FALSE, "dsl_destroy_snapshot(%s, B_TRUE) = %d",
	fullname, error);
	}

	error = dsl_destroy_head(clonename);
	if (error)
	fatal(B_FALSE, "dsl_destroy_head(%s) = %d", clonename, error);

	error = dmu_objset_hold(fullname, FTAG, &origin);
	if (error != ENOENT)
	fatal(B_FALSE, "dmu_objset_hold(%s) = %d", fullname, error);

	/*
	* Create snapshot, add temporary hold, verify that we can't
	* destroy a held snapshot, mark for deferred destroy,
	* release hold, verify snapshot was destroyed.
	*/
	error = dmu_objset_snapshot_one(osname, snapname);
	if (error) {
	if (error == ENOSPC) {
	ztest_record_enospc("dmu_objset_snapshot");
	goto out;
	}
	fatal(B_FALSE, "dmu_objset_snapshot(%s) = %d", fullname, error);
	}

	holds = fnvlist_alloc();
	fnvlist_add_string(holds, fullname, tag);
	error = dsl_dataset_user_hold(holds, 0, NULL);
	fnvlist_free(holds);

	if (error == ENOSPC) {
	ztest_record_enospc("dsl_dataset_user_hold");
	goto out;
	} else if (error) {
	fatal(B_FALSE, "dsl_dataset_user_hold(%s, %s) = %u",
	fullname, tag, error);
	}

	error = dsl_destroy_snapshot(fullname, B_FALSE);
	if (error != EBUSY) {
	fatal(B_FALSE, "dsl_destroy_snapshot(%s, B_FALSE) = %d",
	fullname, error);
	}

	error = dsl_destroy_snapshot(fullname, B_TRUE);
	if (error) {
	fatal(B_FALSE, "dsl_destroy_snapshot(%s, B_TRUE) = %d",
	fullname, error);
	}

	error = user_release_one(fullname, tag);
	if (error)
	fatal(B_FALSE, "user_release_one(%s, %s) = %d",
	fullname, tag, error);

	VERIFY3U(dmu_objset_hold(fullname, FTAG, &origin), ==, ENOENT);

	out:
	(void) pthread_rwlock_unlock(&ztest_name_lock);
	}

	/*
	* Inject random faults into the on-disk data.
	*/
	void
	ztest_fault_inject(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	ztest_shared_t *zs = ztest_shared;
	spa_t *spa = ztest_spa;
	int fd;
	uint64_t offset;
	uint64_t leaves;
	uint64_t bad = 0x1990c0ffeedecadeull;
	uint64_t top, leaf;
	char *path0;
	char *pathrand;
	size_t fsize;
	int bshift = SPA_MAXBLOCKSHIFT + 2;
	int iters = 1000;
	int maxfaults;
	int mirror_save;
	vdev_t *vd0 = NULL;
	uint64_t guid0 = 0;
	boolean_t islog = B_FALSE;

	path0 = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
	pathrand = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);

	mutex_enter(&ztest_vdev_lock);

	/*
	* Device removal is in progress, fault injection must be disabled
	* until it completes and the pool is scrubbed. The fault injection
	* strategy for damaging blocks does not take in to account evacuated
	* blocks which may have already been damaged.
	*/
	if (ztest_device_removal_active) {
	mutex_exit(&ztest_vdev_lock);
	goto out;
	}

	maxfaults = MAXFAULTS(zs);
	leaves = MAX(zs->zs_mirrors, 1) * ztest_opts.zo_raid_children;
	mirror_save = zs->zs_mirrors;
	mutex_exit(&ztest_vdev_lock);

	ASSERT3U(leaves, >=, 1);

	/*
	* While ztest is running the number of leaves will not change. This
	* is critical for the fault injection logic as it determines where
	* errors can be safely injected such that they are always repairable.
	*
	* When restarting ztest a different number of leaves may be requested
	* which will shift the regions to be damaged. This is fine as long
	* as the pool has been scrubbed prior to using the new mapping.
	* Failure to do can result in non-repairable damage being injected.
	*/
	if (ztest_pool_scrubbed == B_FALSE)
	goto out;

	/*
	* Grab the name lock as reader. There are some operations
	* which don't like to have their vdevs changed while
	* they are in progress (i.e. spa_change_guid). Those
	* operations will have grabbed the name lock as writer.
	*/
	(void) pthread_rwlock_rdlock(&ztest_name_lock);

	/*
	* We need SCL_STATE here because we're going to look at vd0->vdev_tsd.
	*/
	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);

	if (ztest_random(2) == 0) {
	/*
	* Inject errors on a normal data device or slog device.
	*/
	top = ztest_random_vdev_top(spa, B_TRUE);
	leaf = ztest_random(leaves) + zs->zs_splits;

	/*
	* Generate paths to the first leaf in this top-level vdev,
	* and to the random leaf we selected. We'll induce transient
	* write failures and random online/offline activity on leaf 0,
	* and we'll write random garbage to the randomly chosen leaf.
	*/
	(void) snprintf(path0, MAXPATHLEN, ztest_dev_template,
	ztest_opts.zo_dir, ztest_opts.zo_pool,
	top * leaves + zs->zs_splits);
	(void) snprintf(pathrand, MAXPATHLEN, ztest_dev_template,
	ztest_opts.zo_dir, ztest_opts.zo_pool,
	top * leaves + leaf);

	vd0 = vdev_lookup_by_path(spa->spa_root_vdev, path0);
	if (vd0 != NULL && vd0->vdev_top->vdev_islog)
	islog = B_TRUE;

	/*
	* If the top-level vdev needs to be resilvered
	* then we only allow faults on the device that is
	* resilvering.
	*/
	if (vd0 != NULL && maxfaults != 1 &&
	(!vdev_resilver_needed(vd0->vdev_top, NULL, NULL) \|\|
	vd0->vdev_resilver_txg != 0)) {
	/*
	* Make vd0 explicitly claim to be unreadable,
	* or unwritable, or reach behind its back
	* and close the underlying fd. We can do this if
	* maxfaults == 0 because we'll fail and reexecute,
	* and we can do it if maxfaults >= 2 because we'll
	* have enough redundancy. If maxfaults == 1, the
	* combination of this with injection of random data
	* corruption below exceeds the pool's fault tolerance.
	*/
	vdev_file_t *vf = vd0->vdev_tsd;

	zfs_dbgmsg("injecting fault to vdev %llu; maxfaults=%d",
	(long long)vd0->vdev_id, (int)maxfaults);

	if (vf != NULL && ztest_random(3) == 0) {
	(void) close(vf->vf_file->f_fd);
	vf->vf_file->f_fd = -1;
	} else if (ztest_random(2) == 0) {
	vd0->vdev_cant_read = B_TRUE;
	} else {
	vd0->vdev_cant_write = B_TRUE;
	}
	guid0 = vd0->vdev_guid;
	}
	} else {
	/*
	* Inject errors on an l2cache device.
	*/
	spa_aux_vdev_t *sav = &spa->spa_l2cache;

	if (sav->sav_count == 0) {
	spa_config_exit(spa, SCL_STATE, FTAG);
	(void) pthread_rwlock_unlock(&ztest_name_lock);
	goto out;
	}
	vd0 = sav->sav_vdevs[ztest_random(sav->sav_count)];
	guid0 = vd0->vdev_guid;
	(void) strlcpy(path0, vd0->vdev_path, MAXPATHLEN);
	(void) strlcpy(pathrand, vd0->vdev_path, MAXPATHLEN);

	leaf = 0;
	leaves = 1;
	maxfaults = INT_MAX; /* no limit on cache devices */
	}

	spa_config_exit(spa, SCL_STATE, FTAG);
	(void) pthread_rwlock_unlock(&ztest_name_lock);

	/*
	* If we can tolerate two or more faults, or we're dealing
	* with a slog, randomly online/offline vd0.
	*/
	if ((maxfaults >= 2 \|\| islog) && guid0 != 0) {
	if (ztest_random(10) < 6) {
	int flags = (ztest_random(2) == 0 ?
	ZFS_OFFLINE_TEMPORARY : 0);

	/*
	* We have to grab the zs_name_lock as writer to
	* prevent a race between offlining a slog and
	* destroying a dataset. Offlining the slog will
	* grab a reference on the dataset which may cause
	* dsl_destroy_head() to fail with EBUSY thus
	* leaving the dataset in an inconsistent state.
	*/
	if (islog)
	(void) pthread_rwlock_wrlock(&ztest_name_lock);

	VERIFY3U(vdev_offline(spa, guid0, flags), !=, EBUSY);

	if (islog)
	(void) pthread_rwlock_unlock(&ztest_name_lock);
	} else {
	/*
	* Ideally we would like to be able to randomly
	* call vdev_[on\|off]line without holding locks
	* to force unpredictable failures but the side
	* effects of vdev_[on\|off]line prevent us from
	* doing so. We grab the ztest_vdev_lock here to
	* prevent a race between injection testing and
	* aux_vdev removal.
	*/
	mutex_enter(&ztest_vdev_lock);
	(void) vdev_online(spa, guid0, 0, NULL);
	mutex_exit(&ztest_vdev_lock);
	}
	}

	if (maxfaults == 0)
	goto out;

	/*
	* We have at least single-fault tolerance, so inject data corruption.
	*/
	fd = open(pathrand, O_RDWR);

	if (fd == -1) /* we hit a gap in the device namespace */
	goto out;

	fsize = lseek(fd, 0, SEEK_END);

	while (--iters != 0) {
	/*
	* The offset must be chosen carefully to ensure that
	* we do not inject a given logical block with errors
	* on two different leaf devices, because ZFS can not
	* tolerate that (if maxfaults==1).
	*
	* To achieve this we divide each leaf device into
	* chunks of size (# leaves * SPA_MAXBLOCKSIZE * 4).
	* Each chunk is further divided into error-injection
	* ranges (can accept errors) and clear ranges (we do
	* not inject errors in those). Each error-injection
	* range can accept errors only for a single leaf vdev.
	* Error-injection ranges are separated by clear ranges.
	*
	* For example, with 3 leaves, each chunk looks like:
	* 0 to 32M: injection range for leaf 0
	* 32M to 64M: clear range - no injection allowed
	* 64M to 96M: injection range for leaf 1
	* 96M to 128M: clear range - no injection allowed
	* 128M to 160M: injection range for leaf 2
	* 160M to 192M: clear range - no injection allowed
	*
	* Each clear range must be large enough such that a
	* single block cannot straddle it. This way a block
	* can't be a target in two different injection ranges
	* (on different leaf vdevs).
	*/
	offset = ztest_random(fsize / (leaves << bshift)) *
	(leaves << bshift) + (leaf << bshift) +
	(ztest_random(1ULL << (bshift - 1)) & -8ULL);

	/*
	* Only allow damage to the labels at one end of the vdev.
	*
	* If all labels are damaged, the device will be totally
	* inaccessible, which will result in loss of data,
	* because we also damage (parts of) the other side of
	* the mirror/raidz.
	*
	* Additionally, we will always have both an even and an
	* odd label, so that we can handle crashes in the
	* middle of vdev_config_sync().
	*/
	if ((leaf & 1) == 0 && offset < VDEV_LABEL_START_SIZE)
	continue;

	/*
	* The two end labels are stored at the "end" of the disk, but
	* the end of the disk (vdev_psize) is aligned to
	* sizeof (vdev_label_t).
	*/
	uint64_t psize = P2ALIGN(fsize, sizeof (vdev_label_t));
	if ((leaf & 1) == 1 &&
	offset + sizeof (bad) > psize - VDEV_LABEL_END_SIZE)
	continue;

	mutex_enter(&ztest_vdev_lock);
	if (mirror_save != zs->zs_mirrors) {
	mutex_exit(&ztest_vdev_lock);
	(void) close(fd);
	goto out;
	}

	if (pwrite(fd, &bad, sizeof (bad), offset) != sizeof (bad))
	fatal(B_TRUE,
	"can't inject bad word at 0x%"PRIx64" in %s",
	offset, pathrand);

	mutex_exit(&ztest_vdev_lock);

	if (ztest_opts.zo_verbose >= 7)
	(void) printf("injected bad word into %s,"
	" offset 0x%"PRIx64"\n", pathrand, offset);
	}

	(void) close(fd);
	out:
	umem_free(path0, MAXPATHLEN);
	umem_free(pathrand, MAXPATHLEN);
	}

	/*
	* By design ztest will never inject uncorrectable damage in to the pool.
	* Issue a scrub, wait for it to complete, and verify there is never any
	* persistent damage.
	*
	* Only after a full scrub has been completed is it safe to start injecting
	* data corruption. See the comment in zfs_fault_inject().
	*/
	static int
	ztest_scrub_impl(spa_t *spa)
	{
	int error = spa_scan(spa, POOL_SCAN_SCRUB);
	if (error)
	return (error);

	while (dsl_scan_scrubbing(spa_get_dsl(spa)))
	txg_wait_synced(spa_get_dsl(spa), 0);

	if (spa_approx_errlog_size(spa) > 0)
	return (ECKSUM);

	ztest_pool_scrubbed = B_TRUE;

	return (0);
	}

	/*
	* Scrub the pool.
	*/
	void
	ztest_scrub(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	spa_t *spa = ztest_spa;
	int error;

	/*
	* Scrub in progress by device removal.
	*/
	if (ztest_device_removal_active)
	return;

	/*
	* Start a scrub, wait a moment, then force a restart.
	*/
	(void) spa_scan(spa, POOL_SCAN_SCRUB);
	(void) poll(NULL, 0, 100);

	error = ztest_scrub_impl(spa);
	if (error == EBUSY)
	error = 0;
	ASSERT0(error);
	}

	/*
	* Change the guid for the pool.
	*/
	void
	ztest_reguid(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	spa_t *spa = ztest_spa;
	uint64_t orig, load;
	int error;
	ztest_shared_t *zs = ztest_shared;

	if (ztest_opts.zo_mmp_test)
	return;

	orig = spa_guid(spa);
	load = spa_load_guid(spa);

	(void) pthread_rwlock_wrlock(&ztest_name_lock);
	error = spa_change_guid(spa);
	zs->zs_guid = spa_guid(spa);
	(void) pthread_rwlock_unlock(&ztest_name_lock);

	if (error != 0)
	return;

	if (ztest_opts.zo_verbose >= 4) {
	(void) printf("Changed guid old %"PRIu64" -> %"PRIu64"\n",
	orig, spa_guid(spa));
	}

	VERIFY3U(orig, !=, spa_guid(spa));
	VERIFY3U(load, ==, spa_load_guid(spa));
	}

	void
	ztest_blake3(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	hrtime_t end = gethrtime() + NANOSEC;
	zio_cksum_salt_t salt;
	void *salt_ptr = &salt.zcs_bytes;
	struct abd abd_data, abd_meta;
	void buf, templ;
	int i, *ptr;
	uint32_t size;
	BLAKE3_CTX ctx;
	const zfs_impl_t *blake3 = zfs_impl_get_ops("blake3");

	size = ztest_random_blocksize();
	buf = umem_alloc(size, UMEM_NOFAIL);
	abd_data = abd_alloc(size, B_FALSE);
	abd_meta = abd_alloc(size, B_TRUE);

	for (i = 0, ptr = buf; i < size / sizeof (*ptr); i++, ptr++)
	*ptr = ztest_random(UINT_MAX);
	memset(salt_ptr, 'A', 32);

	abd_copy_from_buf_off(abd_data, buf, 0, size);
	abd_copy_from_buf_off(abd_meta, buf, 0, size);

	while (gethrtime() <= end) {
	int run_count = 100;
	zio_cksum_t zc_ref1, zc_ref2;
	zio_cksum_t zc_res1, zc_res2;

	void *ref1 = &zc_ref1;
	void *ref2 = &zc_ref2;
	void *res1 = &zc_res1;
	void *res2 = &zc_res2;

	/* BLAKE3_KEY_LEN = 32 */
	VERIFY0(blake3->setname("generic"));
	templ = abd_checksum_blake3_tmpl_init(&salt);
	Blake3_InitKeyed(&ctx, salt_ptr);
	Blake3_Update(&ctx, buf, size);
	Blake3_Final(&ctx, ref1);
	zc_ref2 = zc_ref1;
	ZIO_CHECKSUM_BSWAP(&zc_ref2);
	abd_checksum_blake3_tmpl_free(templ);

	VERIFY0(blake3->setname("cycle"));
	while (run_count-- > 0) {

	/* Test current implementation */
	Blake3_InitKeyed(&ctx, salt_ptr);
	Blake3_Update(&ctx, buf, size);
	Blake3_Final(&ctx, res1);
	zc_res2 = zc_res1;
	ZIO_CHECKSUM_BSWAP(&zc_res2);

	VERIFY0(memcmp(ref1, res1, 32));
	VERIFY0(memcmp(ref2, res2, 32));

	/* Test ABD - data */
	templ = abd_checksum_blake3_tmpl_init(&salt);
	abd_checksum_blake3_native(abd_data, size,
	templ, &zc_res1);
	abd_checksum_blake3_byteswap(abd_data, size,
	templ, &zc_res2);

	VERIFY0(memcmp(ref1, res1, 32));
	VERIFY0(memcmp(ref2, res2, 32));

	/* Test ABD - metadata */
	abd_checksum_blake3_native(abd_meta, size,
	templ, &zc_res1);
	abd_checksum_blake3_byteswap(abd_meta, size,
	templ, &zc_res2);
	abd_checksum_blake3_tmpl_free(templ);

	VERIFY0(memcmp(ref1, res1, 32));
	VERIFY0(memcmp(ref2, res2, 32));

	}
	}

	abd_free(abd_data);
	abd_free(abd_meta);
	umem_free(buf, size);
	}

	void
	ztest_fletcher(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	hrtime_t end = gethrtime() + NANOSEC;

	while (gethrtime() <= end) {
	int run_count = 100;
	void *buf;
	struct abd abd_data, abd_meta;
	uint32_t size;
	int *ptr;
	int i;
	zio_cksum_t zc_ref;
	zio_cksum_t zc_ref_byteswap;

	size = ztest_random_blocksize();

	buf = umem_alloc(size, UMEM_NOFAIL);
	abd_data = abd_alloc(size, B_FALSE);
	abd_meta = abd_alloc(size, B_TRUE);

	for (i = 0, ptr = buf; i < size / sizeof (*ptr); i++, ptr++)
	*ptr = ztest_random(UINT_MAX);

	abd_copy_from_buf_off(abd_data, buf, 0, size);
	abd_copy_from_buf_off(abd_meta, buf, 0, size);

	VERIFY0(fletcher_4_impl_set("scalar"));
	fletcher_4_native(buf, size, NULL, &zc_ref);
	fletcher_4_byteswap(buf, size, NULL, &zc_ref_byteswap);

	VERIFY0(fletcher_4_impl_set("cycle"));
	while (run_count-- > 0) {
	zio_cksum_t zc;
	zio_cksum_t zc_byteswap;

	fletcher_4_byteswap(buf, size, NULL, &zc_byteswap);
	fletcher_4_native(buf, size, NULL, &zc);

	VERIFY0(memcmp(&zc, &zc_ref, sizeof (zc)));
	VERIFY0(memcmp(&zc_byteswap, &zc_ref_byteswap,
	sizeof (zc_byteswap)));

	/* Test ABD - data */
	abd_fletcher_4_byteswap(abd_data, size, NULL,
	&zc_byteswap);
	abd_fletcher_4_native(abd_data, size, NULL, &zc);

	VERIFY0(memcmp(&zc, &zc_ref, sizeof (zc)));
	VERIFY0(memcmp(&zc_byteswap, &zc_ref_byteswap,
	sizeof (zc_byteswap)));

	/* Test ABD - metadata */
	abd_fletcher_4_byteswap(abd_meta, size, NULL,
	&zc_byteswap);
	abd_fletcher_4_native(abd_meta, size, NULL, &zc);

	VERIFY0(memcmp(&zc, &zc_ref, sizeof (zc)));
	VERIFY0(memcmp(&zc_byteswap, &zc_ref_byteswap,
	sizeof (zc_byteswap)));

	}

	umem_free(buf, size);
	abd_free(abd_data);
	abd_free(abd_meta);
	}
	}

	void
	ztest_fletcher_incr(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	void *buf;
	size_t size;
	int *ptr;
	int i;
	zio_cksum_t zc_ref;
	zio_cksum_t zc_ref_bswap;

	hrtime_t end = gethrtime() + NANOSEC;

	while (gethrtime() <= end) {
	int run_count = 100;

	size = ztest_random_blocksize();
	buf = umem_alloc(size, UMEM_NOFAIL);

	for (i = 0, ptr = buf; i < size / sizeof (*ptr); i++, ptr++)
	*ptr = ztest_random(UINT_MAX);

	VERIFY0(fletcher_4_impl_set("scalar"));
	fletcher_4_native(buf, size, NULL, &zc_ref);
	fletcher_4_byteswap(buf, size, NULL, &zc_ref_bswap);

	VERIFY0(fletcher_4_impl_set("cycle"));

	while (run_count-- > 0) {
	zio_cksum_t zc;
	zio_cksum_t zc_bswap;
	size_t pos = 0;

	ZIO_SET_CHECKSUM(&zc, 0, 0, 0, 0);
	ZIO_SET_CHECKSUM(&zc_bswap, 0, 0, 0, 0);

	while (pos < size) {
	size_t inc = 64 * ztest_random(size / 67);
	/* sometimes add few bytes to test non-simd */
	if (ztest_random(100) < 10)
	inc += P2ALIGN(ztest_random(64),
	sizeof (uint32_t));

	if (inc > (size - pos))
	inc = size - pos;

	fletcher_4_incremental_native(buf + pos, inc,
	&zc);
	fletcher_4_incremental_byteswap(buf + pos, inc,
	&zc_bswap);

	pos += inc;
	}

	VERIFY3U(pos, ==, size);

	VERIFY(ZIO_CHECKSUM_EQUAL(zc, zc_ref));
	VERIFY(ZIO_CHECKSUM_EQUAL(zc_bswap, zc_ref_bswap));

	/*
	* verify if incremental on the whole buffer is
	* equivalent to non-incremental version
	*/
	ZIO_SET_CHECKSUM(&zc, 0, 0, 0, 0);
	ZIO_SET_CHECKSUM(&zc_bswap, 0, 0, 0, 0);

	fletcher_4_incremental_native(buf, size, &zc);
	fletcher_4_incremental_byteswap(buf, size, &zc_bswap);

	VERIFY(ZIO_CHECKSUM_EQUAL(zc, zc_ref));
	VERIFY(ZIO_CHECKSUM_EQUAL(zc_bswap, zc_ref_bswap));
	}

	umem_free(buf, size);
	}
	}

	static int
	ztest_set_global_vars(void)
	{
	for (size_t i = 0; i < ztest_opts.zo_gvars_count; i++) {
	char *kv = ztest_opts.zo_gvars[i];
	VERIFY3U(strlen(kv), <=, ZO_GVARS_MAX_ARGLEN);
	VERIFY3U(strlen(kv), >, 0);
	int err = set_global_var(kv);
	if (ztest_opts.zo_verbose > 0) {
	(void) printf("setting global var %s ... %s\n", kv,
	err ? "failed" : "ok");
	}
	if (err != 0) {
	(void) fprintf(stderr,
	"failed to set global var '%s'\n", kv);
	return (err);
	}
	}
	return (0);
	}

	static char **
	ztest_global_vars_to_zdb_args(void)
	{
	char *args = calloc(2ztest_opts.zo_gvars_count + 1, sizeof (char *));
	char **cur = args;
	if (args == NULL)
	return (NULL);
	for (size_t i = 0; i < ztest_opts.zo_gvars_count; i++) {
	cur++ = (char )"-o";
	*cur++ = ztest_opts.zo_gvars[i];
	}
	ASSERT3P(cur, ==, &args[2*ztest_opts.zo_gvars_count]);
	*cur = NULL;
	return (args);
	}

	/* The end of strings is indicated by a NULL element */
	static char *
	join_strings(char *strings, const char sep)
	{
	size_t totallen = 0;
	for (char *sp = strings; sp != NULL; sp++) {
	totallen += strlen(*sp);
	totallen += strlen(sep);
	}
	if (totallen > 0) {
	ASSERT(totallen >= strlen(sep));
	totallen -= strlen(sep);
	}

	size_t buflen = totallen + 1;
	char o = umem_alloc(buflen, UMEM_NOFAIL); / trailing 0 byte */
	o[0] = '\0';
	for (char *sp = strings; sp != NULL; sp++) {
	size_t would;
	would = strlcat(o, *sp, buflen);
	VERIFY3U(would, <, buflen);
	if (*(sp+1) == NULL) {
	break;
	}
	would = strlcat(o, sep, buflen);
	VERIFY3U(would, <, buflen);
	}
	ASSERT3S(strlen(o), ==, totallen);
	return (o);
	}

	static int
	ztest_check_path(char *path)
	{
	struct stat s;
	/* return true on success */
	return (!stat(path, &s));
	}

	static void
	ztest_get_zdb_bin(char *bin, int len)
	{
	char *zdb_path;
	/*
	* Try to use $ZDB and in-tree zdb path. If not successful, just
	* let popen to search through PATH.
	*/
	if ((zdb_path = getenv("ZDB"))) {
	strlcpy(bin, zdb_path, len); /* In env */
	if (!ztest_check_path(bin)) {
	ztest_dump_core = 0;
	fatal(B_TRUE, "invalid ZDB '%s'", bin);
	}
	return;
	}

	VERIFY3P(realpath(getexecname(), bin), !=, NULL);
	if (strstr(bin, ".libs/ztest")) {
	strstr(bin, ".libs/ztest")[0] = '\0'; /* In-tree */
	strcat(bin, "zdb");
	if (ztest_check_path(bin))
	return;
	}
	strcpy(bin, "zdb");
	}

	static vdev_t *
	ztest_random_concrete_vdev_leaf(vdev_t *vd)
	{
	if (vd == NULL)
	return (NULL);

	if (vd->vdev_children == 0)
	return (vd);

	vdev_t *eligible[vd->vdev_children];
	int eligible_idx = 0, i;
	for (i = 0; i < vd->vdev_children; i++) {
	vdev_t *cvd = vd->vdev_child[i];
	if (cvd->vdev_top->vdev_removing)
	continue;
	if (cvd->vdev_children > 0 \|\|
	(vdev_is_concrete(cvd) && !cvd->vdev_detached)) {
	eligible[eligible_idx++] = cvd;
	}
	}
	VERIFY3S(eligible_idx, >, 0);

	uint64_t child_no = ztest_random(eligible_idx);
	return (ztest_random_concrete_vdev_leaf(eligible[child_no]));
	}

	void
	ztest_initialize(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	spa_t *spa = ztest_spa;
	int error = 0;

	mutex_enter(&ztest_vdev_lock);

	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);

	/* Random leaf vdev */
	vdev_t *rand_vd = ztest_random_concrete_vdev_leaf(spa->spa_root_vdev);
	if (rand_vd == NULL) {
	spa_config_exit(spa, SCL_VDEV, FTAG);
	mutex_exit(&ztest_vdev_lock);
	return;
	}

	/*
	* The random vdev we've selected may change as soon as we
	* drop the spa_config_lock. We create local copies of things
	* we're interested in.
	*/
	uint64_t guid = rand_vd->vdev_guid;
	char *path = strdup(rand_vd->vdev_path);
	boolean_t active = rand_vd->vdev_initialize_thread != NULL;

	zfs_dbgmsg("vd %px, guid %llu", rand_vd, (u_longlong_t)guid);
	spa_config_exit(spa, SCL_VDEV, FTAG);

	uint64_t cmd = ztest_random(POOL_INITIALIZE_FUNCS);

	nvlist_t *vdev_guids = fnvlist_alloc();
	nvlist_t *vdev_errlist = fnvlist_alloc();
	fnvlist_add_uint64(vdev_guids, path, guid);
	error = spa_vdev_initialize(spa, vdev_guids, cmd, vdev_errlist);
	fnvlist_free(vdev_guids);
	fnvlist_free(vdev_errlist);

	switch (cmd) {
	case POOL_INITIALIZE_CANCEL:
	if (ztest_opts.zo_verbose >= 4) {
	(void) printf("Cancel initialize %s", path);
	if (!active)
	(void) printf(" failed (no initialize active)");
	(void) printf("\n");
	}
	break;
	case POOL_INITIALIZE_START:
	if (ztest_opts.zo_verbose >= 4) {
	(void) printf("Start initialize %s", path);
	if (active && error == 0)
	(void) printf(" failed (already active)");
	else if (error != 0)
	(void) printf(" failed (error %d)", error);
	(void) printf("\n");
	}
	break;
	case POOL_INITIALIZE_SUSPEND:
	if (ztest_opts.zo_verbose >= 4) {
	(void) printf("Suspend initialize %s", path);
	if (!active)
	(void) printf(" failed (no initialize active)");
	(void) printf("\n");
	}
	break;
	}
	free(path);
	mutex_exit(&ztest_vdev_lock);
	}

	void
	ztest_trim(ztest_ds_t *zd, uint64_t id)
	{
	(void) zd, (void) id;
	spa_t *spa = ztest_spa;
	int error = 0;

	mutex_enter(&ztest_vdev_lock);

	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);

	/* Random leaf vdev */
	vdev_t *rand_vd = ztest_random_concrete_vdev_leaf(spa->spa_root_vdev);
	if (rand_vd == NULL) {
	spa_config_exit(spa, SCL_VDEV, FTAG);
	mutex_exit(&ztest_vdev_lock);
	return;
	}

	/*
	* The random vdev we've selected may change as soon as we
	* drop the spa_config_lock. We create local copies of things
	* we're interested in.
	*/
	uint64_t guid = rand_vd->vdev_guid;
	char *path = strdup(rand_vd->vdev_path);
	boolean_t active = rand_vd->vdev_trim_thread != NULL;

	zfs_dbgmsg("vd %p, guid %llu", rand_vd, (u_longlong_t)guid);
	spa_config_exit(spa, SCL_VDEV, FTAG);

	uint64_t cmd = ztest_random(POOL_TRIM_FUNCS);
	uint64_t rate = 1 << ztest_random(30);
	boolean_t partial = (ztest_random(5) > 0);
	boolean_t secure = (ztest_random(5) > 0);

	nvlist_t *vdev_guids = fnvlist_alloc();
	nvlist_t *vdev_errlist = fnvlist_alloc();
	fnvlist_add_uint64(vdev_guids, path, guid);
	error = spa_vdev_trim(spa, vdev_guids, cmd, rate, partial,
	secure, vdev_errlist);
	fnvlist_free(vdev_guids);
	fnvlist_free(vdev_errlist);

	switch (cmd) {
	case POOL_TRIM_CANCEL:
	if (ztest_opts.zo_verbose >= 4) {
	(void) printf("Cancel TRIM %s", path);
	if (!active)
	(void) printf(" failed (no TRIM active)");
	(void) printf("\n");
	}
	break;
	case POOL_TRIM_START:
	if (ztest_opts.zo_verbose >= 4) {
	(void) printf("Start TRIM %s", path);
	if (active && error == 0)
	(void) printf(" failed (already active)");
	else if (error != 0)
	(void) printf(" failed (error %d)", error);
	(void) printf("\n");
	}
	break;
	case POOL_TRIM_SUSPEND:
	if (ztest_opts.zo_verbose >= 4) {
	(void) printf("Suspend TRIM %s", path);
	if (!active)
	(void) printf(" failed (no TRIM active)");
	(void) printf("\n");
	}
	break;
	}
	free(path);
	mutex_exit(&ztest_vdev_lock);
	}

	/*
	* Verify pool integrity by running zdb.
	*/
	static void
	ztest_run_zdb(uint64_t guid)
	{
	int status;
	char *bin;
	char *zdb;
	char *zbuf;
	const int len = MAXPATHLEN + MAXNAMELEN + 20;
	FILE *fp;

	bin = umem_alloc(len, UMEM_NOFAIL);
	zdb = umem_alloc(len, UMEM_NOFAIL);
	zbuf = umem_alloc(1024, UMEM_NOFAIL);

	ztest_get_zdb_bin(bin, len);

	char **set_gvars_args = ztest_global_vars_to_zdb_args();
	if (set_gvars_args == NULL) {
	fatal(B_FALSE, "Failed to allocate memory in "
	"ztest_global_vars_to_zdb_args(). Cannot run zdb.\n");
	}
	char *set_gvars_args_joined = join_strings(set_gvars_args, " ");
	free(set_gvars_args);

	size_t would = snprintf(zdb, len,
	"%s -bcc%s%s -G -d -Y -e -y %s -p %s %"PRIu64,
	bin,
	ztest_opts.zo_verbose >= 3 ? "s" : "",
	ztest_opts.zo_verbose >= 4 ? "v" : "",
	set_gvars_args_joined,
	ztest_opts.zo_dir,
	guid);
	ASSERT3U(would, <, len);

	umem_free(set_gvars_args_joined, strlen(set_gvars_args_joined) + 1);

	if (ztest_opts.zo_verbose >= 5)
	(void) printf("Executing %s\n", zdb);

	fp = popen(zdb, "r");

	while (fgets(zbuf, 1024, fp) != NULL)
	if (ztest_opts.zo_verbose >= 3)
	(void) printf("%s", zbuf);

	status = pclose(fp);

	if (status == 0)
	goto out;

	ztest_dump_core = 0;
	if (WIFEXITED(status))
	fatal(B_FALSE, "'%s' exit code %d", zdb, WEXITSTATUS(status));
	else
	fatal(B_FALSE, "'%s' died with signal %d",
	zdb, WTERMSIG(status));
	out:
	umem_free(bin, len);
	umem_free(zdb, len);
	umem_free(zbuf, 1024);
	}

	static void
	ztest_walk_pool_directory(const char *header)
	{
	spa_t *spa = NULL;

	if (ztest_opts.zo_verbose >= 6)
	(void) puts(header);

	mutex_enter(&spa_namespace_lock);
	while ((spa = spa_next(spa)) != NULL)
	if (ztest_opts.zo_verbose >= 6)
	(void) printf("\t%s\n", spa_name(spa));
	mutex_exit(&spa_namespace_lock);
	}

	static void
	ztest_spa_import_export(char oldname, char newname)
	{
	nvlist_t config, newconfig;
	uint64_t pool_guid;
	spa_t *spa;
	int error;

	if (ztest_opts.zo_verbose >= 4) {
	(void) printf("import/export: old = %s, new = %s\n",
	oldname, newname);
	}

	/*
	* Clean up from previous runs.
	*/
	(void) spa_destroy(newname);

	/*
	* Get the pool's configuration and guid.
	*/
	VERIFY0(spa_open(oldname, &spa, FTAG));

	/*
	* Kick off a scrub to tickle scrub/export races.
	*/
	if (ztest_random(2) == 0)
	(void) spa_scan(spa, POOL_SCAN_SCRUB);

	pool_guid = spa_guid(spa);
	spa_close(spa, FTAG);

	ztest_walk_pool_directory("pools before export");

	/*
	* Export it.
	*/
	VERIFY0(spa_export(oldname, &config, B_FALSE, B_FALSE));

	ztest_walk_pool_directory("pools after export");

	/*
	* Try to import it.
	*/
	newconfig = spa_tryimport(config);
	ASSERT3P(newconfig, !=, NULL);
	fnvlist_free(newconfig);

	/*
	* Import it under the new name.
	*/
	error = spa_import(newname, config, NULL, 0);
	if (error != 0) {
	dump_nvlist(config, 0);
	fatal(B_FALSE, "couldn't import pool %s as %s: error %u",
	oldname, newname, error);
	}

	ztest_walk_pool_directory("pools after import");

	/*
	* Try to import it again -- should fail with EEXIST.
	*/
	VERIFY3U(EEXIST, ==, spa_import(newname, config, NULL, 0));

	/*
	* Try to import it under a different name -- should fail with EEXIST.
	*/
	VERIFY3U(EEXIST, ==, spa_import(oldname, config, NULL, 0));

	/*
	* Verify that the pool is no longer visible under the old name.
	*/
	VERIFY3U(ENOENT, ==, spa_open(oldname, &spa, FTAG));

	/*
	* Verify that we can open and close the pool using the new name.
	*/
	VERIFY0(spa_open(newname, &spa, FTAG));
	ASSERT3U(pool_guid, ==, spa_guid(spa));
	spa_close(spa, FTAG);

	fnvlist_free(config);
	}

	static void
	ztest_resume(spa_t *spa)
	{
	if (spa_suspended(spa) && ztest_opts.zo_verbose >= 6)
	(void) printf("resuming from suspended state\n");
	spa_vdev_state_enter(spa, SCL_NONE);
	vdev_clear(spa, NULL);
	(void) spa_vdev_state_exit(spa, NULL, 0);
	(void) zio_resume(spa);
	}

	static __attribute__((noreturn)) void
	ztest_resume_thread(void *arg)
	{
	spa_t *spa = arg;

	while (!ztest_exiting) {
	if (spa_suspended(spa))
	ztest_resume(spa);
	(void) poll(NULL, 0, 100);

	/*
	* Periodically change the zfs_compressed_arc_enabled setting.
	*/
	if (ztest_random(10) == 0)
	zfs_compressed_arc_enabled = ztest_random(2);

	/*
	* Periodically change the zfs_abd_scatter_enabled setting.
	*/
	if (ztest_random(10) == 0)
	zfs_abd_scatter_enabled = ztest_random(2);
	}

	thread_exit();
	}

	static __attribute__((noreturn)) void
	ztest_deadman_thread(void *arg)
	{
	ztest_shared_t *zs = arg;
	spa_t *spa = ztest_spa;
	hrtime_t delay, overdue, last_run = gethrtime();

	delay = (zs->zs_thread_stop - zs->zs_thread_start) +
	MSEC2NSEC(zfs_deadman_synctime_ms);

	while (!ztest_exiting) {
	/*
	* Wait for the delay timer while checking occasionally
	* if we should stop.
	*/
	if (gethrtime() < last_run + delay) {
	(void) poll(NULL, 0, 1000);
	continue;
	}

	/*
	* If the pool is suspended then fail immediately. Otherwise,
	* check to see if the pool is making any progress. If
	* vdev_deadman() discovers that there hasn't been any recent
	* I/Os then it will end up aborting the tests.
	*/
	if (spa_suspended(spa) \|\| spa->spa_root_vdev == NULL) {
	fatal(B_FALSE,
	"aborting test after %llu seconds because "
	"pool has transitioned to a suspended state.",
	(u_longlong_t)zfs_deadman_synctime_ms / 1000);
	}
	vdev_deadman(spa->spa_root_vdev, FTAG);

	/*
	* If the process doesn't complete within a grace period of
	* zfs_deadman_synctime_ms over the expected finish time,
	* then it may be hung and is terminated.
	*/
	overdue = zs->zs_proc_stop + MSEC2NSEC(zfs_deadman_synctime_ms);
	if (gethrtime() > overdue) {
	fatal(B_FALSE,
	"aborting test after %llu seconds because "
	"the process is overdue for termination.",
	(gethrtime() - zs->zs_proc_start) / NANOSEC);
	}

	(void) printf("ztest has been running for %lld seconds\n",
	(gethrtime() - zs->zs_proc_start) / NANOSEC);

	last_run = gethrtime();
	delay = MSEC2NSEC(zfs_deadman_checktime_ms);
	}

	thread_exit();
	}

	static void
	ztest_execute(int test, ztest_info_t *zi, uint64_t id)
	{
	ztest_ds_t *zd = &ztest_ds[id % ztest_opts.zo_datasets];
	ztest_shared_callstate_t *zc = ZTEST_GET_SHARED_CALLSTATE(test);
	hrtime_t functime = gethrtime();
	int i;

	for (i = 0; i < zi->zi_iters; i++)
	zi->zi_func(zd, id);

	functime = gethrtime() - functime;

	atomic_add_64(&zc->zc_count, 1);
	atomic_add_64(&zc->zc_time, functime);

	if (ztest_opts.zo_verbose >= 4)
	(void) printf("%6.2f sec in %s\n",
	(double)functime / NANOSEC, zi->zi_funcname);
	}

	static __attribute__((noreturn)) void
	ztest_thread(void *arg)
	{
	int rand;
	uint64_t id = (uintptr_t)arg;
	ztest_shared_t *zs = ztest_shared;
	uint64_t call_next;
	hrtime_t now;
	ztest_info_t *zi;
	ztest_shared_callstate_t *zc;

	while ((now = gethrtime()) < zs->zs_thread_stop) {
	/*
	* See if it's time to force a crash.
	*/
	if (now > zs->zs_thread_kill)
	ztest_kill(zs);

	/*
	* If we're getting ENOSPC with some regularity, stop.
	*/
	if (zs->zs_enospc_count > 10)
	break;

	/*
	* Pick a random function to execute.
	*/
	rand = ztest_random(ZTEST_FUNCS);
	zi = &ztest_info[rand];
	zc = ZTEST_GET_SHARED_CALLSTATE(rand);
	call_next = zc->zc_next;

	if (now >= call_next &&
	atomic_cas_64(&zc->zc_next, call_next, call_next +
	ztest_random(2 * zi->zi_interval[0] + 1)) == call_next) {
	ztest_execute(rand, zi, id);
	}
	}

	thread_exit();
	}

	static void
	ztest_dataset_name(char dsname, const char pool, int d)
	{
	(void) snprintf(dsname, ZFS_MAX_DATASET_NAME_LEN, "%s/ds_%d", pool, d);
	}

	static void
	ztest_dataset_destroy(int d)
	{
	char name[ZFS_MAX_DATASET_NAME_LEN];
	int t;

	ztest_dataset_name(name, ztest_opts.zo_pool, d);

	if (ztest_opts.zo_verbose >= 3)
	(void) printf("Destroying %s to free up space\n", name);

	/*
	* Cleanup any non-standard clones and snapshots. In general,
	* ztest thread t operates on dataset (t % zopt_datasets),
	* so there may be more than one thing to clean up.
	*/
	for (t = d; t < ztest_opts.zo_threads;
	t += ztest_opts.zo_datasets)
	ztest_dsl_dataset_cleanup(name, t);

	(void) dmu_objset_find(name, ztest_objset_destroy_cb, NULL,
	DS_FIND_SNAPSHOTS \| DS_FIND_CHILDREN);
	}

	static void
	ztest_dataset_dirobj_verify(ztest_ds_t *zd)
	{
	uint64_t usedobjs, dirobjs, scratch;

	/*
	* ZTEST_DIROBJ is the object directory for the entire dataset.
	* Therefore, the number of objects in use should equal the
	* number of ZTEST_DIROBJ entries, +1 for ZTEST_DIROBJ itself.
	* If not, we have an object leak.
	*
	* Note that we can only check this in ztest_dataset_open(),
	* when the open-context and syncing-context values agree.
	* That's because zap_count() returns the open-context value,
	* while dmu_objset_space() returns the rootbp fill count.
	*/
	VERIFY0(zap_count(zd->zd_os, ZTEST_DIROBJ, &dirobjs));
	dmu_objset_space(zd->zd_os, &scratch, &scratch, &usedobjs, &scratch);
	ASSERT3U(dirobjs + 1, ==, usedobjs);
	}

	static int
	ztest_dataset_open(int d)
	{
	ztest_ds_t *zd = &ztest_ds[d];
	uint64_t committed_seq = ZTEST_GET_SHARED_DS(d)->zd_seq;
	objset_t *os;
	zilog_t *zilog;
	char name[ZFS_MAX_DATASET_NAME_LEN];
	int error;

	ztest_dataset_name(name, ztest_opts.zo_pool, d);

	(void) pthread_rwlock_rdlock(&ztest_name_lock);

	error = ztest_dataset_create(name);
	if (error == ENOSPC) {
	(void) pthread_rwlock_unlock(&ztest_name_lock);
	ztest_record_enospc(FTAG);
	return (error);
	}
	ASSERT(error == 0 \|\| error == EEXIST);

	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE,
	B_TRUE, zd, &os));
	(void) pthread_rwlock_unlock(&ztest_name_lock);

	ztest_zd_init(zd, ZTEST_GET_SHARED_DS(d), os);

	zilog = zd->zd_zilog;

	if (zilog->zl_header->zh_claim_lr_seq != 0 &&
	zilog->zl_header->zh_claim_lr_seq < committed_seq)
	fatal(B_FALSE, "missing log records: "
	"claimed %"PRIu64" < committed %"PRIu64"",
	zilog->zl_header->zh_claim_lr_seq, committed_seq);

	ztest_dataset_dirobj_verify(zd);

	zil_replay(os, zd, ztest_replay_vector);

	ztest_dataset_dirobj_verify(zd);

	if (ztest_opts.zo_verbose >= 6)
	(void) printf("%s replay %"PRIu64" blocks, "
	"%"PRIu64" records, seq %"PRIu64"\n",
	zd->zd_name,
	zilog->zl_parse_blk_count,
	zilog->zl_parse_lr_count,
	zilog->zl_replaying_seq);

	zilog = zil_open(os, ztest_get_data, NULL);

	if (zilog->zl_replaying_seq != 0 &&
	zilog->zl_replaying_seq < committed_seq)
	fatal(B_FALSE, "missing log records: "
	"replayed %"PRIu64" < committed %"PRIu64"",
	zilog->zl_replaying_seq, committed_seq);

	return (0);
	}

	static void
	ztest_dataset_close(int d)
	{
	ztest_ds_t *zd = &ztest_ds[d];

	zil_close(zd->zd_zilog);
	dmu_objset_disown(zd->zd_os, B_TRUE, zd);

	ztest_zd_fini(zd);
	}

	static int
	ztest_replay_zil_cb(const char name, void arg)
	{
	(void) arg;
	objset_t *os;
	ztest_ds_t *zdtmp;

	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_ANY, B_TRUE,
	B_TRUE, FTAG, &os));

	zdtmp = umem_alloc(sizeof (ztest_ds_t), UMEM_NOFAIL);

	ztest_zd_init(zdtmp, NULL, os);
	zil_replay(os, zdtmp, ztest_replay_vector);
	ztest_zd_fini(zdtmp);

	if (dmu_objset_zil(os)->zl_parse_lr_count != 0 &&
	ztest_opts.zo_verbose >= 6) {
	zilog_t *zilog = dmu_objset_zil(os);

	(void) printf("%s replay %"PRIu64" blocks, "
	"%"PRIu64" records, seq %"PRIu64"\n",
	name,
	zilog->zl_parse_blk_count,
	zilog->zl_parse_lr_count,
	zilog->zl_replaying_seq);
	}

	umem_free(zdtmp, sizeof (ztest_ds_t));

	dmu_objset_disown(os, B_TRUE, FTAG);
	return (0);
	}

	static void
	ztest_freeze(void)
	{
	ztest_ds_t *zd = &ztest_ds[0];
	spa_t *spa;
	int numloops = 0;

	if (ztest_opts.zo_verbose >= 3)
	(void) printf("testing spa_freeze()...\n");

	kernel_init(SPA_MODE_READ \| SPA_MODE_WRITE);
	VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
	VERIFY0(ztest_dataset_open(0));
	ztest_spa = spa;

	/*
	* Force the first log block to be transactionally allocated.
	* We have to do this before we freeze the pool -- otherwise
	* the log chain won't be anchored.
	*/
	while (BP_IS_HOLE(&zd->zd_zilog->zl_header->zh_log)) {
	ztest_dmu_object_alloc_free(zd, 0);
	zil_commit(zd->zd_zilog, 0);
	}

	txg_wait_synced(spa_get_dsl(spa), 0);

	/*
	* Freeze the pool. This stops spa_sync() from doing anything,
	* so that the only way to record changes from now on is the ZIL.
	*/
	spa_freeze(spa);

	/*
	* Because it is hard to predict how much space a write will actually
	* require beforehand, we leave ourselves some fudge space to write over
	* capacity.
	*/
	uint64_t capacity = metaslab_class_get_space(spa_normal_class(spa)) / 2;

	/*
	* Run tests that generate log records but don't alter the pool config
	* or depend on DSL sync tasks (snapshots, objset create/destroy, etc).
	* We do a txg_wait_synced() after each iteration to force the txg
	* to increase well beyond the last synced value in the uberblock.
	* The ZIL should be OK with that.
	*
	* Run a random number of times less than zo_maxloops and ensure we do
	* not run out of space on the pool.
	*/
	while (ztest_random(10) != 0 &&
	numloops++ < ztest_opts.zo_maxloops &&
	metaslab_class_get_alloc(spa_normal_class(spa)) < capacity) {
	ztest_od_t od;
	ztest_od_init(&od, 0, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, 0);
	VERIFY0(ztest_object_init(zd, &od, sizeof (od), B_FALSE));
	ztest_io(zd, od.od_object,
	ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT);
	txg_wait_synced(spa_get_dsl(spa), 0);
	}

	/*
	* Commit all of the changes we just generated.
	*/
	zil_commit(zd->zd_zilog, 0);
	txg_wait_synced(spa_get_dsl(spa), 0);

	/*
	* Close our dataset and close the pool.
	*/
	ztest_dataset_close(0);
	spa_close(spa, FTAG);
	kernel_fini();

	/*
	* Open and close the pool and dataset to induce log replay.
	*/
	kernel_init(SPA_MODE_READ \| SPA_MODE_WRITE);
	VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
	ASSERT3U(spa_freeze_txg(spa), ==, UINT64_MAX);
	VERIFY0(ztest_dataset_open(0));
	ztest_spa = spa;
	txg_wait_synced(spa_get_dsl(spa), 0);
	ztest_dataset_close(0);
	ztest_reguid(NULL, 0);

	spa_close(spa, FTAG);
	kernel_fini();
	}

	static void
	ztest_import_impl(void)
	{
	importargs_t args = { 0 };
	nvlist_t *cfg = NULL;
	int nsearch = 1;
	char *searchdirs[nsearch];
	int flags = ZFS_IMPORT_MISSING_LOG;

	searchdirs[0] = ztest_opts.zo_dir;
	args.paths = nsearch;
	args.path = searchdirs;
	args.can_be_active = B_FALSE;

	libpc_handle_t lpch = {
	.lpc_lib_handle = NULL,
	.lpc_ops = &libzpool_config_ops,
	.lpc_printerr = B_TRUE
	};
	VERIFY0(zpool_find_config(&lpch, ztest_opts.zo_pool, &cfg, &args));
	VERIFY0(spa_import(ztest_opts.zo_pool, cfg, NULL, flags));
	fnvlist_free(cfg);
	}

	/*
	* Import a storage pool with the given name.
	*/
	static void
	ztest_import(ztest_shared_t *zs)
	{
	spa_t *spa;

	mutex_init(&ztest_vdev_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&ztest_checkpoint_lock, NULL, MUTEX_DEFAULT, NULL);
	VERIFY0(pthread_rwlock_init(&ztest_name_lock, NULL));

	kernel_init(SPA_MODE_READ \| SPA_MODE_WRITE);

	ztest_import_impl();

	VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
	zs->zs_metaslab_sz =
	1ULL << spa->spa_root_vdev->vdev_child[0]->vdev_ms_shift;
	zs->zs_guid = spa_guid(spa);
	spa_close(spa, FTAG);

	kernel_fini();

	if (!ztest_opts.zo_mmp_test) {
	ztest_run_zdb(zs->zs_guid);
	ztest_freeze();
	ztest_run_zdb(zs->zs_guid);
	}

	(void) pthread_rwlock_destroy(&ztest_name_lock);
	mutex_destroy(&ztest_vdev_lock);
	mutex_destroy(&ztest_checkpoint_lock);
	}

	/*
	* Kick off threads to run tests on all datasets in parallel.
	*/
	static void
	ztest_run(ztest_shared_t *zs)
	{
	spa_t *spa;
	objset_t *os;
	kthread_t resume_thread, deadman_thread;
	kthread_t **run_threads;
	uint64_t object;
	int error;
	int t, d;

	ztest_exiting = B_FALSE;

	/*
	* Initialize parent/child shared state.
	*/
	mutex_init(&ztest_vdev_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&ztest_checkpoint_lock, NULL, MUTEX_DEFAULT, NULL);
	VERIFY0(pthread_rwlock_init(&ztest_name_lock, NULL));

	zs->zs_thread_start = gethrtime();
	zs->zs_thread_stop =
	zs->zs_thread_start + ztest_opts.zo_passtime * NANOSEC;
	zs->zs_thread_stop = MIN(zs->zs_thread_stop, zs->zs_proc_stop);
	zs->zs_thread_kill = zs->zs_thread_stop;
	if (ztest_random(100) < ztest_opts.zo_killrate) {
	zs->zs_thread_kill -=
	ztest_random(ztest_opts.zo_passtime * NANOSEC);
	}

	mutex_init(&zcl.zcl_callbacks_lock, NULL, MUTEX_DEFAULT, NULL);

	list_create(&zcl.zcl_callbacks, sizeof (ztest_cb_data_t),
	offsetof(ztest_cb_data_t, zcd_node));

	/*
	* Open our pool. It may need to be imported first depending on
	* what tests were running when the previous pass was terminated.
	*/
	kernel_init(SPA_MODE_READ \| SPA_MODE_WRITE);
	error = spa_open(ztest_opts.zo_pool, &spa, FTAG);
	if (error) {
	VERIFY3S(error, ==, ENOENT);
	ztest_import_impl();
	VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
	zs->zs_metaslab_sz =
	1ULL << spa->spa_root_vdev->vdev_child[0]->vdev_ms_shift;
	}

	metaslab_preload_limit = ztest_random(20) + 1;
	ztest_spa = spa;

	VERIFY0(vdev_raidz_impl_set("cycle"));

	dmu_objset_stats_t dds;
	VERIFY0(ztest_dmu_objset_own(ztest_opts.zo_pool,
	DMU_OST_ANY, B_TRUE, B_TRUE, FTAG, &os));
	dsl_pool_config_enter(dmu_objset_pool(os), FTAG);
	dmu_objset_fast_stat(os, &dds);
	dsl_pool_config_exit(dmu_objset_pool(os), FTAG);
	dmu_objset_disown(os, B_TRUE, FTAG);

	/*
	* Create a thread to periodically resume suspended I/O.
	*/
	resume_thread = thread_create(NULL, 0, ztest_resume_thread,
	spa, 0, NULL, TS_RUN \| TS_JOINABLE, defclsyspri);

	/*
	* Create a deadman thread and set to panic if we hang.
	*/
	deadman_thread = thread_create(NULL, 0, ztest_deadman_thread,
	zs, 0, NULL, TS_RUN \| TS_JOINABLE, defclsyspri);

	spa->spa_deadman_failmode = ZIO_FAILURE_MODE_PANIC;

	/*
	* Verify that we can safely inquire about any object,
	* whether it's allocated or not. To make it interesting,
	* we probe a 5-wide window around each power of two.
	* This hits all edge cases, including zero and the max.
	*/
	for (t = 0; t < 64; t++) {
	for (d = -5; d <= 5; d++) {
	error = dmu_object_info(spa->spa_meta_objset,
	(1ULL << t) + d, NULL);
	ASSERT(error == 0 \|\| error == ENOENT \|\|
	error == EINVAL);
	}
	}

	/*
	* If we got any ENOSPC errors on the previous run, destroy something.
	*/
	if (zs->zs_enospc_count != 0) {
	int d = ztest_random(ztest_opts.zo_datasets);
	ztest_dataset_destroy(d);
	}
	zs->zs_enospc_count = 0;

	/*
	* If we were in the middle of ztest_device_removal() and were killed
	* we need to ensure the removal and scrub complete before running
	* any tests that check ztest_device_removal_active. The removal will
	* be restarted automatically when the spa is opened, but we need to
	* initiate the scrub manually if it is not already in progress. Note
	* that we always run the scrub whenever an indirect vdev exists
	* because we have no way of knowing for sure if ztest_device_removal()
	* fully completed its scrub before the pool was reimported.
	*/
	if (spa->spa_removing_phys.sr_state == DSS_SCANNING \|\|
	spa->spa_removing_phys.sr_prev_indirect_vdev != -1) {
	while (spa->spa_removing_phys.sr_state == DSS_SCANNING)
	txg_wait_synced(spa_get_dsl(spa), 0);

	error = ztest_scrub_impl(spa);
	if (error == EBUSY)
	error = 0;
	ASSERT0(error);
	}

	run_threads = umem_zalloc(ztest_opts.zo_threads * sizeof (kthread_t *),
	UMEM_NOFAIL);

	if (ztest_opts.zo_verbose >= 4)
	(void) printf("starting main threads...\n");

	/*
	* Replay all logs of all datasets in the pool. This is primarily for
	* temporary datasets which wouldn't otherwise get replayed, which
	* can trigger failures when attempting to offline a SLOG in
	* ztest_fault_inject().
	*/
	(void) dmu_objset_find(ztest_opts.zo_pool, ztest_replay_zil_cb,
	NULL, DS_FIND_CHILDREN);

	/*
	* Kick off all the tests that run in parallel.
	*/
	for (t = 0; t < ztest_opts.zo_threads; t++) {
	if (t < ztest_opts.zo_datasets && ztest_dataset_open(t) != 0) {
	umem_free(run_threads, ztest_opts.zo_threads *
	sizeof (kthread_t *));
	return;
	}

	run_threads[t] = thread_create(NULL, 0, ztest_thread,
	(void *)(uintptr_t)t, 0, NULL, TS_RUN \| TS_JOINABLE,
	defclsyspri);
	}

	/*
	* Wait for all of the tests to complete.
	*/
	for (t = 0; t < ztest_opts.zo_threads; t++)
	VERIFY0(thread_join(run_threads[t]));

	/*
	* Close all datasets. This must be done after all the threads
	* are joined so we can be sure none of the datasets are in-use
	* by any of the threads.
	*/
	for (t = 0; t < ztest_opts.zo_threads; t++) {
	if (t < ztest_opts.zo_datasets)
	ztest_dataset_close(t);
	}

	txg_wait_synced(spa_get_dsl(spa), 0);

	zs->zs_alloc = metaslab_class_get_alloc(spa_normal_class(spa));
	zs->zs_space = metaslab_class_get_space(spa_normal_class(spa));

	umem_free(run_threads, ztest_opts.zo_threads * sizeof (kthread_t *));

	/* Kill the resume and deadman threads */
	ztest_exiting = B_TRUE;
	VERIFY0(thread_join(resume_thread));
	VERIFY0(thread_join(deadman_thread));
	ztest_resume(spa);

	/*
	* Right before closing the pool, kick off a bunch of async I/O;
	* spa_close() should wait for it to complete.
	*/
	for (object = 1; object < 50; object++) {
	dmu_prefetch(spa->spa_meta_objset, object, 0, 0, 1ULL << 20,
	ZIO_PRIORITY_SYNC_READ);
	}

	/* Verify that at least one commit cb was called in a timely fashion */
	if (zc_cb_counter >= ZTEST_COMMIT_CB_MIN_REG)
	VERIFY0(zc_min_txg_delay);

	spa_close(spa, FTAG);

	/*
	* Verify that we can loop over all pools.
	*/
	mutex_enter(&spa_namespace_lock);
	for (spa = spa_next(NULL); spa != NULL; spa = spa_next(spa))
	if (ztest_opts.zo_verbose > 3)
	(void) printf("spa_next: found %s\n", spa_name(spa));
	mutex_exit(&spa_namespace_lock);

	/*
	* Verify that we can export the pool and reimport it under a
	* different name.
	*/
	if ((ztest_random(2) == 0) && !ztest_opts.zo_mmp_test) {
	char name[ZFS_MAX_DATASET_NAME_LEN];
	(void) snprintf(name, sizeof (name), "%s_import",
	ztest_opts.zo_pool);
	ztest_spa_import_export(ztest_opts.zo_pool, name);
	ztest_spa_import_export(name, ztest_opts.zo_pool);
	}

	kernel_fini();

	list_destroy(&zcl.zcl_callbacks);
	mutex_destroy(&zcl.zcl_callbacks_lock);
	(void) pthread_rwlock_destroy(&ztest_name_lock);
	mutex_destroy(&ztest_vdev_lock);
	mutex_destroy(&ztest_checkpoint_lock);
	}

	static void
	print_time(hrtime_t t, char *timebuf)
	{
	hrtime_t s = t / NANOSEC;
	hrtime_t m = s / 60;
	hrtime_t h = m / 60;
	hrtime_t d = h / 24;

	s -= m * 60;
	m -= h * 60;
	h -= d * 24;

	timebuf[0] = '\0';

	if (d)
	(void) sprintf(timebuf,
	"%llud%02lluh%02llum%02llus", d, h, m, s);
	else if (h)
	(void) sprintf(timebuf, "%lluh%02llum%02llus", h, m, s);
	else if (m)
	(void) sprintf(timebuf, "%llum%02llus", m, s);
	else
	(void) sprintf(timebuf, "%llus", s);
	}

	static nvlist_t *
	make_random_props(void)
	{
	nvlist_t *props;

	props = fnvlist_alloc();

	if (ztest_random(2) == 0)
	return (props);

	fnvlist_add_uint64(props,
	zpool_prop_to_name(ZPOOL_PROP_AUTOREPLACE), 1);

	return (props);
	}

	/*
	* Create a storage pool with the given name and initial vdev size.
	* Then test spa_freeze() functionality.
	*/
	static void
	ztest_init(ztest_shared_t *zs)
	{
	spa_t *spa;
	nvlist_t nvroot, props;
	int i;

	mutex_init(&ztest_vdev_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&ztest_checkpoint_lock, NULL, MUTEX_DEFAULT, NULL);
	VERIFY0(pthread_rwlock_init(&ztest_name_lock, NULL));

	kernel_init(SPA_MODE_READ \| SPA_MODE_WRITE);

	/*
	* Create the storage pool.
	*/
	(void) spa_destroy(ztest_opts.zo_pool);
	ztest_shared->zs_vdev_next_leaf = 0;
	zs->zs_splits = 0;
	zs->zs_mirrors = ztest_opts.zo_mirrors;
	nvroot = make_vdev_root(NULL, NULL, NULL, ztest_opts.zo_vdev_size, 0,
	NULL, ztest_opts.zo_raid_children, zs->zs_mirrors, 1);
	props = make_random_props();

	/*
	* We don't expect the pool to suspend unless maxfaults == 0,
	* in which case ztest_fault_inject() temporarily takes away
	* the only valid replica.
	*/
	fnvlist_add_uint64(props,
	zpool_prop_to_name(ZPOOL_PROP_FAILUREMODE),
	MAXFAULTS(zs) ? ZIO_FAILURE_MODE_PANIC : ZIO_FAILURE_MODE_WAIT);

	for (i = 0; i < SPA_FEATURES; i++) {
	char *buf;

	if (!spa_feature_table[i].fi_zfs_mod_supported)
	continue;

	/*
	* 75% chance of using the log space map feature. We want ztest
	* to exercise both the code paths that use the log space map
	* feature and the ones that don't.
	*/
	if (i == SPA_FEATURE_LOG_SPACEMAP && ztest_random(4) == 0)
	continue;

	VERIFY3S(-1, !=, asprintf(&buf, "feature@%s",
	spa_feature_table[i].fi_uname));
	fnvlist_add_uint64(props, buf, 0);
	free(buf);
	}

	VERIFY0(spa_create(ztest_opts.zo_pool, nvroot, props, NULL, NULL));
	fnvlist_free(nvroot);
	fnvlist_free(props);

	VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
	zs->zs_metaslab_sz =
	1ULL << spa->spa_root_vdev->vdev_child[0]->vdev_ms_shift;
	zs->zs_guid = spa_guid(spa);
	spa_close(spa, FTAG);

	kernel_fini();

	if (!ztest_opts.zo_mmp_test) {
	ztest_run_zdb(zs->zs_guid);
	ztest_freeze();
	ztest_run_zdb(zs->zs_guid);
	}

	(void) pthread_rwlock_destroy(&ztest_name_lock);
	mutex_destroy(&ztest_vdev_lock);
	mutex_destroy(&ztest_checkpoint_lock);
	}

	static void
	setup_data_fd(void)
	{
	static char ztest_name_data[] = "/tmp/ztest.data.XXXXXX";

	ztest_fd_data = mkstemp(ztest_name_data);
	ASSERT3S(ztest_fd_data, >=, 0);
	(void) unlink(ztest_name_data);
	}

	static int
	shared_data_size(ztest_shared_hdr_t *hdr)
	{
	int size;

	size = hdr->zh_hdr_size;
	size += hdr->zh_opts_size;
	size += hdr->zh_size;
	size += hdr->zh_stats_size * hdr->zh_stats_count;
	size += hdr->zh_ds_size * hdr->zh_ds_count;

	return (size);
	}

	static void
	setup_hdr(void)
	{
	int size;
	ztest_shared_hdr_t *hdr;

	hdr = (void )mmap(0, P2ROUNDUP(sizeof (hdr), getpagesize()),
	PROT_READ \| PROT_WRITE, MAP_SHARED, ztest_fd_data, 0);
	ASSERT3P(hdr, !=, MAP_FAILED);

	VERIFY0(ftruncate(ztest_fd_data, sizeof (ztest_shared_hdr_t)));

	hdr->zh_hdr_size = sizeof (ztest_shared_hdr_t);
	hdr->zh_opts_size = sizeof (ztest_shared_opts_t);
	hdr->zh_size = sizeof (ztest_shared_t);
	hdr->zh_stats_size = sizeof (ztest_shared_callstate_t);
	hdr->zh_stats_count = ZTEST_FUNCS;
	hdr->zh_ds_size = sizeof (ztest_shared_ds_t);
	hdr->zh_ds_count = ztest_opts.zo_datasets;

	size = shared_data_size(hdr);
	VERIFY0(ftruncate(ztest_fd_data, size));

	(void) munmap((caddr_t)hdr, P2ROUNDUP(sizeof (*hdr), getpagesize()));
	}

	static void
	setup_data(void)
	{
	int size, offset;
	ztest_shared_hdr_t *hdr;
	uint8_t *buf;

	hdr = (void )mmap(0, P2ROUNDUP(sizeof (hdr), getpagesize()),
	PROT_READ, MAP_SHARED, ztest_fd_data, 0);
	ASSERT3P(hdr, !=, MAP_FAILED);

	size = shared_data_size(hdr);

	(void) munmap((caddr_t)hdr, P2ROUNDUP(sizeof (*hdr), getpagesize()));
	hdr = ztest_shared_hdr = (void *)mmap(0, P2ROUNDUP(size, getpagesize()),
	PROT_READ \| PROT_WRITE, MAP_SHARED, ztest_fd_data, 0);
	ASSERT3P(hdr, !=, MAP_FAILED);
	buf = (uint8_t *)hdr;

	offset = hdr->zh_hdr_size;
	ztest_shared_opts = (void *)&buf[offset];
	offset += hdr->zh_opts_size;
	ztest_shared = (void *)&buf[offset];
	offset += hdr->zh_size;
	ztest_shared_callstate = (void *)&buf[offset];
	offset += hdr->zh_stats_size * hdr->zh_stats_count;
	ztest_shared_ds = (void *)&buf[offset];
	}

	static boolean_t
	exec_child(char cmd, char libpath, boolean_t ignorekill, int *statusp)
	{
	pid_t pid;
	int status;
	char *cmdbuf = NULL;

	pid = fork();

	if (cmd == NULL) {
	cmdbuf = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
	(void) strlcpy(cmdbuf, getexecname(), MAXPATHLEN);
	cmd = cmdbuf;
	}

	if (pid == -1)
	fatal(B_TRUE, "fork failed");

	if (pid == 0) { /* child */
	char fd_data_str[12];

	VERIFY3S(11, >=,
	snprintf(fd_data_str, 12, "%d", ztest_fd_data));
	VERIFY0(setenv("ZTEST_FD_DATA", fd_data_str, 1));

	if (libpath != NULL) {
	const char *curlp = getenv("LD_LIBRARY_PATH");
	if (curlp == NULL)
	VERIFY0(setenv("LD_LIBRARY_PATH", libpath, 1));
	else {
	char *newlp = NULL;
	VERIFY3S(-1, !=,
	asprintf(&newlp, "%s:%s", libpath, curlp));
	VERIFY0(setenv("LD_LIBRARY_PATH", newlp, 1));
	free(newlp);
	}
	}
	(void) execl(cmd, cmd, (char *)NULL);
	ztest_dump_core = B_FALSE;
	fatal(B_TRUE, "exec failed: %s", cmd);
	}

	if (cmdbuf != NULL) {
	umem_free(cmdbuf, MAXPATHLEN);
	cmd = NULL;
	}

	while (waitpid(pid, &status, 0) != pid)
	continue;
	if (statusp != NULL)
	*statusp = status;

	if (WIFEXITED(status)) {
	if (WEXITSTATUS(status) != 0) {
	(void) fprintf(stderr, "child exited with code %d\n",
	WEXITSTATUS(status));
	exit(2);
	}
	return (B_FALSE);
	} else if (WIFSIGNALED(status)) {
	if (!ignorekill \|\| WTERMSIG(status) != SIGKILL) {
	(void) fprintf(stderr, "child died with signal %d\n",
	WTERMSIG(status));
	exit(3);
	}
	return (B_TRUE);
	} else {
	(void) fprintf(stderr, "something strange happened to child\n");
	exit(4);
	}
	}

	static void
	ztest_run_init(void)
	{
	int i;

	ztest_shared_t *zs = ztest_shared;

	/*
	* Blow away any existing copy of zpool.cache
	*/
	(void) remove(spa_config_path);

	if (ztest_opts.zo_init == 0) {
	if (ztest_opts.zo_verbose >= 1)
	(void) printf("Importing pool %s\n",
	ztest_opts.zo_pool);
	ztest_import(zs);
	return;
	}

	/*
	* Create and initialize our storage pool.
	*/
	for (i = 1; i <= ztest_opts.zo_init; i++) {
	memset(zs, 0, sizeof (*zs));
	if (ztest_opts.zo_verbose >= 3 &&
	ztest_opts.zo_init != 1) {
	(void) printf("ztest_init(), pass %d\n", i);
	}
	ztest_init(zs);
	}
	}

	int
	main(int argc, char **argv)
	{
	int kills = 0;
	int iters = 0;
	int older = 0;
	int newer = 0;
	ztest_shared_t *zs;
	ztest_info_t *zi;
	ztest_shared_callstate_t *zc;
	char timebuf[100];
	char numbuf[NN_NUMBUF_SZ];
	char *cmd;
	boolean_t hasalt;
	int f, err;
	char *fd_data_str = getenv("ZTEST_FD_DATA");
	struct sigaction action;

	(void) setvbuf(stdout, NULL, _IOLBF, 0);

	dprintf_setup(&argc, argv);
	zfs_deadman_synctime_ms = 300000;
	zfs_deadman_checktime_ms = 30000;
	/*
	* As two-word space map entries may not come up often (especially
	* if pool and vdev sizes are small) we want to force at least some
	* of them so the feature get tested.
	*/
	zfs_force_some_double_word_sm_entries = B_TRUE;

	/*
	* Verify that even extensively damaged split blocks with many
	* segments can be reconstructed in a reasonable amount of time
	* when reconstruction is known to be possible.
	*
	* Note: the lower this value is, the more damage we inflict, and
	* the more time ztest spends in recovering that damage. We chose
	* to induce damage 1/100th of the time so recovery is tested but
	* not so frequently that ztest doesn't get to test other code paths.
	*/
	zfs_reconstruct_indirect_damage_fraction = 100;

	action.sa_handler = sig_handler;
	sigemptyset(&action.sa_mask);
	action.sa_flags = 0;

	if (sigaction(SIGSEGV, &action, NULL) < 0) {
	(void) fprintf(stderr, "ztest: cannot catch SIGSEGV: %s.\n",
	strerror(errno));
	exit(EXIT_FAILURE);
	}

	if (sigaction(SIGABRT, &action, NULL) < 0) {
	(void) fprintf(stderr, "ztest: cannot catch SIGABRT: %s.\n",
	strerror(errno));
	exit(EXIT_FAILURE);
	}

	/*
	* Force random_get_bytes() to use /dev/urandom in order to prevent
	* ztest from needlessly depleting the system entropy pool.
	*/
	random_path = "/dev/urandom";
	ztest_fd_rand = open(random_path, O_RDONLY \| O_CLOEXEC);
	ASSERT3S(ztest_fd_rand, >=, 0);

	if (!fd_data_str) {
	process_options(argc, argv);

	setup_data_fd();
	setup_hdr();
	setup_data();
	memcpy(ztest_shared_opts, &ztest_opts,
	sizeof (*ztest_shared_opts));
	} else {
	ztest_fd_data = atoi(fd_data_str);
	setup_data();
	memcpy(&ztest_opts, ztest_shared_opts, sizeof (ztest_opts));
	}
	ASSERT3U(ztest_opts.zo_datasets, ==, ztest_shared_hdr->zh_ds_count);

	err = ztest_set_global_vars();
	if (err != 0 && !fd_data_str) {
	/* error message done by ztest_set_global_vars */
	exit(EXIT_FAILURE);
	} else {
	/* children should not be spawned if setting gvars fails */
	VERIFY3S(err, ==, 0);
	}

	/* Override location of zpool.cache */
	VERIFY3S(asprintf((char **)&spa_config_path, "%s/zpool.cache",
	ztest_opts.zo_dir), !=, -1);

	ztest_ds = umem_alloc(ztest_opts.zo_datasets * sizeof (ztest_ds_t),
	UMEM_NOFAIL);
	zs = ztest_shared;

	if (fd_data_str) {
	metaslab_force_ganging = ztest_opts.zo_metaslab_force_ganging;
	metaslab_df_alloc_threshold =
	zs->zs_metaslab_df_alloc_threshold;

	if (zs->zs_do_init)
	ztest_run_init();
	else
	ztest_run(zs);
	exit(0);
	}

	hasalt = (strlen(ztest_opts.zo_alt_ztest) != 0);

	if (ztest_opts.zo_verbose >= 1) {
	(void) printf("%"PRIu64" vdevs, %d datasets, %d threads,"
	"%d %s disks, %"PRIu64" seconds...\n\n",
	ztest_opts.zo_vdevs,
	ztest_opts.zo_datasets,
	ztest_opts.zo_threads,
	ztest_opts.zo_raid_children,
	ztest_opts.zo_raid_type,
	ztest_opts.zo_time);
	}

	cmd = umem_alloc(MAXNAMELEN, UMEM_NOFAIL);
	(void) strlcpy(cmd, getexecname(), MAXNAMELEN);

	zs->zs_do_init = B_TRUE;
	if (strlen(ztest_opts.zo_alt_ztest) != 0) {
	if (ztest_opts.zo_verbose >= 1) {
	(void) printf("Executing older ztest for "
	"initialization: %s\n", ztest_opts.zo_alt_ztest);
	}
	VERIFY(!exec_child(ztest_opts.zo_alt_ztest,
	ztest_opts.zo_alt_libpath, B_FALSE, NULL));
	} else {
	VERIFY(!exec_child(NULL, NULL, B_FALSE, NULL));
	}
	zs->zs_do_init = B_FALSE;

	zs->zs_proc_start = gethrtime();
	zs->zs_proc_stop = zs->zs_proc_start + ztest_opts.zo_time * NANOSEC;

	for (f = 0; f < ZTEST_FUNCS; f++) {
	zi = &ztest_info[f];
	zc = ZTEST_GET_SHARED_CALLSTATE(f);
	if (zs->zs_proc_start + zi->zi_interval[0] > zs->zs_proc_stop)
	zc->zc_next = UINT64_MAX;
	else
	zc->zc_next = zs->zs_proc_start +
	ztest_random(2 * zi->zi_interval[0] + 1);
	}

	/*
	* Run the tests in a loop. These tests include fault injection
	* to verify that self-healing data works, and forced crashes
	* to verify that we never lose on-disk consistency.
	*/
	while (gethrtime() < zs->zs_proc_stop) {
	int status;
	boolean_t killed;

	/*
	* Initialize the workload counters for each function.
	*/
	for (f = 0; f < ZTEST_FUNCS; f++) {
	zc = ZTEST_GET_SHARED_CALLSTATE(f);
	zc->zc_count = 0;
	zc->zc_time = 0;
	}

	/* Set the allocation switch size */
	zs->zs_metaslab_df_alloc_threshold =
	ztest_random(zs->zs_metaslab_sz / 4) + 1;

	if (!hasalt \|\| ztest_random(2) == 0) {
	if (hasalt && ztest_opts.zo_verbose >= 1) {
	(void) printf("Executing newer ztest: %s\n",
	cmd);
	}
	newer++;
	killed = exec_child(cmd, NULL, B_TRUE, &status);
	} else {
	if (hasalt && ztest_opts.zo_verbose >= 1) {
	(void) printf("Executing older ztest: %s\n",
	ztest_opts.zo_alt_ztest);
	}
	older++;
	killed = exec_child(ztest_opts.zo_alt_ztest,
	ztest_opts.zo_alt_libpath, B_TRUE, &status);
	}

	if (killed)
	kills++;
	iters++;

	if (ztest_opts.zo_verbose >= 1) {
	hrtime_t now = gethrtime();

	now = MIN(now, zs->zs_proc_stop);
	print_time(zs->zs_proc_stop - now, timebuf);
	nicenum(zs->zs_space, numbuf, sizeof (numbuf));

	(void) printf("Pass %3d, %8s, %3"PRIu64" ENOSPC, "
	"%4.1f%% of %5s used, %3.0f%% done, %8s to go\n",
	iters,
	WIFEXITED(status) ? "Complete" : "SIGKILL",
	zs->zs_enospc_count,
	100.0 * zs->zs_alloc / zs->zs_space,
	numbuf,
	100.0 * (now - zs->zs_proc_start) /
	(ztest_opts.zo_time * NANOSEC), timebuf);
	}

	if (ztest_opts.zo_verbose >= 2) {
	(void) printf("\nWorkload summary:\n\n");
	(void) printf("%7s %9s %s\n",
	"Calls", "Time", "Function");
	(void) printf("%7s %9s %s\n",
	"-----", "----", "--------");
	for (f = 0; f < ZTEST_FUNCS; f++) {
	zi = &ztest_info[f];
	zc = ZTEST_GET_SHARED_CALLSTATE(f);
	print_time(zc->zc_time, timebuf);
	(void) printf("%7"PRIu64" %9s %s\n",
	zc->zc_count, timebuf,
	zi->zi_funcname);
	}
	(void) printf("\n");
	}

	if (!ztest_opts.zo_mmp_test)
	ztest_run_zdb(zs->zs_guid);
	}

	if (ztest_opts.zo_verbose >= 1) {
	if (hasalt) {
	(void) printf("%d runs of older ztest: %s\n", older,
	ztest_opts.zo_alt_ztest);
	(void) printf("%d runs of newer ztest: %s\n", newer,
	cmd);
	}
	(void) printf("%d killed, %d completed, %.0f%% kill rate\n",
	kills, iters - kills, (100.0 * kills) / MAX(1, iters));
	}

	umem_free(cmd, MAXNAMELEN);

	return (0);
	}
	diff --git a/sys/contrib/openzfs/config/Substfiles.am b/sys/contrib/openzfs/config/Substfiles.am
	index 38e870b2f501..2459637abe6e 100644
	--- a/sys/contrib/openzfs/config/Substfiles.am
	+++ b/sys/contrib/openzfs/config/Substfiles.am
	@@ -1,46 +1,47 @@
	subst_sed_cmd = \
	-e 's\|@abs_top_srcdir[@]\|$(abs_top_srcdir)\|g' \
	-e 's\|@bindir[@]\|$(bindir)\|g' \
	-e 's\|@datadir[@]\|$(datadir)\|g' \
	-e 's\|@initconfdir[@]\|$(initconfdir)\|g' \
	-e 's\|@initdir[@]\|$(initdir)\|g' \
	-e 's\|@mounthelperdir[@]\|$(mounthelperdir)\|g' \
	-e 's\|@pammoduledir[@]\|$(pammoduledir)\|g' \
	-e 's\|@runstatedir[@]\|$(runstatedir)\|g' \
	-e 's\|@sbindir[@]\|$(sbindir)\|g' \
	-e 's\|@sysconfdir[@]\|$(sysconfdir)\|g' \
	-e 's\|@systemdgeneratordir[@]\|$(systemdgeneratordir)\|g' \
	-e 's\|@systemdunitdir[@]\|$(systemdunitdir)\|g' \
	-e 's\|@udevdir[@]\|$(udevdir)\|g' \
	-e 's\|@udevruledir[@]\|$(udevruledir)\|g' \
	-e 's\|@zfsexecdir[@]\|$(zfsexecdir)\|g' \
	\
	-e 's\|@ASAN_ENABLED[@]\|$(ASAN_ENABLED)\|g' \
	-e 's\|@DEFAULT_INIT_NFS_SERVER[@]\|$(DEFAULT_INIT_NFS_SERVER)\|g' \
	-e 's\|@DEFAULT_INIT_SHELL[@]\|$(DEFAULT_INIT_SHELL)\|g' \
	+ -e 's\|@IS_SYSV_RC[@]\|$(IS_SYSV_RC)\|g' \
	-e 's\|@LIBFETCH_DYNAMIC[@]\|$(LIBFETCH_DYNAMIC)\|g' \
	-e 's\|@LIBFETCH_SONAME[@]\|$(LIBFETCH_SONAME)\|g' \
	-e 's\|@PYTHON[@]\|$(PYTHON)\|g' \
	-e 's\|@PYTHON_SHEBANG[@]\|$(PYTHON_SHEBANG)\|g' \
	-e 's\|@UBSAN_ENABLED[@]\|$(UBSAN_ENABLED)\|g' \
	-e 's\|@VERSION[@]\|$(VERSION)\|g'

	define SUBST
	$(1) : $(2)$(1).in Makefile;
	$$(AM_V_GEN)set -e; \
	$$(MKDIR_P) $$(@D); \
	$$(RM) $$@~; \
	$$(SED) $$(subst_sed_cmd) $$< >$$@~; \
	if grep -E '@[a-zA-Z0-9_]+@' $$@~ >&2; then \
	echo "Undefined substitution" >&2; \
	exit 1; \
	fi; \
	[ -x $$< ] && chmod +x $$@~; \
	mv -f $$@~ $$@
	endef

	SUBSTFILES =
	CLEANFILES += $(SUBSTFILES)
	dist_noinst_DATA += $(SUBSTFILES:=.in)

	-$(call SUBST,%,)
	+$(SUBSTFILES): $(call SUBST,%,)
	diff --git a/sys/contrib/openzfs/config/always-pyzfs.m4 b/sys/contrib/openzfs/config/always-pyzfs.m4
	index 9b123b1b2db1..98c1cc230205 100644
	--- a/sys/contrib/openzfs/config/always-pyzfs.m4
	+++ b/sys/contrib/openzfs/config/always-pyzfs.m4
	@@ -1,129 +1,130 @@
	dnl #
	dnl # ZFS_AC_PYTHON_MODULE(module_name, [action-if-true], [action-if-false])
	dnl #
	dnl # Checks for Python module. Freely inspired by AX_PYTHON_MODULE
	dnl # https://www.gnu.org/software/autoconf-archive/ax_python_module.html
	dnl # Required by ZFS_AC_CONFIG_ALWAYS_PYZFS.
	dnl #
	AC_DEFUN([ZFS_AC_PYTHON_MODULE], [
	PYTHON_NAME=${PYTHON##*/}
	AC_MSG_CHECKING([for $PYTHON_NAME module: $1])
	AS_IF([$PYTHON -c "import $1" 2>/dev/null], [
	AC_MSG_RESULT(yes)
	m4_ifvaln([$2], [$2])
	], [
	AC_MSG_RESULT(no)
	m4_ifvaln([$3], [$3])
	])
	])

	dnl #
	dnl # Determines if pyzfs can be built, requires Python 3.6 or later.
	dnl #
	AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [
	AC_ARG_ENABLE([pyzfs],
	AS_HELP_STRING([--enable-pyzfs],
	[install libzfs_core python bindings @<:@default=check@:>@]),
	[enable_pyzfs=$enableval],
	[enable_pyzfs=check])

	dnl #
	dnl # Packages for pyzfs specifically enabled/disabled.
	dnl #
	AS_IF([test "x$enable_pyzfs" != xcheck], [
	AS_IF([test "x$enable_pyzfs" = xyes], [
	DEFINE_PYZFS='--with pyzfs'
	], [
	DEFINE_PYZFS='--without pyzfs'
	])
	], [
	AS_IF([test "$PYTHON" != :], [
	DEFINE_PYZFS=''
	], [
	enable_pyzfs=no
	DEFINE_PYZFS='--without pyzfs'
	])
	])
	AC_SUBST(DEFINE_PYZFS)

	dnl #
	dnl # Autodetection disables pyzfs if kernel or srpm config
	dnl #
	AS_IF([test "x$enable_pyzfs" = xcheck], [
	AS_IF([test "x$ZFS_CONFIG" = xkernel -o "x$ZFS_CONFIG" = xsrpm ], [
	enable_pyzfs=no
	AC_MSG_NOTICE([Disabling pyzfs for kernel/srpm config])
	])
	])

	dnl #
	dnl # Python "packaging" (or, failing that, "distlib") module is required to build and install pyzfs
	dnl #
	AS_IF([test "x$enable_pyzfs" = xcheck -o "x$enable_pyzfs" = xyes], [
	ZFS_AC_PYTHON_MODULE([packaging], [], [
	ZFS_AC_PYTHON_MODULE([distlib], [], [
	AS_IF([test "x$enable_pyzfs" = xyes], [
	AC_MSG_ERROR("Python $PYTHON_VERSION packaging and distlib modules are not installed")
	], [test "x$enable_pyzfs" != xno], [
	enable_pyzfs=no
	])
	])
	])
	])

	dnl #
	dnl # Require python3-devel libraries
	dnl #
	AS_IF([test "x$enable_pyzfs" = xcheck -o "x$enable_pyzfs" = xyes], [
	AS_CASE([$PYTHON_VERSION],
	[3.*], [PYTHON_REQUIRED_VERSION=">= '3.6.0'"],
	[AC_MSG_ERROR("Python $PYTHON_VERSION unknown")]
	)

	- AX_PYTHON_DEVEL([$PYTHON_REQUIRED_VERSION], [
	- AS_IF([test "x$enable_pyzfs" = xyes], [
	- AC_MSG_ERROR("Python $PYTHON_REQUIRED_VERSION development library is not installed")
	- ], [test "x$enable_pyzfs" != xno], [
	+ AS_IF([test "x$enable_pyzfs" = xyes], [
	+ AX_PYTHON_DEVEL([$PYTHON_REQUIRED_VERSION])
	+ ], [
	+ AX_PYTHON_DEVEL([$PYTHON_REQUIRED_VERSION], [true])
	+ AS_IF([test "x$ax_python_devel_found" = xno], [
	enable_pyzfs=no
	])
	])
	])

	dnl #
	dnl # Python "setuptools" module is required to build and install pyzfs
	dnl #
	AS_IF([test "x$enable_pyzfs" = xcheck -o "x$enable_pyzfs" = xyes], [
	ZFS_AC_PYTHON_MODULE([setuptools], [], [
	AS_IF([test "x$enable_pyzfs" = xyes], [
	AC_MSG_ERROR("Python $PYTHON_VERSION setuptools is not installed")
	], [test "x$enable_pyzfs" != xno], [
	enable_pyzfs=no
	])
	])
	])

	dnl #
	dnl # Python "cffi" module is required to run pyzfs
	dnl #
	AS_IF([test "x$enable_pyzfs" = xcheck -o "x$enable_pyzfs" = xyes], [
	ZFS_AC_PYTHON_MODULE([cffi], [], [
	AS_IF([test "x$enable_pyzfs" = xyes], [
	AC_MSG_ERROR("Python $PYTHON_VERSION cffi is not installed")
	], [test "x$enable_pyzfs" != xno], [
	enable_pyzfs=no
	])
	])
	])

	dnl #
	dnl # Set enable_pyzfs to 'yes' if every check passed
	dnl #
	AS_IF([test "x$enable_pyzfs" = xcheck], [enable_pyzfs=yes])

	AM_CONDITIONAL([PYZFS_ENABLED], [test "x$enable_pyzfs" = xyes])
	AC_SUBST([PYZFS_ENABLED], [$enable_pyzfs])
	AC_SUBST(pythonsitedir, [$PYTHON_SITE_PKG])

	AC_MSG_CHECKING([whether to enable pyzfs: ])
	AC_MSG_RESULT($enable_pyzfs)
	])
	diff --git a/sys/contrib/openzfs/config/ax_python_devel.m4 b/sys/contrib/openzfs/config/ax_python_devel.m4
	index f6d4b01444d6..1f480db6d233 100644
	--- a/sys/contrib/openzfs/config/ax_python_devel.m4
	+++ b/sys/contrib/openzfs/config/ax_python_devel.m4
	@@ -1,351 +1,468 @@
	# ===========================================================================
	# https://www.gnu.org/software/autoconf-archive/ax_python_devel.html
	# ===========================================================================
	#
	# SYNOPSIS
	#
	-# AX_PYTHON_DEVEL([version], [action-if-not-found])
	+# AX_PYTHON_DEVEL([version[,optional]])
	#
	# DESCRIPTION
	#
	# Note: Defines as a precious variable "PYTHON_VERSION". Don't override it
	# in your configure.ac.
	#
	-# Note: this is a slightly modified version of the original AX_PYTHON_DEVEL
	-# macro which accepts an additional [action-if-not-found] argument. This
	-# allow to detect if Python development is available without aborting the
	-# configure phase with an hard error in case it is not.
	-#
	# This macro checks for Python and tries to get the include path to
	# 'Python.h'. It provides the $(PYTHON_CPPFLAGS) and $(PYTHON_LIBS) output
	# variables. It also exports $(PYTHON_EXTRA_LIBS) and
	# $(PYTHON_EXTRA_LDFLAGS) for embedding Python in your code.
	#
	# You can search for some particular version of Python by passing a
	# parameter to this macro, for example ">= '2.3.1'", or "== '2.4'". Please
	# note that you have to pass also an operator along with the version to
	# match, and pay special attention to the single quotes surrounding the
	# version number. Don't use "PYTHON_VERSION" for this: that environment
	# variable is declared as precious and thus reserved for the end-user.
	#
	+# By default this will fail if it does not detect a development version of
	+# python. If you want it to continue, set optional to true, like
	+# AX_PYTHON_DEVEL([], [true]). The ax_python_devel_found variable will be
	+# "no" if it fails.
	+#
	# This macro should work for all versions of Python >= 2.1.0. As an end
	# user, you can disable the check for the python version by setting the
	# PYTHON_NOVERSIONCHECK environment variable to something else than the
	# empty string.
	#
	# If you need to use this macro for an older Python version, please
	# contact the authors. We're always open for feedback.
	#
	# LICENSE
	#
	# Copyright (c) 2009 Sebastian Huber <sebastian-huber@web.de>
	# Copyright (c) 2009 Alan W. Irwin
	# Copyright (c) 2009 Rafael Laboissiere <rafael@laboissiere.net>
	# Copyright (c) 2009 Andrew Collier
	# Copyright (c) 2009 Matteo Settenvini <matteo@member.fsf.org>
	# Copyright (c) 2009 Horst Knorr <hk_classes@knoda.org>
	# Copyright (c) 2013 Daniel Mullner <muellner@math.stanford.edu>
	-# Copyright (c) 2018 loli10K <ezomori.nozomu@gmail.com>
	#
	# This program is free software: you can redistribute it and/or modify it
	# under the terms of the GNU General Public License as published by the
	# Free Software Foundation, either version 3 of the License, or (at your
	# option) any later version.
	#
	# This program is distributed in the hope that it will be useful, but
	# WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
	# Public License for more details.
	#
	# You should have received a copy of the GNU General Public License along
	# with this program. If not, see <https://www.gnu.org/licenses/>.
	#
	# As a special exception, the respective Autoconf Macro's copyright owner
	# gives unlimited permission to copy, distribute and modify the configure
	# scripts that are the output of Autoconf when processing the Macro. You
	# need not follow the terms of the GNU General Public License when using
	# or distributing such scripts, even though portions of the text of the
	# Macro appear in them. The GNU General Public License (GPL) does govern
	# all other use of the material that constitutes the Autoconf Macro.
	#
	# This special exception to the GPL applies to versions of the Autoconf
	# Macro released by the Autoconf Archive. When you make and distribute a
	# modified version of the Autoconf Macro, you may extend this special
	# exception to the GPL to apply to your modified version as well.

	-#serial 21
	+#serial 36

	AU_ALIAS([AC_PYTHON_DEVEL], [AX_PYTHON_DEVEL])
	AC_DEFUN([AX_PYTHON_DEVEL],[
	+ # Get whether it's optional
	+ if test -z "$2"; then
	+ ax_python_devel_optional=false
	+ else
	+ ax_python_devel_optional=$2
	+ fi
	+ ax_python_devel_found=yes
	+
	#
	# Allow the use of a (user set) custom python version
	#
	AC_ARG_VAR([PYTHON_VERSION],[The installed Python
	version to use, for example '2.3'. This string
	will be appended to the Python interpreter
	canonical name.])

	AC_PATH_PROG([PYTHON],[python[$PYTHON_VERSION]])
	if test -z "$PYTHON"; then
	- m4_ifvaln([$2],[$2],[
	- AC_MSG_ERROR([Cannot find python$PYTHON_VERSION in your system path])
	- PYTHON_VERSION=""
	- ])
	+ AC_MSG_WARN([Cannot find python$PYTHON_VERSION in your system path])
	+ if ! $ax_python_devel_optional; then
	+ AC_MSG_ERROR([Giving up, python development not available])
	+ fi
	+ ax_python_devel_found=no
	+ PYTHON_VERSION=""
	fi

	- #
	- # Check for a version of Python >= 2.1.0
	- #
	- AC_MSG_CHECKING([for a version of Python >= '2.1.0'])
	- ac_supports_python_ver=`$PYTHON -c "import sys; \
	+ if test $ax_python_devel_found = yes; then
	+ #
	+ # Check for a version of Python >= 2.1.0
	+ #
	+ AC_MSG_CHECKING([for a version of Python >= '2.1.0'])
	+ ac_supports_python_ver=`$PYTHON -c "import sys; \
	ver = sys.version.split ()[[0]]; \
	print (ver >= '2.1.0')"`
	- if test "$ac_supports_python_ver" != "True"; then
	+ if test "$ac_supports_python_ver" != "True"; then
	if test -z "$PYTHON_NOVERSIONCHECK"; then
	AC_MSG_RESULT([no])
	- AC_MSG_FAILURE([
	+ AC_MSG_WARN([
	This version of the AC@&t@_PYTHON_DEVEL macro
	doesn't work properly with versions of Python before
	2.1.0. You may need to re-run configure, setting the
	variables PYTHON_CPPFLAGS, PYTHON_LIBS, PYTHON_SITE_PKG,
	PYTHON_EXTRA_LIBS and PYTHON_EXTRA_LDFLAGS by hand.
	Moreover, to disable this check, set PYTHON_NOVERSIONCHECK
	to something else than an empty string.
	])
	+ if ! $ax_python_devel_optional; then
	+ AC_MSG_FAILURE([Giving up])
	+ fi
	+ ax_python_devel_found=no
	+ PYTHON_VERSION=""
	else
	AC_MSG_RESULT([skip at user request])
	fi
	- else
	+ else
	AC_MSG_RESULT([yes])
	+ fi
	fi

	- #
	- # If the macro parameter ``version'' is set, honour it.
	- # A Python shim class, VPy, is used to implement correct version comparisons via
	- # string expressions, since e.g. a naive textual ">= 2.7.3" won't work for
	- # Python 2.7.10 (the ".1" being evaluated as less than ".3").
	- #
	- if test -n "$1"; then
	+ if test $ax_python_devel_found = yes; then
	+ #
	+ # If the macro parameter ``version'' is set, honour it.
	+ # A Python shim class, VPy, is used to implement correct version comparisons via
	+ # string expressions, since e.g. a naive textual ">= 2.7.3" won't work for
	+ # Python 2.7.10 (the ".1" being evaluated as less than ".3").
	+ #
	+ if test -n "$1"; then
	AC_MSG_CHECKING([for a version of Python $1])
	cat << EOF > ax_python_devel_vpy.py
	class VPy:
	def vtup(self, s):
	return tuple(map(int, s.strip().replace("rc", ".").split(".")))
	def __init__(self):
	import sys
	- self.vpy = tuple(sys.version_info)
	+ self.vpy = tuple(sys.version_info)[[:3]]
	def __eq__(self, s):
	return self.vpy == self.vtup(s)
	def __ne__(self, s):
	return self.vpy != self.vtup(s)
	def __lt__(self, s):
	return self.vpy < self.vtup(s)
	def __gt__(self, s):
	return self.vpy > self.vtup(s)
	def __le__(self, s):
	return self.vpy <= self.vtup(s)
	def __ge__(self, s):
	return self.vpy >= self.vtup(s)
	EOF
	ac_supports_python_ver=`$PYTHON -c "import ax_python_devel_vpy; \
	ver = ax_python_devel_vpy.VPy(); \
	print (ver $1)"`
	rm -rf ax_python_devel_vpy.py __pycache__/ax_python_devel_vpy.py
	if test "$ac_supports_python_ver" = "True"; then
	AC_MSG_RESULT([yes])
	else
	AC_MSG_RESULT([no])
	- AC_MSG_ERROR([this package requires Python $1.
	+ AC_MSG_WARN([this package requires Python $1.
	If you have it installed, but it isn't the default Python
	interpreter in your system path, please pass the PYTHON_VERSION
	variable to configure. See ``configure --help'' for reference.
	])
	+ if ! $ax_python_devel_optional; then
	+ AC_MSG_ERROR([Giving up])
	+ fi
	+ ax_python_devel_found=no
	PYTHON_VERSION=""
	fi
	+ fi
	fi

	- #
	- # Check for Python include path
	- #
	- #
	- AC_MSG_CHECKING([for Python include path])
	- if test -z "$PYTHON_CPPFLAGS"; then
	- python_path=`$PYTHON -c "import sysconfig; \
	- print (sysconfig.get_path('include'));"`
	- plat_python_path=`$PYTHON -c "import sysconfig; \
	- print (sysconfig.get_path('platinclude'));"`
	+ if test $ax_python_devel_found = yes; then
	+ #
	+ # Check if you have distutils, else fail
	+ #
	+ AC_MSG_CHECKING([for the sysconfig Python package])
	+ ac_sysconfig_result=`$PYTHON -c "import sysconfig" 2>&1`
	+ if test $? -eq 0; then
	+ AC_MSG_RESULT([yes])
	+ IMPORT_SYSCONFIG="import sysconfig"
	+ else
	+ AC_MSG_RESULT([no])
	+
	+ AC_MSG_CHECKING([for the distutils Python package])
	+ ac_sysconfig_result=`$PYTHON -c "from distutils import sysconfig" 2>&1`
	+ if test $? -eq 0; then
	+ AC_MSG_RESULT([yes])
	+ IMPORT_SYSCONFIG="from distutils import sysconfig"
	+ else
	+ AC_MSG_WARN([cannot import Python module "distutils".
	+Please check your Python installation. The error was:
	+$ac_sysconfig_result])
	+ if ! $ax_python_devel_optional; then
	+ AC_MSG_ERROR([Giving up])
	+ fi
	+ ax_python_devel_found=no
	+ PYTHON_VERSION=""
	+ fi
	+ fi
	+ fi
	+
	+ if test $ax_python_devel_found = yes; then
	+ #
	+ # Check for Python include path
	+ #
	+ AC_MSG_CHECKING([for Python include path])
	+ if test -z "$PYTHON_CPPFLAGS"; then
	+ if test "$IMPORT_SYSCONFIG" = "import sysconfig"; then
	+ # sysconfig module has different functions
	+ python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \
	+ print (sysconfig.get_path ('include'));"`
	+ plat_python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \
	+ print (sysconfig.get_path ('platinclude'));"`
	+ else
	+ # old distutils way
	+ python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \
	+ print (sysconfig.get_python_inc ());"`
	+ plat_python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \
	+ print (sysconfig.get_python_inc (plat_specific=1));"`
	+ fi
	if test -n "${python_path}"; then
	if test "${plat_python_path}" != "${python_path}"; then
	python_path="-I$python_path -I$plat_python_path"
	else
	python_path="-I$python_path"
	fi
	fi
	PYTHON_CPPFLAGS=$python_path
	- fi
	- AC_MSG_RESULT([$PYTHON_CPPFLAGS])
	- AC_SUBST([PYTHON_CPPFLAGS])
	+ fi
	+ AC_MSG_RESULT([$PYTHON_CPPFLAGS])
	+ AC_SUBST([PYTHON_CPPFLAGS])

	- #
	- # Check for Python library path
	- #
	- AC_MSG_CHECKING([for Python library path])
	- if test -z "$PYTHON_LIBS"; then
	+ #
	+ # Check for Python library path
	+ #
	+ AC_MSG_CHECKING([for Python library path])
	+ if test -z "$PYTHON_LIBS"; then
	# (makes two attempts to ensure we've got a version number
	# from the interpreter)
	ac_python_version=`cat<<EOD \| $PYTHON -

	# join all versioning strings, on some systems
	# major/minor numbers could be in different list elements
	from sysconfig import *
	e = get_config_var('VERSION')
	if e is not None:
	print(e)
	EOD`

	if test -z "$ac_python_version"; then
	if test -n "$PYTHON_VERSION"; then
	ac_python_version=$PYTHON_VERSION
	else
	ac_python_version=`$PYTHON -c "import sys; \
	- print ('.'.join(sys.version.split('.')[[:2]]))"`
	+ print ("%d.%d" % sys.version_info[[:2]])"`
	fi
	fi

	# Make the versioning information available to the compiler
	AC_DEFINE_UNQUOTED([HAVE_PYTHON], ["$ac_python_version"],
	[If available, contains the Python version number currently in use.])

	# First, the library directory:
	ac_python_libdir=`cat<<EOD \| $PYTHON -

	# There should be only one
	-import sysconfig
	+$IMPORT_SYSCONFIG
	e = sysconfig.get_config_var('LIBDIR')
	if e is not None:
	print (e)
	EOD`

	# Now, for the library:
	ac_python_library=`cat<<EOD \| $PYTHON -

	-import sysconfig
	+$IMPORT_SYSCONFIG
	c = sysconfig.get_config_vars()
	if 'LDVERSION' in c:
	print ('python'+c[['LDVERSION']])
	else:
	print ('python'+c[['VERSION']])
	EOD`

	# This small piece shamelessly adapted from PostgreSQL python macro;
	# credits goes to momjian, I think. I'd like to put the right name
	# in the credits, if someone can point me in the right direction... ?
	#
	if test -n "$ac_python_libdir" -a -n "$ac_python_library"
	then
	# use the official shared library
	ac_python_library=`echo "$ac_python_library" \| sed "s/^lib//"`
	PYTHON_LIBS="-L$ac_python_libdir -l$ac_python_library"
	else
	# old way: use libpython from python_configdir
	ac_python_libdir=`$PYTHON -c \
	- "import sysconfig; \
	+ "from sysconfig import get_python_lib as f; \
	import os; \
	- print (os.path.join(sysconfig.get_path('platstdlib'), 'config'));"`
	+ print (os.path.join(f(plat_specific=1, standard_lib=1), 'config'));"`
	PYTHON_LIBS="-L$ac_python_libdir -lpython$ac_python_version"
	fi

	if test -z "PYTHON_LIBS"; then
	- m4_ifvaln([$2],[$2],[
	- AC_MSG_ERROR([
	+ AC_MSG_WARN([
	Cannot determine location of your Python DSO. Please check it was installed with
	dynamic libraries enabled, or try setting PYTHON_LIBS by hand.
	- ])
	])
	+ if ! $ax_python_devel_optional; then
	+ AC_MSG_ERROR([Giving up])
	+ fi
	+ ax_python_devel_found=no
	+ PYTHON_VERSION=""
	fi
	+ fi
	fi
	- AC_MSG_RESULT([$PYTHON_LIBS])
	- AC_SUBST([PYTHON_LIBS])

	- #
	- # Check for site packages
	- #
	- AC_MSG_CHECKING([for Python site-packages path])
	- if test -z "$PYTHON_SITE_PKG"; then
	- PYTHON_SITE_PKG=`$PYTHON -c "import distutils.sysconfig; \
	- print (distutils.sysconfig.get_python_lib(0,0));" 2>/dev/null \|\| \
	- $PYTHON -c "import sysconfig; \
	- print (sysconfig.get_path('purelib'));"`
	- fi
	- AC_MSG_RESULT([$PYTHON_SITE_PKG])
	- AC_SUBST([PYTHON_SITE_PKG])
	+ if test $ax_python_devel_found = yes; then
	+ AC_MSG_RESULT([$PYTHON_LIBS])
	+ AC_SUBST([PYTHON_LIBS])

	- #
	- # libraries which must be linked in when embedding
	- #
	- AC_MSG_CHECKING(python extra libraries)
	- if test -z "$PYTHON_EXTRA_LIBS"; then
	- PYTHON_EXTRA_LIBS=`$PYTHON -c "import sysconfig; \
	+ #
	+ # Check for site packages
	+ #
	+ AC_MSG_CHECKING([for Python site-packages path])
	+ if test -z "$PYTHON_SITE_PKG"; then
	+ if test "$IMPORT_SYSCONFIG" = "import sysconfig"; then
	+ PYTHON_SITE_PKG=`$PYTHON -c "
	+$IMPORT_SYSCONFIG;
	+if hasattr(sysconfig, 'get_default_scheme'):
	+ scheme = sysconfig.get_default_scheme()
	+else:
	+ scheme = sysconfig._get_default_scheme()
	+if scheme == 'posix_local':
	+ # Debian's default scheme installs to /usr/local/ but we want to find headers in /usr/
	+ scheme = 'posix_prefix'
	+prefix = '$prefix'
	+if prefix == 'NONE':
	+ prefix = '$ac_default_prefix'
	+sitedir = sysconfig.get_path('purelib', scheme, vars={'base': prefix})
	+print(sitedir)"`
	+ else
	+ # distutils.sysconfig way
	+ PYTHON_SITE_PKG=`$PYTHON -c "$IMPORT_SYSCONFIG; \
	+ print (sysconfig.get_python_lib(0,0));"`
	+ fi
	+ fi
	+ AC_MSG_RESULT([$PYTHON_SITE_PKG])
	+ AC_SUBST([PYTHON_SITE_PKG])
	+
	+ #
	+ # Check for platform-specific site packages
	+ #
	+ AC_MSG_CHECKING([for Python platform specific site-packages path])
	+ if test -z "$PYTHON_PLATFORM_SITE_PKG"; then
	+ if test "$IMPORT_SYSCONFIG" = "import sysconfig"; then
	+ PYTHON_PLATFORM_SITE_PKG=`$PYTHON -c "
	+$IMPORT_SYSCONFIG;
	+if hasattr(sysconfig, 'get_default_scheme'):
	+ scheme = sysconfig.get_default_scheme()
	+else:
	+ scheme = sysconfig._get_default_scheme()
	+if scheme == 'posix_local':
	+ # Debian's default scheme installs to /usr/local/ but we want to find headers in /usr/
	+ scheme = 'posix_prefix'
	+prefix = '$prefix'
	+if prefix == 'NONE':
	+ prefix = '$ac_default_prefix'
	+sitedir = sysconfig.get_path('platlib', scheme, vars={'platbase': prefix})
	+print(sitedir)"`
	+ else
	+ # distutils.sysconfig way
	+ PYTHON_PLATFORM_SITE_PKG=`$PYTHON -c "$IMPORT_SYSCONFIG; \
	+ print (sysconfig.get_python_lib(1,0));"`
	+ fi
	+ fi
	+ AC_MSG_RESULT([$PYTHON_PLATFORM_SITE_PKG])
	+ AC_SUBST([PYTHON_PLATFORM_SITE_PKG])
	+
	+ #
	+ # libraries which must be linked in when embedding
	+ #
	+ AC_MSG_CHECKING(python extra libraries)
	+ if test -z "$PYTHON_EXTRA_LIBS"; then
	+ PYTHON_EXTRA_LIBS=`$PYTHON -c "$IMPORT_SYSCONFIG; \
	conf = sysconfig.get_config_var; \
	print (conf('LIBS') + ' ' + conf('SYSLIBS'))"`
	- fi
	- AC_MSG_RESULT([$PYTHON_EXTRA_LIBS])
	- AC_SUBST(PYTHON_EXTRA_LIBS)
	+ fi
	+ AC_MSG_RESULT([$PYTHON_EXTRA_LIBS])
	+ AC_SUBST(PYTHON_EXTRA_LIBS)

	- #
	- # linking flags needed when embedding
	- #
	- AC_MSG_CHECKING(python extra linking flags)
	- if test -z "$PYTHON_EXTRA_LDFLAGS"; then
	- PYTHON_EXTRA_LDFLAGS=`$PYTHON -c "import sysconfig; \
	+ #
	+ # linking flags needed when embedding
	+ #
	+ AC_MSG_CHECKING(python extra linking flags)
	+ if test -z "$PYTHON_EXTRA_LDFLAGS"; then
	+ PYTHON_EXTRA_LDFLAGS=`$PYTHON -c "$IMPORT_SYSCONFIG; \
	conf = sysconfig.get_config_var; \
	print (conf('LINKFORSHARED'))"`
	- fi
	- AC_MSG_RESULT([$PYTHON_EXTRA_LDFLAGS])
	- AC_SUBST(PYTHON_EXTRA_LDFLAGS)
	+ # Hack for macos, it sticks this in here.
	+ PYTHON_EXTRA_LDFLAGS=`echo $PYTHON_EXTRA_LDFLAGS \| sed 's/CoreFoundation.*$/CoreFoundation/'`
	+ fi
	+ AC_MSG_RESULT([$PYTHON_EXTRA_LDFLAGS])
	+ AC_SUBST(PYTHON_EXTRA_LDFLAGS)

	- #
	- # final check to see if everything compiles alright
	- #
	- AC_MSG_CHECKING([consistency of all components of python development environment])
	- # save current global flags
	- ac_save_LIBS="$LIBS"
	- ac_save_LDFLAGS="$LDFLAGS"
	- ac_save_CPPFLAGS="$CPPFLAGS"
	- LIBS="$ac_save_LIBS $PYTHON_LIBS $PYTHON_EXTRA_LIBS $PYTHON_EXTRA_LIBS"
	- LDFLAGS="$ac_save_LDFLAGS $PYTHON_EXTRA_LDFLAGS"
	- CPPFLAGS="$ac_save_CPPFLAGS $PYTHON_CPPFLAGS"
	- AC_LANG_PUSH([C])
	- AC_LINK_IFELSE([
	+ #
	+ # final check to see if everything compiles alright
	+ #
	+ AC_MSG_CHECKING([consistency of all components of python development environment])
	+ # save current global flags
	+ ac_save_LIBS="$LIBS"
	+ ac_save_LDFLAGS="$LDFLAGS"
	+ ac_save_CPPFLAGS="$CPPFLAGS"
	+ LIBS="$ac_save_LIBS $PYTHON_LIBS $PYTHON_EXTRA_LIBS"
	+ LDFLAGS="$ac_save_LDFLAGS $PYTHON_EXTRA_LDFLAGS"
	+ CPPFLAGS="$ac_save_CPPFLAGS $PYTHON_CPPFLAGS"
	+ AC_LANG_PUSH([C])
	+ AC_LINK_IFELSE([
	AC_LANG_PROGRAM([[#include <Python.h>]],
	[[Py_Initialize();]])
	],[pythonexists=yes],[pythonexists=no])
	- AC_LANG_POP([C])
	- # turn back to default flags
	- CPPFLAGS="$ac_save_CPPFLAGS"
	- LIBS="$ac_save_LIBS"
	- LDFLAGS="$ac_save_LDFLAGS"
	+ AC_LANG_POP([C])
	+ # turn back to default flags
	+ CPPFLAGS="$ac_save_CPPFLAGS"
	+ LIBS="$ac_save_LIBS"
	+ LDFLAGS="$ac_save_LDFLAGS"

	- AC_MSG_RESULT([$pythonexists])
	+ AC_MSG_RESULT([$pythonexists])

	- if test ! "x$pythonexists" = "xyes"; then
	- m4_ifvaln([$2],[$2],[
	- AC_MSG_FAILURE([
	+ if test ! "x$pythonexists" = "xyes"; then
	+ AC_MSG_WARN([
	Could not link test program to Python. Maybe the main Python library has been
	installed in some non-standard library path. If so, pass it to configure,
	via the LIBS environment variable.
	Example: ./configure LIBS="-L/usr/non-standard-path/python/lib"
	============================================================================
	ERROR!
	You probably have to install the development version of the Python package
	for your distribution. The exact name of this package varies among them.
	============================================================================
	- ])
	- PYTHON_VERSION=""
	- ])
	+ ])
	+ if ! $ax_python_devel_optional; then
	+ AC_MSG_ERROR([Giving up])
	+ fi
	+ ax_python_devel_found=no
	+ PYTHON_VERSION=""
	+ fi
	fi

	#
	# all done!
	#
	])
	diff --git a/sys/contrib/openzfs/config/kernel-blk-queue.m4 b/sys/contrib/openzfs/config/kernel-blk-queue.m4
	index bb5903b313eb..15dbe1c7dff0 100644
	--- a/sys/contrib/openzfs/config/kernel-blk-queue.m4
	+++ b/sys/contrib/openzfs/config/kernel-blk-queue.m4
	@@ -1,418 +1,433 @@
	dnl #
	dnl # 2.6.39 API change,
	dnl # blk_start_plug() and blk_finish_plug()
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE_PLUG], [
	ZFS_LINUX_TEST_SRC([blk_plug], [
	#include <linux/blkdev.h>
	],[
	struct blk_plug plug __attribute__ ((unused));

	blk_start_plug(&plug);
	blk_finish_plug(&plug);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_PLUG], [
	AC_MSG_CHECKING([whether struct blk_plug is available])
	ZFS_LINUX_TEST_RESULT([blk_plug], [
	AC_MSG_RESULT(yes)
	],[
	ZFS_LINUX_TEST_ERROR([blk_plug])
	])
	])

	dnl #
	dnl # 2.6.32 - 4.11: statically allocated bdi in request_queue
	dnl # 4.12: dynamically allocated bdi in request_queue
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE_BDI], [
	ZFS_LINUX_TEST_SRC([blk_queue_bdi], [
	#include <linux/blkdev.h>
	],[
	struct request_queue q;
	struct backing_dev_info bdi;
	q.backing_dev_info = &bdi;
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_BDI], [
	AC_MSG_CHECKING([whether blk_queue bdi is dynamic])
	ZFS_LINUX_TEST_RESULT([blk_queue_bdi], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLK_QUEUE_BDI_DYNAMIC, 1,
	[blk queue backing_dev_info is dynamic])
	],[
	AC_MSG_RESULT(no)
	])
	])

	dnl #
	dnl # 5.9: added blk_queue_update_readahead(),
	dnl # 5.15: renamed to disk_update_readahead()
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE_UPDATE_READAHEAD], [
	ZFS_LINUX_TEST_SRC([blk_queue_update_readahead], [
	#include <linux/blkdev.h>
	],[
	struct request_queue q;
	blk_queue_update_readahead(&q);
	])

	ZFS_LINUX_TEST_SRC([disk_update_readahead], [
	#include <linux/blkdev.h>
	],[
	struct gendisk disk;
	disk_update_readahead(&disk);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_UPDATE_READAHEAD], [
	AC_MSG_CHECKING([whether blk_queue_update_readahead() exists])
	ZFS_LINUX_TEST_RESULT([blk_queue_update_readahead], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLK_QUEUE_UPDATE_READAHEAD, 1,
	[blk_queue_update_readahead() exists])
	],[
	AC_MSG_RESULT(no)

	AC_MSG_CHECKING([whether disk_update_readahead() exists])
	ZFS_LINUX_TEST_RESULT([disk_update_readahead], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_DISK_UPDATE_READAHEAD, 1,
	[disk_update_readahead() exists])
	],[
	AC_MSG_RESULT(no)
	])
	])
	])

	dnl #
	dnl # 5.19: bdev_max_discard_sectors() available
	dnl # 2.6.32: blk_queue_discard() available
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE_DISCARD], [
	ZFS_LINUX_TEST_SRC([bdev_max_discard_sectors], [
	#include <linux/blkdev.h>
	],[
	struct block_device *bdev __attribute__ ((unused)) = NULL;
	unsigned int error __attribute__ ((unused));

	error = bdev_max_discard_sectors(bdev);
	])

	ZFS_LINUX_TEST_SRC([blk_queue_discard], [
	#include <linux/blkdev.h>
	],[
	struct request_queue r;
	struct request_queue *q = &r;
	int value __attribute__ ((unused));
	memset(q, 0, sizeof(r));
	value = blk_queue_discard(q);
	],[-Wframe-larger-than=8192])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_DISCARD], [
	AC_MSG_CHECKING([whether bdev_max_discard_sectors() is available])
	ZFS_LINUX_TEST_RESULT([bdev_max_discard_sectors], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BDEV_MAX_DISCARD_SECTORS, 1,
	[bdev_max_discard_sectors() is available])
	],[
	AC_MSG_RESULT(no)

	AC_MSG_CHECKING([whether blk_queue_discard() is available])
	ZFS_LINUX_TEST_RESULT([blk_queue_discard], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLK_QUEUE_DISCARD, 1,
	[blk_queue_discard() is available])
	],[
	ZFS_LINUX_TEST_ERROR([blk_queue_discard])
	])
	])
	])

	dnl #
	dnl # 5.19: bdev_max_secure_erase_sectors() available
	dnl # 4.8: blk_queue_secure_erase() available
	dnl # 2.6.36: blk_queue_secdiscard() available
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE_SECURE_ERASE], [
	ZFS_LINUX_TEST_SRC([bdev_max_secure_erase_sectors], [
	#include <linux/blkdev.h>
	],[
	struct block_device *bdev __attribute__ ((unused)) = NULL;
	unsigned int error __attribute__ ((unused));

	error = bdev_max_secure_erase_sectors(bdev);
	])

	ZFS_LINUX_TEST_SRC([blk_queue_secure_erase], [
	#include <linux/blkdev.h>
	],[
	struct request_queue r;
	struct request_queue *q = &r;
	int value __attribute__ ((unused));
	memset(q, 0, sizeof(r));
	value = blk_queue_secure_erase(q);
	],[-Wframe-larger-than=8192])

	ZFS_LINUX_TEST_SRC([blk_queue_secdiscard], [
	#include <linux/blkdev.h>
	],[
	struct request_queue r;
	struct request_queue *q = &r;
	int value __attribute__ ((unused));
	memset(q, 0, sizeof(r));
	value = blk_queue_secdiscard(q);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_SECURE_ERASE], [
	AC_MSG_CHECKING([whether bdev_max_secure_erase_sectors() is available])
	ZFS_LINUX_TEST_RESULT([bdev_max_secure_erase_sectors], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BDEV_MAX_SECURE_ERASE_SECTORS, 1,
	[bdev_max_secure_erase_sectors() is available])
	],[
	AC_MSG_RESULT(no)

	AC_MSG_CHECKING([whether blk_queue_secure_erase() is available])
	ZFS_LINUX_TEST_RESULT([blk_queue_secure_erase], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLK_QUEUE_SECURE_ERASE, 1,
	[blk_queue_secure_erase() is available])
	],[
	AC_MSG_RESULT(no)

	AC_MSG_CHECKING([whether blk_queue_secdiscard() is available])
	ZFS_LINUX_TEST_RESULT([blk_queue_secdiscard], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLK_QUEUE_SECDISCARD, 1,
	[blk_queue_secdiscard() is available])
	],[
	ZFS_LINUX_TEST_ERROR([blk_queue_secure_erase])
	])
	])
	])
	])

	dnl #
	dnl # 4.16 API change,
	dnl # Introduction of blk_queue_flag_set and blk_queue_flag_clear
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE_FLAG_SET], [
	ZFS_LINUX_TEST_SRC([blk_queue_flag_set], [
	#include <linux/kernel.h>
	#include <linux/blkdev.h>
	],[
	struct request_queue *q = NULL;
	blk_queue_flag_set(0, q);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_FLAG_SET], [
	AC_MSG_CHECKING([whether blk_queue_flag_set() exists])
	ZFS_LINUX_TEST_RESULT([blk_queue_flag_set], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLK_QUEUE_FLAG_SET, 1,
	[blk_queue_flag_set() exists])
	],[
	AC_MSG_RESULT(no)
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE_FLAG_CLEAR], [
	ZFS_LINUX_TEST_SRC([blk_queue_flag_clear], [
	#include <linux/kernel.h>
	#include <linux/blkdev.h>
	],[
	struct request_queue *q = NULL;
	blk_queue_flag_clear(0, q);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_FLAG_CLEAR], [
	AC_MSG_CHECKING([whether blk_queue_flag_clear() exists])
	ZFS_LINUX_TEST_RESULT([blk_queue_flag_clear], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLK_QUEUE_FLAG_CLEAR, 1,
	[blk_queue_flag_clear() exists])
	],[
	AC_MSG_RESULT(no)
	])
	])

	dnl #
	dnl # 2.6.36 API change,
	dnl # Added blk_queue_flush() interface, while the previous interface
	dnl # was available to all the new one is GPL-only. Thus in addition to
	dnl # detecting if this function is available we determine if it is
	dnl # GPL-only. If the GPL-only interface is there we implement our own
	dnl # compatibility function, otherwise we use the function. The hope
	dnl # is that long term this function will be opened up.
	dnl #
	dnl # 4.7 API change,
	dnl # Replace blk_queue_flush with blk_queue_write_cache
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE_FLUSH], [
	ZFS_LINUX_TEST_SRC([blk_queue_flush], [
	#include <linux/blkdev.h>
	], [
	struct request_queue *q __attribute__ ((unused)) = NULL;
	(void) blk_queue_flush(q, REQ_FLUSH);
	], [], [ZFS_META_LICENSE])

	ZFS_LINUX_TEST_SRC([blk_queue_write_cache], [
	#include <linux/kernel.h>
	#include <linux/blkdev.h>
	], [
	struct request_queue *q __attribute__ ((unused)) = NULL;
	blk_queue_write_cache(q, true, true);
	], [], [ZFS_META_LICENSE])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_FLUSH], [
	AC_MSG_CHECKING([whether blk_queue_flush() is available])
	ZFS_LINUX_TEST_RESULT([blk_queue_flush], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLK_QUEUE_FLUSH, 1,
	[blk_queue_flush() is available])

	AC_MSG_CHECKING([whether blk_queue_flush() is GPL-only])
	ZFS_LINUX_TEST_RESULT([blk_queue_flush_license], [
	AC_MSG_RESULT(no)
	],[
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLK_QUEUE_FLUSH_GPL_ONLY, 1,
	[blk_queue_flush() is GPL-only])
	])
	],[
	AC_MSG_RESULT(no)
	])

	dnl #
	dnl # 4.7 API change
	dnl # Replace blk_queue_flush with blk_queue_write_cache
	dnl #
	AC_MSG_CHECKING([whether blk_queue_write_cache() exists])
	ZFS_LINUX_TEST_RESULT([blk_queue_write_cache], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLK_QUEUE_WRITE_CACHE, 1,
	[blk_queue_write_cache() exists])

	AC_MSG_CHECKING([whether blk_queue_write_cache() is GPL-only])
	ZFS_LINUX_TEST_RESULT([blk_queue_write_cache_license], [
	AC_MSG_RESULT(no)
	],[
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLK_QUEUE_WRITE_CACHE_GPL_ONLY, 1,
	[blk_queue_write_cache() is GPL-only])
	])
	],[
	AC_MSG_RESULT(no)
	])
	])

	dnl #
	dnl # 2.6.34 API change
	dnl # blk_queue_max_hw_sectors() replaces blk_queue_max_sectors().
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE_MAX_HW_SECTORS], [
	ZFS_LINUX_TEST_SRC([blk_queue_max_hw_sectors], [
	#include <linux/blkdev.h>
	], [
	struct request_queue *q __attribute__ ((unused)) = NULL;
	(void) blk_queue_max_hw_sectors(q, BLK_SAFE_MAX_SECTORS);
	], [])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_MAX_HW_SECTORS], [
	AC_MSG_CHECKING([whether blk_queue_max_hw_sectors() is available])
	ZFS_LINUX_TEST_RESULT([blk_queue_max_hw_sectors], [
	AC_MSG_RESULT(yes)
	],[
	ZFS_LINUX_TEST_ERROR([blk_queue_max_hw_sectors])
	])
	])

	dnl #
	dnl # 2.6.34 API change
	dnl # blk_queue_max_segments() consolidates blk_queue_max_hw_segments()
	dnl # and blk_queue_max_phys_segments().
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE_MAX_SEGMENTS], [
	ZFS_LINUX_TEST_SRC([blk_queue_max_segments], [
	#include <linux/blkdev.h>
	], [
	struct request_queue *q __attribute__ ((unused)) = NULL;
	(void) blk_queue_max_segments(q, BLK_MAX_SEGMENTS);
	], [])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_MAX_SEGMENTS], [
	AC_MSG_CHECKING([whether blk_queue_max_segments() is available])
	ZFS_LINUX_TEST_RESULT([blk_queue_max_segments], [
	AC_MSG_RESULT(yes)
	], [
	ZFS_LINUX_TEST_ERROR([blk_queue_max_segments])
	])
	])

	dnl #
	dnl # See if kernel supports block multi-queue and blk_status_t.
	dnl # blk_status_t represents the new status codes introduced in the 4.13
	dnl # kernel patch:
	dnl #
	dnl # block: introduce new block status code type
	dnl #
	dnl # We do not currently support the "old" block multi-queue interfaces from
	dnl # prior kernels.
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_MQ], [
	ZFS_LINUX_TEST_SRC([blk_mq], [
	#include <linux/blk-mq.h>
	], [
	struct blk_mq_tag_set tag_set __attribute__ ((unused)) = {0};
	(void) blk_mq_alloc_tag_set(&tag_set);
	return BLK_STS_OK;
	], [])
	+ ZFS_LINUX_TEST_SRC([blk_mq_rq_hctx], [
	+ #include <linux/blk-mq.h>
	+ #include <linux/blkdev.h>
	+ ], [
	+ struct request rq = {0};
	+ struct blk_mq_hw_ctx *hctx = NULL;
	+ rq.mq_hctx = hctx;
	+ ], [])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLK_MQ], [
	AC_MSG_CHECKING([whether block multiqueue with blk_status_t is available])
	ZFS_LINUX_TEST_RESULT([blk_mq], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLK_MQ, 1, [block multiqueue is available])
	+ AC_MSG_CHECKING([whether block multiqueue hardware context is cached in struct request])
	+ ZFS_LINUX_TEST_RESULT([blk_mq_rq_hctx], [
	+ AC_MSG_RESULT(yes)
	+ AC_DEFINE(HAVE_BLK_MQ_RQ_HCTX, 1, [block multiqueue hardware context is cached in struct request])
	+ ], [
	+ AC_MSG_RESULT(no)
	+ ])
	], [
	AC_MSG_RESULT(no)
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE], [
	ZFS_AC_KERNEL_SRC_BLK_QUEUE_PLUG
	ZFS_AC_KERNEL_SRC_BLK_QUEUE_BDI
	ZFS_AC_KERNEL_SRC_BLK_QUEUE_UPDATE_READAHEAD
	ZFS_AC_KERNEL_SRC_BLK_QUEUE_DISCARD
	ZFS_AC_KERNEL_SRC_BLK_QUEUE_SECURE_ERASE
	ZFS_AC_KERNEL_SRC_BLK_QUEUE_FLAG_SET
	ZFS_AC_KERNEL_SRC_BLK_QUEUE_FLAG_CLEAR
	ZFS_AC_KERNEL_SRC_BLK_QUEUE_FLUSH
	ZFS_AC_KERNEL_SRC_BLK_QUEUE_MAX_HW_SECTORS
	ZFS_AC_KERNEL_SRC_BLK_QUEUE_MAX_SEGMENTS
	ZFS_AC_KERNEL_SRC_BLK_MQ
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE], [
	ZFS_AC_KERNEL_BLK_QUEUE_PLUG
	ZFS_AC_KERNEL_BLK_QUEUE_BDI
	ZFS_AC_KERNEL_BLK_QUEUE_UPDATE_READAHEAD
	ZFS_AC_KERNEL_BLK_QUEUE_DISCARD
	ZFS_AC_KERNEL_BLK_QUEUE_SECURE_ERASE
	ZFS_AC_KERNEL_BLK_QUEUE_FLAG_SET
	ZFS_AC_KERNEL_BLK_QUEUE_FLAG_CLEAR
	ZFS_AC_KERNEL_BLK_QUEUE_FLUSH
	ZFS_AC_KERNEL_BLK_QUEUE_MAX_HW_SECTORS
	ZFS_AC_KERNEL_BLK_QUEUE_MAX_SEGMENTS
	ZFS_AC_KERNEL_BLK_MQ
	])
	diff --git a/sys/contrib/openzfs/config/kernel-blkdev.m4 b/sys/contrib/openzfs/config/kernel-blkdev.m4
	index c5a353ca9203..b6ce1e1cf083 100644
	--- a/sys/contrib/openzfs/config/kernel-blkdev.m4
	+++ b/sys/contrib/openzfs/config/kernel-blkdev.m4
	@@ -1,687 +1,778 @@
	dnl #
	dnl # 2.6.38 API change,
	dnl # Added blkdev_get_by_path()
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_GET_BY_PATH], [
	ZFS_LINUX_TEST_SRC([blkdev_get_by_path], [
	#include <linux/fs.h>
	#include <linux/blkdev.h>
	], [
	struct block_device *bdev __attribute__ ((unused)) = NULL;
	const char *path = "path";
	fmode_t mode = 0;
	void *holder = NULL;

	bdev = blkdev_get_by_path(path, mode, holder);
	])
	])

	dnl #
	dnl # 6.5.x API change,
	dnl # blkdev_get_by_path() takes 4 args
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_GET_BY_PATH_4ARG], [
	ZFS_LINUX_TEST_SRC([blkdev_get_by_path_4arg], [
	#include <linux/fs.h>
	#include <linux/blkdev.h>
	], [
	struct block_device *bdev __attribute__ ((unused)) = NULL;
	const char *path = "path";
	fmode_t mode = 0;
	void *holder = NULL;
	struct blk_holder_ops h;

	bdev = blkdev_get_by_path(path, mode, holder, &h);
	])
	])

	dnl #
	dnl # 6.8.x API change
	dnl # bdev_open_by_path() replaces blkdev_get_by_path()
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_OPEN_BY_PATH], [
	ZFS_LINUX_TEST_SRC([bdev_open_by_path], [
	#include <linux/fs.h>
	#include <linux/blkdev.h>
	], [
	struct bdev_handle *bdh __attribute__ ((unused)) = NULL;
	const char *path = "path";
	fmode_t mode = 0;
	void *holder = NULL;
	struct blk_holder_ops h;

	bdh = bdev_open_by_path(path, mode, holder, &h);
	])
	])

	+dnl #
	+dnl # 6.9.x API change
	+dnl # bdev_file_open_by_path() replaced bdev_open_by_path(),
	+dnl # and returns struct file*
	+dnl #
	+AC_DEFUN([ZFS_AC_KERNEL_SRC_BDEV_FILE_OPEN_BY_PATH], [
	+ ZFS_LINUX_TEST_SRC([bdev_file_open_by_path], [
	+ #include <linux/fs.h>
	+ #include <linux/blkdev.h>
	+ ], [
	+ struct file *file __attribute__ ((unused)) = NULL;
	+ const char *path = "path";
	+ fmode_t mode = 0;
	+ void *holder = NULL;
	+ struct blk_holder_ops h;
	+
	+ file = bdev_file_open_by_path(path, mode, holder, &h);
	+ ])
	+])
	+
	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_GET_BY_PATH], [
	AC_MSG_CHECKING([whether blkdev_get_by_path() exists and takes 3 args])
	ZFS_LINUX_TEST_RESULT([blkdev_get_by_path], [
	AC_MSG_RESULT(yes)
	], [
	AC_MSG_RESULT(no)
	AC_MSG_CHECKING([whether blkdev_get_by_path() exists and takes 4 args])
	ZFS_LINUX_TEST_RESULT([blkdev_get_by_path_4arg], [
	AC_DEFINE(HAVE_BLKDEV_GET_BY_PATH_4ARG, 1,
	[blkdev_get_by_path() exists and takes 4 args])
	AC_MSG_RESULT(yes)
	], [
	AC_MSG_RESULT(no)
	AC_MSG_CHECKING([whether bdev_open_by_path() exists])
	ZFS_LINUX_TEST_RESULT([bdev_open_by_path], [
	AC_DEFINE(HAVE_BDEV_OPEN_BY_PATH, 1,
	[bdev_open_by_path() exists])
	AC_MSG_RESULT(yes)
	], [
	- ZFS_LINUX_TEST_ERROR([blkdev_get_by_path()])
	+ AC_MSG_RESULT(no)
	+ AC_MSG_CHECKING([whether bdev_file_open_by_path() exists])
	+ ZFS_LINUX_TEST_RESULT([bdev_file_open_by_path], [
	+ AC_DEFINE(HAVE_BDEV_FILE_OPEN_BY_PATH, 1,
	+ [bdev_file_open_by_path() exists])
	+ AC_MSG_RESULT(yes)
	+ ], [
	+ AC_MSG_RESULT(no)
	+ ZFS_LINUX_TEST_ERROR([blkdev_get_by_path()])
	+ ])
	])
	])
	])
	])

	dnl #
	dnl # 6.5.x API change
	dnl # blk_mode_t was added as a type to supercede some places where fmode_t
	dnl # is used
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_BLK_MODE_T], [
	ZFS_LINUX_TEST_SRC([blk_mode_t], [
	#include <linux/fs.h>
	#include <linux/blkdev.h>
	], [
	blk_mode_t m __attribute((unused)) = (blk_mode_t)0;
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_BLK_MODE_T], [
	AC_MSG_CHECKING([whether blk_mode_t is defined])
	ZFS_LINUX_TEST_RESULT([blk_mode_t], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLK_MODE_T, 1, [blk_mode_t is defined])
	], [
	AC_MSG_RESULT(no)
	])
	])

	dnl #
	dnl # 2.6.38 API change,
	dnl # Added blkdev_put()
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_PUT], [
	ZFS_LINUX_TEST_SRC([blkdev_put], [
	#include <linux/fs.h>
	#include <linux/blkdev.h>
	], [
	struct block_device *bdev = NULL;
	fmode_t mode = 0;

	blkdev_put(bdev, mode);
	])
	])

	dnl #
	dnl # 6.5.x API change.
	dnl # blkdev_put() takes (void* holder) as arg 2
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_PUT_HOLDER], [
	ZFS_LINUX_TEST_SRC([blkdev_put_holder], [
	#include <linux/fs.h>
	#include <linux/blkdev.h>
	], [
	struct block_device *bdev = NULL;
	void *holder = NULL;

	blkdev_put(bdev, holder);
	])
	])

	dnl #
	dnl # 6.8.x API change
	dnl # bdev_release() replaces blkdev_put()
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_RELEASE], [
	ZFS_LINUX_TEST_SRC([bdev_release], [
	#include <linux/fs.h>
	#include <linux/blkdev.h>
	], [
	struct bdev_handle *bdh = NULL;
	bdev_release(bdh);
	])
	])

	+dnl #
	+dnl # 6.9.x API change
	+dnl #
	+dnl # bdev_release() now private, but because bdev_file_open_by_path() returns
	+dnl # struct file*, we can just use fput(). So the blkdev_put test no longer
	+dnl # fails if not found.
	+dnl #
	+
	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_PUT], [
	AC_MSG_CHECKING([whether blkdev_put() exists])
	ZFS_LINUX_TEST_RESULT([blkdev_put], [
	AC_MSG_RESULT(yes)
	+ AC_DEFINE(HAVE_BLKDEV_PUT, 1, [blkdev_put() exists])
	], [
	AC_MSG_RESULT(no)
	AC_MSG_CHECKING([whether blkdev_put() accepts void* as arg 2])
	ZFS_LINUX_TEST_RESULT([blkdev_put_holder], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLKDEV_PUT_HOLDER, 1,
	[blkdev_put() accepts void* as arg 2])
	], [
	AC_MSG_RESULT(no)
	AC_MSG_CHECKING([whether bdev_release() exists])
	ZFS_LINUX_TEST_RESULT([bdev_release], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BDEV_RELEASE, 1,
	[bdev_release() exists])
	], [
	- ZFS_LINUX_TEST_ERROR([blkdev_put()])
	+ AC_MSG_RESULT(no)
	])
	])
	])
	])

	dnl #
	dnl # 4.1 API, exported blkdev_reread_part() symbol, back ported to the
	dnl # 3.10.0 CentOS 7.x enterprise kernels.
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_REREAD_PART], [
	ZFS_LINUX_TEST_SRC([blkdev_reread_part], [
	#include <linux/fs.h>
	#include <linux/blkdev.h>
	], [
	struct block_device *bdev = NULL;
	int error;

	error = blkdev_reread_part(bdev);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_REREAD_PART], [
	AC_MSG_CHECKING([whether blkdev_reread_part() exists])
	ZFS_LINUX_TEST_RESULT([blkdev_reread_part], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLKDEV_REREAD_PART, 1,
	[blkdev_reread_part() exists])
	], [
	AC_MSG_RESULT(no)
	])
	])

	dnl #
	dnl # check_disk_change() was removed in 5.10
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_CHECK_DISK_CHANGE], [
	ZFS_LINUX_TEST_SRC([check_disk_change], [
	#include <linux/fs.h>
	#include <linux/blkdev.h>
	], [
	struct block_device *bdev = NULL;
	bool error;

	error = check_disk_change(bdev);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_CHECK_DISK_CHANGE], [
	AC_MSG_CHECKING([whether check_disk_change() exists])
	ZFS_LINUX_TEST_RESULT([check_disk_change], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_CHECK_DISK_CHANGE, 1,
	[check_disk_change() exists])
	], [
	AC_MSG_RESULT(no)
	])
	])

	dnl #
	dnl # 6.5.x API change
	dnl # disk_check_media_change() was added
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_DISK_CHECK_MEDIA_CHANGE], [
	ZFS_LINUX_TEST_SRC([disk_check_media_change], [
	#include <linux/fs.h>
	#include <linux/blkdev.h>
	], [
	struct block_device *bdev = NULL;
	bool error;

	error = disk_check_media_change(bdev->bd_disk);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_DISK_CHECK_MEDIA_CHANGE], [
	AC_MSG_CHECKING([whether disk_check_media_change() exists])
	ZFS_LINUX_TEST_RESULT([disk_check_media_change], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_DISK_CHECK_MEDIA_CHANGE, 1,
	[disk_check_media_change() exists])
	], [
	AC_MSG_RESULT(no)
	])
	])

	dnl #
	dnl # bdev_kobj() is introduced from 5.12
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_KOBJ], [
	ZFS_LINUX_TEST_SRC([bdev_kobj], [
	#include <linux/fs.h>
	#include <linux/blkdev.h>
	#include <linux/kobject.h>
	], [
	struct block_device *bdev = NULL;
	struct kobject *disk_kobj;
	disk_kobj = bdev_kobj(bdev);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_BDEV_KOBJ], [
	AC_MSG_CHECKING([whether bdev_kobj() exists])
	ZFS_LINUX_TEST_RESULT([bdev_kobj], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BDEV_KOBJ, 1,
	[bdev_kobj() exists])
	], [
	AC_MSG_RESULT(no)
	])
	])

	dnl #
	dnl # part_to_dev() was removed in 5.12
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_PART_TO_DEV], [
	ZFS_LINUX_TEST_SRC([part_to_dev], [
	#include <linux/fs.h>
	#include <linux/blkdev.h>
	], [
	struct hd_struct *p = NULL;
	struct device *pdev;
	pdev = part_to_dev(p);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_PART_TO_DEV], [
	AC_MSG_CHECKING([whether part_to_dev() exists])
	ZFS_LINUX_TEST_RESULT([part_to_dev], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_PART_TO_DEV, 1,
	[part_to_dev() exists])
	], [
	AC_MSG_RESULT(no)
	])
	])

	dnl #
	dnl # 5.10 API, check_disk_change() is removed, in favor of
	dnl # bdev_check_media_change(), which doesn't force revalidation
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_CHECK_MEDIA_CHANGE], [
	ZFS_LINUX_TEST_SRC([bdev_check_media_change], [
	#include <linux/fs.h>
	#include <linux/blkdev.h>
	], [
	struct block_device *bdev = NULL;
	int error;

	error = bdev_check_media_change(bdev);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_BDEV_CHECK_MEDIA_CHANGE], [
	AC_MSG_CHECKING([whether bdev_check_media_change() exists])
	ZFS_LINUX_TEST_RESULT([bdev_check_media_change], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BDEV_CHECK_MEDIA_CHANGE, 1,
	[bdev_check_media_change() exists])
	], [
	AC_MSG_RESULT(no)
	])
	])

	dnl #
	dnl # 2.6.22 API change
	dnl # Single argument invalidate_bdev()
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_INVALIDATE_BDEV], [
	ZFS_LINUX_TEST_SRC([invalidate_bdev], [
	#include <linux/buffer_head.h>
	#include <linux/blkdev.h>
	],[
	struct block_device *bdev = NULL;
	invalidate_bdev(bdev);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_INVALIDATE_BDEV], [
	AC_MSG_CHECKING([whether invalidate_bdev() exists])
	ZFS_LINUX_TEST_RESULT([invalidate_bdev], [
	AC_MSG_RESULT(yes)
	],[
	ZFS_LINUX_TEST_ERROR([invalidate_bdev()])
	])
	])

	dnl #
	dnl # 5.11 API, lookup_bdev() takes dev_t argument.
	dnl # 2.6.27 API, lookup_bdev() was first exported.
	dnl # 4.4.0-6.21 API, lookup_bdev() on Ubuntu takes mode argument.
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_LOOKUP_BDEV], [
	ZFS_LINUX_TEST_SRC([lookup_bdev_devt], [
	#include <linux/blkdev.h>
	], [
	int error __attribute__ ((unused));
	const char path[] = "/example/path";
	dev_t dev;

	error = lookup_bdev(path, &dev);
	])

	ZFS_LINUX_TEST_SRC([lookup_bdev_1arg], [
	#include <linux/fs.h>
	#include <linux/blkdev.h>
	], [
	struct block_device *bdev __attribute__ ((unused));
	const char path[] = "/example/path";

	bdev = lookup_bdev(path);
	])

	ZFS_LINUX_TEST_SRC([lookup_bdev_mode], [
	#include <linux/fs.h>
	], [
	struct block_device *bdev __attribute__ ((unused));
	const char path[] = "/example/path";

	bdev = lookup_bdev(path, FMODE_READ);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_LOOKUP_BDEV], [
	AC_MSG_CHECKING([whether lookup_bdev() wants dev_t arg])
	ZFS_LINUX_TEST_RESULT_SYMBOL([lookup_bdev_devt],
	[lookup_bdev], [fs/block_dev.c], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_DEVT_LOOKUP_BDEV, 1,
	[lookup_bdev() wants dev_t arg])
	], [
	AC_MSG_RESULT(no)

	AC_MSG_CHECKING([whether lookup_bdev() wants 1 arg])
	ZFS_LINUX_TEST_RESULT_SYMBOL([lookup_bdev_1arg],
	[lookup_bdev], [fs/block_dev.c], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_1ARG_LOOKUP_BDEV, 1,
	[lookup_bdev() wants 1 arg])
	], [
	AC_MSG_RESULT(no)

	AC_MSG_CHECKING([whether lookup_bdev() wants mode arg])
	ZFS_LINUX_TEST_RESULT_SYMBOL([lookup_bdev_mode],
	[lookup_bdev], [fs/block_dev.c], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_MODE_LOOKUP_BDEV, 1,
	[lookup_bdev() wants mode arg])
	], [
	ZFS_LINUX_TEST_ERROR([lookup_bdev()])
	])
	])
	])
	])

	dnl #
	dnl # 2.6.30 API change
	dnl #
	dnl # The bdev_physical_block_size() interface was added to provide a way
	dnl # to determine the smallest write which can be performed without a
	dnl # read-modify-write operation.
	dnl #
	dnl # Unfortunately, this interface isn't entirely reliable because
	dnl # drives are sometimes known to misreport this value.
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_PHYSICAL_BLOCK_SIZE], [
	ZFS_LINUX_TEST_SRC([bdev_physical_block_size], [
	#include <linux/blkdev.h>
	],[
	struct block_device *bdev __attribute__ ((unused)) = NULL;
	bdev_physical_block_size(bdev);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_BDEV_PHYSICAL_BLOCK_SIZE], [
	AC_MSG_CHECKING([whether bdev_physical_block_size() is available])
	ZFS_LINUX_TEST_RESULT([bdev_physical_block_size], [
	AC_MSG_RESULT(yes)
	],[
	ZFS_LINUX_TEST_ERROR([bdev_physical_block_size()])
	])
	])

	dnl #
	dnl # 2.6.30 API change
	dnl # Added bdev_logical_block_size().
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_LOGICAL_BLOCK_SIZE], [
	ZFS_LINUX_TEST_SRC([bdev_logical_block_size], [
	#include <linux/blkdev.h>
	],[
	struct block_device *bdev __attribute__ ((unused)) = NULL;
	bdev_logical_block_size(bdev);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_BDEV_LOGICAL_BLOCK_SIZE], [
	AC_MSG_CHECKING([whether bdev_logical_block_size() is available])
	ZFS_LINUX_TEST_RESULT([bdev_logical_block_size], [
	AC_MSG_RESULT(yes)
	],[
	ZFS_LINUX_TEST_ERROR([bdev_logical_block_size()])
	])
	])

	dnl #
	dnl # 5.11 API change
	dnl # Added bdev_whole() helper.
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_WHOLE], [
	ZFS_LINUX_TEST_SRC([bdev_whole], [
	#include <linux/blkdev.h>
	],[
	struct block_device *bdev = NULL;
	bdev = bdev_whole(bdev);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_BDEV_WHOLE], [
	AC_MSG_CHECKING([whether bdev_whole() is available])
	ZFS_LINUX_TEST_RESULT([bdev_whole], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BDEV_WHOLE, 1, [bdev_whole() is available])
	],[
	AC_MSG_RESULT(no)
	])
	])

	dnl #
	dnl # 5.20 API change,
	dnl # Removed bdevname(), snprintf(.., %pg) should be used.
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_BDEVNAME], [
	ZFS_LINUX_TEST_SRC([bdevname], [
	#include <linux/fs.h>
	#include <linux/blkdev.h>
	], [
	struct block_device *bdev __attribute__ ((unused)) = NULL;
	char path[BDEVNAME_SIZE];

	(void) bdevname(bdev, path);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_BDEVNAME], [
	AC_MSG_CHECKING([whether bdevname() exists])
	ZFS_LINUX_TEST_RESULT([bdevname], [
	AC_DEFINE(HAVE_BDEVNAME, 1, [bdevname() is available])
	AC_MSG_RESULT(yes)
	], [
	AC_MSG_RESULT(no)
	])
	])

	dnl #
	-dnl # 5.19 API: blkdev_issue_secure_erase()
	-dnl # 4.7 API: __blkdev_issue_discard(..., BLKDEV_DISCARD_SECURE)
	-dnl # 3.10 API: blkdev_issue_discard(..., BLKDEV_DISCARD_SECURE)
	+dnl # TRIM support: discard and secure erase. We make use of asynchronous
	+dnl # functions when available.
	dnl #
	-AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_ISSUE_SECURE_ERASE], [
	- ZFS_LINUX_TEST_SRC([blkdev_issue_secure_erase], [
	+dnl # 3.10:
	+dnl # sync discard: blkdev_issue_discard(..., 0)
	+dnl # sync erase: blkdev_issue_discard(..., BLKDEV_DISCARD_SECURE)
	+dnl # async discard: [not available]
	+dnl # async erase: [not available]
	+dnl #
	+dnl # 4.7:
	+dnl # sync discard: blkdev_issue_discard(..., 0)
	+dnl # sync erase: blkdev_issue_discard(..., BLKDEV_DISCARD_SECURE)
	+dnl # async discard: __blkdev_issue_discard(..., 0)
	+dnl # async erase: __blkdev_issue_discard(..., BLKDEV_DISCARD_SECURE)
	+dnl #
	+dnl # 5.19:
	+dnl # sync discard: blkdev_issue_discard(...)
	+dnl # sync erase: blkdev_issue_secure_erase(...)
	+dnl # async discard: __blkdev_issue_discard(...)
	+dnl # async erase: [not available]
	+dnl #
	+AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_ISSUE_DISCARD], [
	+ ZFS_LINUX_TEST_SRC([blkdev_issue_discard_noflags], [
	#include <linux/blkdev.h>
	],[
	struct block_device *bdev = NULL;
	sector_t sector = 0;
	sector_t nr_sects = 0;
	int error __attribute__ ((unused));

	- error = blkdev_issue_secure_erase(bdev,
	+ error = blkdev_issue_discard(bdev,
	sector, nr_sects, GFP_KERNEL);
	])
	+ ZFS_LINUX_TEST_SRC([blkdev_issue_discard_flags], [
	+ #include <linux/blkdev.h>
	+ ],[
	+ struct block_device *bdev = NULL;
	+ sector_t sector = 0;
	+ sector_t nr_sects = 0;
	+ unsigned long flags = 0;
	+ int error __attribute__ ((unused));

	+ error = blkdev_issue_discard(bdev,
	+ sector, nr_sects, GFP_KERNEL, flags);
	+ ])
	+ ZFS_LINUX_TEST_SRC([blkdev_issue_discard_async_noflags], [
	+ #include <linux/blkdev.h>
	+ ],[
	+ struct block_device *bdev = NULL;
	+ sector_t sector = 0;
	+ sector_t nr_sects = 0;
	+ struct bio *biop = NULL;
	+ int error __attribute__ ((unused));
	+
	+ error = __blkdev_issue_discard(bdev,
	+ sector, nr_sects, GFP_KERNEL, &biop);
	+ ])
	ZFS_LINUX_TEST_SRC([blkdev_issue_discard_async_flags], [
	#include <linux/blkdev.h>
	],[
	struct block_device *bdev = NULL;
	sector_t sector = 0;
	sector_t nr_sects = 0;
	unsigned long flags = 0;
	struct bio *biop = NULL;
	int error __attribute__ ((unused));

	error = __blkdev_issue_discard(bdev,
	sector, nr_sects, GFP_KERNEL, flags, &biop);
	])
	-
	- ZFS_LINUX_TEST_SRC([blkdev_issue_discard_flags], [
	+ ZFS_LINUX_TEST_SRC([blkdev_issue_secure_erase], [
	#include <linux/blkdev.h>
	],[
	struct block_device *bdev = NULL;
	sector_t sector = 0;
	sector_t nr_sects = 0;
	- unsigned long flags = 0;
	int error __attribute__ ((unused));

	- error = blkdev_issue_discard(bdev,
	- sector, nr_sects, GFP_KERNEL, flags);
	+ error = blkdev_issue_secure_erase(bdev,
	+ sector, nr_sects, GFP_KERNEL);
	])
	])

	-AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_ISSUE_SECURE_ERASE], [
	+AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_ISSUE_DISCARD], [
	+ AC_MSG_CHECKING([whether blkdev_issue_discard() is available])
	+ ZFS_LINUX_TEST_RESULT([blkdev_issue_discard_noflags], [
	+ AC_MSG_RESULT(yes)
	+ AC_DEFINE(HAVE_BLKDEV_ISSUE_DISCARD_NOFLAGS, 1,
	+ [blkdev_issue_discard() is available])
	+ ],[
	+ AC_MSG_RESULT(no)
	+ ])
	+ AC_MSG_CHECKING([whether blkdev_issue_discard(flags) is available])
	+ ZFS_LINUX_TEST_RESULT([blkdev_issue_discard_flags], [
	+ AC_MSG_RESULT(yes)
	+ AC_DEFINE(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS, 1,
	+ [blkdev_issue_discard(flags) is available])
	+ ],[
	+ AC_MSG_RESULT(no)
	+ ])
	+ AC_MSG_CHECKING([whether __blkdev_issue_discard() is available])
	+ ZFS_LINUX_TEST_RESULT([blkdev_issue_discard_async_noflags], [
	+ AC_MSG_RESULT(yes)
	+ AC_DEFINE(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_NOFLAGS, 1,
	+ [__blkdev_issue_discard() is available])
	+ ],[
	+ AC_MSG_RESULT(no)
	+ ])
	+ AC_MSG_CHECKING([whether __blkdev_issue_discard(flags) is available])
	+ ZFS_LINUX_TEST_RESULT([blkdev_issue_discard_async_flags], [
	+ AC_MSG_RESULT(yes)
	+ AC_DEFINE(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS, 1,
	+ [__blkdev_issue_discard(flags) is available])
	+ ],[
	+ AC_MSG_RESULT(no)
	+ ])
	AC_MSG_CHECKING([whether blkdev_issue_secure_erase() is available])
	ZFS_LINUX_TEST_RESULT([blkdev_issue_secure_erase], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLKDEV_ISSUE_SECURE_ERASE, 1,
	[blkdev_issue_secure_erase() is available])
	],[
	AC_MSG_RESULT(no)
	-
	- AC_MSG_CHECKING([whether __blkdev_issue_discard() is available])
	- ZFS_LINUX_TEST_RESULT([blkdev_issue_discard_async_flags], [
	- AC_MSG_RESULT(yes)
	- AC_DEFINE(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC, 1,
	- [__blkdev_issue_discard() is available])
	- ],[
	- AC_MSG_RESULT(no)
	-
	- AC_MSG_CHECKING([whether blkdev_issue_discard() is available])
	- ZFS_LINUX_TEST_RESULT([blkdev_issue_discard_flags], [
	- AC_MSG_RESULT(yes)
	- AC_DEFINE(HAVE_BLKDEV_ISSUE_DISCARD, 1,
	- [blkdev_issue_discard() is available])
	- ],[
	- ZFS_LINUX_TEST_ERROR([blkdev_issue_discard()])
	- ])
	- ])
	])
	])

	dnl #
	dnl # 5.13 API change
	dnl # blkdev_get_by_path() no longer handles ERESTARTSYS
	dnl #
	dnl # Unfortunately we're forced to rely solely on the kernel version
	dnl # number in order to determine the expected behavior. This was an
	dnl # internal change to blkdev_get_by_dev(), see commit a8ed1a0607.
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_GET_ERESTARTSYS], [
	AC_MSG_CHECKING([whether blkdev_get_by_path() handles ERESTARTSYS])
	AS_VERSION_COMPARE([$LINUX_VERSION], [5.13.0], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_BLKDEV_GET_ERESTARTSYS, 1,
	[blkdev_get_by_path() handles ERESTARTSYS])
	],[
	AC_MSG_RESULT(no)
	],[
	AC_MSG_RESULT(no)
	])
	])

	dnl #
	dnl # 6.5.x API change
	dnl # BLK_STS_NEXUS replaced with BLK_STS_RESV_CONFLICT
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_BLK_STS_RESV_CONFLICT], [
	ZFS_LINUX_TEST_SRC([blk_sts_resv_conflict], [
	#include <linux/blkdev.h>
	],[
	blk_status_t s __attribute__ ((unused)) = BLK_STS_RESV_CONFLICT;
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_BLK_STS_RESV_CONFLICT], [
	AC_MSG_CHECKING([whether BLK_STS_RESV_CONFLICT is defined])
	ZFS_LINUX_TEST_RESULT([blk_sts_resv_conflict], [
	AC_DEFINE(HAVE_BLK_STS_RESV_CONFLICT, 1, [BLK_STS_RESV_CONFLICT is defined])
	AC_MSG_RESULT(yes)
	], [
	AC_MSG_RESULT(no)
	])
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV], [
	ZFS_AC_KERNEL_SRC_BLKDEV_GET_BY_PATH
	ZFS_AC_KERNEL_SRC_BLKDEV_GET_BY_PATH_4ARG
	ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_OPEN_BY_PATH
	+ ZFS_AC_KERNEL_SRC_BDEV_FILE_OPEN_BY_PATH
	ZFS_AC_KERNEL_SRC_BLKDEV_PUT
	ZFS_AC_KERNEL_SRC_BLKDEV_PUT_HOLDER
	ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_RELEASE
	ZFS_AC_KERNEL_SRC_BLKDEV_REREAD_PART
	ZFS_AC_KERNEL_SRC_BLKDEV_INVALIDATE_BDEV
	ZFS_AC_KERNEL_SRC_BLKDEV_LOOKUP_BDEV
	ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_LOGICAL_BLOCK_SIZE
	ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_PHYSICAL_BLOCK_SIZE
	ZFS_AC_KERNEL_SRC_BLKDEV_CHECK_DISK_CHANGE
	ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_CHECK_MEDIA_CHANGE
	ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_WHOLE
	ZFS_AC_KERNEL_SRC_BLKDEV_BDEVNAME
	- ZFS_AC_KERNEL_SRC_BLKDEV_ISSUE_SECURE_ERASE
	+ ZFS_AC_KERNEL_SRC_BLKDEV_ISSUE_DISCARD
	ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_KOBJ
	ZFS_AC_KERNEL_SRC_BLKDEV_PART_TO_DEV
	ZFS_AC_KERNEL_SRC_BLKDEV_DISK_CHECK_MEDIA_CHANGE
	ZFS_AC_KERNEL_SRC_BLKDEV_BLK_STS_RESV_CONFLICT
	ZFS_AC_KERNEL_SRC_BLKDEV_BLK_MODE_T
	])

	AC_DEFUN([ZFS_AC_KERNEL_BLKDEV], [
	ZFS_AC_KERNEL_BLKDEV_GET_BY_PATH
	ZFS_AC_KERNEL_BLKDEV_PUT
	ZFS_AC_KERNEL_BLKDEV_REREAD_PART
	ZFS_AC_KERNEL_BLKDEV_INVALIDATE_BDEV
	ZFS_AC_KERNEL_BLKDEV_LOOKUP_BDEV
	ZFS_AC_KERNEL_BLKDEV_BDEV_LOGICAL_BLOCK_SIZE
	ZFS_AC_KERNEL_BLKDEV_BDEV_PHYSICAL_BLOCK_SIZE
	ZFS_AC_KERNEL_BLKDEV_CHECK_DISK_CHANGE
	ZFS_AC_KERNEL_BLKDEV_BDEV_CHECK_MEDIA_CHANGE
	ZFS_AC_KERNEL_BLKDEV_BDEV_WHOLE
	ZFS_AC_KERNEL_BLKDEV_BDEVNAME
	ZFS_AC_KERNEL_BLKDEV_GET_ERESTARTSYS
	- ZFS_AC_KERNEL_BLKDEV_ISSUE_SECURE_ERASE
	+ ZFS_AC_KERNEL_BLKDEV_ISSUE_DISCARD
	ZFS_AC_KERNEL_BLKDEV_BDEV_KOBJ
	ZFS_AC_KERNEL_BLKDEV_PART_TO_DEV
	ZFS_AC_KERNEL_BLKDEV_DISK_CHECK_MEDIA_CHANGE
	ZFS_AC_KERNEL_BLKDEV_BLK_STS_RESV_CONFLICT
	ZFS_AC_KERNEL_BLKDEV_BLK_MODE_T
	])
	diff --git a/sys/contrib/openzfs/config/kernel-filemap.m4 b/sys/contrib/openzfs/config/kernel-filemap.m4
	index 745928168f92..0b7da828d299 100644
	--- a/sys/contrib/openzfs/config/kernel-filemap.m4
	+++ b/sys/contrib/openzfs/config/kernel-filemap.m4
	@@ -1,26 +1,27 @@
	dnl #
	dnl # filemap_range_has_page was not available till 4.13
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_FILEMAP], [
	ZFS_LINUX_TEST_SRC([filemap_range_has_page], [
	#include <linux/fs.h>
	+ #include <linux/pagemap.h>
	],[
	struct address_space *mapping = NULL;
	loff_t lstart = 0;
	loff_t lend = 0;
	bool ret __attribute__ ((unused));

	ret = filemap_range_has_page(mapping, lstart, lend);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_FILEMAP], [
	AC_MSG_CHECKING([whether filemap_range_has_page() is available])
	ZFS_LINUX_TEST_RESULT([filemap_range_has_page], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_FILEMAP_RANGE_HAS_PAGE, 1,
	[filemap_range_has_page() is available])
	],[
	AC_MSG_RESULT(no)
	])
	])
	diff --git a/sys/contrib/openzfs/config/kernel-make-request-fn.m4 b/sys/contrib/openzfs/config/kernel-make-request-fn.m4
	index 4d20dd45c4a1..9813ad2fb3f3 100644
	--- a/sys/contrib/openzfs/config/kernel-make-request-fn.m4
	+++ b/sys/contrib/openzfs/config/kernel-make-request-fn.m4
	@@ -1,180 +1,213 @@
	dnl #
	dnl # Check for make_request_fn interface.
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_MAKE_REQUEST_FN], [
	ZFS_LINUX_TEST_SRC([make_request_fn_void], [
	#include <linux/blkdev.h>
	static void make_request(struct request_queue *q,
	struct bio *bio) { return; }
	],[
	blk_queue_make_request(NULL, &make_request);
	])

	ZFS_LINUX_TEST_SRC([make_request_fn_blk_qc_t], [
	#include <linux/blkdev.h>
	static blk_qc_t make_request(struct request_queue *q,
	struct bio *bio) { return (BLK_QC_T_NONE); }
	],[
	blk_queue_make_request(NULL, &make_request);
	])

	ZFS_LINUX_TEST_SRC([blk_alloc_queue_request_fn], [
	#include <linux/blkdev.h>
	static blk_qc_t make_request(struct request_queue *q,
	struct bio *bio) { return (BLK_QC_T_NONE); }
	],[
	struct request_queue *q __attribute__ ((unused));
	q = blk_alloc_queue(make_request, NUMA_NO_NODE);
	])

	ZFS_LINUX_TEST_SRC([blk_alloc_queue_request_fn_rh], [
	#include <linux/blkdev.h>
	static blk_qc_t make_request(struct request_queue *q,
	struct bio *bio) { return (BLK_QC_T_NONE); }
	],[
	struct request_queue *q __attribute__ ((unused));
	q = blk_alloc_queue_rh(make_request, NUMA_NO_NODE);
	])

	ZFS_LINUX_TEST_SRC([block_device_operations_submit_bio], [
	#include <linux/blkdev.h>
	],[
	struct block_device_operations o;
	o.submit_bio = NULL;
	])

	ZFS_LINUX_TEST_SRC([blk_alloc_disk], [
	#include <linux/blkdev.h>
	],[
	struct gendisk *disk __attribute__ ((unused));
	disk = blk_alloc_disk(NUMA_NO_NODE);
	])

	+ ZFS_LINUX_TEST_SRC([blk_alloc_disk_2arg], [
	+ #include <linux/blkdev.h>
	+ ],[
	+ struct queue_limits *lim = NULL;
	+ struct gendisk *disk __attribute__ ((unused));
	+ disk = blk_alloc_disk(lim, NUMA_NO_NODE);
	+ ])
	+
	ZFS_LINUX_TEST_SRC([blk_cleanup_disk], [
	#include <linux/blkdev.h>
	],[
	struct gendisk *disk __attribute__ ((unused));
	blk_cleanup_disk(disk);
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_MAKE_REQUEST_FN], [
	dnl # Checked as part of the blk_alloc_queue_request_fn test
	dnl #
	dnl # Linux 5.9 API Change
	dnl # make_request_fn was moved into block_device_operations->submit_bio
	dnl #
	AC_MSG_CHECKING([whether submit_bio is member of struct block_device_operations])
	ZFS_LINUX_TEST_RESULT([block_device_operations_submit_bio], [
	AC_MSG_RESULT(yes)

	AC_DEFINE(HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS, 1,
	[submit_bio is member of struct block_device_operations])

	dnl #
	dnl # Linux 5.14 API Change:
	dnl # blk_alloc_queue() + alloc_disk() combo replaced by
	dnl # a single call to blk_alloc_disk().
	dnl #
	AC_MSG_CHECKING([whether blk_alloc_disk() exists])
	ZFS_LINUX_TEST_RESULT([blk_alloc_disk], [
	AC_MSG_RESULT(yes)
	AC_DEFINE([HAVE_BLK_ALLOC_DISK], 1, [blk_alloc_disk() exists])

	dnl #
	dnl # 5.20 API change,
	dnl # Removed blk_cleanup_disk(), put_disk() should be used.
	dnl #
	AC_MSG_CHECKING([whether blk_cleanup_disk() exists])
	ZFS_LINUX_TEST_RESULT([blk_cleanup_disk], [
	AC_MSG_RESULT(yes)
	AC_DEFINE([HAVE_BLK_CLEANUP_DISK], 1,
	[blk_cleanup_disk() exists])
	], [
	AC_MSG_RESULT(no)
	])
	], [
	AC_MSG_RESULT(no)
	])
	+
	+ dnl #
	+ dnl # Linux 6.9 API Change:
	+ dnl # blk_alloc_queue() takes a nullable queue_limits arg.
	+ dnl #
	+ AC_MSG_CHECKING([whether blk_alloc_disk() exists and takes 2 args])
	+ ZFS_LINUX_TEST_RESULT([blk_alloc_disk_2arg], [
	+ AC_MSG_RESULT(yes)
	+ AC_DEFINE([HAVE_BLK_ALLOC_DISK_2ARG], 1, [blk_alloc_disk() exists and takes 2 args])
	+
	+ dnl #
	+ dnl # 5.20 API change,
	+ dnl # Removed blk_cleanup_disk(), put_disk() should be used.
	+ dnl #
	+ AC_MSG_CHECKING([whether blk_cleanup_disk() exists])
	+ ZFS_LINUX_TEST_RESULT([blk_cleanup_disk], [
	+ AC_MSG_RESULT(yes)
	+ AC_DEFINE([HAVE_BLK_CLEANUP_DISK], 1,
	+ [blk_cleanup_disk() exists])
	+ ], [
	+ AC_MSG_RESULT(no)
	+ ])
	+ ], [
	+ AC_MSG_RESULT(no)
	+ ])
	],[
	AC_MSG_RESULT(no)

	dnl # Checked as part of the blk_alloc_queue_request_fn test
	dnl #
	dnl # Linux 5.7 API Change
	dnl # blk_alloc_queue() expects request function.
	dnl #
	AC_MSG_CHECKING([whether blk_alloc_queue() expects request function])
	ZFS_LINUX_TEST_RESULT([blk_alloc_queue_request_fn], [
	AC_MSG_RESULT(yes)

	dnl # This is currently always the case.
	AC_MSG_CHECKING([whether make_request_fn() returns blk_qc_t])
	AC_MSG_RESULT(yes)

	AC_DEFINE(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN, 1,
	[blk_alloc_queue() expects request function])
	AC_DEFINE(MAKE_REQUEST_FN_RET, blk_qc_t,
	[make_request_fn() return type])
	AC_DEFINE(HAVE_MAKE_REQUEST_FN_RET_QC, 1,
	[Noting that make_request_fn() returns blk_qc_t])
	],[
	dnl #
	dnl # CentOS Stream 4.18.0-257 API Change
	dnl # The Linux 5.7 blk_alloc_queue() change was back-
	dnl # ported and the symbol renamed blk_alloc_queue_rh().
	dnl # As of this kernel version they're not providing
	dnl # any compatibility code in the kernel for this.
	dnl #
	ZFS_LINUX_TEST_RESULT([blk_alloc_queue_request_fn_rh], [
	AC_MSG_RESULT(yes)

	dnl # This is currently always the case.
	AC_MSG_CHECKING([whether make_request_fn_rh() returns blk_qc_t])
	AC_MSG_RESULT(yes)

	AC_DEFINE(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN_RH, 1,
	[blk_alloc_queue_rh() expects request function])
	AC_DEFINE(MAKE_REQUEST_FN_RET, blk_qc_t,
	[make_request_fn() return type])
	AC_DEFINE(HAVE_MAKE_REQUEST_FN_RET_QC, 1,
	[Noting that make_request_fn() returns blk_qc_t])
	],[
	AC_MSG_RESULT(no)

	dnl #
	dnl # Linux 3.2 API Change
	dnl # make_request_fn returns void.
	dnl #
	AC_MSG_CHECKING(
	[whether make_request_fn() returns void])
	ZFS_LINUX_TEST_RESULT([make_request_fn_void], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(MAKE_REQUEST_FN_RET, void,
	[make_request_fn() return type])
	AC_DEFINE(HAVE_MAKE_REQUEST_FN_RET_VOID, 1,
	[Noting that make_request_fn() returns void])
	],[
	AC_MSG_RESULT(no)

	dnl #
	dnl # Linux 4.4 API Change
	dnl # make_request_fn returns blk_qc_t.
	dnl #
	AC_MSG_CHECKING(
	[whether make_request_fn() returns blk_qc_t])
	ZFS_LINUX_TEST_RESULT([make_request_fn_blk_qc_t], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(MAKE_REQUEST_FN_RET, blk_qc_t,
	[make_request_fn() return type])
	AC_DEFINE(HAVE_MAKE_REQUEST_FN_RET_QC, 1,
	[Noting that make_request_fn() ]
	[returns blk_qc_t])
	],[
	ZFS_LINUX_TEST_ERROR([make_request_fn])
	])
	])
	])
	])
	])
	])
	diff --git a/sys/contrib/openzfs/config/kernel-mm-page-size.m4 b/sys/contrib/openzfs/config/kernel-mm-page-size.m4
	new file mode 100644
	index 000000000000..d5ebd926986a
	--- /dev/null
	+++ b/sys/contrib/openzfs/config/kernel-mm-page-size.m4
	@@ -0,0 +1,17 @@
	+AC_DEFUN([ZFS_AC_KERNEL_SRC_MM_PAGE_SIZE], [
	+ ZFS_LINUX_TEST_SRC([page_size], [
	+ #include <linux/mm.h>
	+ ],[
	+ unsigned long s;
	+ s = page_size(NULL);
	+ ])
	+])
	+AC_DEFUN([ZFS_AC_KERNEL_MM_PAGE_SIZE], [
	+ AC_MSG_CHECKING([whether page_size() is available])
	+ ZFS_LINUX_TEST_RESULT([page_size], [
	+ AC_MSG_RESULT(yes)
	+ AC_DEFINE(HAVE_MM_PAGE_SIZE, 1, [page_size() is available])
	+ ],[
	+ AC_MSG_RESULT(no)
	+ ])
	+])
	diff --git a/sys/contrib/openzfs/config/kernel-vfs-file_range.m4 b/sys/contrib/openzfs/config/kernel-vfs-file_range.m4
	index cc96404d8bbe..8a5cbe2eeeed 100644
	--- a/sys/contrib/openzfs/config/kernel-vfs-file_range.m4
	+++ b/sys/contrib/openzfs/config/kernel-vfs-file_range.m4
	@@ -1,164 +1,191 @@
	dnl #
	dnl # The *_file_range APIs have a long history:
	dnl #
	dnl # 2.6.29: BTRFS_IOC_CLONE and BTRFS_IOC_CLONE_RANGE ioctl introduced
	dnl # 3.12: BTRFS_IOC_FILE_EXTENT_SAME ioctl introduced
	dnl #
	dnl # 4.5: copy_file_range() syscall introduced, added to VFS
	dnl # 4.5: BTRFS_IOC_CLONE and BTRFS_IOC_CLONE_RANGE renamed to FICLONE ands
	dnl # FICLONERANGE, added to VFS as clone_file_range()
	dnl # 4.5: BTRFS_IOC_FILE_EXTENT_SAME renamed to FIDEDUPERANGE, added to VFS
	dnl # as dedupe_file_range()
	dnl #
	dnl # 4.20: VFS clone_file_range() and dedupe_file_range() replaced by
	dnl # remap_file_range()
	dnl #
	dnl # 5.3: VFS copy_file_range() expected to do its own fallback,
	dnl # generic_copy_file_range() added to support it
	dnl #
	+dnl # 6.8: generic_copy_file_range() removed, replaced by
	+dnl # splice_copy_file_range()
	+dnl #
	AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_COPY_FILE_RANGE], [
	ZFS_LINUX_TEST_SRC([vfs_copy_file_range], [
	#include <linux/fs.h>

	static ssize_t test_copy_file_range(struct file *src_file,
	loff_t src_off, struct file *dst_file, loff_t dst_off,
	size_t len, unsigned int flags) {
	(void) src_file; (void) src_off;
	(void) dst_file; (void) dst_off;
	(void) len; (void) flags;
	return (0);
	}

	static const struct file_operations
	fops __attribute__ ((unused)) = {
	.copy_file_range = test_copy_file_range,
	};
	],[])
	])
	AC_DEFUN([ZFS_AC_KERNEL_VFS_COPY_FILE_RANGE], [
	AC_MSG_CHECKING([whether fops->copy_file_range() is available])
	ZFS_LINUX_TEST_RESULT([vfs_copy_file_range], [
	AC_MSG_RESULT([yes])
	AC_DEFINE(HAVE_VFS_COPY_FILE_RANGE, 1,
	[fops->copy_file_range() is available])
	],[
	AC_MSG_RESULT([no])
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_GENERIC_COPY_FILE_RANGE], [
	ZFS_LINUX_TEST_SRC([generic_copy_file_range], [
	#include <linux/fs.h>
	], [
	struct file *src_file __attribute__ ((unused)) = NULL;
	loff_t src_off __attribute__ ((unused)) = 0;
	struct file *dst_file __attribute__ ((unused)) = NULL;
	loff_t dst_off __attribute__ ((unused)) = 0;
	size_t len __attribute__ ((unused)) = 0;
	unsigned int flags __attribute__ ((unused)) = 0;
	generic_copy_file_range(src_file, src_off, dst_file, dst_off,
	len, flags);
	])
	])
	AC_DEFUN([ZFS_AC_KERNEL_VFS_GENERIC_COPY_FILE_RANGE], [
	AC_MSG_CHECKING([whether generic_copy_file_range() is available])
	ZFS_LINUX_TEST_RESULT_SYMBOL([generic_copy_file_range],
	[generic_copy_file_range], [fs/read_write.c], [
	AC_MSG_RESULT(yes)
	AC_DEFINE(HAVE_VFS_GENERIC_COPY_FILE_RANGE, 1,
	[generic_copy_file_range() is available])
	],[
	AC_MSG_RESULT(no)
	])
	])

	+AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_SPLICE_COPY_FILE_RANGE], [
	+ ZFS_LINUX_TEST_SRC([splice_copy_file_range], [
	+ #include <linux/splice.h>
	+ ], [
	+ struct file *src_file __attribute__ ((unused)) = NULL;
	+ loff_t src_off __attribute__ ((unused)) = 0;
	+ struct file *dst_file __attribute__ ((unused)) = NULL;
	+ loff_t dst_off __attribute__ ((unused)) = 0;
	+ size_t len __attribute__ ((unused)) = 0;
	+ splice_copy_file_range(src_file, src_off, dst_file, dst_off,
	+ len);
	+ ])
	+])
	+AC_DEFUN([ZFS_AC_KERNEL_VFS_SPLICE_COPY_FILE_RANGE], [
	+ AC_MSG_CHECKING([whether splice_copy_file_range() is available])
	+ ZFS_LINUX_TEST_RESULT([splice_copy_file_range], [
	+ AC_MSG_RESULT(yes)
	+ AC_DEFINE(HAVE_VFS_SPLICE_COPY_FILE_RANGE, 1,
	+ [splice_copy_file_range() is available])
	+ ],[
	+ AC_MSG_RESULT(no)
	+ ])
	+])
	+
	AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_CLONE_FILE_RANGE], [
	ZFS_LINUX_TEST_SRC([vfs_clone_file_range], [
	#include <linux/fs.h>

	static int test_clone_file_range(struct file *src_file,
	loff_t src_off, struct file *dst_file, loff_t dst_off,
	u64 len) {
	(void) src_file; (void) src_off;
	(void) dst_file; (void) dst_off;
	(void) len;
	return (0);
	}

	static const struct file_operations
	fops __attribute__ ((unused)) = {
	.clone_file_range = test_clone_file_range,
	};
	],[])
	])
	AC_DEFUN([ZFS_AC_KERNEL_VFS_CLONE_FILE_RANGE], [
	AC_MSG_CHECKING([whether fops->clone_file_range() is available])
	ZFS_LINUX_TEST_RESULT([vfs_clone_file_range], [
	AC_MSG_RESULT([yes])
	AC_DEFINE(HAVE_VFS_CLONE_FILE_RANGE, 1,
	[fops->clone_file_range() is available])
	],[
	AC_MSG_RESULT([no])
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_DEDUPE_FILE_RANGE], [
	ZFS_LINUX_TEST_SRC([vfs_dedupe_file_range], [
	#include <linux/fs.h>

	static int test_dedupe_file_range(struct file *src_file,
	loff_t src_off, struct file *dst_file, loff_t dst_off,
	u64 len) {
	(void) src_file; (void) src_off;
	(void) dst_file; (void) dst_off;
	(void) len;
	return (0);
	}

	static const struct file_operations
	fops __attribute__ ((unused)) = {
	.dedupe_file_range = test_dedupe_file_range,
	};
	],[])
	])
	AC_DEFUN([ZFS_AC_KERNEL_VFS_DEDUPE_FILE_RANGE], [
	AC_MSG_CHECKING([whether fops->dedupe_file_range() is available])
	ZFS_LINUX_TEST_RESULT([vfs_dedupe_file_range], [
	AC_MSG_RESULT([yes])
	AC_DEFINE(HAVE_VFS_DEDUPE_FILE_RANGE, 1,
	[fops->dedupe_file_range() is available])
	],[
	AC_MSG_RESULT([no])
	])
	])

	AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_REMAP_FILE_RANGE], [
	ZFS_LINUX_TEST_SRC([vfs_remap_file_range], [
	#include <linux/fs.h>

	static loff_t test_remap_file_range(struct file *src_file,
	loff_t src_off, struct file *dst_file, loff_t dst_off,
	loff_t len, unsigned int flags) {
	(void) src_file; (void) src_off;
	(void) dst_file; (void) dst_off;
	(void) len; (void) flags;
	return (0);
	}

	static const struct file_operations
	fops __attribute__ ((unused)) = {
	.remap_file_range = test_remap_file_range,
	};
	],[])
	])

	AC_DEFUN([ZFS_AC_KERNEL_VFS_REMAP_FILE_RANGE], [
	AC_MSG_CHECKING([whether fops->remap_file_range() is available])
	ZFS_LINUX_TEST_RESULT([vfs_remap_file_range], [
	AC_MSG_RESULT([yes])
	AC_DEFINE(HAVE_VFS_REMAP_FILE_RANGE, 1,
	[fops->remap_file_range() is available])
	],[
	AC_MSG_RESULT([no])
	])
	])
	diff --git a/sys/contrib/openzfs/config/kernel.m4 b/sys/contrib/openzfs/config/kernel.m4
	index e3f8645774c5..548905ccd04d 100644
	--- a/sys/contrib/openzfs/config/kernel.m4
	+++ b/sys/contrib/openzfs/config/kernel.m4
	@@ -1,1042 +1,1046 @@
	dnl #
	dnl # Default ZFS kernel configuration
	dnl #
	AC_DEFUN([ZFS_AC_CONFIG_KERNEL], [
	AM_COND_IF([BUILD_LINUX], [
	dnl # Setup the kernel build environment.
	ZFS_AC_KERNEL
	ZFS_AC_QAT

	dnl # Sanity checks for module building and CONFIG_* defines
	ZFS_AC_KERNEL_CONFIG_DEFINED
	ZFS_AC_MODULE_SYMVERS

	dnl # Sequential ZFS_LINUX_TRY_COMPILE tests
	ZFS_AC_KERNEL_FPU_HEADER
	ZFS_AC_KERNEL_OBJTOOL_HEADER
	ZFS_AC_KERNEL_WAIT_QUEUE_ENTRY_T
	ZFS_AC_KERNEL_MISC_MINOR
	ZFS_AC_KERNEL_DECLARE_EVENT_CLASS

	dnl # Parallel ZFS_LINUX_TEST_SRC / ZFS_LINUX_TEST_RESULT tests
	ZFS_AC_KERNEL_TEST_SRC
	ZFS_AC_KERNEL_TEST_RESULT

	AS_IF([test "$LINUX_OBJ" != "$LINUX"], [
	KERNEL_MAKE="$KERNEL_MAKE O=$LINUX_OBJ"
	])

	AC_SUBST(KERNEL_MAKE)
	])
	])

	dnl #
	dnl # Generate and compile all of the kernel API test cases to determine
	dnl # which interfaces are available. By invoking the kernel build system
	dnl # only once the compilation can be done in parallel significantly
	dnl # speeding up the process.
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [
	ZFS_AC_KERNEL_SRC_OBJTOOL
	ZFS_AC_KERNEL_SRC_GLOBAL_PAGE_STATE
	ZFS_AC_KERNEL_SRC_ACCESS_OK_TYPE
	ZFS_AC_KERNEL_SRC_PDE_DATA
	ZFS_AC_KERNEL_SRC_FALLOCATE
	ZFS_AC_KERNEL_SRC_FADVISE
	ZFS_AC_KERNEL_SRC_GENERIC_FADVISE
	ZFS_AC_KERNEL_SRC_2ARGS_ZLIB_DEFLATE_WORKSPACESIZE
	ZFS_AC_KERNEL_SRC_RWSEM
	ZFS_AC_KERNEL_SRC_SCHED
	ZFS_AC_KERNEL_SRC_USLEEP_RANGE
	ZFS_AC_KERNEL_SRC_KMEM_CACHE
	ZFS_AC_KERNEL_SRC_KVMALLOC
	ZFS_AC_KERNEL_SRC_VMALLOC_PAGE_KERNEL
	ZFS_AC_KERNEL_SRC_WAIT
	ZFS_AC_KERNEL_SRC_INODE_TIMES
	ZFS_AC_KERNEL_SRC_INODE_LOCK
	ZFS_AC_KERNEL_SRC_GROUP_INFO_GID
	ZFS_AC_KERNEL_SRC_RW
	ZFS_AC_KERNEL_SRC_TIMER_SETUP
	ZFS_AC_KERNEL_SRC_SUPER_USER_NS
	ZFS_AC_KERNEL_SRC_PROC_OPERATIONS
	ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS
	ZFS_AC_KERNEL_SRC_BIO
	ZFS_AC_KERNEL_SRC_BLKDEV
	ZFS_AC_KERNEL_SRC_BLK_QUEUE
	ZFS_AC_KERNEL_SRC_GENHD_FLAGS
	ZFS_AC_KERNEL_SRC_REVALIDATE_DISK
	ZFS_AC_KERNEL_SRC_GET_DISK_RO
	ZFS_AC_KERNEL_SRC_GENERIC_READLINK_GLOBAL
	ZFS_AC_KERNEL_SRC_DISCARD_GRANULARITY
	ZFS_AC_KERNEL_SRC_INODE_OWNER_OR_CAPABLE
	ZFS_AC_KERNEL_SRC_XATTR
	ZFS_AC_KERNEL_SRC_ACL
	ZFS_AC_KERNEL_SRC_INODE_SETATTR
	ZFS_AC_KERNEL_SRC_INODE_GETATTR
	ZFS_AC_KERNEL_SRC_INODE_SET_FLAGS
	ZFS_AC_KERNEL_SRC_INODE_SET_IVERSION
	ZFS_AC_KERNEL_SRC_SHOW_OPTIONS
	ZFS_AC_KERNEL_SRC_FILE_INODE
	ZFS_AC_KERNEL_SRC_FILE_DENTRY
	ZFS_AC_KERNEL_SRC_FSYNC
	ZFS_AC_KERNEL_SRC_AIO_FSYNC
	ZFS_AC_KERNEL_SRC_EVICT_INODE
	ZFS_AC_KERNEL_SRC_DIRTY_INODE
	ZFS_AC_KERNEL_SRC_SHRINKER
	ZFS_AC_KERNEL_SRC_MKDIR
	ZFS_AC_KERNEL_SRC_LOOKUP_FLAGS
	ZFS_AC_KERNEL_SRC_CREATE
	ZFS_AC_KERNEL_SRC_PERMISSION
	ZFS_AC_KERNEL_SRC_GET_LINK
	ZFS_AC_KERNEL_SRC_PUT_LINK
	ZFS_AC_KERNEL_SRC_TMPFILE
	ZFS_AC_KERNEL_SRC_AUTOMOUNT
	ZFS_AC_KERNEL_SRC_ENCODE_FH_WITH_INODE
	ZFS_AC_KERNEL_SRC_COMMIT_METADATA
	ZFS_AC_KERNEL_SRC_CLEAR_INODE
	ZFS_AC_KERNEL_SRC_SETATTR_PREPARE
	ZFS_AC_KERNEL_SRC_INSERT_INODE_LOCKED
	ZFS_AC_KERNEL_SRC_DENTRY
	ZFS_AC_KERNEL_SRC_DENTRY_ALIAS_D_U
	ZFS_AC_KERNEL_SRC_TRUNCATE_SETSIZE
	ZFS_AC_KERNEL_SRC_SECURITY_INODE
	ZFS_AC_KERNEL_SRC_FST_MOUNT
	ZFS_AC_KERNEL_SRC_BDI
	ZFS_AC_KERNEL_SRC_SET_NLINK
	ZFS_AC_KERNEL_SRC_SGET
	ZFS_AC_KERNEL_SRC_LSEEK_EXECUTE
	ZFS_AC_KERNEL_SRC_VFS_FILEMAP_DIRTY_FOLIO
	ZFS_AC_KERNEL_SRC_VFS_READ_FOLIO
	ZFS_AC_KERNEL_SRC_VFS_GETATTR
	ZFS_AC_KERNEL_SRC_VFS_FSYNC_2ARGS
	ZFS_AC_KERNEL_SRC_VFS_ITERATE
	ZFS_AC_KERNEL_SRC_VFS_DIRECT_IO
	ZFS_AC_KERNEL_SRC_VFS_READPAGES
	ZFS_AC_KERNEL_SRC_VFS_SET_PAGE_DIRTY_NOBUFFERS
	ZFS_AC_KERNEL_SRC_VFS_RW_ITERATE
	ZFS_AC_KERNEL_SRC_VFS_GENERIC_WRITE_CHECKS
	ZFS_AC_KERNEL_SRC_VFS_IOV_ITER
	ZFS_AC_KERNEL_SRC_VFS_COPY_FILE_RANGE
	ZFS_AC_KERNEL_SRC_VFS_GENERIC_COPY_FILE_RANGE
	+ ZFS_AC_KERNEL_SRC_VFS_SPLICE_COPY_FILE_RANGE
	ZFS_AC_KERNEL_SRC_VFS_REMAP_FILE_RANGE
	ZFS_AC_KERNEL_SRC_VFS_CLONE_FILE_RANGE
	ZFS_AC_KERNEL_SRC_VFS_DEDUPE_FILE_RANGE
	ZFS_AC_KERNEL_SRC_VFS_FILE_OPERATIONS_EXTEND
	ZFS_AC_KERNEL_SRC_KMAP_ATOMIC_ARGS
	ZFS_AC_KERNEL_SRC_FOLLOW_DOWN_ONE
	ZFS_AC_KERNEL_SRC_MAKE_REQUEST_FN
	ZFS_AC_KERNEL_SRC_GENERIC_IO_ACCT
	ZFS_AC_KERNEL_SRC_FPU
	ZFS_AC_KERNEL_SRC_FMODE_T
	ZFS_AC_KERNEL_SRC_KUIDGID_T
	ZFS_AC_KERNEL_SRC_KUID_HELPERS
	ZFS_AC_KERNEL_SRC_RENAME
	ZFS_AC_KERNEL_SRC_CURRENT_TIME
	ZFS_AC_KERNEL_SRC_USERNS_CAPABILITIES
	ZFS_AC_KERNEL_SRC_IN_COMPAT_SYSCALL
	ZFS_AC_KERNEL_SRC_KTIME
	ZFS_AC_KERNEL_SRC_TOTALRAM_PAGES_FUNC
	ZFS_AC_KERNEL_SRC_TOTALHIGH_PAGES
	ZFS_AC_KERNEL_SRC_KSTRTOUL
	ZFS_AC_KERNEL_SRC_PERCPU
	ZFS_AC_KERNEL_SRC_CPU_HOTPLUG
	ZFS_AC_KERNEL_SRC_GENERIC_FILLATTR
	ZFS_AC_KERNEL_SRC_MKNOD
	ZFS_AC_KERNEL_SRC_SYMLINK
	ZFS_AC_KERNEL_SRC_BIO_MAX_SEGS
	ZFS_AC_KERNEL_SRC_SIGNAL_STOP
	ZFS_AC_KERNEL_SRC_SIGINFO
	ZFS_AC_KERNEL_SRC_SYSFS
	ZFS_AC_KERNEL_SRC_SET_SPECIAL_STATE
	ZFS_AC_KERNEL_SRC_STANDALONE_LINUX_STDARG
	ZFS_AC_KERNEL_SRC_STRLCPY
	ZFS_AC_KERNEL_SRC_STRSCPY
	ZFS_AC_KERNEL_SRC_PAGEMAP_FOLIO_WAIT_BIT
	ZFS_AC_KERNEL_SRC_ADD_DISK
	ZFS_AC_KERNEL_SRC_KTHREAD
	ZFS_AC_KERNEL_SRC_ZERO_PAGE
	ZFS_AC_KERNEL_SRC___COPY_FROM_USER_INATOMIC
	ZFS_AC_KERNEL_SRC_USER_NS_COMMON_INUM
	ZFS_AC_KERNEL_SRC_IDMAP_MNT_API
	ZFS_AC_KERNEL_SRC_IDMAP_NO_USERNS
	ZFS_AC_KERNEL_SRC_IATTR_VFSID
	ZFS_AC_KERNEL_SRC_FILEMAP
	ZFS_AC_KERNEL_SRC_WRITEPAGE_T
	ZFS_AC_KERNEL_SRC_RECLAIMED
	ZFS_AC_KERNEL_SRC_REGISTER_SYSCTL_TABLE
	ZFS_AC_KERNEL_SRC_COPY_SPLICE_READ
	ZFS_AC_KERNEL_SRC_SYNC_BDEV
	+ ZFS_AC_KERNEL_SRC_MM_PAGE_SIZE
	case "$host_cpu" in
	powerpc*)
	ZFS_AC_KERNEL_SRC_CPU_HAS_FEATURE
	ZFS_AC_KERNEL_SRC_FLUSH_DCACHE_PAGE
	;;
	riscv*)
	ZFS_AC_KERNEL_SRC_FLUSH_DCACHE_PAGE
	;;
	esac

	AC_MSG_CHECKING([for available kernel interfaces])
	ZFS_LINUX_TEST_COMPILE_ALL([kabi])
	AC_MSG_RESULT([done])
	])

	dnl #
	dnl # Check results of kernel interface tests.
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [
	ZFS_AC_KERNEL_ACCESS_OK_TYPE
	ZFS_AC_KERNEL_GLOBAL_PAGE_STATE
	ZFS_AC_KERNEL_OBJTOOL
	ZFS_AC_KERNEL_PDE_DATA
	ZFS_AC_KERNEL_FALLOCATE
	ZFS_AC_KERNEL_FADVISE
	ZFS_AC_KERNEL_GENERIC_FADVISE
	ZFS_AC_KERNEL_2ARGS_ZLIB_DEFLATE_WORKSPACESIZE
	ZFS_AC_KERNEL_RWSEM
	ZFS_AC_KERNEL_SCHED
	ZFS_AC_KERNEL_USLEEP_RANGE
	ZFS_AC_KERNEL_KMEM_CACHE
	ZFS_AC_KERNEL_KVMALLOC
	ZFS_AC_KERNEL_VMALLOC_PAGE_KERNEL
	ZFS_AC_KERNEL_WAIT
	ZFS_AC_KERNEL_INODE_TIMES
	ZFS_AC_KERNEL_INODE_LOCK
	ZFS_AC_KERNEL_GROUP_INFO_GID
	ZFS_AC_KERNEL_RW
	ZFS_AC_KERNEL_TIMER_SETUP
	ZFS_AC_KERNEL_SUPER_USER_NS
	ZFS_AC_KERNEL_PROC_OPERATIONS
	ZFS_AC_KERNEL_BLOCK_DEVICE_OPERATIONS
	ZFS_AC_KERNEL_BIO
	ZFS_AC_KERNEL_BLKDEV
	ZFS_AC_KERNEL_BLK_QUEUE
	ZFS_AC_KERNEL_GENHD_FLAGS
	ZFS_AC_KERNEL_REVALIDATE_DISK
	ZFS_AC_KERNEL_GET_DISK_RO
	ZFS_AC_KERNEL_GENERIC_READLINK_GLOBAL
	ZFS_AC_KERNEL_DISCARD_GRANULARITY
	ZFS_AC_KERNEL_INODE_OWNER_OR_CAPABLE
	ZFS_AC_KERNEL_XATTR
	ZFS_AC_KERNEL_ACL
	ZFS_AC_KERNEL_INODE_SETATTR
	ZFS_AC_KERNEL_INODE_GETATTR
	ZFS_AC_KERNEL_INODE_SET_FLAGS
	ZFS_AC_KERNEL_INODE_SET_IVERSION
	ZFS_AC_KERNEL_SHOW_OPTIONS
	ZFS_AC_KERNEL_FILE_INODE
	ZFS_AC_KERNEL_FILE_DENTRY
	ZFS_AC_KERNEL_FSYNC
	ZFS_AC_KERNEL_AIO_FSYNC
	ZFS_AC_KERNEL_EVICT_INODE
	ZFS_AC_KERNEL_DIRTY_INODE
	ZFS_AC_KERNEL_SHRINKER
	ZFS_AC_KERNEL_MKDIR
	ZFS_AC_KERNEL_LOOKUP_FLAGS
	ZFS_AC_KERNEL_CREATE
	ZFS_AC_KERNEL_PERMISSION
	ZFS_AC_KERNEL_GET_LINK
	ZFS_AC_KERNEL_PUT_LINK
	ZFS_AC_KERNEL_TMPFILE
	ZFS_AC_KERNEL_AUTOMOUNT
	ZFS_AC_KERNEL_ENCODE_FH_WITH_INODE
	ZFS_AC_KERNEL_COMMIT_METADATA
	ZFS_AC_KERNEL_CLEAR_INODE
	ZFS_AC_KERNEL_SETATTR_PREPARE
	ZFS_AC_KERNEL_INSERT_INODE_LOCKED
	ZFS_AC_KERNEL_DENTRY
	ZFS_AC_KERNEL_DENTRY_ALIAS_D_U
	ZFS_AC_KERNEL_TRUNCATE_SETSIZE
	ZFS_AC_KERNEL_SECURITY_INODE
	ZFS_AC_KERNEL_FST_MOUNT
	ZFS_AC_KERNEL_BDI
	ZFS_AC_KERNEL_SET_NLINK
	ZFS_AC_KERNEL_SGET
	ZFS_AC_KERNEL_LSEEK_EXECUTE
	ZFS_AC_KERNEL_VFS_FILEMAP_DIRTY_FOLIO
	ZFS_AC_KERNEL_VFS_READ_FOLIO
	ZFS_AC_KERNEL_VFS_GETATTR
	ZFS_AC_KERNEL_VFS_FSYNC_2ARGS
	ZFS_AC_KERNEL_VFS_ITERATE
	ZFS_AC_KERNEL_VFS_DIRECT_IO
	ZFS_AC_KERNEL_VFS_READPAGES
	ZFS_AC_KERNEL_VFS_SET_PAGE_DIRTY_NOBUFFERS
	ZFS_AC_KERNEL_VFS_RW_ITERATE
	ZFS_AC_KERNEL_VFS_GENERIC_WRITE_CHECKS
	ZFS_AC_KERNEL_VFS_IOV_ITER
	ZFS_AC_KERNEL_VFS_COPY_FILE_RANGE
	ZFS_AC_KERNEL_VFS_GENERIC_COPY_FILE_RANGE
	+ ZFS_AC_KERNEL_VFS_SPLICE_COPY_FILE_RANGE
	ZFS_AC_KERNEL_VFS_REMAP_FILE_RANGE
	ZFS_AC_KERNEL_VFS_CLONE_FILE_RANGE
	ZFS_AC_KERNEL_VFS_DEDUPE_FILE_RANGE
	ZFS_AC_KERNEL_VFS_FILE_OPERATIONS_EXTEND
	ZFS_AC_KERNEL_KMAP_ATOMIC_ARGS
	ZFS_AC_KERNEL_FOLLOW_DOWN_ONE
	ZFS_AC_KERNEL_MAKE_REQUEST_FN
	ZFS_AC_KERNEL_GENERIC_IO_ACCT
	ZFS_AC_KERNEL_FPU
	ZFS_AC_KERNEL_FMODE_T
	ZFS_AC_KERNEL_KUIDGID_T
	ZFS_AC_KERNEL_KUID_HELPERS
	ZFS_AC_KERNEL_RENAME
	ZFS_AC_KERNEL_CURRENT_TIME
	ZFS_AC_KERNEL_USERNS_CAPABILITIES
	ZFS_AC_KERNEL_IN_COMPAT_SYSCALL
	ZFS_AC_KERNEL_KTIME
	ZFS_AC_KERNEL_TOTALRAM_PAGES_FUNC
	ZFS_AC_KERNEL_TOTALHIGH_PAGES
	ZFS_AC_KERNEL_KSTRTOUL
	ZFS_AC_KERNEL_PERCPU
	ZFS_AC_KERNEL_CPU_HOTPLUG
	ZFS_AC_KERNEL_GENERIC_FILLATTR
	ZFS_AC_KERNEL_MKNOD
	ZFS_AC_KERNEL_SYMLINK
	ZFS_AC_KERNEL_BIO_MAX_SEGS
	ZFS_AC_KERNEL_SIGNAL_STOP
	ZFS_AC_KERNEL_SIGINFO
	ZFS_AC_KERNEL_SYSFS
	ZFS_AC_KERNEL_SET_SPECIAL_STATE
	ZFS_AC_KERNEL_STANDALONE_LINUX_STDARG
	ZFS_AC_KERNEL_STRLCPY
	ZFS_AC_KERNEL_STRSCPY
	ZFS_AC_KERNEL_PAGEMAP_FOLIO_WAIT_BIT
	ZFS_AC_KERNEL_ADD_DISK
	ZFS_AC_KERNEL_KTHREAD
	ZFS_AC_KERNEL_ZERO_PAGE
	ZFS_AC_KERNEL___COPY_FROM_USER_INATOMIC
	ZFS_AC_KERNEL_USER_NS_COMMON_INUM
	ZFS_AC_KERNEL_IDMAP_MNT_API
	ZFS_AC_KERNEL_IDMAP_NO_USERNS
	ZFS_AC_KERNEL_IATTR_VFSID
	ZFS_AC_KERNEL_FILEMAP
	ZFS_AC_KERNEL_WRITEPAGE_T
	ZFS_AC_KERNEL_RECLAIMED
	ZFS_AC_KERNEL_REGISTER_SYSCTL_TABLE
	ZFS_AC_KERNEL_COPY_SPLICE_READ
	ZFS_AC_KERNEL_SYNC_BDEV
	+ ZFS_AC_KERNEL_MM_PAGE_SIZE
	case "$host_cpu" in
	powerpc*)
	ZFS_AC_KERNEL_CPU_HAS_FEATURE
	ZFS_AC_KERNEL_FLUSH_DCACHE_PAGE
	;;
	riscv*)
	ZFS_AC_KERNEL_FLUSH_DCACHE_PAGE
	;;
	esac
	])

	dnl #
	dnl # Detect name used for Module.symvers file in kernel
	dnl #
	AC_DEFUN([ZFS_AC_MODULE_SYMVERS], [
	modpost=$LINUX/scripts/Makefile.modpost
	AC_MSG_CHECKING([kernel file name for module symbols])
	AS_IF([test "x$enable_linux_builtin" != xyes -a -f "$modpost"], [
	AS_IF([grep -q Modules.symvers $modpost], [
	LINUX_SYMBOLS=Modules.symvers
	], [
	LINUX_SYMBOLS=Module.symvers
	])

	AS_IF([test ! -f "$LINUX_OBJ/$LINUX_SYMBOLS"], [
	AC_MSG_ERROR([
	*** Please make sure the kernel devel package for your distribution
	*** is installed. If you are building with a custom kernel, make sure
	*** the kernel is configured, built, and the '--with-linux=PATH'
	*** configure option refers to the location of the kernel source.
	])
	])
	], [
	LINUX_SYMBOLS=NONE
	])
	AC_MSG_RESULT($LINUX_SYMBOLS)
	AC_SUBST(LINUX_SYMBOLS)
	])

	dnl #
	dnl # Detect the kernel to be built against
	dnl #
	dnl # Most modern Linux distributions have separate locations for bare
	dnl # source (source) and prebuilt (build) files. Additionally, there are
	dnl # `source` and `build` symlinks in `/lib/modules/$(KERNEL_VERSION)`
	dnl # pointing to them. The directory search order is now:
	dnl #
	dnl # - `configure` command line values if both `--with-linux` and
	dnl # `--with-linux-obj` were defined
	dnl #
	dnl # - If only `--with-linux` was defined, `--with-linux-obj` is assumed
	dnl # to have the same value as `--with-linux`
	dnl #
	dnl # - If neither `--with-linux` nor `--with-linux-obj` were defined
	dnl # autodetection is used:
	dnl #
	dnl # - `/lib/modules/$(uname -r)/{source,build}` respectively, if exist.
	dnl #
	dnl # - If only `/lib/modules/$(uname -r)/build` exists, it is assumed
	dnl # to be both source and build directory.
	dnl #
	dnl # - The first directory in `/lib/modules` with the highest version
	dnl # number according to `sort -V` which contains both `source` and
	dnl # `build` symlinks/directories. If module directory contains only
	dnl # `build` component, it is assumed to be both source and build
	dnl # directory.
	dnl #
	dnl # - Last resort: the first directory matching `/usr/src/kernels/*`
	dnl # and `/usr/src/linux-*` with the highest version number according
	dnl # to `sort -V` is assumed to be both source and build directory.
	dnl #
	AC_DEFUN([ZFS_AC_KERNEL], [
	AC_ARG_WITH([linux],
	AS_HELP_STRING([--with-linux=PATH],
	[Path to kernel source]),
	[kernelsrc="$withval"])

	AC_ARG_WITH(linux-obj,
	AS_HELP_STRING([--with-linux-obj=PATH],
	[Path to kernel build objects]),
	[kernelbuild="$withval"])

	AC_MSG_CHECKING([kernel source and build directories])
	AS_IF([test -n "$kernelsrc" && test -z "$kernelbuild"], [
	kernelbuild="$kernelsrc"
	], [test -z "$kernelsrc"], [
	AS_IF([test -e "/lib/modules/$(uname -r)/source" && \
	test -e "/lib/modules/$(uname -r)/build"], [
	src="/lib/modules/$(uname -r)/source"
	build="/lib/modules/$(uname -r)/build"
	], [test -e "/lib/modules/$(uname -r)/build"], [
	build="/lib/modules/$(uname -r)/build"
	src="$build"
	], [
	src=

	for d in $(ls -1d /lib/modules/* 2>/dev/null \| sort -Vr); do
	if test -e "$d/source" && test -e "$d/build"; then
	src="$d/source"
	build="$d/build"
	break
	fi

	if test -e "$d/build"; then
	src="$d/build"
	build="$d/build"
	break
	fi
	done

	# the least reliable method
	if test -z "$src"; then
	src=$(ls -1d /usr/src/kernels/* /usr/src/linux-* \
	2>/dev/null \| grep -v obj \| sort -Vr \| head -1)
	build="$src"
	fi
	])

	AS_IF([test -n "$src" && test -e "$src"], [
	kernelsrc=$(readlink -e "$src")
	], [
	kernelsrc="[Not found]"
	])
	AS_IF([test -n "$build" && test -e "$build"], [
	kernelbuild=$(readlink -e "$build")
	], [
	kernelbuild="[Not found]"
	])
	], [
	AS_IF([test "$kernelsrc" = "NONE"], [
	kernsrcver=NONE
	])
	withlinux=yes
	])

	AC_MSG_RESULT([done])
	AC_MSG_CHECKING([kernel source directory])
	AC_MSG_RESULT([$kernelsrc])
	AC_MSG_CHECKING([kernel build directory])
	AC_MSG_RESULT([$kernelbuild])
	AS_IF([test ! -d "$kernelsrc" \|\| test ! -d "$kernelbuild"], [
	AC_MSG_ERROR([
	*** Please make sure the kernel devel package for your distribution
	*** is installed and then try again. If that fails, you can specify the
	*** location of the kernel source and build with the '--with-linux=PATH' and
	*** '--with-linux-obj=PATH' options respectively.])
	])

	AC_MSG_CHECKING([kernel source version])
	utsrelease1=$kernelbuild/include/linux/version.h
	utsrelease2=$kernelbuild/include/linux/utsrelease.h
	utsrelease3=$kernelbuild/include/generated/utsrelease.h
	AS_IF([test -r $utsrelease1 && grep -qF UTS_RELEASE $utsrelease1], [
	utsrelease=$utsrelease1
	], [test -r $utsrelease2 && grep -qF UTS_RELEASE $utsrelease2], [
	utsrelease=$utsrelease2
	], [test -r $utsrelease3 && grep -qF UTS_RELEASE $utsrelease3], [
	utsrelease=$utsrelease3
	])

	AS_IF([test -n "$utsrelease"], [
	kernsrcver=$($AWK '/UTS_RELEASE/ { gsub(/"/, "", $[3]); print $[3] }' $utsrelease)
	AS_IF([test -z "$kernsrcver"], [
	AC_MSG_RESULT([Not found])
	AC_MSG_ERROR([
	*** Cannot determine kernel version.
	])
	])
	], [
	AC_MSG_RESULT([Not found])
	if test "x$enable_linux_builtin" != xyes; then
	AC_MSG_ERROR([
	*** Cannot find UTS_RELEASE definition.
	])
	else
	AC_MSG_ERROR([
	*** Cannot find UTS_RELEASE definition.
	*** Please run 'make prepare' inside the kernel source tree.])
	fi
	])

	AC_MSG_RESULT([$kernsrcver])

	AS_VERSION_COMPARE([$kernsrcver], [$ZFS_META_KVER_MIN], [
	AC_MSG_ERROR([
	*** Cannot build against kernel version $kernsrcver.
	*** The minimum supported kernel version is $ZFS_META_KVER_MIN.
	])
	])

	LINUX=${kernelsrc}
	LINUX_OBJ=${kernelbuild}
	LINUX_VERSION=${kernsrcver}

	AC_SUBST(LINUX)
	AC_SUBST(LINUX_OBJ)
	AC_SUBST(LINUX_VERSION)
	])

	dnl #
	dnl # Detect the QAT module to be built against, QAT provides hardware
	dnl # acceleration for data compression:
	dnl #
	dnl # https://01.org/intel-quickassist-technology
	dnl #
	dnl # 1) Download and install QAT driver from the above link
	dnl # 2) Start QAT driver in your system:
	dnl # service qat_service start
	dnl # 3) Enable QAT in ZFS, e.g.:
	dnl # ./configure --with-qat=<qat-driver-path>/QAT1.6
	dnl # make
	dnl # 4) Set GZIP compression in ZFS dataset:
	dnl # zfs set compression = gzip <dataset>
	dnl #
	dnl # Then the data written to this ZFS pool is compressed by QAT accelerator
	dnl # automatically, and de-compressed by QAT when read from the pool.
	dnl #
	dnl # 1) Get QAT hardware statistics with:
	dnl # cat /proc/icp_dh895xcc_dev/qat
	dnl # 2) To disable QAT:
	dnl # insmod zfs.ko zfs_qat_disable=1
	dnl #
	AC_DEFUN([ZFS_AC_QAT], [
	AC_ARG_WITH([qat],
	AS_HELP_STRING([--with-qat=PATH],
	[Path to qat source]),
	AS_IF([test "$withval" = "yes"],
	AC_MSG_ERROR([--with-qat=PATH requires a PATH]),
	[qatsrc="$withval"]))

	AC_ARG_WITH([qat-obj],
	AS_HELP_STRING([--with-qat-obj=PATH],
	[Path to qat build objects]),
	[qatbuild="$withval"])

	AS_IF([test ! -z "${qatsrc}"], [
	AC_MSG_CHECKING([qat source directory])
	AC_MSG_RESULT([$qatsrc])
	QAT_SRC="${qatsrc}/quickassist"
	AS_IF([ test ! -e "$QAT_SRC/include/cpa.h"], [
	AC_MSG_ERROR([
	*** Please make sure the qat driver package is installed
	*** and specify the location of the qat source with the
	*** '--with-qat=PATH' option then try again. Failed to
	*** find cpa.h in:
	${QAT_SRC}/include])
	])
	])

	AS_IF([test ! -z "${qatsrc}"], [
	AC_MSG_CHECKING([qat build directory])
	AS_IF([test -z "$qatbuild"], [
	qatbuild="${qatsrc}/build"
	])

	AC_MSG_RESULT([$qatbuild])
	QAT_OBJ=${qatbuild}
	AS_IF([ ! test -e "$QAT_OBJ/icp_qa_al.ko" && ! test -e "$QAT_OBJ/qat_api.ko"], [
	AC_MSG_ERROR([
	*** Please make sure the qat driver is installed then try again.
	*** Failed to find icp_qa_al.ko or qat_api.ko in:
	$QAT_OBJ])
	])

	AC_SUBST(QAT_SRC)
	AC_SUBST(QAT_OBJ)

	AC_DEFINE(HAVE_QAT, 1,
	[qat is enabled and existed])
	])

	dnl #
	dnl # Detect the name used for the QAT Module.symvers file.
	dnl #
	AS_IF([test ! -z "${qatsrc}"], [
	AC_MSG_CHECKING([qat file for module symbols])
	QAT_SYMBOLS=$QAT_SRC/lookaside/access_layer/src/Module.symvers

	AS_IF([test -r $QAT_SYMBOLS], [
	AC_MSG_RESULT([$QAT_SYMBOLS])
	AC_SUBST(QAT_SYMBOLS)
	],[
	AC_MSG_ERROR([
	*** Please make sure the qat driver is installed then try again.
	*** Failed to find Module.symvers in:
	$QAT_SYMBOLS
	])
	])
	])
	])

	dnl #
	dnl # ZFS_LINUX_CONFTEST_H
	dnl #
	AC_DEFUN([ZFS_LINUX_CONFTEST_H], [
	test -d build/$2 \|\| mkdir -p build/$2
	cat - <<_ACEOF >build/$2/$2.h
	$1
	_ACEOF
	])

	dnl #
	dnl # ZFS_LINUX_CONFTEST_C
	dnl #
	AC_DEFUN([ZFS_LINUX_CONFTEST_C], [
	test -d build/$2 \|\| mkdir -p build/$2
	cat confdefs.h - <<_ACEOF >build/$2/$2.c
	$1
	_ACEOF
	])

	dnl #
	dnl # ZFS_LINUX_CONFTEST_MAKEFILE
	dnl #
	dnl # $1 - test case name
	dnl # $2 - add to top-level Makefile
	dnl # $3 - additional build flags
	dnl #
	AC_DEFUN([ZFS_LINUX_CONFTEST_MAKEFILE], [
	test -d build \|\| mkdir -p build
	test -d build/$1 \|\| mkdir -p build/$1

	file=build/$1/Makefile

	dnl # Example command line to manually build source.
	cat - <<_ACEOF >$file
	# Example command line to manually build source
	# make modules -C $LINUX_OBJ $ARCH_UM M=$PWD/build/$1

	ccflags-y := -Werror $FRAME_LARGER_THAN
	_ACEOF

	dnl # Additional custom CFLAGS as requested.
	m4_ifval($3, [echo "ccflags-y += $3" >>$file], [])

	dnl # Test case source
	echo "obj-m := $1.o" >>$file

	AS_IF([test "x$2" = "xyes"], [echo "obj-m += $1/" >>build/Makefile], [])
	])

	dnl #
	dnl # ZFS_LINUX_TEST_PROGRAM(C)([PROLOGUE], [BODY])
	dnl #
	m4_define([ZFS_LINUX_TEST_PROGRAM], [
	#include <linux/module.h>
	$1

	int
	main (void)
	{
	$2
	;
	return 0;
	}

	MODULE_DESCRIPTION("conftest");
	MODULE_AUTHOR(ZFS_META_AUTHOR);
	MODULE_VERSION(ZFS_META_VERSION "-" ZFS_META_RELEASE);
	MODULE_LICENSE($3);
	])

	dnl #
	dnl # ZFS_LINUX_TEST_REMOVE
	dnl #
	dnl # Removes the specified test source and results.
	dnl #
	AC_DEFUN([ZFS_LINUX_TEST_REMOVE], [
	test -d build/$1 && rm -Rf build/$1
	test -f build/Makefile && sed '/$1/d' build/Makefile
	])

	dnl #
	dnl # ZFS_LINUX_COMPILE
	dnl #
	dnl # $1 - build dir
	dnl # $2 - test command
	dnl # $3 - pass command
	dnl # $4 - fail command
	dnl # $5 - set KBUILD_MODPOST_NOFINAL='yes'
	dnl # $6 - set KBUILD_MODPOST_WARN='yes'
	dnl #
	dnl # Used internally by ZFS_LINUX_TEST_{COMPILE,MODPOST}
	dnl #
	AC_DEFUN([ZFS_LINUX_COMPILE], [
	AC_ARG_VAR([KERNEL_CC], [C compiler for
	building kernel modules])
	AC_ARG_VAR([KERNEL_LD], [Linker for
	building kernel modules])
	AC_ARG_VAR([KERNEL_LLVM], [Binary option to
	build kernel modules with LLVM/CLANG toolchain])
	AC_TRY_COMMAND([
	KBUILD_MODPOST_NOFINAL="$5" KBUILD_MODPOST_WARN="$6"
	make modules -k -j$TEST_JOBS ${KERNEL_CC:+CC=$KERNEL_CC}
	${KERNEL_LD:+LD=$KERNEL_LD} ${KERNEL_LLVM:+LLVM=$KERNEL_LLVM}
	CONFIG_MODULES=y CFLAGS_MODULE=-DCONFIG_MODULES
	-C $LINUX_OBJ $ARCH_UM M=$PWD/$1 >$1/build.log 2>&1])
	AS_IF([AC_TRY_COMMAND([$2])], [$3], [$4])
	])

	dnl #
	dnl # ZFS_LINUX_TEST_COMPILE
	dnl #
	dnl # Perform a full compile excluding the final modpost phase.
	dnl #
	AC_DEFUN([ZFS_LINUX_TEST_COMPILE], [
	ZFS_LINUX_COMPILE([$2], [test -f $2/build.log], [
	mv $2/Makefile $2/Makefile.compile.$1
	mv $2/build.log $2/build.log.$1
	],[
	AC_MSG_ERROR([
	*** Unable to compile test source to determine kernel interfaces.])
	], [yes], [])
	])

	dnl #
	dnl # ZFS_LINUX_TEST_MODPOST
	dnl #
	dnl # Perform a full compile including the modpost phase. This may
	dnl # be an incremental build if the objects have already been built.
	dnl #
	AC_DEFUN([ZFS_LINUX_TEST_MODPOST], [
	ZFS_LINUX_COMPILE([$2], [test -f $2/build.log], [
	mv $2/Makefile $2/Makefile.modpost.$1
	cat $2/build.log >>build/build.log.$1
	],[
	AC_MSG_ERROR([
	*** Unable to modpost test source to determine kernel interfaces.])
	], [], [yes])
	])

	dnl #
	dnl # Perform the compilation of the test cases in two phases.
	dnl #
	dnl # Phase 1) attempt to build the object files for all of the tests
	dnl # defined by the ZFS_LINUX_TEST_SRC macro. But do not
	dnl # perform the final modpost stage.
	dnl #
	dnl # Phase 2) disable all tests which failed the initial compilation,
	dnl # then invoke the final modpost step for the remaining tests.
	dnl #
	dnl # This allows us efficiently build the test cases in parallel while
	dnl # remaining resilient to build failures which are expected when
	dnl # detecting the available kernel interfaces.
	dnl #
	dnl # The maximum allowed parallelism can be controlled by setting the
	dnl # TEST_JOBS environment variable. Otherwise, it default to $(nproc).
	dnl #
	AC_DEFUN([ZFS_LINUX_TEST_COMPILE_ALL], [
	dnl # Phase 1 - Compilation only, final linking is skipped.
	ZFS_LINUX_TEST_COMPILE([$1], [build])

	dnl #
	dnl # Phase 2 - When building external modules disable test cases
	dnl # which failed to compile and invoke modpost to verify the
	dnl # final linking.
	dnl #
	dnl # Test names suffixed with '_license' call modpost independently
	dnl # to ensure that a single incompatibility does not result in the
	dnl # modpost phase exiting early. This check is not performed on
	dnl # every symbol since the majority are compatible and doing so
	dnl # would significantly slow down this phase.
	dnl #
	dnl # When configuring for builtin (--enable-linux-builtin)
	dnl # fake the linking step artificially create the expected .ko
	dnl # files for tests which did compile. This is required for
	dnl # kernels which do not have loadable module support or have
	dnl # not yet been built.
	dnl #
	AS_IF([test "x$enable_linux_builtin" = "xno"], [
	for dir in $(awk '/^obj-m/ { print [$]3 }' \
	build/Makefile.compile.$1); do
	name=${dir%/}
	AS_IF([test -f build/$name/$name.o], [
	AS_IF([test "${name##*_}" = "license"], [
	ZFS_LINUX_TEST_MODPOST([$1],
	[build/$name])
	echo "obj-n += $dir" >>build/Makefile
	], [
	echo "obj-m += $dir" >>build/Makefile
	])
	], [
	echo "obj-n += $dir" >>build/Makefile
	])
	done

	ZFS_LINUX_TEST_MODPOST([$1], [build])
	], [
	for dir in $(awk '/^obj-m/ { print [$]3 }' \
	build/Makefile.compile.$1); do
	name=${dir%/}
	AS_IF([test -f build/$name/$name.o], [
	touch build/$name/$name.ko
	])
	done
	])
	])

	dnl #
	dnl # ZFS_LINUX_TEST_SRC
	dnl #
	dnl # $1 - name
	dnl # $2 - global
	dnl # $3 - source
	dnl # $4 - extra cflags
	dnl # $5 - check license-compatibility
	dnl #
	dnl # Check if the test source is buildable at all and then if it is
	dnl # license compatible.
	dnl #
	dnl # N.B because all of the test cases are compiled in parallel they
	dnl # must never depend on the results of previous tests. Each test
	dnl # needs to be entirely independent.
	dnl #
	AC_DEFUN([ZFS_LINUX_TEST_SRC], [
	ZFS_LINUX_CONFTEST_C([ZFS_LINUX_TEST_PROGRAM([[$2]], [[$3]],
	[["Dual BSD/GPL"]])], [$1])
	ZFS_LINUX_CONFTEST_MAKEFILE([$1], [yes], [$4])

	AS_IF([ test -n "$5" ], [
	ZFS_LINUX_CONFTEST_C([ZFS_LINUX_TEST_PROGRAM(
	[[$2]], [[$3]], [[$5]])], [$1_license])
	ZFS_LINUX_CONFTEST_MAKEFILE([$1_license], [yes], [$4])
	])
	])

	dnl #
	dnl # ZFS_LINUX_TEST_RESULT
	dnl #
	dnl # $1 - name of a test source (ZFS_LINUX_TEST_SRC)
	dnl # $2 - run on success (valid .ko generated)
	dnl # $3 - run on failure (unable to compile)
	dnl #
	AC_DEFUN([ZFS_LINUX_TEST_RESULT], [
	AS_IF([test -d build/$1], [
	AS_IF([test -f build/$1/$1.ko], [$2], [$3])
	], [
	AC_MSG_ERROR([
	*** No matching source for the "$1" test, check that
	*** both the test source and result macros refer to the same name.
	])
	])
	])

	dnl #
	dnl # ZFS_LINUX_TEST_ERROR
	dnl #
	dnl # Generic error message which can be used when none of the expected
	dnl # kernel interfaces were detected.
	dnl #
	AC_DEFUN([ZFS_LINUX_TEST_ERROR], [
	AC_MSG_ERROR([
	*** None of the expected "$1" interfaces were detected.
	*** This may be because your kernel version is newer than what is
	*** supported, or you are using a patched custom kernel with
	*** incompatible modifications.
	***
	*** ZFS Version: $ZFS_META_ALIAS
	*** Compatible Kernels: $ZFS_META_KVER_MIN - $ZFS_META_KVER_MAX
	])
	])

	dnl #
	dnl # ZFS_LINUX_TEST_RESULT_SYMBOL
	dnl #
	dnl # Like ZFS_LINUX_TEST_RESULT except ZFS_CHECK_SYMBOL_EXPORT is called to
	dnl # verify symbol exports, unless --enable-linux-builtin was provided to
	dnl # configure.
	dnl #
	AC_DEFUN([ZFS_LINUX_TEST_RESULT_SYMBOL], [
	AS_IF([ ! test -f build/$1/$1.ko], [
	$5
	], [
	AS_IF([test "x$enable_linux_builtin" != "xyes"], [
	ZFS_CHECK_SYMBOL_EXPORT([$2], [$3], [$4], [$5])
	], [
	$4
	])
	])
	])

	dnl #
	dnl # ZFS_LINUX_COMPILE_IFELSE
	dnl #
	AC_DEFUN([ZFS_LINUX_COMPILE_IFELSE], [
	ZFS_LINUX_TEST_REMOVE([conftest])

	m4_ifvaln([$1], [ZFS_LINUX_CONFTEST_C([$1], [conftest])])
	m4_ifvaln([$5], [ZFS_LINUX_CONFTEST_H([$5], [conftest])],
	[ZFS_LINUX_CONFTEST_H([], [conftest])])

	ZFS_LINUX_CONFTEST_MAKEFILE([conftest], [no],
	[m4_ifvaln([$5], [-I$PWD/build/conftest], [])])
	ZFS_LINUX_COMPILE([build/conftest], [$2], [$3], [$4], [], [])
	])

	dnl #
	dnl # ZFS_LINUX_TRY_COMPILE
	dnl #
	dnl # $1 - global
	dnl # $2 - source
	dnl # $3 - run on success (valid .ko generated)
	dnl # $4 - run on failure (unable to compile)
	dnl #
	dnl # When configuring as builtin (--enable-linux-builtin) for kernels
	dnl # without loadable module support (CONFIG_MODULES=n) only the object
	dnl # file is created. See ZFS_LINUX_TEST_COMPILE_ALL for details.
	dnl #
	AC_DEFUN([ZFS_LINUX_TRY_COMPILE], [
	AS_IF([test "x$enable_linux_builtin" = "xyes"], [
	ZFS_LINUX_COMPILE_IFELSE(
	[ZFS_LINUX_TEST_PROGRAM([[$1]], [[$2]],
	[[ZFS_META_LICENSE]])],
	[test -f build/conftest/conftest.o], [$3], [$4])
	], [
	ZFS_LINUX_COMPILE_IFELSE(
	[ZFS_LINUX_TEST_PROGRAM([[$1]], [[$2]],
	[[ZFS_META_LICENSE]])],
	[test -f build/conftest/conftest.ko], [$3], [$4])
	])
	])

	dnl #
	dnl # ZFS_CHECK_SYMBOL_EXPORT
	dnl #
	dnl # Check if a symbol is exported on not by consulting the symbols
	dnl # file, or optionally the source code.
	dnl #
	AC_DEFUN([ZFS_CHECK_SYMBOL_EXPORT], [
	grep -q -E '[[[:space:]]]$1[[[:space:]]]' \
	$LINUX_OBJ/$LINUX_SYMBOLS 2>/dev/null
	rc=$?
	if test $rc -ne 0; then
	export=0
	for file in $2; do
	grep -q -E "EXPORT_SYMBOL.*($1)" \
	"$LINUX/$file" 2>/dev/null
	rc=$?
	if test $rc -eq 0; then
	export=1
	break;
	fi
	done
	if test $export -eq 0; then :
	$4
	else :
	$3
	fi
	else :
	$3
	fi
	])

	dnl #
	dnl # ZFS_LINUX_TRY_COMPILE_SYMBOL
	dnl #
	dnl # Like ZFS_LINUX_TRY_COMPILER except ZFS_CHECK_SYMBOL_EXPORT is called
	dnl # to verify symbol exports, unless --enable-linux-builtin was provided
	dnl # to configure.
	dnl #
	AC_DEFUN([ZFS_LINUX_TRY_COMPILE_SYMBOL], [
	ZFS_LINUX_TRY_COMPILE([$1], [$2], [rc=0], [rc=1])
	if test $rc -ne 0; then :
	$6
	else
	if test "x$enable_linux_builtin" != xyes; then
	ZFS_CHECK_SYMBOL_EXPORT([$3], [$4], [rc=0], [rc=1])
	fi
	if test $rc -ne 0; then :
	$6
	else :
	$5
	fi
	fi
	])

	dnl #
	dnl # ZFS_LINUX_TRY_COMPILE_HEADER
	dnl # like ZFS_LINUX_TRY_COMPILE, except the contents conftest.h are
	dnl # provided via the fifth parameter
	dnl #
	AC_DEFUN([ZFS_LINUX_TRY_COMPILE_HEADER], [
	AS_IF([test "x$enable_linux_builtin" = "xyes"], [
	ZFS_LINUX_COMPILE_IFELSE(
	[ZFS_LINUX_TEST_PROGRAM([[$1]], [[$2]],
	[[ZFS_META_LICENSE]])],
	[test -f build/conftest/conftest.o], [$3], [$4], [$5])
	], [
	ZFS_LINUX_COMPILE_IFELSE(
	[ZFS_LINUX_TEST_PROGRAM([[$1]], [[$2]],
	[[ZFS_META_LICENSE]])],
	[test -f build/conftest/conftest.ko], [$3], [$4], [$5])
	])
	])

	dnl #
	dnl # AS_VERSION_COMPARE_LE
	dnl # like AS_VERSION_COMPARE_LE, but runs $3 if (and only if) $1 <= $2
	dnl # AS_VERSION_COMPARE_LE (version-1, version-2, [action-if-less-or-equal], [action-if-greater])
	dnl #
	AC_DEFUN([AS_VERSION_COMPARE_LE], [
	AS_VERSION_COMPARE([$1], [$2], [$3], [$3], [$4])
	])

	dnl #
	dnl # ZFS_LINUX_REQUIRE_API
	dnl # like ZFS_LINUX_TEST_ERROR, except only fails if the kernel is
	dnl # at least some specified version.
	dnl #
	AC_DEFUN([ZFS_LINUX_REQUIRE_API], [
	AS_VERSION_COMPARE_LE([$2], [$kernsrcver], [
	AC_MSG_ERROR([
	*** None of the expected "$1" interfaces were detected. This
	*** interface is expected for kernels version "$2" and above.
	*** This may be because your kernel version is newer than what is
	*** supported, or you are using a patched custom kernel with
	*** incompatible modifications. Newer kernels may have incompatible
	*** APIs.
	***
	*** ZFS Version: $ZFS_META_ALIAS
	*** Compatible Kernels: $ZFS_META_KVER_MIN - $ZFS_META_KVER_MAX
	])
	], [
	AC_MSG_RESULT(no)
	])
	])
	diff --git a/sys/contrib/openzfs/config/zfs-build.m4 b/sys/contrib/openzfs/config/zfs-build.m4
	index 5f36569fe25b..bb5a85d815d1 100644
	--- a/sys/contrib/openzfs/config/zfs-build.m4
	+++ b/sys/contrib/openzfs/config/zfs-build.m4
	@@ -1,647 +1,649 @@
	AC_DEFUN([ZFS_AC_LICENSE], [
	AC_MSG_CHECKING([zfs author])
	AC_MSG_RESULT([$ZFS_META_AUTHOR])

	AC_MSG_CHECKING([zfs license])
	AC_MSG_RESULT([$ZFS_META_LICENSE])
	])

	AC_DEFUN([ZFS_AC_DEBUG_ENABLE], [
	DEBUG_CFLAGS="-Werror"
	DEBUG_CPPFLAGS="-DDEBUG -UNDEBUG"
	DEBUG_LDFLAGS=""
	DEBUG_ZFS="_with_debug"
	WITH_DEBUG="true"
	AC_DEFINE(ZFS_DEBUG, 1, [zfs debugging enabled])

	KERNEL_DEBUG_CFLAGS="-Werror"
	KERNEL_DEBUG_CPPFLAGS="-DDEBUG -UNDEBUG"
	])

	AC_DEFUN([ZFS_AC_DEBUG_DISABLE], [
	DEBUG_CFLAGS=""
	DEBUG_CPPFLAGS="-UDEBUG -DNDEBUG"
	DEBUG_LDFLAGS=""
	DEBUG_ZFS="_without_debug"
	WITH_DEBUG=""

	KERNEL_DEBUG_CFLAGS=""
	KERNEL_DEBUG_CPPFLAGS="-UDEBUG -DNDEBUG"
	])

	dnl #
	dnl # When debugging is enabled:
	dnl # - Enable all ASSERTs (-DDEBUG)
	dnl # - Promote all compiler warnings to errors (-Werror)
	dnl #
	dnl # (If INVARIANTS is detected, we need to force DEBUG, or strange panics
	dnl # can ensue.)
	dnl #
	AC_DEFUN([ZFS_AC_DEBUG], [
	AC_MSG_CHECKING([whether assertion support will be enabled])
	AC_ARG_ENABLE([debug],
	[AS_HELP_STRING([--enable-debug],
	[Enable compiler and code assertions @<:@default=no@:>@])],
	[],
	[enable_debug=no])

	AS_CASE(["x$enable_debug"],
	["xyes"],
	[ZFS_AC_DEBUG_ENABLE],
	["xno"],
	[ZFS_AC_DEBUG_DISABLE],
	[AC_MSG_ERROR([Unknown option $enable_debug])])

	AS_CASE(["x$enable_invariants"],
	["xyes"],
	[],
	["xno"],
	[],
	[ZFS_AC_DEBUG_INVARIANTS_DETECT])

	AS_CASE(["x$enable_invariants"],
	["xyes"],
	[ZFS_AC_DEBUG_ENABLE],
	["xno"],
	[],
	[AC_MSG_ERROR([Unknown option $enable_invariants])])

	AC_SUBST(DEBUG_CFLAGS)
	AC_SUBST(DEBUG_CPPFLAGS)
	AC_SUBST(DEBUG_LDFLAGS)
	AC_SUBST(DEBUG_ZFS)
	AC_SUBST(WITH_DEBUG)

	AC_SUBST(KERNEL_DEBUG_CFLAGS)
	AC_SUBST(KERNEL_DEBUG_CPPFLAGS)

	AC_MSG_RESULT([$enable_debug])
	])

	AC_DEFUN([ZFS_AC_DEBUGINFO_ENABLE], [
	DEBUG_CFLAGS="$DEBUG_CFLAGS -g -fno-inline $NO_IPA_SRA"

	KERNEL_DEBUG_CFLAGS="$KERNEL_DEBUG_CFLAGS -fno-inline $KERNEL_NO_IPA_SRA"
	KERNEL_MAKE="$KERNEL_MAKE CONFIG_DEBUG_INFO=y"

	DEBUGINFO_ZFS="_with_debuginfo"
	])

	AC_DEFUN([ZFS_AC_DEBUGINFO_DISABLE], [
	DEBUGINFO_ZFS="_without_debuginfo"
	])

	AC_DEFUN([ZFS_AC_DEBUGINFO], [
	AC_MSG_CHECKING([whether debuginfo support will be forced])
	AC_ARG_ENABLE([debuginfo],
	[AS_HELP_STRING([--enable-debuginfo],
	[Force generation of debuginfo @<:@default=no@:>@])],
	[],
	[enable_debuginfo=no])

	AS_CASE(["x$enable_debuginfo"],
	["xyes"],
	[ZFS_AC_DEBUGINFO_ENABLE],
	["xno"],
	[ZFS_AC_DEBUGINFO_DISABLE],
	[AC_MSG_ERROR([Unknown option $enable_debuginfo])])

	AC_SUBST(DEBUG_CFLAGS)
	AC_SUBST(DEBUGINFO_ZFS)

	AC_SUBST(KERNEL_DEBUG_CFLAGS)
	AC_SUBST(KERNEL_MAKE)

	AC_MSG_RESULT([$enable_debuginfo])
	])

	dnl #
	dnl # Disabled by default, provides basic memory tracking. Track the total
	dnl # number of bytes allocated with kmem_alloc() and freed with kmem_free().
	dnl # Then at module unload time if any bytes were leaked it will be reported
	dnl # on the console.
	dnl #
	AC_DEFUN([ZFS_AC_DEBUG_KMEM], [
	AC_MSG_CHECKING([whether basic kmem accounting is enabled])
	AC_ARG_ENABLE([debug-kmem],
	[AS_HELP_STRING([--enable-debug-kmem],
	[Enable basic kmem accounting @<:@default=no@:>@])],
	[],
	[enable_debug_kmem=no])

	AS_IF([test "x$enable_debug_kmem" = xyes], [
	KERNEL_DEBUG_CPPFLAGS="${KERNEL_DEBUG_CPPFLAGS} -DDEBUG_KMEM"
	DEBUG_KMEM_ZFS="_with_debug_kmem"
	], [
	DEBUG_KMEM_ZFS="_without_debug_kmem"
	])

	AC_SUBST(KERNEL_DEBUG_CPPFLAGS)
	AC_SUBST(DEBUG_KMEM_ZFS)

	AC_MSG_RESULT([$enable_debug_kmem])
	])

	dnl #
	dnl # Disabled by default, provides detailed memory tracking. This feature
	dnl # also requires --enable-debug-kmem to be set. When enabled not only will
	dnl # total bytes be tracked but also the location of every kmem_alloc() and
	dnl # kmem_free(). When the module is unloaded a list of all leaked addresses
	dnl # and where they were allocated will be dumped to the console. Enabling
	dnl # this feature has a significant impact on performance but it makes finding
	dnl # memory leaks straight forward.
	dnl #
	AC_DEFUN([ZFS_AC_DEBUG_KMEM_TRACKING], [
	AC_MSG_CHECKING([whether detailed kmem tracking is enabled])
	AC_ARG_ENABLE([debug-kmem-tracking],
	[AS_HELP_STRING([--enable-debug-kmem-tracking],
	[Enable detailed kmem tracking @<:@default=no@:>@])],
	[],
	[enable_debug_kmem_tracking=no])

	AS_IF([test "x$enable_debug_kmem_tracking" = xyes], [
	KERNEL_DEBUG_CPPFLAGS="${KERNEL_DEBUG_CPPFLAGS} -DDEBUG_KMEM_TRACKING"
	DEBUG_KMEM_TRACKING_ZFS="_with_debug_kmem_tracking"
	], [
	DEBUG_KMEM_TRACKING_ZFS="_without_debug_kmem_tracking"
	])

	AC_SUBST(KERNEL_DEBUG_CPPFLAGS)
	AC_SUBST(DEBUG_KMEM_TRACKING_ZFS)

	AC_MSG_RESULT([$enable_debug_kmem_tracking])
	])

	AC_DEFUN([ZFS_AC_DEBUG_INVARIANTS_DETECT_FREEBSD], [
	AS_IF([sysctl -n kern.conftxt \| grep -Fqx $'options\tINVARIANTS'],
	[enable_invariants="yes"],
	[enable_invariants="no"])
	])

	AC_DEFUN([ZFS_AC_DEBUG_INVARIANTS_DETECT], [
	AM_COND_IF([BUILD_FREEBSD],
	[ZFS_AC_DEBUG_INVARIANTS_DETECT_FREEBSD],
	[enable_invariants="no"])
	])

	dnl #
	dnl # Detected for the running kernel by default, enables INVARIANTS features
	dnl # in the FreeBSD kernel module. This feature must be used when building
	dnl # for a FreeBSD kernel with "options INVARIANTS" in the KERNCONF and must
	dnl # not be used when the INVARIANTS option is absent.
	dnl #
	AC_DEFUN([ZFS_AC_DEBUG_INVARIANTS], [
	AC_MSG_CHECKING([whether FreeBSD kernel INVARIANTS checks are enabled])
	AC_ARG_ENABLE([invariants],
	[AS_HELP_STRING([--enable-invariants],
	[Enable FreeBSD kernel INVARIANTS checks [[default: detect]]])],
	[], [ZFS_AC_DEBUG_INVARIANTS_DETECT])

	AS_IF([test "x$enable_invariants" = xyes],
	[WITH_INVARIANTS="true"],
	[WITH_INVARIANTS=""])
	AC_SUBST(WITH_INVARIANTS)

	AC_MSG_RESULT([$enable_invariants])
	])

	AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [
	AX_COUNT_CPUS([])
	AC_SUBST(CPU_COUNT)

	ZFS_AC_CONFIG_ALWAYS_CC_NO_CLOBBERED
	ZFS_AC_CONFIG_ALWAYS_CC_INFINITE_RECURSION
	ZFS_AC_CONFIG_ALWAYS_KERNEL_CC_INFINITE_RECURSION
	ZFS_AC_CONFIG_ALWAYS_CC_IMPLICIT_FALLTHROUGH
	ZFS_AC_CONFIG_ALWAYS_CC_FRAME_LARGER_THAN
	ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_TRUNCATION
	ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_ZERO_LENGTH
	ZFS_AC_CONFIG_ALWAYS_CC_FORMAT_OVERFLOW
	ZFS_AC_CONFIG_ALWAYS_CC_NO_OMIT_FRAME_POINTER
	ZFS_AC_CONFIG_ALWAYS_CC_NO_IPA_SRA
	ZFS_AC_CONFIG_ALWAYS_KERNEL_CC_NO_IPA_SRA
	ZFS_AC_CONFIG_ALWAYS_CC_ASAN
	ZFS_AC_CONFIG_ALWAYS_CC_UBSAN
	ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD
	ZFS_AC_CONFIG_ALWAYS_SYSTEM
	ZFS_AC_CONFIG_ALWAYS_ARCH
	ZFS_AC_CONFIG_ALWAYS_PYTHON
	ZFS_AC_CONFIG_ALWAYS_PYZFS
	ZFS_AC_CONFIG_ALWAYS_SED
	ZFS_AC_CONFIG_ALWAYS_CPPCHECK
	ZFS_AC_CONFIG_ALWAYS_SHELLCHECK
	ZFS_AC_CONFIG_ALWAYS_PARALLEL
	])

	AC_DEFUN([ZFS_AC_CONFIG], [

	dnl # Remove the previous build test directory.
	rm -Rf build

	ZFS_CONFIG=all
	AC_ARG_WITH([config],
	AS_HELP_STRING([--with-config=CONFIG],
	[Config file 'kernel\|user\|all\|srpm']),
	[ZFS_CONFIG="$withval"])
	AC_ARG_ENABLE([linux-builtin],
	[AS_HELP_STRING([--enable-linux-builtin],
	[Configure for builtin in-tree kernel modules @<:@default=no@:>@])],
	[],
	[enable_linux_builtin=no])

	AC_MSG_CHECKING([zfs config])
	AC_MSG_RESULT([$ZFS_CONFIG]);
	AC_SUBST(ZFS_CONFIG)

	ZFS_AC_CONFIG_ALWAYS

	AM_COND_IF([BUILD_LINUX], [
	AC_ARG_VAR([TEST_JOBS], [simultaneous jobs during configure])
	if test "x$ac_cv_env_TEST_JOBS_set" != "xset"; then
	TEST_JOBS=$CPU_COUNT
	fi
	AC_SUBST(TEST_JOBS)
	])

	ZFS_INIT_SYSV=
	ZFS_INIT_SYSTEMD=
	ZFS_WANT_MODULES_LOAD_D=

	case "$ZFS_CONFIG" in
	kernel) ZFS_AC_CONFIG_KERNEL ;;
	user) ZFS_AC_CONFIG_USER ;;
	all) ZFS_AC_CONFIG_USER
	ZFS_AC_CONFIG_KERNEL ;;
	dist) ;;
	srpm) ;;
	*)
	AC_MSG_RESULT([Error!])
	AC_MSG_ERROR([Bad value "$ZFS_CONFIG" for --with-config,
	user kernel\|user\|all\|srpm]) ;;
	esac

	AM_CONDITIONAL([INIT_SYSV], [test "x$ZFS_INIT_SYSV" = "xyes"])
	AM_CONDITIONAL([INIT_SYSTEMD], [test "x$ZFS_INIT_SYSTEMD" = "xyes"])
	AM_CONDITIONAL([WANT_MODULES_LOAD_D], [test "x$ZFS_WANT_MODULES_LOAD_D" = "xyes"])

	AM_CONDITIONAL([CONFIG_USER],
	[test "$ZFS_CONFIG" = user -o "$ZFS_CONFIG" = all])
	AM_CONDITIONAL([CONFIG_KERNEL],
	[test "$ZFS_CONFIG" = kernel -o "$ZFS_CONFIG" = all] &&
	[test "x$enable_linux_builtin" != xyes ])
	AM_CONDITIONAL([CONFIG_QAT],
	[test "$ZFS_CONFIG" = kernel -o "$ZFS_CONFIG" = all] &&
	[test "x$qatsrc" != x ])
	AM_CONDITIONAL([WANT_DEVNAME2DEVID], [test "x$user_libudev" = xyes ])
	AM_CONDITIONAL([WANT_MMAP_LIBAIO], [test "x$user_libaio" = xyes ])
	AM_CONDITIONAL([PAM_ZFS_ENABLED], [test "x$enable_pam" = xyes])
	])

	dnl #
	dnl # Check for rpm+rpmbuild to build RPM packages. If these tools
	dnl # are missing it is non-fatal but you will not be able to build
	dnl # RPM packages and will be warned if you try too.
	dnl #
	dnl # By default the generic spec file will be used because it requires
	dnl # minimal dependencies. Distribution specific spec files can be
	dnl # placed under the 'rpm/<distribution>' directory and enabled using
	dnl # the --with-spec=<distribution> configure option.
	dnl #
	AC_DEFUN([ZFS_AC_RPM], [
	RPM=rpm
	RPMBUILD=rpmbuild

	AC_MSG_CHECKING([whether $RPM is available])
	AS_IF([tmp=$($RPM --version 2>/dev/null)], [
	RPM_VERSION=$(echo $tmp \| $AWK '/RPM/ { print $[3] }')
	HAVE_RPM=yes
	AC_MSG_RESULT([$HAVE_RPM ($RPM_VERSION)])
	],[
	HAVE_RPM=no
	AC_MSG_RESULT([$HAVE_RPM])
	])

	AC_MSG_CHECKING([whether $RPMBUILD is available])
	AS_IF([tmp=$($RPMBUILD --version 2>/dev/null)], [
	RPMBUILD_VERSION=$(echo $tmp \| $AWK '/RPM/ { print $[3] }')
	HAVE_RPMBUILD=yes
	AC_MSG_RESULT([$HAVE_RPMBUILD ($RPMBUILD_VERSION)])
	],[
	HAVE_RPMBUILD=no
	AC_MSG_RESULT([$HAVE_RPMBUILD])
	])

	RPM_DEFINE_COMMON='--define "$(DEBUG_ZFS) 1"'
	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(DEBUGINFO_ZFS) 1"'
	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(DEBUG_KMEM_ZFS) 1"'
	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(DEBUG_KMEM_TRACKING_ZFS) 1"'
	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(ASAN_ZFS) 1"'
	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(UBSAN_ZFS) 1"'

	AS_IF([test "x$enable_debuginfo" = xyes], [
	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "__strip /bin/true"'
	])

	RPM_DEFINE_UTIL=' --define "_initconfdir $(initconfdir)"'

	dnl # Make the next three RPM_DEFINE_UTIL additions conditional, since
	dnl # their values may not be set when running:
	dnl #
	dnl # ./configure --with-config=srpm
	dnl #
	AS_IF([test -n "$dracutdir" ], [
	RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' --define "_dracutdir $(dracutdir)"'
	])
	AS_IF([test -n "$udevdir" ], [
	RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' --define "_udevdir $(udevdir)"'
	])
	AS_IF([test -n "$udevruledir" ], [
	RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' --define "_udevruledir $(udevruledir)"'
	])
	AS_IF([test -n "$bashcompletiondir" ], [
	RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' --define "_bashcompletiondir $(bashcompletiondir)"'
	])
	RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' $(DEFINE_SYSTEMD)'
	RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' $(DEFINE_PYZFS)'
	RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' $(DEFINE_PAM)'
	RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' $(DEFINE_PYTHON_VERSION)'
	RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' $(DEFINE_PYTHON_PKG_VERSION)'

	dnl # Override default lib directory on Debian/Ubuntu systems. The
	dnl # provided /usr/lib/rpm/platform/<arch>/macros files do not
	dnl # specify the correct path for multiarch systems as described
	dnl # by the packaging guidelines.
	dnl #
	dnl # https://wiki.ubuntu.com/MultiarchSpec
	dnl # https://wiki.debian.org/Multiarch/Implementation
	dnl #
	AS_IF([test "$DEFAULT_PACKAGE" = "deb"], [
	MULTIARCH_LIBDIR="lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)"
	RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' --define "_lib $(MULTIARCH_LIBDIR)"'
	AC_SUBST(MULTIARCH_LIBDIR)
	])

	dnl # Make RPM_DEFINE_KMOD additions conditional on CONFIG_KERNEL,
	dnl # since the values will not be set otherwise. The spec files
	dnl # provide defaults for them.
	dnl #
	RPM_DEFINE_KMOD='--define "_wrong_version_format_terminate_build 0"'
	AM_COND_IF([CONFIG_KERNEL], [
	RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernels $(LINUX_VERSION)"'
	RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "ksrc $(LINUX)"'
	RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kobj $(LINUX_OBJ)"'
	RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernel_cc KERNEL_CC=$(KERNEL_CC)"'
	RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernel_ld KERNEL_LD=$(KERNEL_LD)"'
	RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernel_llvm KERNEL_LLVM=$(KERNEL_LLVM)"'
	])

	RPM_DEFINE_DKMS=''

	SRPM_DEFINE_COMMON='--define "build_src_rpm 1"'
	SRPM_DEFINE_UTIL=
	SRPM_DEFINE_KMOD=
	SRPM_DEFINE_DKMS=

	RPM_SPEC_DIR="rpm/generic"
	AC_ARG_WITH([spec],
	AS_HELP_STRING([--with-spec=SPEC],
	[Spec files 'generic\|redhat']),
	[RPM_SPEC_DIR="rpm/$withval"])

	AC_MSG_CHECKING([whether spec files are available])
	AC_MSG_RESULT([yes ($RPM_SPEC_DIR/*.spec.in)])

	AC_SUBST(HAVE_RPM)
	AC_SUBST(RPM)
	AC_SUBST(RPM_VERSION)

	AC_SUBST(HAVE_RPMBUILD)
	AC_SUBST(RPMBUILD)
	AC_SUBST(RPMBUILD_VERSION)

	AC_SUBST(RPM_SPEC_DIR)
	AC_SUBST(RPM_DEFINE_UTIL)
	AC_SUBST(RPM_DEFINE_KMOD)
	AC_SUBST(RPM_DEFINE_DKMS)
	AC_SUBST(RPM_DEFINE_COMMON)
	AC_SUBST(SRPM_DEFINE_UTIL)
	AC_SUBST(SRPM_DEFINE_KMOD)
	AC_SUBST(SRPM_DEFINE_DKMS)
	AC_SUBST(SRPM_DEFINE_COMMON)
	])

	dnl #
	dnl # Check for dpkg+dpkg-buildpackage to build DEB packages. If these
	dnl # tools are missing it is non-fatal but you will not be able to build
	dnl # DEB packages and will be warned if you try too.
	dnl #
	AC_DEFUN([ZFS_AC_DPKG], [
	DPKG=dpkg
	DPKGBUILD=dpkg-buildpackage

	AC_MSG_CHECKING([whether $DPKG is available])
	AS_IF([tmp=$($DPKG --version 2>/dev/null)], [
	DPKG_VERSION=$(echo $tmp \| $AWK '/Debian/ { print $[7] }')
	HAVE_DPKG=yes
	AC_MSG_RESULT([$HAVE_DPKG ($DPKG_VERSION)])
	],[
	HAVE_DPKG=no
	AC_MSG_RESULT([$HAVE_DPKG])
	])

	AC_MSG_CHECKING([whether $DPKGBUILD is available])
	AS_IF([tmp=$($DPKGBUILD --version 2>/dev/null)], [
	DPKGBUILD_VERSION=$(echo $tmp \| \
	$AWK '/Debian/ { print $[4] }' \| cut -f-4 -d'.')
	HAVE_DPKGBUILD=yes
	AC_MSG_RESULT([$HAVE_DPKGBUILD ($DPKGBUILD_VERSION)])
	],[
	HAVE_DPKGBUILD=no
	AC_MSG_RESULT([$HAVE_DPKGBUILD])
	])

	AC_SUBST(HAVE_DPKG)
	AC_SUBST(DPKG)
	AC_SUBST(DPKG_VERSION)

	AC_SUBST(HAVE_DPKGBUILD)
	AC_SUBST(DPKGBUILD)
	AC_SUBST(DPKGBUILD_VERSION)
	AC_SUBST([CFGOPTS], ["$CFGOPTS"])
	])

	dnl #
	dnl # Until native packaging for various different packing systems
	dnl # can be added the least we can do is attempt to use alien to
	dnl # convert the RPM packages to the needed package type. This is
	dnl # a hack but so far it has worked reasonable well.
	dnl #
	AC_DEFUN([ZFS_AC_ALIEN], [
	ALIEN=alien

	AC_MSG_CHECKING([whether $ALIEN is available])
	AS_IF([tmp=$($ALIEN --version 2>/dev/null)], [
	ALIEN_VERSION=$(echo $tmp \| $AWK '{ print $[3] }')
	ALIEN_MAJOR=$(echo ${ALIEN_VERSION} \| $AWK -F'.' '{ print $[1] }')
	ALIEN_MINOR=$(echo ${ALIEN_VERSION} \| $AWK -F'.' '{ print $[2] }')
	ALIEN_POINT=$(echo ${ALIEN_VERSION} \| $AWK -F'.' '{ print $[3] }')
	HAVE_ALIEN=yes
	AC_MSG_RESULT([$HAVE_ALIEN ($ALIEN_VERSION)])
	],[
	HAVE_ALIEN=no
	AC_MSG_RESULT([$HAVE_ALIEN])
	])

	AC_SUBST(HAVE_ALIEN)
	AC_SUBST(ALIEN)
	AC_SUBST(ALIEN_VERSION)
	AC_SUBST(ALIEN_MAJOR)
	AC_SUBST(ALIEN_MINOR)
	AC_SUBST(ALIEN_POINT)
	])

	dnl #
	dnl # Using the VENDOR tag from config.guess set the default
	dnl # package type for 'make pkg': (rpm \| deb \| tgz)
	dnl #
	AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [
	AC_MSG_CHECKING([os distribution])
	AC_ARG_WITH([vendor],
	[AS_HELP_STRING([--with-vendor],
	[Distribution vendor @<:@default=check@:>@])],
	[with_vendor=$withval],
	[with_vendor=check])
	AS_IF([test "x$with_vendor" = "xcheck"],[
	if test -f /etc/toss-release ; then
	VENDOR=toss ;
	elif test -f /etc/fedora-release ; then
	VENDOR=fedora ;
	elif test -f /etc/redhat-release ; then
	VENDOR=redhat ;
	elif test -f /etc/gentoo-release ; then
	VENDOR=gentoo ;
	elif test -f /etc/arch-release ; then
	VENDOR=arch ;
	elif test -f /etc/SuSE-release ; then
	VENDOR=sles ;
	elif test -f /etc/slackware-version ; then
	VENDOR=slackware ;
	elif test -f /etc/lunar.release ; then
	VENDOR=lunar ;
	elif test -f /etc/lsb-release ; then
	VENDOR=ubuntu ;
	elif test -f /etc/debian_version ; then
	VENDOR=debian ;
	elif test -f /etc/alpine-release ; then
	VENDOR=alpine ;
	elif test -f /bin/freebsd-version ; then
	VENDOR=freebsd ;
	elif test -f /etc/openEuler-release ; then
	VENDOR=openeuler ;
	else
	VENDOR= ;
	fi],
	[ test "x${with_vendor}" != x],[
	VENDOR="$with_vendor" ],
	[ VENDOR= ; ]
	)
	AC_MSG_RESULT([$VENDOR])
	AC_SUBST(VENDOR)

	AC_MSG_CHECKING([default package type])
	case "$VENDOR" in
	toss) DEFAULT_PACKAGE=rpm ;;
	redhat) DEFAULT_PACKAGE=rpm ;;
	fedora) DEFAULT_PACKAGE=rpm ;;
	gentoo) DEFAULT_PACKAGE=tgz ;;
	alpine) DEFAULT_PACKAGE=tgz ;;
	arch) DEFAULT_PACKAGE=tgz ;;
	sles) DEFAULT_PACKAGE=rpm ;;
	slackware) DEFAULT_PACKAGE=tgz ;;
	lunar) DEFAULT_PACKAGE=tgz ;;
	ubuntu) DEFAULT_PACKAGE=deb ;;
	debian) DEFAULT_PACKAGE=deb ;;
	freebsd) DEFAULT_PACKAGE=pkg ;;
	openeuler) DEFAULT_PACKAGE=rpm ;;
	*) DEFAULT_PACKAGE=rpm ;;
	esac
	AC_MSG_RESULT([$DEFAULT_PACKAGE])
	AC_SUBST(DEFAULT_PACKAGE)

	AC_MSG_CHECKING([default init directory])
	case "$VENDOR" in
	freebsd) initdir=$sysconfdir/rc.d ;;
	*) initdir=$sysconfdir/init.d;;
	esac
	AC_MSG_RESULT([$initdir])
	AC_SUBST(initdir)

	AC_MSG_CHECKING([default shell])
	case "$VENDOR" in
	- gentoo) DEFAULT_INIT_SHELL="/sbin/openrc-run";;
	- alpine) DEFAULT_INIT_SHELL="/sbin/openrc-run";;
	- *) DEFAULT_INIT_SHELL="/bin/sh" ;;
	+ gentoo\|alpine) DEFAULT_INIT_SHELL=/sbin/openrc-run
	+ IS_SYSV_RC=false ;;
	+ *) DEFAULT_INIT_SHELL=/bin/sh
	+ IS_SYSV_RC=true ;;
	esac

	AC_MSG_RESULT([$DEFAULT_INIT_SHELL])
	AC_SUBST(DEFAULT_INIT_SHELL)
	+ AC_SUBST(IS_SYSV_RC)

	AC_MSG_CHECKING([default nfs server init script])
	AS_IF([test "$VENDOR" = "debian"],
	[DEFAULT_INIT_NFS_SERVER="nfs-kernel-server"],
	[DEFAULT_INIT_NFS_SERVER="nfs"]
	)
	AC_MSG_RESULT([$DEFAULT_INIT_NFS_SERVER])
	AC_SUBST(DEFAULT_INIT_NFS_SERVER)

	AC_MSG_CHECKING([default init config directory])
	case "$VENDOR" in
	alpine) initconfdir=/etc/conf.d ;;
	gentoo) initconfdir=/etc/conf.d ;;
	toss) initconfdir=/etc/sysconfig ;;
	redhat) initconfdir=/etc/sysconfig ;;
	fedora) initconfdir=/etc/sysconfig ;;
	sles) initconfdir=/etc/sysconfig ;;
	openeuler) initconfdir=/etc/sysconfig ;;
	ubuntu) initconfdir=/etc/default ;;
	debian) initconfdir=/etc/default ;;
	freebsd) initconfdir=$sysconfdir/rc.conf.d;;
	*) initconfdir=/etc/default ;;
	esac
	AC_MSG_RESULT([$initconfdir])
	AC_SUBST(initconfdir)

	AC_MSG_CHECKING([whether initramfs-tools is available])
	if test -d /usr/share/initramfs-tools ; then
	RPM_DEFINE_INITRAMFS='--define "_initramfs 1"'
	AC_MSG_RESULT([yes])
	else
	RPM_DEFINE_INITRAMFS=''
	AC_MSG_RESULT([no])
	fi
	AC_SUBST(RPM_DEFINE_INITRAMFS)

	AC_MSG_CHECKING([default bash completion directory])
	case "$VENDOR" in
	ubuntu) bashcompletiondir=/usr/share/bash-completion/completions ;;
	debian) bashcompletiondir=/usr/share/bash-completion/completions ;;
	freebsd) bashcompletiondir=$sysconfdir/bash_completion.d;;
	gentoo) bashcompletiondir=/usr/share/bash-completion/completions ;;
	*) bashcompletiondir=/etc/bash_completion.d ;;
	esac
	AC_MSG_RESULT([$bashcompletiondir])
	AC_SUBST(bashcompletiondir)

	])

	dnl #
	dnl # Default ZFS package configuration
	dnl #
	AC_DEFUN([ZFS_AC_PACKAGE], [
	ZFS_AC_DEFAULT_PACKAGE
	AS_IF([test x$VENDOR != xfreebsd], [
	ZFS_AC_RPM
	ZFS_AC_DPKG
	ZFS_AC_ALIEN
	])
	])
	diff --git a/sys/contrib/openzfs/contrib/debian/control b/sys/contrib/openzfs/contrib/debian/control
	index 98beb900d0fa..e56fbf0f1c93 100644
	--- a/sys/contrib/openzfs/contrib/debian/control
	+++ b/sys/contrib/openzfs/contrib/debian/control
	@@ -1,326 +1,326 @@
	Source: openzfs-linux
	Section: contrib/kernel
	Priority: optional
	Maintainer: ZFS on Linux specific mailing list <zfs-discuss@list.zfsonlinux.org>
	Build-Depends: debhelper-compat (= 12),
	dh-python,
	dh-sequence-dkms \| dkms (>> 2.1.1.2-5),
	libaio-dev,
	libblkid-dev,
	libcurl4-openssl-dev,
	libelf-dev,
	libpam0g-dev,
	libssl-dev \| libssl1.0-dev,
	libtool,
	libudev-dev,
	lsb-release,
	po-debconf,
	python3-all-dev,
	python3-cffi,
	python3-setuptools,
	python3-sphinx,
	uuid-dev,
	zlib1g-dev
	Standards-Version: 4.5.1
	Homepage: https://openzfs.org/
	Vcs-Git: https://github.com/openzfs/zfs.git
	Vcs-Browser: https://github.com/openzfs/zfs
	Rules-Requires-Root: no
	XS-Autobuild: yes

	Package: openzfs-libnvpair3
	Section: contrib/libs
	Architecture: linux-any
	Depends: ${misc:Depends}, ${shlibs:Depends}
	Breaks: libnvpair1, libnvpair3
	Replaces: libnvpair1, libnvpair3, libnvpair3linux
	Conflicts: libnvpair3linux
	Description: Solaris name-value library for Linux
	This library provides routines for packing and unpacking nv pairs for
	transporting data across process boundaries, transporting between
	kernel and userland, and possibly saving onto disk files.

	Package: openzfs-libpam-zfs
	Section: contrib/admin
	Architecture: linux-any
	Depends: libpam-runtime, ${misc:Depends}, ${shlibs:Depends}
	Replaces: libpam-zfs
	Conflicts: libpam-zfs
	Description: PAM module for managing encryption keys for ZFS
	OpenZFS is a storage platform that encompasses the functionality of
	traditional filesystems and volume managers. It supports data checksums,
	compression, encryption, snapshots, and more.
	.
	This provides a Pluggable Authentication Module (PAM) that automatically
	unlocks encrypted ZFS datasets upon login.

	Package: openzfs-libuutil3
	Section: contrib/libs
	Architecture: linux-any
	Depends: ${misc:Depends}, ${shlibs:Depends}
	Breaks: libuutil1, libuutil3
	Replaces: libuutil1, libuutil3, libuutil3linux
	Conflicts: libuutil3linux
	Description: Solaris userland utility library for Linux
	This library provides a variety of glue functions for ZFS on Linux:
	* libspl: The Solaris Porting Layer userland library, which provides APIs
	that make it possible to run Solaris user code in a Linux environment
	with relatively minimal modification.
	* libavl: The Adelson-Velskii Landis balanced binary tree manipulation
	library.
	* libefi: The Extensible Firmware Interface library for GUID disk
	partitioning.
	* libshare: NFS, SMB, and iSCSI service integration for ZFS.

	Package: openzfs-libzfs-dev
	Section: contrib/libdevel
	Architecture: linux-any
	Depends: libssl-dev \| libssl1.0-dev,
	openzfs-libnvpair3 (= ${binary:Version}),
	openzfs-libuutil3 (= ${binary:Version}),
	openzfs-libzfs4 (= ${binary:Version}),
	openzfs-libzfsbootenv1 (= ${binary:Version}),
	openzfs-libzpool5 (= ${binary:Version}),
	${misc:Depends}
	Replaces: libzfslinux-dev
	Conflicts: libzfslinux-dev
	Provides: libnvpair-dev, libuutil-dev
	Description: OpenZFS filesystem development files for Linux
	Header files and static libraries for compiling software against
	libraries of OpenZFS filesystem.
	.
	This package includes the development files of libnvpair3, libuutil3,
	libzpool5 and libzfs4.

	Package: openzfs-libzfs4
	Section: contrib/libs
	Architecture: linux-any
	Depends: ${misc:Depends}, ${shlibs:Depends}
	# The libcurl4 is loaded through dlopen("libcurl.so.4").
	# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=988521
	Recommends: libcurl4
	Breaks: libzfs2, libzfs4
	Replaces: libzfs2, libzfs4, libzfs4linux
	Conflicts: libzfs4linux
	Description: OpenZFS filesystem library for Linux - general support
	OpenZFS is a storage platform that encompasses the functionality of
	traditional filesystems and volume managers. It supports data checksums,
	compression, encryption, snapshots, and more.
	.
	The OpenZFS library provides support for managing OpenZFS filesystems.

	Package: openzfs-libzfsbootenv1
	Section: contrib/libs
	Architecture: linux-any
	Depends: ${misc:Depends}, ${shlibs:Depends}
	Breaks: libzfs2, libzfs4
	Replaces: libzfs2, libzfs4, libzfsbootenv1linux
	Conflicts: libzfsbootenv1linux
	Description: OpenZFS filesystem library for Linux - label info support
	OpenZFS is a storage platform that encompasses the functionality of
	traditional filesystems and volume managers. It supports data checksums,
	compression, encryption, snapshots, and more.
	.
	The zfsbootenv library provides support for modifying ZFS label information.

	Package: openzfs-libzpool5
	Section: contrib/libs
	Architecture: linux-any
	Depends: ${misc:Depends}, ${shlibs:Depends}
	Breaks: libzpool2, libzpool5
	Replaces: libzpool2, libzpool5, libzpool5linux
	Conflicts: libzpool5linux
	Description: OpenZFS pool library for Linux
	OpenZFS is a storage platform that encompasses the functionality of
	traditional filesystems and volume managers. It supports data checksums,
	compression, encryption, snapshots, and more.
	.
	This zpool library provides support for managing zpools.

	Package: openzfs-python3-pyzfs
	Section: contrib/python
	Architecture: linux-any
	Depends: python3-cffi,
	openzfs-zfsutils (= ${binary:Version}),
	${misc:Depends},
	${python3:Depends}
	Replaces: python3-pyzfs
	Conflicts: python3-pyzfs
	Description: wrapper for libzfs_core C library
	libzfs_core is intended to be a stable interface for programmatic
	administration of ZFS. This wrapper provides one-to-one wrappers for
	libzfs_core API functions, but the signatures and types are more natural to
	Python.
	.
	nvlists are wrapped as dictionaries or lists depending on their usage.
	Some parameters have default values depending on typical use for
	increased convenience. Enumerations and bit flags become strings and lists
	of strings in Python. Errors are reported as exceptions rather than integer
	errno-style error codes. The wrapper takes care to provide one-to-many
	mapping of the error codes to the exceptions by interpreting a context
	in which the error code is produced.

	Package: openzfs-pyzfs-doc
	Section: contrib/doc
	Architecture: all
	Depends: ${misc:Depends}, ${sphinxdoc:Depends}
	Recommends: openzfs-python3-pyzfs
	Replaces: pyzfs-doc
	Conflicts: pyzfs-doc
	Description: wrapper for libzfs_core C library (documentation)
	libzfs_core is intended to be a stable interface for programmatic
	administration of ZFS. This wrapper provides one-to-one wrappers for
	libzfs_core API functions, but the signatures and types are more natural to
	Python.
	.
	nvlists are wrapped as dictionaries or lists depending on their usage.
	Some parameters have default values depending on typical use for
	increased convenience. Enumerations and bit flags become strings and lists
	of strings in Python. Errors are reported as exceptions rather than integer
	errno-style error codes. The wrapper takes care to provide one-to-many
	mapping of the error codes to the exceptions by interpreting a context
	in which the error code is produced.
	.
	This package contains the documentation.

	Package: openzfs-zfs-dkms
	Architecture: all
	Depends: dkms (>> 2.1.1.2-5),
	file,
	libc6-dev \| libc-dev,
	lsb-release,
	- python3-distutils \| libpython3-stdlib (<< 3.6.4),
	+ python3 (>> 3.12) \| python3-distutils \| libpython3-stdlib (<< 3.6.4),
	${misc:Depends},
	${perl:Depends}
	Recommends: openzfs-zfs-zed, openzfs-zfsutils (>= ${source:Version}), ${linux:Recommends}
	# suggests debhelper because e.g. `dkms mkdeb -m zfs -v 0.8.2` needs dh_testdir (#909183)
	Suggests: debhelper
	Breaks: spl-dkms (<< 0.8.0~rc1)
	Replaces: spl-dkms, zfs-dkms
	Provides: openzfs-zfs-modules
	Description: OpenZFS filesystem kernel modules for Linux
	OpenZFS is a storage platform that encompasses the functionality of
	traditional filesystems and volume managers. It supports data checksums,
	compression, encryption, snapshots, and more.
	.
	This DKMS package includes the SPA, DMU, ZVOL, and ZPL components of
	OpenZFS.

	Package: openzfs-zfs-initramfs
	Architecture: all
	Depends: busybox-initramfs \| busybox-static \| busybox,
	initramfs-tools,
	openzfs-zfs-modules \| openzfs-zfs-dkms,
	openzfs-zfsutils (>= ${source:Version}),
	${misc:Depends}
	Breaks: zfsutils-linux (<= 0.7.11-2)
	Replaces: zfsutils-linux (<= 0.7.11-2), zfs-initramfs
	Conflicts: zfs-initramfs
	Description: OpenZFS root filesystem capabilities for Linux - initramfs
	OpenZFS is a storage platform that encompasses the functionality of
	traditional filesystems and volume managers. It supports data checksums,
	compression, encryption, snapshots, and more.
	.
	This package adds OpenZFS to the system initramfs with a hook
	for the initramfs-tools infrastructure.

	Package: openzfs-zfs-dracut
	Architecture: all
	Depends: dracut,
	openzfs-zfs-modules \| openzfs-zfs-dkms,
	openzfs-zfsutils (>= ${source:Version}),
	${misc:Depends}
	Conflicts: zfs-dracut
	Replaces: zfs-dracut
	Description: OpenZFS root filesystem capabilities for Linux - dracut
	OpenZFS is a storage platform that encompasses the functionality of
	traditional filesystems and volume managers. It supports data checksums,
	compression, encryption, snapshots, and more.
	.
	This package adds OpenZFS to the system initramfs with a hook
	for the dracut infrastructure.

	Package: openzfs-zfsutils
	Section: contrib/admin
	Architecture: linux-any
	Pre-Depends: ${misc:Pre-Depends}
	Depends: openzfs-libnvpair3 (= ${binary:Version}),
	openzfs-libuutil3 (= ${binary:Version}),
	openzfs-libzfs4 (= ${binary:Version}),
	openzfs-libzpool5 (= ${binary:Version}),
	python3,
	${misc:Depends},
	${shlibs:Depends}
	Recommends: lsb-base, openzfs-zfs-modules \| openzfs-zfs-dkms, openzfs-zfs-zed
	Breaks: openrc,
	spl (<< 0.7.9-2),
	spl-dkms (<< 0.8.0~rc1),
	openzfs-zfs-dkms (<< ${source:Version}),
	openzfs-zfs-dkms (>> ${source:Version}...)
	Replaces: spl (<< 0.7.9-2), spl-dkms, zfsutils-linux
	Conflicts: zfs, zfs-fuse, zfsutils-linux
	Suggests: nfs-kernel-server,
	samba-common-bin (>= 3.0.23),
	openzfs-zfs-initramfs \| openzfs-zfs-dracut
	Provides: openzfsutils
	Description: command-line tools to manage OpenZFS filesystems
	OpenZFS is a storage platform that encompasses the functionality of
	traditional filesystems and volume managers. It supports data checksums,
	compression, encryption, snapshots, and more.
	.
	This package provides the zfs and zpool commands to create and administer
	OpenZFS filesystems.

	Package: openzfs-zfs-zed
	Section: contrib/admin
	Architecture: linux-any
	Pre-Depends: ${misc:Pre-Depends}
	Depends: openzfs-zfs-modules \| openzfs-zfs-dkms,
	openzfs-zfsutils (>= ${binary:Version}),
	${misc:Depends},
	${shlibs:Depends}
	Conflicts: zfs, zfs-zed
	Replaces: zfs-zed
	Description: OpenZFS Event Daemon
	OpenZFS is a storage platform that encompasses the functionality of
	traditional filesystems and volume managers. It supports data checksums,
	compression, encryption, snapshots, and more.
	.
	ZED (ZFS Event Daemon) monitors events generated by the ZFS kernel
	module. When a zevent (ZFS Event) is posted, ZED will run any ZEDLETs
	(ZFS Event Daemon Linkage for Executable Tasks) that have been enabled
	for the corresponding zevent class.
	.
	This package provides the OpenZFS Event Daemon (zed).

	Package: openzfs-zfs-test
	Section: contrib/admin
	Architecture: linux-any
	Depends: acl,
	attr,
	bc,
	fio,
	ksh,
	lsscsi,
	mdadm,
	parted,
	python3,
	openzfs-python3-pyzfs,
	sudo,
	sysstat,
	openzfs-zfs-modules \| openzfs-zfs-dkms,
	openzfs-zfsutils (>=${binary:Version}),
	${misc:Depends},
	${shlibs:Depends}
	Recommends: nfs-kernel-server
	Breaks: zfsutils-linux (<= 0.7.9-2)
	Replaces: zfsutils-linux (<= 0.7.9-2), zfs-test
	Conflicts: zutils, zfs-test
	Description: OpenZFS test infrastructure and support scripts
	OpenZFS is a storage platform that encompasses the functionality of
	traditional filesystems and volume managers. It supports data checksums,
	compression, encryption, snapshots, and more.
	.
	This package provides the OpenZFS test infrastructure for destructively
	testing and validating a system using OpenZFS. It is entirely optional
	and should only be installed and used in test environments.
	diff --git a/sys/contrib/openzfs/etc/init.d/README.md b/sys/contrib/openzfs/etc/init.d/README.md
	index 2de05042ce63..da780fdc1222 100644
	--- a/sys/contrib/openzfs/etc/init.d/README.md
	+++ b/sys/contrib/openzfs/etc/init.d/README.md
	@@ -1,75 +1,71 @@
	DESCRIPTION
	These script were written with the primary intention of being portable and
	usable on as many systems as possible.

	This is, in practice, usually not possible. But the intention is there.
	And it is a good one.

	They have been tested successfully on:

	- * Debian GNU/Linux Wheezy
	- * Debian GNU/Linux Jessie
	- * Ubuntu Trusty
	- * CentOS 6.0
	- * CentOS 6.6
	+ * Debian GNU/Linux Bookworm
	* Gentoo

	SUPPORT
	If you find that they don't work for your platform, please report this
	at the OpenZFS issue tracker at https://github.com/openzfs/zfs/issues.

	Please include:

	* Distribution name
	* Distribution version
	* Where to find an install CD image
	* Architecture

	If you have code to share that fixes the problem, that is much better.
	But please remember to try your best keep portability in mind. If you
	suspect that what you're writing/modifying won't work on anything else
	than your distribution, please make sure to put that code in appropriate
	if/else/fi code.

	It currently MUST be bash (or fully compatible) for this to work.

	If you're making your own distribution and you want the scripts to
	work on that, the biggest problem you'll (probably) have is the part
	at the beginning of the "zfs-functions" file which sets up the
	logging output.

	INSTALLING INIT SCRIPT LINKS
	To setup the init script links in /etc/rc?.d manually on a Debian GNU/Linux
	(or derived) system, run the following commands (the order is important!):

	update-rc.d zfs-import start 07 S . stop 07 0 1 6 .
	update-rc.d zfs-load-key start 02 2 3 4 5 . stop 06 0 1 6 .
	update-rc.d zfs-mount start 02 S . stop 06 0 1 6 .
	update-rc.d zfs-zed start 07 2 3 4 5 . stop 08 0 1 6 .
	update-rc.d zfs-share start 27 2 3 4 5 . stop 05 0 1 6 .

	To do the same on RedHat, Fedora and/or CentOS:

	chkconfig zfs-import
	chkconfig zfs-load-key
	chkconfig zfs-mount
	chkconfig zfs-zed
	chkconfig zfs-share

	On Gentoo:

	rc-update add zfs-import boot
	rc-update add zfs-load-key boot
	rc-update add zfs-mount boot
	rc-update add zfs-zed default
	rc-update add zfs-share default

	The idea here is to make sure all of the ZFS filesystems, including possibly
	separate datasets like /var, are mounted before anything else is started.

	Then, ZED, which depends on /var, can be started. It will consume and act
	on events that occurred before it started. ZED may also play a role in
	sharing filesystems in the future, so it is important to start before the
	'share' service.

	Finally, we share filesystems configured with the share\* property.
	diff --git a/sys/contrib/openzfs/etc/init.d/zfs-import.in b/sys/contrib/openzfs/etc/init.d/zfs-import.in
	index a9a0604f81ac..ff169eb96d86 100755
	--- a/sys/contrib/openzfs/etc/init.d/zfs-import.in
	+++ b/sys/contrib/openzfs/etc/init.d/zfs-import.in
	@@ -1,338 +1,338 @@
	#!@DEFAULT_INIT_SHELL@
	# shellcheck disable=SC2154
	#
	# zfs-import This script will import ZFS pools
	#
	# chkconfig: 2345 01 99
	# description: This script will perform a verbatim import of ZFS pools
	# during system boot.
	# probe: true
	#
	### BEGIN INIT INFO
	# Provides: zfs-import
	# Required-Start: mtab
	# Required-Stop: $local_fs mtab
	# Default-Start: S
	# Default-Stop: 0 1 6
	# X-Start-Before: checkfs
	# X-Stop-After: zfs-mount
	# Short-Description: Import ZFS pools
	# Description: Run the `zpool import` command.
	### END INIT INFO
	#
	# NOTE: Not having '$local_fs' on Required-Start but only on Required-Stop
	# is on purpose. If we have '$local_fs' in both (and X-Start-Before=checkfs)
	# we get conflicts - import needs to be started extremely early,
	# but not stopped too late.
	#
	# Released under the 2-clause BSD license.
	#
	# This script is based on debian/zfsutils.zfs.init from the
	# Debian GNU/kFreeBSD zfsutils 8.1-3 package, written by Aurelien Jarno.

	# Source the common init script
	. @sysconfdir@/zfs/zfs-functions

	# ----------------------------------------------------

	do_depend()
	{
	before swap
	after sysfs udev
	keyword -lxc -openvz -prefix -vserver
	}

	# Use the zpool cache file to import pools
	do_verbatim_import()
	{
	if [ -f "$ZPOOL_CACHE" ]
	then
	zfs_action "Importing ZFS pool(s)" \
	"$ZPOOL" import -c "$ZPOOL_CACHE" -N -a
	fi
	}

	# Support function to get a list of all pools, separated with ';'
	find_pools()
	{
	local pools

	pools=$("$@" 2> /dev/null \| \
	sed -Ee '/pool:\|^[a-zA-Z0-9]/!d' -e 's@.*: @@' \| \
	sort \| \
	tr '\n' ';')

	echo "${pools%%;}" # Return without the last ';'.
	}

	# Find and import all visible pools, even exported ones
	do_import_all_visible()
	{
	local already_imported available_pools pool npools
	local exception dir ZPOOL_IMPORT_PATH RET=0 r=1

	# In case not shutdown cleanly.
	# shellcheck disable=SC2154
	[ -n "$init" ] && rm -f /etc/dfs/sharetab

	# Just simplify code later on.
	if [ -n "$USE_DISK_BY_ID" ] && [ "$USE_DISK_BY_ID" != 'yes' ]
	then
	# It's something, but not 'yes' so it's no good to us.
	unset USE_DISK_BY_ID
	fi

	# Find list of already imported pools.
	already_imported=$(find_pools "$ZPOOL" list -H -oname)
	available_pools=$(find_pools "$ZPOOL" import)

	# Just in case - seen it happen (that a pool isn't visible/found
	# with a simple "zpool import" but only when using the "-d"
	# option or setting ZPOOL_IMPORT_PATH).
	if [ -d "/dev/disk/by-id" ]
	then
	npools=$(find_pools "$ZPOOL" import -d /dev/disk/by-id)
	if [ -n "$npools" ]
	then
	# Because we have found extra pool(s) here, which wasn't
	# found 'normally', we need to force USE_DISK_BY_ID to
	# make sure we're able to actually import it/them later.
	USE_DISK_BY_ID='yes'

	if [ -n "$available_pools" ]
	then
	# Filter out duplicates (pools found with the simpl
	# "zpool import" but which is also found with the
	# "zpool import -d ...").
	npools=$(echo "$npools" \| sed "s,$available_pools,,")

	# Add the list to the existing list of
	# available pools
	available_pools="$available_pools;$npools"
	else
	available_pools="$npools"
	fi
	fi
	fi

	# Filter out any exceptions...
	if [ -n "$ZFS_POOL_EXCEPTIONS" ]
	then
	local found=""
	local apools=""
	OLD_IFS="$IFS" ; IFS=";"

	for pool in $available_pools
	do
	for exception in $ZFS_POOL_EXCEPTIONS
	do
	[ "$pool" = "$exception" ] && continue 2
	found="$pool"
	done

	if [ -n "$found" ]
	then
	if [ -n "$apools" ]
	then
	apools="$apools;$pool"
	else
	apools="$pool"
	fi
	fi
	done

	IFS="$OLD_IFS"
	available_pools="$apools"
	fi

	# For backwards compatibility, make sure that ZPOOL_IMPORT_PATH is set
	# to something we can use later with the real import(s). We want to
	# make sure we find all by* dirs, BUT by-vdev should be first (if it
	# exists).
	if [ -n "$USE_DISK_BY_ID" ] && [ -z "$ZPOOL_IMPORT_PATH" ]
	then
	local dirs
	dirs="$(for dir in $(echo /dev/disk/by-*)
	do
	# Ignore by-vdev here - we want it first!
	echo "$dir" \| grep -q /by-vdev && continue
	[ ! -d "$dir" ] && continue

	printf "%s" "$dir:"
	done \| sed 's,:$,,g')"

	if [ -d "/dev/disk/by-vdev" ]
	then
	# Add by-vdev at the beginning.
	ZPOOL_IMPORT_PATH="/dev/disk/by-vdev:"
	fi

	# Help with getting LUKS partitions etc imported.
	if [ -d "/dev/mapper" ]; then
	if [ -n "$ZPOOL_IMPORT_PATH" ]; then
	ZPOOL_IMPORT_PATH="$ZPOOL_IMPORT_PATH:/dev/mapper:"
	else
	ZPOOL_IMPORT_PATH="/dev/mapper:"
	fi
	fi

	# ... and /dev at the very end, just for good measure.
	ZPOOL_IMPORT_PATH="$ZPOOL_IMPORT_PATH$dirs:/dev"
	fi

	# Needs to be exported for "zpool" to catch it.
	[ -n "$ZPOOL_IMPORT_PATH" ] && export ZPOOL_IMPORT_PATH

	# Mount all available pools (except those set in ZFS_POOL_EXCEPTIONS.
	#
	# If not interactive (run from init - variable init='/sbin/init')
	# we get ONE line for all pools being imported, with just a dot
	# as status for each pool.
	# Example: Importing ZFS pool(s)... [OK]
	#
	# If it IS interactive (started from the shell manually), then we
	# get one line per pool importing.
	# Example: Importing ZFS pool pool1 [OK]
	# Importing ZFS pool pool2 [OK]
	# [etc]
	[ -n "$init" ] && zfs_log_begin_msg "Importing ZFS pool(s)"
	OLD_IFS="$IFS" ; IFS=";"
	for pool in $available_pools
	do
	[ -z "$pool" ] && continue

	# We have pools that haven't been imported - import them
	if [ -n "$init" ]
	then
	# Not interactive - a dot for each pool.
	# Except on Gentoo where this doesn't work.
	zfs_log_progress_msg "."
	else
	# Interactive - one 'Importing ...' line per pool
	zfs_log_begin_msg "Importing ZFS pool $pool"
	fi

	# Import by using ZPOOL_IMPORT_PATH (either set above or in
	# the config file) _or_ with the 'built in' default search
	# paths. This is the preferred way.
	# shellcheck disable=SC2086
	"$ZPOOL" import -N ${ZPOOL_IMPORT_OPTS} "$pool" 2> /dev/null
	r="$?" ; RET=$((RET + r))
	if [ "$r" -eq 0 ]
	then
	# Output success and process the next pool
	[ -z "$init" ] && zfs_log_end_msg 0
	continue
	fi
	# We don't want a fail msg here, we're going to try import
	# using the cache file soon and that might succeed.
	[ ! -f "$ZPOOL_CACHE" ] && zfs_log_end_msg "$RET"

	if [ "$r" -gt 0 ] && [ -f "$ZPOOL_CACHE" ]
	then
	# Failed to import without a cache file. Try WITH...
	if [ -z "$init" ] && check_boolean "$VERBOSE_MOUNT"
	then
	# Interactive + Verbose = more information
	zfs_log_progress_msg " using cache file"
	fi

	# shellcheck disable=SC2086
	"$ZPOOL" import -c "$ZPOOL_CACHE" -N ${ZPOOL_IMPORT_OPTS} \
	"$pool" 2> /dev/null
	r="$?" ; RET=$((RET + r))
	if [ "$r" -eq 0 ]
	then
	[ -z "$init" ] && zfs_log_end_msg 0
	continue 3 # Next pool
	fi
	zfs_log_end_msg "$RET"
	fi
	done
	[ -n "$init" ] && zfs_log_end_msg "$RET"

	IFS="$OLD_IFS"
	[ -n "$already_imported" ] && [ -z "$available_pools" ] && return 0

	return "$RET"
	}

	do_import()
	{
	if check_boolean "$ZPOOL_IMPORT_ALL_VISIBLE"
	then
	do_import_all_visible
	else
	# This is the default option
	do_verbatim_import
	fi
	}

	# Output the status and list of pools
	do_status()
	{
	check_module_loaded "zfs" \|\| exit 0

	"$ZPOOL" status && echo "" && "$ZPOOL" list
	}

	do_start()
	{
	if check_boolean "$VERBOSE_MOUNT"
	then
	zfs_log_begin_msg "Checking if ZFS userspace tools present"
	fi

	if checksystem
	then
	check_boolean "$VERBOSE_MOUNT" && zfs_log_end_msg 0

	check_boolean "$VERBOSE_MOUNT" && \
	zfs_log_begin_msg "Loading kernel ZFS infrastructure"

	if ! load_module "zfs"
	then
	check_boolean "$VERBOSE_MOUNT" && zfs_log_end_msg 1
	return 5
	fi
	check_boolean "$VERBOSE_MOUNT" && zfs_log_end_msg 0

	do_import && udev_trigger # just to make sure we get zvols.

	return 0
	else
	return 1
	fi
	}

	# ----------------------------------------------------

	-if [ ! -e /sbin/openrc-run ]
	+if @IS_SYSV_RC@
	then
	case "$1" in
	start)
	do_start
	;;
	stop)
	# no-op
	;;
	status)
	do_status
	;;
	force-reload\|condrestart\|reload\|restart)
	# no-op
	;;
	*)
	[ -n "$1" ] && echo "Error: Unknown command $1."
	echo "Usage: $0 {start\|status}"
	exit 3
	;;
	esac

	exit $?
	else
	# Create wrapper functions since Gentoo don't use the case part.
	depend() { do_depend; }
	start() { do_start; }
	status() { do_status; }
	fi
	diff --git a/sys/contrib/openzfs/etc/init.d/zfs-load-key.in b/sys/contrib/openzfs/etc/init.d/zfs-load-key.in
	index 53c7766b793a..27dfeeb0bcc5 100755
	--- a/sys/contrib/openzfs/etc/init.d/zfs-load-key.in
	+++ b/sys/contrib/openzfs/etc/init.d/zfs-load-key.in
	@@ -1,132 +1,132 @@
	#!@DEFAULT_INIT_SHELL@
	# shellcheck disable=SC2154
	#
	# zfs-load-key This script will load/unload the zfs filesystems keys.
	#
	# chkconfig: 2345 06 99
	# description: This script will load or unload the zfs filesystems keys during
	# system boot/shutdown. Only filesystems with key path set
	# in keylocation property. See the zfs(8) man page for details.
	# probe: true
	#
	### BEGIN INIT INFO
	# Provides: zfs-load-key
	# Required-Start: $local_fs zfs-import
	# Required-Stop: $local_fs zfs-import
	# Default-Start: 2 3 4 5
	# Default-Stop: 0 1 6
	# X-Start-Before: zfs-mount
	# X-Stop-After: zfs-zed
	# Short-Description: Load ZFS keys for filesystems and volumes
	# Description: Run the `zfs load-key` or `zfs unload-key` commands.
	### END INIT INFO
	#
	# Released under the 2-clause BSD license.
	#
	# This script is based on debian/zfsutils.zfs.init from the
	# Debian GNU/kFreeBSD zfsutils 8.1-3 package, written by Aurelien Jarno.

	# Source the common init script
	. @sysconfdir@/zfs/zfs-functions

	# ----------------------------------------------------

	do_depend()
	{
	# bootmisc will log to /var which may be a different zfs than root.
	before bootmisc logger zfs-mount

	after zfs-import sysfs
	keyword -lxc -openvz -prefix -vserver
	}

	# Load keys for all datasets/filesystems
	do_load_keys()
	{
	zfs_log_begin_msg "Load ZFS filesystem(s) keys"

	"$ZFS" list -Ho name,encryptionroot,keystatus,keylocation \|
	while IFS=" " read -r name encryptionroot keystatus keylocation; do
	if [ "$encryptionroot" != "-" ] &&
	[ "$name" = "$encryptionroot" ] &&
	[ "$keystatus" = "unavailable" ] &&
	[ "$keylocation" != "prompt" ] &&
	[ "$keylocation" != "none" ]
	then
	zfs_action "Load key for $encryptionroot" \
	"$ZFS" load-key "$encryptionroot"
	fi
	done

	zfs_log_end_msg 0

	return 0
	}

	# Unload keys for all datasets/filesystems
	do_unload_keys()
	{
	zfs_log_begin_msg "Unload ZFS filesystem(s) key"

	"$ZFS" list -Ho name,encryptionroot,keystatus \| sed '1!G;h;$!d' \|
	while IFS=" " read -r name encryptionroot keystatus; do
	if [ "$encryptionroot" != "-" ] &&
	[ "$name" = "$encryptionroot" ] &&
	[ "$keystatus" = "available" ]
	then
	zfs_action "Unload key for $encryptionroot" \
	"$ZFS" unload-key "$encryptionroot"
	fi
	done

	zfs_log_end_msg 0

	return 0
	}

	do_start()
	{
	check_boolean "$ZFS_LOAD_KEY" \|\| exit 0

	check_module_loaded "zfs" \|\| exit 0

	do_load_keys
	}

	do_stop()
	{
	check_boolean "$ZFS_UNLOAD_KEY" \|\| exit 0

	check_module_loaded "zfs" \|\| exit 0

	do_unload_keys
	}

	# ----------------------------------------------------

	-if [ ! -e /sbin/openrc-run ]
	+if @IS_SYSV_RC@
	then
	case "$1" in
	start)
	do_start
	;;
	stop)
	do_stop
	;;
	force-reload\|condrestart\|reload\|restart\|status)
	# no-op
	;;
	*)
	[ -n "$1" ] && echo "Error: Unknown command $1."
	echo "Usage: $0 {start\|stop}"
	exit 3
	;;
	esac

	exit $?
	else
	# Create wrapper functions since Gentoo don't use the case part.
	depend() { do_depend; }
	start() { do_start; }
	stop() { do_stop; }
	fi
	diff --git a/sys/contrib/openzfs/etc/init.d/zfs-mount.in b/sys/contrib/openzfs/etc/init.d/zfs-mount.in
	index a0825f19fcdd..6a3ca5f86908 100755
	--- a/sys/contrib/openzfs/etc/init.d/zfs-mount.in
	+++ b/sys/contrib/openzfs/etc/init.d/zfs-mount.in
	@@ -1,142 +1,142 @@
	#!@DEFAULT_INIT_SHELL@
	# shellcheck disable=SC2154
	#
	# zfs-mount This script will mount/umount the zfs filesystems.
	#
	# chkconfig: 2345 06 99
	# description: This script will mount/umount the zfs filesystems during
	# system boot/shutdown. Configuration of which filesystems
	# should be mounted is handled by the zfs 'mountpoint' and
	# 'canmount' properties. See the zfs(8) man page for details.
	# It is also responsible for all userspace zfs services.
	# probe: true
	#
	### BEGIN INIT INFO
	# Provides: zfs-mount
	# Required-Start: zfs-import
	# Required-Stop: $local_fs zfs-import
	# Default-Start: S
	# Default-Stop: 0 1 6
	# X-Start-Before: mountall
	# X-Stop-After: zfs-zed
	# Short-Description: Mount ZFS filesystems and volumes
	# Description: Run the `zfs mount -a` or `zfs umount -a` commands.
	### END INIT INFO
	#
	# Released under the 2-clause BSD license.
	#
	# This script is based on debian/zfsutils.zfs.init from the
	# Debian GNU/kFreeBSD zfsutils 8.1-3 package, written by Aurelien Jarno.

	# Source the common init script
	. @sysconfdir@/zfs/zfs-functions

	# ----------------------------------------------------

	chkroot() {
	while read -r _ mp _; do
	if [ "$mp" = "/" ]; then
	return 0
	fi
	done < /proc/self/mounts

	return 1
	}

	do_depend()
	{
	# Try to allow people to mix and match fstab with ZFS in a way that makes sense.
	if [ "$(mountinfo -s /)" = 'zfs' ]
	then
	before localmount
	else
	after localmount
	fi

	# bootmisc will log to /var which may be a different zfs than root.
	before bootmisc logger

	after zfs-import sysfs
	use mtab
	keyword -lxc -openvz -prefix -vserver
	}

	# Mount all datasets/filesystems
	do_mount()
	{
	local verbose overlay

	check_boolean "$VERBOSE_MOUNT" && verbose=v
	check_boolean "$DO_OVERLAY_MOUNTS" && overlay=O

	zfs_action "Mounting ZFS filesystem(s)" \
	"$ZFS" mount "-a$verbose$overlay" "$MOUNT_EXTRA_OPTIONS"

	return 0
	}

	# Unmount all filesystems
	do_unmount()
	{
	# This shouldn't really be necessary, as long as one can get
	# zfs-import to run sufficiently late in the shutdown/reboot process
	# - after unmounting local filesystems. This is just here in case/if
	# this isn't possible.
	zfs_action "Unmounting ZFS filesystems" "$ZFS" unmount -a

	return 0
	}

	do_start()
	{
	check_boolean "$ZFS_MOUNT" \|\| exit 0

	check_module_loaded "zfs" \|\| exit 0

	# Ensure / exists in /proc/self/mounts.
	# This should be handled by rc.sysinit but lets be paranoid.
	if ! chkroot
	then
	mount -f /
	fi

	do_mount
	}

	do_stop()
	{
	check_boolean "$ZFS_UNMOUNT" \|\| exit 0

	check_module_loaded "zfs" \|\| exit 0

	do_unmount
	}

	# ----------------------------------------------------

	-if [ ! -e /sbin/openrc-run ]
	+if @IS_SYSV_RC@
	then
	case "$1" in
	start)
	do_start
	;;
	stop)
	do_stop
	;;
	force-reload\|condrestart\|reload\|restart\|status)
	# no-op
	;;
	*)
	[ -n "$1" ] && echo "Error: Unknown command $1."
	echo "Usage: $0 {start\|stop}"
	exit 3
	;;
	esac

	exit $?
	else
	# Create wrapper functions since Gentoo don't use the case part.
	depend() { do_depend; }
	start() { do_start; }
	stop() { do_stop; }
	fi
	diff --git a/sys/contrib/openzfs/etc/init.d/zfs-share.in b/sys/contrib/openzfs/etc/init.d/zfs-share.in
	index 88978071cbf6..06c59c620b75 100755
	--- a/sys/contrib/openzfs/etc/init.d/zfs-share.in
	+++ b/sys/contrib/openzfs/etc/init.d/zfs-share.in
	@@ -1,84 +1,85 @@
	#!@DEFAULT_INIT_SHELL@
	# shellcheck disable=SC2154
	#
	# zfs-share This script will network share zfs filesystems and volumes.
	#
	# chkconfig: 2345 30 99
	# description: Run the `zfs share -a` or `zfs unshare -a` commands
	# for controlling iSCSI, NFS, or CIFS network shares.
	# probe: true
	#
	### BEGIN INIT INFO
	# Provides: zfs-share
	# Required-Start: $local_fs $network $remote_fs zfs-mount
	# Required-Stop: $local_fs $network $remote_fs zfs-mount
	# Default-Start: 2 3 4 5
	# Default-Stop: 0 1 6
	# Should-Start: iscsi iscsitarget istgt scst @DEFAULT_INIT_NFS_SERVER@ samba samba4 zfs-mount zfs-zed
	# Should-Stop: iscsi iscsitarget istgt scst @DEFAULT_INIT_NFS_SERVER@ samba samba4 zfs-mount zfs-zed
	# Short-Description: Network share ZFS datasets and volumes.
	# Description: Run the `zfs share -a` or `zfs unshare -a` commands
	# for controlling iSCSI, NFS, or CIFS network shares.
	### END INIT INFO
	#
	# Released under the 2-clause BSD license.
	#
	# This script is based on debian/zfsutils.zfs.init from the
	# Debian GNU/kFreeBSD zfsutils 8.1-3 package, written by Aurelien Jarno.

	# Source the common init script
	. @sysconfdir@/zfs/zfs-functions

	# ----------------------------------------------------

	do_depend()
	{
	after sysfs zfs-mount zfs-zed
	keyword -lxc -openvz -prefix -vserver
	}

	do_start()
	{
	check_boolean "$ZFS_SHARE" \|\| exit 0

	check_module_loaded "zfs" \|\| exit 0

	zfs_action "Sharing ZFS filesystems" "$ZFS" share -a
	}

	do_stop()
	{
	check_boolean "$ZFS_UNSHARE" \|\| exit 0

	check_module_loaded "zfs" \|\| exit 0

	zfs_action "Unsharing ZFS filesystems" "$ZFS" unshare -a
	}

	# ----------------------------------------------------

	-if [ ! -e /sbin/openrc-run ]; then
	+if @IS_SYSV_RC@
	+then
	case "$1" in
	start)
	do_start
	;;
	stop)
	do_stop
	;;
	force-reload\|reload\|restart\|status)
	# no-op
	;;
	*)
	[ -n "$1" ] && echo "Error: Unknown command $1."
	echo "Usage: $0 {start\|stop}"
	exit 3
	;;
	esac

	exit $?
	else
	# Create wrapper functions since Gentoo don't use the case part.
	depend() { do_depend; }
	start() { do_start; }
	stop() { do_stop; }
	fi
	diff --git a/sys/contrib/openzfs/etc/init.d/zfs-zed.in b/sys/contrib/openzfs/etc/init.d/zfs-zed.in
	index e9cf8867403c..3d40600cea5d 100755
	--- a/sys/contrib/openzfs/etc/init.d/zfs-zed.in
	+++ b/sys/contrib/openzfs/etc/init.d/zfs-zed.in
	@@ -1,129 +1,130 @@
	#!@DEFAULT_INIT_SHELL@
	# shellcheck disable=SC2154
	#
	# zfs-zed
	#
	# chkconfig: 2345 29 99
	# description: This script will start and stop the ZFS Event Daemon.
	# probe: true
	#
	### BEGIN INIT INFO
	# Provides: zfs-zed
	# Required-Start: zfs-mount
	# Required-Stop: zfs-mount
	# Default-Start: 2 3 4 5
	# Default-Stop: 0 1 6
	# X-Stop-After: zfs-share
	# Short-Description: ZFS Event Daemon
	# Description: zed monitors ZFS events. When a zevent is posted, zed
	# will run any scripts that have been enabled for the
	# corresponding zevent class.
	### END INIT INFO
	#
	# Released under the 2-clause BSD license.
	#
	# This script is based on debian/zfsutils.zfs.init from the
	# Debian GNU/kFreeBSD zfsutils 8.1-3 package, written by Aurelien Jarno.

	# Source the common init script
	. @sysconfdir@/zfs/zfs-functions

	ZED_NAME="zed"
	ZED_PIDFILE="@runstatedir@/$ZED_NAME.pid"

	# shellcheck disable=SC2034
	extra_started_commands="reload"

	# Exit if the package is not installed
	[ -x "$ZED" ] \|\| exit 0

	# ----------------------------------------------------

	do_depend()
	{
	after zfs-mount localmount
	}

	do_start()
	{
	check_module_loaded "zfs" \|\| exit 0

	ZED_ARGS="$ZED_ARGS -p $ZED_PIDFILE"

	zfs_action "Starting ZFS Event Daemon" zfs_daemon_start \
	"$ZED_PIDFILE" "$ZED" "$ZED_ARGS"
	return "$?"
	}

	do_stop()
	{
	local pools
	check_module_loaded "zfs" \|\| exit 0

	zfs_action "Stopping ZFS Event Daemon" zfs_daemon_stop \
	"$ZED_PIDFILE" "$ZED" "$ZED_NAME" \|\| return "$?"

	# Let's see if we have any pools imported
	pools=$("$ZPOOL" list -H -oname)
	if [ -z "$pools" ]
	then
	# No pools imported, it is/should be safe/possible to
	# unload modules.
	zfs_action "Unloading modules" rmmod zfs spl
	return "$?"
	fi
	}

	do_status()
	{
	check_module_loaded "zfs" \|\| exit 0

	zfs_daemon_status "$ZED_PIDFILE" "$ZED" "$ZED_NAME"
	return "$?"
	}

	do_reload()
	{
	check_module_loaded "zfs" \|\| exit 0

	zfs_action "Reloading ZFS Event Daemon" zfs_daemon_reload \
	"$ZED_PIDFILE" "$ZED_NAME"
	return "$?"
	}

	# ----------------------------------------------------

	-if [ ! -e /sbin/openrc-run ]; then
	+if @IS_SYSV_RC@
	+then
	case "$1" in
	start)
	do_start
	;;
	stop)
	do_stop
	;;
	status)
	do_status
	;;
	reload\|force-reload)
	do_reload
	;;
	restart)
	do_stop
	do_start
	;;
	*)
	[ -n "$1" ] && echo "Error: Unknown command $1."
	echo "Usage: $0 {start\|stop\|status\|reload\|restart}"
	exit 1
	;;
	esac

	exit $?
	else
	# Create wrapper functions since Gentoo don't use the case part.
	depend() { do_depend; }
	start() { do_start; }
	stop() { do_stop; }
	status() { do_status; }
	reload() { do_reload; }
	fi
	diff --git a/sys/contrib/openzfs/include/libzfs.h b/sys/contrib/openzfs/include/libzfs.h
	index 770c5e1f201c..2823b8845827 100644
	--- a/sys/contrib/openzfs/include/libzfs.h
	+++ b/sys/contrib/openzfs/include/libzfs.h
	@@ -1,1052 +1,1054 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	- * Copyright (c) 2011, 2022 by Delphix. All rights reserved.
	+ * Copyright (c) 2011, 2024 by Delphix. All rights reserved.
	* Copyright Joyent, Inc.
	* Copyright (c) 2013 Steven Hartland. All rights reserved.
	* Copyright (c) 2016, Intel Corporation.
	* Copyright 2016 Nexenta Systems, Inc.
	* Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
	* Copyright (c) 2019 Datto Inc.
	* Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
	*/

	#ifndef _LIBZFS_H
	#define _LIBZFS_H extern __attribute__((visibility("default")))

	#include <assert.h>
	#include <libshare.h>
	#include <libnvpair.h>
	#include <sys/mnttab.h>
	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/fs/zfs.h>
	#include <sys/avl.h>
	#include <libzfs_core.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*
	* Miscellaneous ZFS constants
	*/
	#define ZFS_MAXPROPLEN MAXPATHLEN
	#define ZPOOL_MAXPROPLEN MAXPATHLEN

	/*
	* libzfs errors
	*/
	typedef enum zfs_error {
	EZFS_SUCCESS = 0, /* no error -- success */
	EZFS_NOMEM = 2000, /* out of memory */
	EZFS_BADPROP, /* invalid property value */
	EZFS_PROPREADONLY, /* cannot set readonly property */
	EZFS_PROPTYPE, /* property does not apply to dataset type */
	EZFS_PROPNONINHERIT, /* property is not inheritable */
	EZFS_PROPSPACE, /* bad quota or reservation */
	EZFS_BADTYPE, /* dataset is not of appropriate type */
	EZFS_BUSY, /* pool or dataset is busy */
	EZFS_EXISTS, /* pool or dataset already exists */
	EZFS_NOENT, /* no such pool or dataset */
	EZFS_BADSTREAM, /* bad backup stream */
	EZFS_DSREADONLY, /* dataset is readonly */
	EZFS_VOLTOOBIG, /* volume is too large for 32-bit system */
	EZFS_INVALIDNAME, /* invalid dataset name */
	EZFS_BADRESTORE, /* unable to restore to destination */
	EZFS_BADBACKUP, /* backup failed */
	EZFS_BADTARGET, /* bad attach/detach/replace target */
	EZFS_NODEVICE, /* no such device in pool */
	EZFS_BADDEV, /* invalid device to add */
	EZFS_NOREPLICAS, /* no valid replicas */
	EZFS_RESILVERING, /* resilvering (healing reconstruction) */
	EZFS_BADVERSION, /* unsupported version */
	EZFS_POOLUNAVAIL, /* pool is currently unavailable */
	EZFS_DEVOVERFLOW, /* too many devices in one vdev */
	EZFS_BADPATH, /* must be an absolute path */
	EZFS_CROSSTARGET, /* rename or clone across pool or dataset */
	EZFS_ZONED, /* used improperly in local zone */
	EZFS_MOUNTFAILED, /* failed to mount dataset */
	EZFS_UMOUNTFAILED, /* failed to unmount dataset */
	EZFS_UNSHARENFSFAILED, /* failed to unshare over nfs */
	EZFS_SHARENFSFAILED, /* failed to share over nfs */
	EZFS_PERM, /* permission denied */
	EZFS_NOSPC, /* out of space */
	EZFS_FAULT, /* bad address */
	EZFS_IO, /* I/O error */
	EZFS_INTR, /* signal received */
	EZFS_ISSPARE, /* device is a hot spare */
	EZFS_INVALCONFIG, /* invalid vdev configuration */
	EZFS_RECURSIVE, /* recursive dependency */
	EZFS_NOHISTORY, /* no history object */
	EZFS_POOLPROPS, /* couldn't retrieve pool props */
	EZFS_POOL_NOTSUP, /* ops not supported for this type of pool */
	EZFS_POOL_INVALARG, /* invalid argument for this pool operation */
	EZFS_NAMETOOLONG, /* dataset name is too long */
	EZFS_OPENFAILED, /* open of device failed */
	EZFS_NOCAP, /* couldn't get capacity */
	EZFS_LABELFAILED, /* write of label failed */
	EZFS_BADWHO, /* invalid permission who */
	EZFS_BADPERM, /* invalid permission */
	EZFS_BADPERMSET, /* invalid permission set name */
	EZFS_NODELEGATION, /* delegated administration is disabled */
	EZFS_UNSHARESMBFAILED, /* failed to unshare over smb */
	EZFS_SHARESMBFAILED, /* failed to share over smb */
	EZFS_BADCACHE, /* bad cache file */
	EZFS_ISL2CACHE, /* device is for the level 2 ARC */
	EZFS_VDEVNOTSUP, /* unsupported vdev type */
	EZFS_NOTSUP, /* ops not supported on this dataset */
	EZFS_ACTIVE_SPARE, /* pool has active shared spare devices */
	EZFS_UNPLAYED_LOGS, /* log device has unplayed logs */
	EZFS_REFTAG_RELE, /* snapshot release: tag not found */
	EZFS_REFTAG_HOLD, /* snapshot hold: tag already exists */
	EZFS_TAGTOOLONG, /* snapshot hold/rele: tag too long */
	EZFS_PIPEFAILED, /* pipe create failed */
	EZFS_THREADCREATEFAILED, /* thread create failed */
	EZFS_POSTSPLIT_ONLINE, /* onlining a disk after splitting it */
	EZFS_SCRUBBING, /* currently scrubbing */
	EZFS_ERRORSCRUBBING, /* currently error scrubbing */
	EZFS_ERRORSCRUB_PAUSED, /* error scrub currently paused */
	EZFS_NO_SCRUB, /* no active scrub */
	EZFS_DIFF, /* general failure of zfs diff */
	EZFS_DIFFDATA, /* bad zfs diff data */
	EZFS_POOLREADONLY, /* pool is in read-only mode */
	EZFS_SCRUB_PAUSED, /* scrub currently paused */
	EZFS_SCRUB_PAUSED_TO_CANCEL, /* scrub currently paused */
	EZFS_ACTIVE_POOL, /* pool is imported on a different system */
	EZFS_CRYPTOFAILED, /* failed to setup encryption */
	EZFS_NO_PENDING, /* cannot cancel, no operation is pending */
	EZFS_CHECKPOINT_EXISTS, /* checkpoint exists */
	EZFS_DISCARDING_CHECKPOINT, /* currently discarding a checkpoint */
	EZFS_NO_CHECKPOINT, /* pool has no checkpoint */
	EZFS_DEVRM_IN_PROGRESS, /* a device is currently being removed */
	EZFS_VDEV_TOO_BIG, /* a device is too big to be used */
	EZFS_IOC_NOTSUPPORTED, /* operation not supported by zfs module */
	EZFS_TOOMANY, /* argument list too long */
	EZFS_INITIALIZING, /* currently initializing */
	EZFS_NO_INITIALIZE, /* no active initialize */
	EZFS_WRONG_PARENT, /* invalid parent dataset (e.g ZVOL) */
	EZFS_TRIMMING, /* currently trimming */
	EZFS_NO_TRIM, /* no active trim */
	EZFS_TRIM_NOTSUP, /* device does not support trim */
	EZFS_NO_RESILVER_DEFER, /* pool doesn't support resilver_defer */
	EZFS_EXPORT_IN_PROGRESS, /* currently exporting the pool */
	EZFS_REBUILDING, /* resilvering (sequential reconstrution) */
	EZFS_VDEV_NOTSUP, /* ops not supported for this type of vdev */
	EZFS_NOT_USER_NAMESPACE, /* a file is not a user namespace */
	EZFS_CKSUM, /* insufficient replicas */
	EZFS_RESUME_EXISTS, /* Resume on existing dataset without force */
	EZFS_SHAREFAILED, /* filesystem share failed */
	+ EZFS_RAIDZ_EXPAND_IN_PROGRESS, /* a raidz is currently expanding */
	+ EZFS_ASHIFT_MISMATCH, /* can't add vdevs with different ashifts */
	EZFS_UNKNOWN
	} zfs_error_t;

	/*
	* The following data structures are all part
	* of the zfs_allow_t data structure which is
	* used for printing 'allow' permissions.
	* It is a linked list of zfs_allow_t's which
	* then contain avl tree's for user/group/sets/...
	* and each one of the entries in those trees have
	* avl tree's for the permissions they belong to and
	* whether they are local,descendent or local+descendent
	* permissions. The AVL trees are used primarily for
	* sorting purposes, but also so that we can quickly find
	* a given user and or permission.
	*/
	typedef struct zfs_perm_node {
	avl_node_t z_node;
	char z_pname[MAXPATHLEN];
	} zfs_perm_node_t;

	typedef struct zfs_allow_node {
	avl_node_t z_node;
	char z_key[MAXPATHLEN]; /* name, such as joe */
	avl_tree_t z_localdescend; /* local+descendent perms */
	avl_tree_t z_local; /* local permissions */
	avl_tree_t z_descend; /* descendent permissions */
	} zfs_allow_node_t;

	typedef struct zfs_allow {
	struct zfs_allow *z_next;
	char z_setpoint[MAXPATHLEN];
	avl_tree_t z_sets;
	avl_tree_t z_crperms;
	avl_tree_t z_user;
	avl_tree_t z_group;
	avl_tree_t z_everyone;
	} zfs_allow_t;

	/*
	* Basic handle types
	*/
	typedef struct zfs_handle zfs_handle_t;
	typedef struct zpool_handle zpool_handle_t;
	typedef struct libzfs_handle libzfs_handle_t;

	_LIBZFS_H int zpool_wait(zpool_handle_t *, zpool_wait_activity_t);
	_LIBZFS_H int zpool_wait_status(zpool_handle_t *, zpool_wait_activity_t,
	boolean_t , boolean_t );

	/*
	* Library initialization
	*/
	_LIBZFS_H libzfs_handle_t *libzfs_init(void);
	_LIBZFS_H void libzfs_fini(libzfs_handle_t *);

	_LIBZFS_H libzfs_handle_t zpool_get_handle(zpool_handle_t );
	_LIBZFS_H libzfs_handle_t zfs_get_handle(zfs_handle_t );

	_LIBZFS_H void libzfs_print_on_error(libzfs_handle_t *, boolean_t);

	_LIBZFS_H void zfs_save_arguments(int argc, char *, char , int);
	_LIBZFS_H int zpool_log_history(libzfs_handle_t , const char );

	_LIBZFS_H int libzfs_errno(libzfs_handle_t *);
	_LIBZFS_H const char *libzfs_error_init(int);
	_LIBZFS_H const char libzfs_error_action(libzfs_handle_t );
	_LIBZFS_H const char libzfs_error_description(libzfs_handle_t );
	_LIBZFS_H int zfs_standard_error(libzfs_handle_t , int, const char );
	_LIBZFS_H void libzfs_mnttab_init(libzfs_handle_t *);
	_LIBZFS_H void libzfs_mnttab_fini(libzfs_handle_t *);
	_LIBZFS_H void libzfs_mnttab_cache(libzfs_handle_t *, boolean_t);
	_LIBZFS_H int libzfs_mnttab_find(libzfs_handle_t , const char ,
	struct mnttab *);
	_LIBZFS_H void libzfs_mnttab_add(libzfs_handle_t , const char ,
	const char , const char );
	_LIBZFS_H void libzfs_mnttab_remove(libzfs_handle_t , const char );

	/*
	* Basic handle functions
	*/
	_LIBZFS_H zpool_handle_t zpool_open(libzfs_handle_t , const char *);
	_LIBZFS_H zpool_handle_t zpool_open_canfail(libzfs_handle_t , const char *);
	_LIBZFS_H void zpool_close(zpool_handle_t *);
	_LIBZFS_H const char zpool_get_name(zpool_handle_t );
	_LIBZFS_H int zpool_get_state(zpool_handle_t *);
	_LIBZFS_H const char *zpool_state_to_name(vdev_state_t, vdev_aux_t);
	_LIBZFS_H const char *zpool_pool_state_to_name(pool_state_t);
	_LIBZFS_H void zpool_free_handles(libzfs_handle_t *);

	/*
	* Iterate over all active pools in the system.
	*/
	typedef int (zpool_iter_f)(zpool_handle_t , void *);
	_LIBZFS_H int zpool_iter(libzfs_handle_t , zpool_iter_f, void );
	_LIBZFS_H boolean_t zpool_skip_pool(const char *);

	/*
	* Functions to create and destroy pools
	*/
	_LIBZFS_H int zpool_create(libzfs_handle_t , const char , nvlist_t *,
	nvlist_t , nvlist_t );
	_LIBZFS_H int zpool_destroy(zpool_handle_t , const char );
	-_LIBZFS_H int zpool_add(zpool_handle_t , nvlist_t );
	+_LIBZFS_H int zpool_add(zpool_handle_t , nvlist_t , boolean_t check_ashift);

	typedef struct splitflags {
	/* do not split, but return the config that would be split off */
	unsigned int dryrun : 1;

	/* after splitting, import the pool */
	unsigned int import : 1;
	int name_flags;
	} splitflags_t;

	typedef struct trimflags {
	/* requested vdevs are for the entire pool */
	boolean_t fullpool;

	/* request a secure trim, requires support from device */
	boolean_t secure;

	/* after starting trim, block until trim completes */
	boolean_t wait;

	/* trim at the requested rate in bytes/second */
	uint64_t rate;
	} trimflags_t;

	/*
	* Functions to manipulate pool and vdev state
	*/
	_LIBZFS_H int zpool_scan(zpool_handle_t *, pool_scan_func_t, pool_scrub_cmd_t);
	_LIBZFS_H int zpool_initialize(zpool_handle_t *, pool_initialize_func_t,
	nvlist_t *);
	_LIBZFS_H int zpool_initialize_wait(zpool_handle_t *, pool_initialize_func_t,
	nvlist_t *);
	_LIBZFS_H int zpool_trim(zpool_handle_t , pool_trim_func_t, nvlist_t ,
	trimflags_t *);

	_LIBZFS_H int zpool_clear(zpool_handle_t , const char , nvlist_t *);
	_LIBZFS_H int zpool_reguid(zpool_handle_t *);
	_LIBZFS_H int zpool_reopen_one(zpool_handle_t , void );

	_LIBZFS_H int zpool_sync_one(zpool_handle_t , void );

	_LIBZFS_H int zpool_vdev_online(zpool_handle_t , const char , int,
	vdev_state_t *);
	_LIBZFS_H int zpool_vdev_offline(zpool_handle_t , const char , boolean_t);
	_LIBZFS_H int zpool_vdev_attach(zpool_handle_t , const char ,
	const char , nvlist_t , int, boolean_t);
	_LIBZFS_H int zpool_vdev_detach(zpool_handle_t , const char );
	_LIBZFS_H int zpool_vdev_remove(zpool_handle_t , const char );
	_LIBZFS_H int zpool_vdev_remove_cancel(zpool_handle_t *);
	_LIBZFS_H int zpool_vdev_indirect_size(zpool_handle_t , const char ,
	uint64_t *);
	_LIBZFS_H int zpool_vdev_split(zpool_handle_t , char , nvlist_t **,
	nvlist_t *, splitflags_t);
	_LIBZFS_H int zpool_vdev_remove_wanted(zpool_handle_t , const char );

	_LIBZFS_H int zpool_vdev_fault(zpool_handle_t *, uint64_t, vdev_aux_t);
	_LIBZFS_H int zpool_vdev_degrade(zpool_handle_t *, uint64_t, vdev_aux_t);
	_LIBZFS_H int zpool_vdev_set_removed_state(zpool_handle_t *, uint64_t,
	vdev_aux_t);

	_LIBZFS_H int zpool_vdev_clear(zpool_handle_t *, uint64_t);

	_LIBZFS_H nvlist_t zpool_find_vdev(zpool_handle_t , const char , boolean_t ,
	boolean_t , boolean_t );
	_LIBZFS_H nvlist_t zpool_find_vdev_by_physpath(zpool_handle_t , const char *,
	boolean_t , boolean_t , boolean_t *);
	_LIBZFS_H int zpool_label_disk(libzfs_handle_t , zpool_handle_t ,
	const char *);
	_LIBZFS_H int zpool_prepare_disk(zpool_handle_t zhp, nvlist_t vdev_nv,
	const char prepare_str, char lines[], int lines_cnt);
	_LIBZFS_H int zpool_prepare_and_label_disk(libzfs_handle_t *hdl,
	zpool_handle_t , const char , nvlist_t vdev_nv, const char prepare_str,
	char *lines[], int lines_cnt);
	_LIBZFS_H char ** zpool_vdev_script_alloc_env(const char *pool_name,
	const char vdev_path, const char vdev_upath,
	const char vdev_enc_sysfs_path, const char opt_key, const char *opt_val);
	_LIBZFS_H void zpool_vdev_script_free_env(char **env);
	_LIBZFS_H uint64_t zpool_vdev_path_to_guid(zpool_handle_t *zhp,
	const char *path);

	_LIBZFS_H const char zpool_get_state_str(zpool_handle_t );

	/*
	* Functions to manage pool properties
	*/
	_LIBZFS_H int zpool_set_prop(zpool_handle_t , const char , const char *);
	_LIBZFS_H int zpool_get_prop(zpool_handle_t , zpool_prop_t, char ,
	size_t proplen, zprop_source_t *, boolean_t literal);
	_LIBZFS_H int zpool_get_userprop(zpool_handle_t , const char , char *,
	size_t proplen, zprop_source_t *);
	_LIBZFS_H uint64_t zpool_get_prop_int(zpool_handle_t *, zpool_prop_t,
	zprop_source_t *);
	_LIBZFS_H int zpool_props_refresh(zpool_handle_t *);

	_LIBZFS_H const char *zpool_prop_to_name(zpool_prop_t);
	_LIBZFS_H const char *zpool_prop_values(zpool_prop_t);

	/*
	* Functions to manage vdev properties
	*/
	_LIBZFS_H int zpool_get_vdev_prop_value(nvlist_t , vdev_prop_t, char , char *,
	size_t, zprop_source_t *, boolean_t);
	_LIBZFS_H int zpool_get_vdev_prop(zpool_handle_t , const char , vdev_prop_t,
	char , char , size_t, zprop_source_t *, boolean_t);
	_LIBZFS_H int zpool_get_all_vdev_props(zpool_handle_t , const char ,
	nvlist_t **);
	_LIBZFS_H int zpool_set_vdev_prop(zpool_handle_t , const char , const char *,
	const char *);

	_LIBZFS_H const char *vdev_prop_to_name(vdev_prop_t);
	_LIBZFS_H const char *vdev_prop_values(vdev_prop_t);
	_LIBZFS_H boolean_t vdev_prop_user(const char *name);
	_LIBZFS_H const char *vdev_prop_column_name(vdev_prop_t);
	_LIBZFS_H boolean_t vdev_prop_align_right(vdev_prop_t);

	/*
	* Pool health statistics.
	*/
	typedef enum {
	/*
	* The following correspond to faults as defined in the (fault.fs.zfs.*)
	* event namespace. Each is associated with a corresponding message ID.
	* This must be kept in sync with the zfs_msgid_table in
	* lib/libzfs/libzfs_status.c.
	*/
	ZPOOL_STATUS_CORRUPT_CACHE, /* corrupt /kernel/drv/zpool.cache */
	ZPOOL_STATUS_MISSING_DEV_R, /* missing device with replicas */
	ZPOOL_STATUS_MISSING_DEV_NR, /* missing device with no replicas */
	ZPOOL_STATUS_CORRUPT_LABEL_R, /* bad device label with replicas */
	ZPOOL_STATUS_CORRUPT_LABEL_NR, /* bad device label with no replicas */
	ZPOOL_STATUS_BAD_GUID_SUM, /* sum of device guids didn't match */
	ZPOOL_STATUS_CORRUPT_POOL, /* pool metadata is corrupted */
	ZPOOL_STATUS_CORRUPT_DATA, /* data errors in user (meta)data */
	ZPOOL_STATUS_FAILING_DEV, /* device experiencing errors */
	ZPOOL_STATUS_VERSION_NEWER, /* newer on-disk version */
	ZPOOL_STATUS_HOSTID_MISMATCH, /* last accessed by another system */
	ZPOOL_STATUS_HOSTID_ACTIVE, /* currently active on another system */
	ZPOOL_STATUS_HOSTID_REQUIRED, /* multihost=on and hostid=0 */
	ZPOOL_STATUS_IO_FAILURE_WAIT, /* failed I/O, failmode 'wait' */
	ZPOOL_STATUS_IO_FAILURE_CONTINUE, /* failed I/O, failmode 'continue' */
	ZPOOL_STATUS_IO_FAILURE_MMP, /* failed MMP, failmode not 'panic' */
	ZPOOL_STATUS_BAD_LOG, /* cannot read log chain(s) */
	ZPOOL_STATUS_ERRATA, /* informational errata available */

	/*
	* If the pool has unsupported features but can still be opened in
	* read-only mode, its status is ZPOOL_STATUS_UNSUP_FEAT_WRITE. If the
	* pool has unsupported features but cannot be opened at all, its
	* status is ZPOOL_STATUS_UNSUP_FEAT_READ.
	*/
	ZPOOL_STATUS_UNSUP_FEAT_READ, /* unsupported features for read */
	ZPOOL_STATUS_UNSUP_FEAT_WRITE, /* unsupported features for write */

	/*
	* These faults have no corresponding message ID. At the time we are
	* checking the status, the original reason for the FMA fault (I/O or
	* checksum errors) has been lost.
	*/
	ZPOOL_STATUS_FAULTED_DEV_R, /* faulted device with replicas */
	ZPOOL_STATUS_FAULTED_DEV_NR, /* faulted device with no replicas */

	/*
	* The following are not faults per se, but still an error possibly
	* requiring administrative attention. There is no corresponding
	* message ID.
	*/
	ZPOOL_STATUS_VERSION_OLDER, /* older legacy on-disk version */
	ZPOOL_STATUS_FEAT_DISABLED, /* supported features are disabled */
	ZPOOL_STATUS_RESILVERING, /* device being resilvered */
	ZPOOL_STATUS_OFFLINE_DEV, /* device offline */
	ZPOOL_STATUS_REMOVED_DEV, /* removed device */
	ZPOOL_STATUS_REBUILDING, /* device being rebuilt */
	ZPOOL_STATUS_REBUILD_SCRUB, /* recommend scrubbing the pool */
	ZPOOL_STATUS_NON_NATIVE_ASHIFT, /* (e.g. 512e dev with ashift of 9) */
	ZPOOL_STATUS_COMPATIBILITY_ERR, /* bad 'compatibility' property */
	ZPOOL_STATUS_INCOMPATIBLE_FEAT, /* feature set outside compatibility */

	/*
	* Finally, the following indicates a healthy pool.
	*/
	ZPOOL_STATUS_OK
	} zpool_status_t;

	_LIBZFS_H zpool_status_t zpool_get_status(zpool_handle_t , const char *,
	zpool_errata_t *);
	_LIBZFS_H zpool_status_t zpool_import_status(nvlist_t , const char *,
	zpool_errata_t *);

	/*
	* Statistics and configuration functions.
	*/
	_LIBZFS_H nvlist_t zpool_get_config(zpool_handle_t , nvlist_t **);
	_LIBZFS_H nvlist_t zpool_get_features(zpool_handle_t );
	_LIBZFS_H int zpool_refresh_stats(zpool_handle_t , boolean_t );
	_LIBZFS_H int zpool_get_errlog(zpool_handle_t , nvlist_t *);

	/*
	* Import and export functions
	*/
	_LIBZFS_H int zpool_export(zpool_handle_t , boolean_t, const char );
	_LIBZFS_H int zpool_export_force(zpool_handle_t , const char );
	_LIBZFS_H int zpool_import(libzfs_handle_t , nvlist_t , const char *,
	char *altroot);
	_LIBZFS_H int zpool_import_props(libzfs_handle_t , nvlist_t , const char *,
	nvlist_t *, int);
	_LIBZFS_H void zpool_print_unsup_feat(nvlist_t *config);

	/*
	* Miscellaneous pool functions
	*/
	struct zfs_cmd;

	_LIBZFS_H const char *const zfs_history_event_names[];

	typedef enum {
	VDEV_NAME_PATH = 1 << 0,
	VDEV_NAME_GUID = 1 << 1,
	VDEV_NAME_FOLLOW_LINKS = 1 << 2,
	VDEV_NAME_TYPE_ID = 1 << 3,
	} vdev_name_t;

	_LIBZFS_H char zpool_vdev_name(libzfs_handle_t , zpool_handle_t , nvlist_t ,
	int name_flags);
	_LIBZFS_H int zpool_upgrade(zpool_handle_t *, uint64_t);
	_LIBZFS_H int zpool_get_history(zpool_handle_t , nvlist_t , uint64_t ,
	boolean_t *);
	_LIBZFS_H int zpool_events_next(libzfs_handle_t , nvlist_t , int , unsigned,
	int);
	_LIBZFS_H int zpool_events_clear(libzfs_handle_t , int );
	_LIBZFS_H int zpool_events_seek(libzfs_handle_t *, uint64_t, int);
	_LIBZFS_H void zpool_obj_to_path_ds(zpool_handle_t *, uint64_t, uint64_t,
	char *, size_t);
	_LIBZFS_H void zpool_obj_to_path(zpool_handle_t , uint64_t, uint64_t, char ,
	size_t);
	_LIBZFS_H int zfs_ioctl(libzfs_handle_t , int, struct zfs_cmd );
	_LIBZFS_H void zpool_explain_recover(libzfs_handle_t , const char , int,
	nvlist_t *);
	_LIBZFS_H int zpool_checkpoint(zpool_handle_t *);
	_LIBZFS_H int zpool_discard_checkpoint(zpool_handle_t *);
	_LIBZFS_H boolean_t zpool_is_draid_spare(const char *);

	/*
	* Basic handle manipulations. These functions do not create or destroy the
	* underlying datasets, only the references to them.
	*/
	_LIBZFS_H zfs_handle_t zfs_open(libzfs_handle_t , const char *, int);
	_LIBZFS_H zfs_handle_t zfs_handle_dup(zfs_handle_t );
	_LIBZFS_H void zfs_close(zfs_handle_t *);
	_LIBZFS_H zfs_type_t zfs_get_type(const zfs_handle_t *);
	_LIBZFS_H zfs_type_t zfs_get_underlying_type(const zfs_handle_t *);
	_LIBZFS_H const char zfs_get_name(const zfs_handle_t );
	_LIBZFS_H zpool_handle_t zfs_get_pool_handle(const zfs_handle_t );
	_LIBZFS_H const char zfs_get_pool_name(const zfs_handle_t );

	/*
	* Property management functions. Some functions are shared with the kernel,
	* and are found in sys/fs/zfs.h.
	*/

	/*
	* zfs dataset property management
	*/
	_LIBZFS_H const char *zfs_prop_default_string(zfs_prop_t);
	_LIBZFS_H uint64_t zfs_prop_default_numeric(zfs_prop_t);
	_LIBZFS_H const char *zfs_prop_column_name(zfs_prop_t);
	_LIBZFS_H boolean_t zfs_prop_align_right(zfs_prop_t);

	_LIBZFS_H nvlist_t zfs_valid_proplist(libzfs_handle_t , zfs_type_t,
	nvlist_t , uint64_t, zfs_handle_t , zpool_handle_t *, boolean_t,
	const char *);

	_LIBZFS_H const char *zfs_prop_to_name(zfs_prop_t);
	_LIBZFS_H int zfs_prop_set(zfs_handle_t , const char , const char *);
	_LIBZFS_H int zfs_prop_set_list(zfs_handle_t , nvlist_t );
	_LIBZFS_H int zfs_prop_set_list_flags(zfs_handle_t , nvlist_t , int);
	_LIBZFS_H int zfs_prop_get(zfs_handle_t , zfs_prop_t, char , size_t,
	zprop_source_t , char , size_t, boolean_t);
	_LIBZFS_H int zfs_prop_get_recvd(zfs_handle_t , const char , char *, size_t,
	boolean_t);
	_LIBZFS_H int zfs_prop_get_numeric(zfs_handle_t , zfs_prop_t, uint64_t ,
	zprop_source_t , char , size_t);
	_LIBZFS_H int zfs_prop_get_userquota_int(zfs_handle_t *zhp,
	const char propname, uint64_t propvalue);
	_LIBZFS_H int zfs_prop_get_userquota(zfs_handle_t zhp, const char propname,
	char *propbuf, int proplen, boolean_t literal);
	_LIBZFS_H int zfs_prop_get_written_int(zfs_handle_t zhp, const char propname,
	uint64_t *propvalue);
	_LIBZFS_H int zfs_prop_get_written(zfs_handle_t zhp, const char propname,
	char *propbuf, int proplen, boolean_t literal);
	_LIBZFS_H int zfs_prop_get_feature(zfs_handle_t zhp, const char propname,
	char *buf, size_t len);
	_LIBZFS_H uint64_t getprop_uint64(zfs_handle_t , zfs_prop_t, const char *);
	_LIBZFS_H uint64_t zfs_prop_get_int(zfs_handle_t *, zfs_prop_t);
	_LIBZFS_H int zfs_prop_inherit(zfs_handle_t , const char , boolean_t);
	_LIBZFS_H const char *zfs_prop_values(zfs_prop_t);
	_LIBZFS_H int zfs_prop_is_string(zfs_prop_t prop);
	_LIBZFS_H nvlist_t zfs_get_all_props(zfs_handle_t );
	_LIBZFS_H nvlist_t zfs_get_user_props(zfs_handle_t );
	_LIBZFS_H nvlist_t zfs_get_recvd_props(zfs_handle_t );
	_LIBZFS_H nvlist_t zfs_get_clones_nvl(zfs_handle_t );

	_LIBZFS_H int zfs_wait_status(zfs_handle_t *, zfs_wait_activity_t,
	boolean_t , boolean_t );

	/*
	* zfs encryption management
	*/
	_LIBZFS_H int zfs_crypto_get_encryption_root(zfs_handle_t , boolean_t ,
	char *);
	_LIBZFS_H int zfs_crypto_create(libzfs_handle_t , char , nvlist_t *,
	nvlist_t , boolean_t stdin_available, uint8_t , uint_t );
	_LIBZFS_H int zfs_crypto_clone_check(libzfs_handle_t , zfs_handle_t , char *,
	nvlist_t *);
	_LIBZFS_H int zfs_crypto_attempt_load_keys(libzfs_handle_t , const char );
	_LIBZFS_H int zfs_crypto_load_key(zfs_handle_t , boolean_t, const char );
	_LIBZFS_H int zfs_crypto_unload_key(zfs_handle_t *);
	_LIBZFS_H int zfs_crypto_rewrap(zfs_handle_t , nvlist_t , boolean_t);

	typedef struct zprop_list {
	int pl_prop;
	char *pl_user_prop;
	struct zprop_list *pl_next;
	boolean_t pl_all;
	size_t pl_width;
	size_t pl_recvd_width;
	boolean_t pl_fixed;
	} zprop_list_t;

	_LIBZFS_H int zfs_expand_proplist(zfs_handle_t , zprop_list_t *, boolean_t,
	boolean_t);
	_LIBZFS_H void zfs_prune_proplist(zfs_handle_t , uint8_t );
	_LIBZFS_H int vdev_expand_proplist(zpool_handle_t , const char ,
	zprop_list_t **);

	#define ZFS_MOUNTPOINT_NONE "none"
	#define ZFS_MOUNTPOINT_LEGACY "legacy"

	#define ZFS_FEATURE_DISABLED "disabled"
	#define ZFS_FEATURE_ENABLED "enabled"
	#define ZFS_FEATURE_ACTIVE "active"

	#define ZFS_UNSUPPORTED_INACTIVE "inactive"
	#define ZFS_UNSUPPORTED_READONLY "readonly"

	/*
	* zpool property management
	*/
	_LIBZFS_H int zpool_expand_proplist(zpool_handle_t , zprop_list_t *,
	zfs_type_t, boolean_t);
	_LIBZFS_H int zpool_prop_get_feature(zpool_handle_t , const char , char *,
	size_t);
	_LIBZFS_H const char *zpool_prop_default_string(zpool_prop_t);
	_LIBZFS_H uint64_t zpool_prop_default_numeric(zpool_prop_t);
	_LIBZFS_H const char *zpool_prop_column_name(zpool_prop_t);
	_LIBZFS_H boolean_t zpool_prop_align_right(zpool_prop_t);

	/*
	* Functions shared by zfs and zpool property management.
	*/
	_LIBZFS_H int zprop_iter(zprop_func func, void *cb, boolean_t show_all,
	boolean_t ordered, zfs_type_t type);
	_LIBZFS_H int zprop_get_list(libzfs_handle_t , char , zprop_list_t **,
	zfs_type_t);
	_LIBZFS_H void zprop_free_list(zprop_list_t *);

	#define ZFS_GET_NCOLS 5

	typedef enum {
	GET_COL_NONE,
	GET_COL_NAME,
	GET_COL_PROPERTY,
	GET_COL_VALUE,
	GET_COL_RECVD,
	GET_COL_SOURCE
	} zfs_get_column_t;

	/*
	* Functions for printing zfs or zpool properties
	*/
	typedef struct vdev_cbdata {
	int cb_name_flags;
	char **cb_names;
	unsigned int cb_names_count;
	} vdev_cbdata_t;

	typedef struct zprop_get_cbdata {
	int cb_sources;
	zfs_get_column_t cb_columns[ZFS_GET_NCOLS];
	int cb_colwidths[ZFS_GET_NCOLS + 1];
	boolean_t cb_scripted;
	boolean_t cb_literal;
	boolean_t cb_first;
	zprop_list_t *cb_proplist;
	zfs_type_t cb_type;
	vdev_cbdata_t cb_vdevs;
	} zprop_get_cbdata_t;

	#define ZFS_SET_NOMOUNT 1

	typedef struct zprop_set_cbdata {
	int cb_flags;
	nvlist_t *cb_proplist;
	} zprop_set_cbdata_t;

	_LIBZFS_H void zprop_print_one_property(const char , zprop_get_cbdata_t ,
	const char , const char , zprop_source_t, const char *,
	const char *);

	/*
	* Iterator functions.
	*/
	#define ZFS_ITER_RECURSE (1 << 0)
	#define ZFS_ITER_ARGS_CAN_BE_PATHS (1 << 1)
	#define ZFS_ITER_PROP_LISTSNAPS (1 << 2)
	#define ZFS_ITER_DEPTH_LIMIT (1 << 3)
	#define ZFS_ITER_RECVD_PROPS (1 << 4)
	#define ZFS_ITER_LITERAL_PROPS (1 << 5)
	#define ZFS_ITER_SIMPLE (1 << 6)

	typedef int (zfs_iter_f)(zfs_handle_t , void *);
	_LIBZFS_H int zfs_iter_root(libzfs_handle_t , zfs_iter_f, void );
	_LIBZFS_H int zfs_iter_children(zfs_handle_t , zfs_iter_f, void );
	_LIBZFS_H int zfs_iter_dependents(zfs_handle_t *, boolean_t, zfs_iter_f,
	void *);
	_LIBZFS_H int zfs_iter_filesystems(zfs_handle_t , zfs_iter_f, void );
	_LIBZFS_H int zfs_iter_snapshots(zfs_handle_t , boolean_t, zfs_iter_f, void ,
	uint64_t, uint64_t);
	_LIBZFS_H int zfs_iter_snapshots_sorted(zfs_handle_t , zfs_iter_f, void ,
	uint64_t, uint64_t);
	_LIBZFS_H int zfs_iter_snapspec(zfs_handle_t , const char , zfs_iter_f,
	void *);
	_LIBZFS_H int zfs_iter_bookmarks(zfs_handle_t , zfs_iter_f, void );

	_LIBZFS_H int zfs_iter_children_v2(zfs_handle_t , int, zfs_iter_f, void );
	_LIBZFS_H int zfs_iter_dependents_v2(zfs_handle_t *, int, boolean_t, zfs_iter_f,
	void *);
	_LIBZFS_H int zfs_iter_filesystems_v2(zfs_handle_t , int, zfs_iter_f, void );
	_LIBZFS_H int zfs_iter_snapshots_v2(zfs_handle_t , int, zfs_iter_f, void ,
	uint64_t, uint64_t);
	_LIBZFS_H int zfs_iter_snapshots_sorted_v2(zfs_handle_t *, int, zfs_iter_f,
	void *, uint64_t, uint64_t);
	_LIBZFS_H int zfs_iter_snapspec_v2(zfs_handle_t , int, const char ,
	zfs_iter_f, void *);
	_LIBZFS_H int zfs_iter_bookmarks_v2(zfs_handle_t , int, zfs_iter_f, void );
	_LIBZFS_H int zfs_iter_mounted(zfs_handle_t , zfs_iter_f, void );

	typedef struct get_all_cb {
	zfs_handle_t **cb_handles;
	size_t cb_alloc;
	size_t cb_used;
	} get_all_cb_t;

	_LIBZFS_H void zfs_foreach_mountpoint(libzfs_handle_t , zfs_handle_t *,
	size_t, zfs_iter_f, void *, boolean_t);
	_LIBZFS_H void libzfs_add_handle(get_all_cb_t , zfs_handle_t );

	/*
	* Functions to create and destroy datasets.
	*/
	_LIBZFS_H int zfs_create(libzfs_handle_t , const char , zfs_type_t,
	nvlist_t *);
	_LIBZFS_H int zfs_create_ancestors(libzfs_handle_t , const char );
	_LIBZFS_H int zfs_destroy(zfs_handle_t *, boolean_t);
	_LIBZFS_H int zfs_destroy_snaps(zfs_handle_t , char , boolean_t);
	_LIBZFS_H int zfs_destroy_snaps_nvl(libzfs_handle_t , nvlist_t , boolean_t);
	_LIBZFS_H int zfs_destroy_snaps_nvl_os(libzfs_handle_t , nvlist_t );
	_LIBZFS_H int zfs_clone(zfs_handle_t , const char , nvlist_t *);
	_LIBZFS_H int zfs_snapshot(libzfs_handle_t , const char , boolean_t,
	nvlist_t *);
	_LIBZFS_H int zfs_snapshot_nvl(libzfs_handle_t hdl, nvlist_t snaps,
	nvlist_t *props);
	_LIBZFS_H int zfs_rollback(zfs_handle_t , zfs_handle_t , boolean_t);

	typedef struct renameflags {
	/* recursive rename */
	unsigned int recursive : 1;

	/* don't unmount file systems */
	unsigned int nounmount : 1;

	/* force unmount file systems */
	unsigned int forceunmount : 1;
	} renameflags_t;

	_LIBZFS_H int zfs_rename(zfs_handle_t , const char , renameflags_t);

	typedef struct sendflags {
	/* Amount of extra information to print. */
	int verbosity;

	/* recursive send (ie, -R) */
	boolean_t replicate;

	/* for recursive send, skip sending missing snapshots */
	boolean_t skipmissing;

	/* for incrementals, do all intermediate snapshots */
	boolean_t doall;

	/* if dataset is a clone, do incremental from its origin */
	boolean_t fromorigin;

	/* field no longer used, maintained for backwards compatibility */
	boolean_t pad;

	/* send properties (ie, -p) */
	boolean_t props;

	/* do not send (no-op, ie. -n) */
	boolean_t dryrun;

	/* parsable verbose output (ie. -P) */
	boolean_t parsable;

	/* show progress (ie. -v) */
	boolean_t progress;

	/* show progress as process title (ie. -V) */
	boolean_t progressastitle;

	/* large blocks (>128K) are permitted */
	boolean_t largeblock;

	/* WRITE_EMBEDDED records of type DATA are permitted */
	boolean_t embed_data;

	/* compressed WRITE records are permitted */
	boolean_t compress;

	/* raw encrypted records are permitted */
	boolean_t raw;

	/* only send received properties (ie. -b) */
	boolean_t backup;

	/* include snapshot holds in send stream */
	boolean_t holds;

	/* stream represents a partially received dataset */
	boolean_t saved;
	} sendflags_t;

	typedef boolean_t (snapfilter_cb_t)(zfs_handle_t , void );

	_LIBZFS_H int zfs_send(zfs_handle_t , const char , const char *,
	sendflags_t , int, snapfilter_cb_t, void , nvlist_t **);
	_LIBZFS_H int zfs_send_one(zfs_handle_t , const char , int, sendflags_t *,
	const char *);
	_LIBZFS_H int zfs_send_progress(zfs_handle_t , int, uint64_t , uint64_t *);
	_LIBZFS_H int zfs_send_resume(libzfs_handle_t , sendflags_t , int outfd,
	const char *);
	_LIBZFS_H int zfs_send_saved(zfs_handle_t , sendflags_t , int, const char *);
	_LIBZFS_H nvlist_t zfs_send_resume_token_to_nvlist(libzfs_handle_t hdl,
	const char *token);

	_LIBZFS_H int zfs_promote(zfs_handle_t *);
	_LIBZFS_H int zfs_hold(zfs_handle_t , const char , const char *,
	boolean_t, int);
	_LIBZFS_H int zfs_hold_nvl(zfs_handle_t , int, nvlist_t );
	_LIBZFS_H int zfs_release(zfs_handle_t , const char , const char *,
	boolean_t);
	_LIBZFS_H int zfs_get_holds(zfs_handle_t , nvlist_t *);
	_LIBZFS_H uint64_t zvol_volsize_to_reservation(zpool_handle_t *, uint64_t,
	nvlist_t *);

	typedef int (zfs_userspace_cb_t)(void arg, const char *domain,
	uid_t rid, uint64_t space);

	_LIBZFS_H int zfs_userspace(zfs_handle_t *, zfs_userquota_prop_t,
	zfs_userspace_cb_t, void *);

	_LIBZFS_H int zfs_get_fsacl(zfs_handle_t , nvlist_t *);
	_LIBZFS_H int zfs_set_fsacl(zfs_handle_t , boolean_t, nvlist_t );

	typedef struct recvflags {
	/* print informational messages (ie, -v was specified) */
	boolean_t verbose;

	/* the destination is a prefix, not the exact fs (ie, -d) */
	boolean_t isprefix;

	/*
	* Only the tail of the sent snapshot path is appended to the
	* destination to determine the received snapshot name (ie, -e).
	*/
	boolean_t istail;

	/* do not actually do the recv, just check if it would work (ie, -n) */
	boolean_t dryrun;

	/* rollback/destroy filesystems as necessary (eg, -F) */
	boolean_t force;

	/* set "canmount=off" on all modified filesystems */
	boolean_t canmountoff;

	/*
	* Mark the file systems as "resumable" and do not destroy them if the
	* receive is interrupted
	*/
	boolean_t resumable;

	/* byteswap flag is used internally; callers need not specify */
	boolean_t byteswap;

	/* do not mount file systems as they are extracted (private) */
	boolean_t nomount;

	/* Was holds flag set in the compound header? */
	boolean_t holds;

	/* skip receive of snapshot holds */
	boolean_t skipholds;

	/* mount the filesystem unless nomount is specified */
	boolean_t domount;

	/* force unmount while recv snapshot (private) */
	boolean_t forceunmount;

	/* use this recv to check (and heal if needed) an existing snapshot */
	boolean_t heal;
	} recvflags_t;

	_LIBZFS_H int zfs_receive(libzfs_handle_t , const char , nvlist_t *,
	recvflags_t , int, avl_tree_t );

	typedef enum diff_flags {
	ZFS_DIFF_PARSEABLE = 1 << 0,
	ZFS_DIFF_TIMESTAMP = 1 << 1,
	ZFS_DIFF_CLASSIFY = 1 << 2,
	ZFS_DIFF_NO_MANGLE = 1 << 3
	} diff_flags_t;

	_LIBZFS_H int zfs_show_diffs(zfs_handle_t , int, const char , const char *,
	int);

	/*
	* Miscellaneous functions.
	*/
	_LIBZFS_H const char *zfs_type_to_name(zfs_type_t);
	_LIBZFS_H void zfs_refresh_properties(zfs_handle_t *);
	_LIBZFS_H int zfs_name_valid(const char *, zfs_type_t);
	_LIBZFS_H zfs_handle_t zfs_path_to_zhandle(libzfs_handle_t , const char *,
	zfs_type_t);
	_LIBZFS_H int zfs_parent_name(zfs_handle_t , char , size_t);
	_LIBZFS_H boolean_t zfs_dataset_exists(libzfs_handle_t , const char ,
	zfs_type_t);
	_LIBZFS_H int zfs_spa_version(zfs_handle_t , int );
	_LIBZFS_H boolean_t zfs_bookmark_exists(const char *path);

	/*
	* Mount support functions.
	*/
	_LIBZFS_H boolean_t is_mounted(libzfs_handle_t , const char special, char **);
	_LIBZFS_H boolean_t zfs_is_mounted(zfs_handle_t , char *);
	_LIBZFS_H int zfs_mount(zfs_handle_t , const char , int);
	_LIBZFS_H int zfs_mount_at(zfs_handle_t , const char , int, const char *);
	_LIBZFS_H int zfs_unmount(zfs_handle_t , const char , int);
	_LIBZFS_H int zfs_unmountall(zfs_handle_t *, int);
	_LIBZFS_H int zfs_mount_delegation_check(void);

	#if defined(__linux__) \|\| defined(__APPLE__)
	_LIBZFS_H int zfs_parse_mount_options(const char *mntopts,
	unsigned long mntflags, unsigned long zfsflags, int sloppy, char *badopt,
	char *mtabopt);
	_LIBZFS_H void zfs_adjust_mount_options(zfs_handle_t zhp, const char mntpoint,
	char mntopts, char mtabopt);
	#endif

	/*
	* Share support functions.
	*
	* enum sa_protocol * lists are terminated with SA_NO_PROTOCOL,
	* NULL means "all/any known to this libzfs".
	*/
	#define SA_NO_PROTOCOL -1

	_LIBZFS_H boolean_t zfs_is_shared(zfs_handle_t zhp, char *where,
	const enum sa_protocol *proto);
	_LIBZFS_H int zfs_share(zfs_handle_t zhp, const enum sa_protocol proto);
	_LIBZFS_H int zfs_unshare(zfs_handle_t zhp, const char mountpoint,
	const enum sa_protocol *proto);
	_LIBZFS_H int zfs_unshareall(zfs_handle_t *zhp,
	const enum sa_protocol *proto);
	_LIBZFS_H void zfs_commit_shares(const enum sa_protocol *proto);
	_LIBZFS_H void zfs_truncate_shares(const enum sa_protocol *proto);

	_LIBZFS_H int zfs_nicestrtonum(libzfs_handle_t , const char , uint64_t *);

	/*
	* Utility functions to run an external process.
	*/
	#define STDOUT_VERBOSE 0x01
	#define STDERR_VERBOSE 0x02
	#define NO_DEFAULT_PATH 0x04 /* Don't use $PATH to lookup the command */

	_LIBZFS_H int libzfs_run_process(const char , char *, int);
	_LIBZFS_H int libzfs_run_process_get_stdout(const char , char [], char *[],
	char *[], int );
	_LIBZFS_H int libzfs_run_process_get_stdout_nopath(const char , char [],
	char [], char [], int );

	_LIBZFS_H void libzfs_free_str_array(char **, int);

	_LIBZFS_H boolean_t libzfs_envvar_is_set(const char *);

	/*
	* Utility functions for zfs version
	*/
	_LIBZFS_H const char *zfs_version_userland(void);
	_LIBZFS_H char *zfs_version_kernel(void);
	_LIBZFS_H int zfs_version_print(void);

	/*
	* Given a device or file, determine if it is part of a pool.
	*/
	_LIBZFS_H int zpool_in_use(libzfs_handle_t , int, pool_state_t , char **,
	boolean_t *);

	/*
	* Label manipulation.
	*/
	_LIBZFS_H int zpool_clear_label(int);
	_LIBZFS_H int zpool_set_bootenv(zpool_handle_t , const nvlist_t );
	_LIBZFS_H int zpool_get_bootenv(zpool_handle_t , nvlist_t *);

	/*
	* Management interfaces for SMB ACL files
	*/

	_LIBZFS_H int zfs_smb_acl_add(libzfs_handle_t , char , char , char );
	_LIBZFS_H int zfs_smb_acl_remove(libzfs_handle_t , char , char , char );
	_LIBZFS_H int zfs_smb_acl_purge(libzfs_handle_t , char , char *);
	_LIBZFS_H int zfs_smb_acl_rename(libzfs_handle_t , char , char , char ,
	char *);

	/*
	* Enable and disable datasets within a pool by mounting/unmounting and
	* sharing/unsharing them.
	*/
	_LIBZFS_H int zpool_enable_datasets(zpool_handle_t , const char , int);
	_LIBZFS_H int zpool_disable_datasets(zpool_handle_t *, boolean_t);
	_LIBZFS_H void zpool_disable_datasets_os(zpool_handle_t *, boolean_t);
	_LIBZFS_H void zpool_disable_volume_os(const char *);

	/*
	* Parse a features file for -o compatibility
	*/
	typedef enum {
	ZPOOL_COMPATIBILITY_OK,
	ZPOOL_COMPATIBILITY_WARNTOKEN,
	ZPOOL_COMPATIBILITY_BADTOKEN,
	ZPOOL_COMPATIBILITY_BADFILE,
	ZPOOL_COMPATIBILITY_NOFILES
	} zpool_compat_status_t;

	_LIBZFS_H zpool_compat_status_t zpool_load_compat(const char *,
	boolean_t , char , size_t);

	#ifdef __FreeBSD__

	/*
	* Attach/detach the given filesystem to/from the given jail.
	*/
	_LIBZFS_H int zfs_jail(zfs_handle_t *zhp, int jailid, int attach);

	/*
	* Set loader options for next boot.
	*/
	_LIBZFS_H int zpool_nextboot(libzfs_handle_t *, uint64_t, uint64_t,
	const char *);

	#endif /* __FreeBSD__ */

	#ifdef __linux__

	/*
	* Add or delete the given filesystem to/from the given user namespace.
	*/
	_LIBZFS_H int zfs_userns(zfs_handle_t zhp, const char nspath, int attach);

	#endif

	#ifdef __cplusplus
	}
	#endif

	#endif /* _LIBZFS_H */
	diff --git a/sys/contrib/openzfs/include/os/freebsd/Makefile.am b/sys/contrib/openzfs/include/os/freebsd/Makefile.am
	index 9819e534b7f6..d4103c2f062a 100644
	--- a/sys/contrib/openzfs/include/os/freebsd/Makefile.am
	+++ b/sys/contrib/openzfs/include/os/freebsd/Makefile.am
	@@ -1,94 +1,94 @@
	noinst_HEADERS = \
	%D%/linux/compiler.h \
	%D%/linux/types.h \
	\
	%D%/spl/acl/acl_common.h \
	\
	- %D%/spl/rpc/xdr.h \
	- \
	%D%/spl/sys/ia32/asm_linkage.h \
	\
	%D%/spl/sys/acl.h \
	%D%/spl/sys/acl_impl.h \
	%D%/spl/sys/atomic.h \
	%D%/spl/sys/byteorder.h \
	%D%/spl/sys/callb.h \
	%D%/spl/sys/ccompat.h \
	%D%/spl/sys/ccompile.h \
	%D%/spl/sys/cmn_err.h \
	%D%/spl/sys/condvar.h \
	%D%/spl/sys/cred.h \
	%D%/spl/sys/ctype.h \
	%D%/spl/sys/debug.h \
	%D%/spl/sys/dirent.h \
	%D%/spl/sys/disp.h \
	%D%/spl/sys/dkio.h \
	%D%/spl/sys/fcntl.h \
	%D%/spl/sys/file.h \
	%D%/spl/sys/freebsd_rwlock.h \
	%D%/spl/sys/idmap.h \
	%D%/spl/sys/inttypes.h \
	%D%/spl/sys/isa_defs.h \
	%D%/spl/sys/kmem.h \
	%D%/spl/sys/kmem_cache.h \
	%D%/spl/sys/kstat.h \
	%D%/spl/sys/list.h \
	%D%/spl/sys/list_impl.h \
	%D%/spl/sys/lock.h \
	%D%/spl/sys/misc.h \
	%D%/spl/sys/mod_os.h \
	%D%/spl/sys/mode.h \
	%D%/spl/sys/mount.h \
	%D%/spl/sys/mutex.h \
	%D%/spl/sys/param.h \
	%D%/spl/sys/policy.h \
	%D%/spl/sys/proc.h \
	%D%/spl/sys/processor.h \
	%D%/spl/sys/procfs_list.h \
	%D%/spl/sys/random.h \
	%D%/spl/sys/rwlock.h \
	%D%/spl/sys/sdt.h \
	%D%/spl/sys/sid.h \
	%D%/spl/sys/sig.h \
	%D%/spl/sys/simd.h \
	%D%/spl/sys/simd_aarch64.h \
	%D%/spl/sys/simd_arm.h \
	%D%/spl/sys/simd_powerpc.h \
	%D%/spl/sys/simd_x86.h \
	%D%/spl/sys/spl_condvar.h \
	%D%/spl/sys/string.h \
	%D%/spl/sys/sunddi.h \
	%D%/spl/sys/sysmacros.h \
	%D%/spl/sys/systeminfo.h \
	%D%/spl/sys/systm.h \
	%D%/spl/sys/taskq.h \
	%D%/spl/sys/thread.h \
	%D%/spl/sys/time.h \
	%D%/spl/sys/timer.h \
	%D%/spl/sys/trace.h \
	%D%/spl/sys/trace_zfs.h \
	%D%/spl/sys/types.h \
	%D%/spl/sys/types32.h \
	%D%/spl/sys/uio.h \
	%D%/spl/sys/uuid.h \
	%D%/spl/sys/vfs.h \
	%D%/spl/sys/vm.h \
	%D%/spl/sys/vmsystm.h \
	%D%/spl/sys/vnode.h \
	%D%/spl/sys/vnode_impl.h \
	%D%/spl/sys/wmsum.h \
	%D%/spl/sys/zmod.h \
	%D%/spl/sys/zone.h \
	\
	+ %D%/zfs/sys/arc_os.h \
	%D%/zfs/sys/freebsd_crypto.h \
	+ %D%/zfs/sys/freebsd_event.h \
	%D%/zfs/sys/vdev_os.h \
	%D%/zfs/sys/zfs_bootenv_os.h \
	%D%/zfs/sys/zfs_context_os.h \
	%D%/zfs/sys/zfs_ctldir.h \
	%D%/zfs/sys/zfs_dir.h \
	%D%/zfs/sys/zfs_ioctl_compat.h \
	%D%/zfs/sys/zfs_vfsops_os.h \
	%D%/zfs/sys/zfs_vnops_os.h \
	%D%/zfs/sys/zfs_znode_impl.h \
	%D%/zfs/sys/zpl.h

	diff --git a/sys/contrib/openzfs/include/os/freebsd/spl/rpc/xdr.h b/sys/contrib/openzfs/include/os/freebsd/spl/rpc/xdr.h
	deleted file mode 100644
	index c98466e9d16a..000000000000
	--- a/sys/contrib/openzfs/include/os/freebsd/spl/rpc/xdr.h
	+++ /dev/null
	@@ -1,71 +0,0 @@
	-/*
	- * Sun RPC is a product of Sun Microsystems, Inc. and is provided for
	- * unrestricted use provided that this legend is included on all tape
	- * media and as a part of the software program in whole or part. Users
	- * may copy or modify Sun RPC without charge, but are not authorized
	- * to license or distribute it to anyone else except as part of a product or
	- * program developed by the user.
	- *
	- * SUN RPC IS PROVIDED AS IS WITH NO WARRANTIES OF ANY KIND INCLUDING THE
	- * WARRANTIES OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR
	- * PURPOSE, OR ARISING FROM A COURSE OF DEALING, USAGE OR TRADE PRACTICE.
	- *
	- * Sun RPC is provided with no support and without any obligation on the
	- * part of Sun Microsystems, Inc. to assist in its use, correction,
	- * modification or enhancement.
	- *
	- * SUN MICROSYSTEMS, INC. SHALL HAVE NO LIABILITY WITH RESPECT TO THE
	- * INFRINGEMENT OF COPYRIGHTS, TRADE SECRETS OR ANY PATENTS BY SUN RPC
	- * OR ANY PART THEREOF.
	- *
	- * In no event will Sun Microsystems, Inc. be liable for any lost revenue
	- * or profits or other special, indirect and consequential damages, even if
	- * Sun has been advised of the possibility of such damages.
	- *
	- * Sun Microsystems, Inc.
	- * 2550 Garcia Avenue
	- * Mountain View, California 94043
	- */
	-
	-#ifndef _OPENSOLARIS_RPC_XDR_H_
	-#define _OPENSOLARIS_RPC_XDR_H_
	-
	-#include <rpc/types.h>
	-#include_next <rpc/xdr.h>
	-
	-#if !defined(_KERNEL) && !defined(_STANDALONE)
	-
	-#include <assert.h>
	-
	-/*
	- * Taken from sys/xdr/xdr_mem.c.
	- *
	- * FreeBSD's userland XDR doesn't implement control method (only the kernel),
	- * but OpenSolaris nvpair still depend on it, so we have to implement it here.
	- */
	-static __inline bool_t
	-xdrmem_control(XDR xdrs, int request, void info)
	-{
	- xdr_bytesrec *xptr;
	-
	- switch (request) {
	- case XDR_GET_BYTES_AVAIL:
	- xptr = (xdr_bytesrec *)info;
	- xptr->xc_is_last_record = TRUE;
	- xptr->xc_num_avail = xdrs->x_handy;
	- return (TRUE);
	- default:
	- assert(!"unexpected request");
	- }
	- return (FALSE);
	-}
	-
	-#undef XDR_CONTROL
	-#define XDR_CONTROL(xdrs, req, op) \
	- (((xdrs)->x_ops->x_control == NULL) ? \
	- xdrmem_control((xdrs), (req), (op)) : \
	- (*(xdrs)->x_ops->x_control)(xdrs, req, op))
	-
	-#endif /* !_KERNEL && !_STANDALONE */
	-
	-#endif /* !_OPENSOLARIS_RPC_XDR_H_ */
	diff --git a/sys/contrib/openzfs/include/os/freebsd/spl/sys/debug.h b/sys/contrib/openzfs/include/os/freebsd/spl/sys/debug.h
	index 3e67cf0e9a7d..785fcf62dd16 100644
	--- a/sys/contrib/openzfs/include/os/freebsd/spl/sys/debug.h
	+++ b/sys/contrib/openzfs/include/os/freebsd/spl/sys/debug.h
	@@ -1,182 +1,194 @@
	/*
	* Copyright (c) 2020 iXsystems, Inc.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	/*
	* Available Solaris debug functions. All of the ASSERT() macros will be
	* compiled out when NDEBUG is defined, this is the default behavior for
	* the SPL. To enable assertions use the --enable-debug with configure.
	* The VERIFY() functions are never compiled out and cannot be disabled.
	*
	* PANIC() - Panic the node and print message.
	* ASSERT() - Assert X is true, if not panic.
	* ASSERT3B() - Assert boolean X OP Y is true, if not panic.
	* ASSERT3S() - Assert signed X OP Y is true, if not panic.
	* ASSERT3U() - Assert unsigned X OP Y is true, if not panic.
	* ASSERT3P() - Assert pointer X OP Y is true, if not panic.
	* ASSERT0() - Assert value is zero, if not panic.
	+ * ASSERT0P() - Assert pointer is null, if not panic.
	* VERIFY() - Verify X is true, if not panic.
	* VERIFY3B() - Verify boolean X OP Y is true, if not panic.
	* VERIFY3S() - Verify signed X OP Y is true, if not panic.
	* VERIFY3U() - Verify unsigned X OP Y is true, if not panic.
	* VERIFY3P() - Verify pointer X OP Y is true, if not panic.
	* VERIFY0() - Verify value is zero, if not panic.
	+ * VERIFY0P() - Verify pointer is null, if not panic.
	*/

	#ifndef _SPL_DEBUG_H
	#define _SPL_DEBUG_H


	/*
	* Common DEBUG functionality.
	*/
	#if defined(__COVERITY__) \|\| defined(__clang_analyzer__)
	__attribute__((__noreturn__))
	#endif
	extern void spl_panic(const char file, const char func, int line,
	const char *fmt, ...) __attribute__((__noreturn__));
	extern void spl_dumpstack(void);

	static inline int
	spl_assert(const char buf, const char file, const char *func, int line)
	{
	spl_panic(file, func, line, "%s", buf);
	return (0);
	}

	#ifndef expect
	#define expect(expr, value) (__builtin_expect((expr), (value)))
	#endif
	#define likely(expr) expect((expr) != 0, 1)
	#define unlikely(expr) expect((expr) != 0, 0)

	#define PANIC(fmt, a...) \
	spl_panic(__FILE__, __FUNCTION__, __LINE__, fmt, ## a)

	#define VERIFY(cond) \
	(void) (unlikely(!(cond)) && \
	spl_assert("VERIFY(" #cond ") failed\n", \
	__FILE__, __FUNCTION__, __LINE__))

	#define VERIFY3B(LEFT, OP, RIGHT) do { \
	const boolean_t _verify3_left = (boolean_t)(LEFT); \
	const boolean_t _verify3_right = (boolean_t)(RIGHT); \
	if (unlikely(!(_verify3_left OP _verify3_right))) \
	spl_panic(__FILE__, __FUNCTION__, __LINE__, \
	"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
	"failed (%d " #OP " %d)\n", \
	- (boolean_t)(_verify3_left), \
	- (boolean_t)(_verify3_right)); \
	+ (boolean_t)_verify3_left, \
	+ (boolean_t)_verify3_right); \
	} while (0)

	#define VERIFY3S(LEFT, OP, RIGHT) do { \
	const int64_t _verify3_left = (int64_t)(LEFT); \
	const int64_t _verify3_right = (int64_t)(RIGHT); \
	if (unlikely(!(_verify3_left OP _verify3_right))) \
	spl_panic(__FILE__, __FUNCTION__, __LINE__, \
	"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
	"failed (%lld " #OP " %lld)\n", \
	- (long long) (_verify3_left), \
	- (long long) (_verify3_right)); \
	+ (long long)_verify3_left, \
	+ (long long)_verify3_right); \
	} while (0)

	#define VERIFY3U(LEFT, OP, RIGHT) do { \
	const uint64_t _verify3_left = (uint64_t)(LEFT); \
	const uint64_t _verify3_right = (uint64_t)(RIGHT); \
	if (unlikely(!(_verify3_left OP _verify3_right))) \
	spl_panic(__FILE__, __FUNCTION__, __LINE__, \
	"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
	"failed (%llu " #OP " %llu)\n", \
	- (unsigned long long) (_verify3_left), \
	- (unsigned long long) (_verify3_right)); \
	+ (unsigned long long)_verify3_left, \
	+ (unsigned long long)_verify3_right); \
	} while (0)

	#define VERIFY3P(LEFT, OP, RIGHT) do { \
	const uintptr_t _verify3_left = (uintptr_t)(LEFT); \
	const uintptr_t _verify3_right = (uintptr_t)(RIGHT); \
	if (unlikely(!(_verify3_left OP _verify3_right))) \
	spl_panic(__FILE__, __FUNCTION__, __LINE__, \
	"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
	- "failed (%px " #OP " %px)\n", \
	- (void *) (_verify3_left), \
	- (void *) (_verify3_right)); \
	+ "failed (%p " #OP " %p)\n", \
	+ (void *)_verify3_left, \
	+ (void *)_verify3_right); \
	} while (0)

	#define VERIFY0(RIGHT) do { \
	- const int64_t _verify3_left = (int64_t)(0); \
	- const int64_t _verify3_right = (int64_t)(RIGHT); \
	- if (unlikely(!(_verify3_left == _verify3_right))) \
	+ const int64_t _verify0_right = (int64_t)(RIGHT); \
	+ if (unlikely(!(0 == _verify0_right))) \
	spl_panic(__FILE__, __FUNCTION__, __LINE__, \
	- "VERIFY0(0 == " #RIGHT ") " \
	+ "VERIFY0(" #RIGHT ") " \
	"failed (0 == %lld)\n", \
	- (long long) (_verify3_right)); \
	+ (long long)_verify0_right); \
	+ } while (0)
	+
	+#define VERIFY0P(RIGHT) do { \
	+ const uintptr_t _verify0_right = (uintptr_t)(RIGHT); \
	+ if (unlikely(!(0 == _verify0_right))) \
	+ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
	+ "VERIFY0P(" #RIGHT ") " \
	+ "failed (NULL == %p)\n", \
	+ (void *)_verify0_right); \
	} while (0)

	/*
	* Debugging disabled (--disable-debug)
	*/
	#ifdef NDEBUG

	#define ASSERT(x) ((void) sizeof ((uintptr_t)(x)))
	#define ASSERT3B(x, y, z) \
	((void) sizeof ((uintptr_t)(x)), (void) sizeof ((uintptr_t)(z)))
	#define ASSERT3S(x, y, z) \
	((void) sizeof ((uintptr_t)(x)), (void) sizeof ((uintptr_t)(z)))
	#define ASSERT3U(x, y, z) \
	((void) sizeof ((uintptr_t)(x)), (void) sizeof ((uintptr_t)(z)))
	#define ASSERT3P(x, y, z) \
	((void) sizeof ((uintptr_t)(x)), (void) sizeof ((uintptr_t)(z)))
	#define ASSERT0(x) ((void) sizeof ((uintptr_t)(x)))
	+#define ASSERT0P(x) ((void) sizeof ((uintptr_t)(x)))
	#define IMPLY(A, B) \
	((void) sizeof ((uintptr_t)(A)), (void) sizeof ((uintptr_t)(B)))
	#define EQUIV(A, B) \
	((void) sizeof ((uintptr_t)(A)), (void) sizeof ((uintptr_t)(B)))

	/*
	* Debugging enabled (--enable-debug)
	*/
	#else

	#define ASSERT3B VERIFY3B
	#define ASSERT3S VERIFY3S
	#define ASSERT3U VERIFY3U
	#define ASSERT3P VERIFY3P
	#define ASSERT0 VERIFY0
	+#define ASSERT0P VERIFY0P
	#define ASSERT VERIFY
	#define IMPLY(A, B) \
	((void)(likely((!(A)) \|\| (B)) \|\| \
	spl_assert("(" #A ") implies (" #B ")", \
	__FILE__, __FUNCTION__, __LINE__)))
	#define EQUIV(A, B) \
	((void)(likely(!!(A) == !!(B)) \|\| \
	spl_assert("(" #A ") is equivalent to (" #B ")", \
	__FILE__, __FUNCTION__, __LINE__)))


	#endif /* NDEBUG */

	#endif /* SPL_DEBUG_H */
	diff --git a/sys/contrib/openzfs/include/os/linux/Makefile.am b/sys/contrib/openzfs/include/os/linux/Makefile.am
	index 3830d198dfff..332569efe361 100644
	--- a/sys/contrib/openzfs/include/os/linux/Makefile.am
	+++ b/sys/contrib/openzfs/include/os/linux/Makefile.am
	@@ -1,116 +1,118 @@
	if CONFIG_KERNEL
	kernel_linuxdir = $(kerneldir)/linux
	kernel_linux_HEADERS = \
	%D%/kernel/linux/blkdev_compat.h \
	%D%/kernel/linux/compiler_compat.h \
	%D%/kernel/linux/dcache_compat.h \
	%D%/kernel/linux/kmap_compat.h \
	+ %D%/kernel/linux/mm_compat.h \
	%D%/kernel/linux/mod_compat.h \
	%D%/kernel/linux/page_compat.h \
	%D%/kernel/linux/percpu_compat.h \
	%D%/kernel/linux/simd.h \
	%D%/kernel/linux/simd_aarch64.h \
	%D%/kernel/linux/simd_arm.h \
	%D%/kernel/linux/simd_powerpc.h \
	%D%/kernel/linux/simd_x86.h \
	%D%/kernel/linux/utsname_compat.h \
	%D%/kernel/linux/vfs_compat.h \
	%D%/kernel/linux/xattr_compat.h

	kernel_sysdir = $(kerneldir)/sys
	kernel_sys_HEADERS = \
	%D%/zfs/sys/policy.h \
	%D%/zfs/sys/trace_acl.h \
	%D%/zfs/sys/trace_arc.h \
	%D%/zfs/sys/trace_common.h \
	%D%/zfs/sys/trace_dbgmsg.h \
	%D%/zfs/sys/trace_dbuf.h \
	%D%/zfs/sys/trace_dmu.h \
	%D%/zfs/sys/trace_dnode.h \
	%D%/zfs/sys/trace_multilist.h \
	%D%/zfs/sys/trace_rrwlock.h \
	%D%/zfs/sys/trace_txg.h \
	%D%/zfs/sys/trace_vdev.h \
	%D%/zfs/sys/trace_zfs.h \
	%D%/zfs/sys/trace_zil.h \
	%D%/zfs/sys/trace_zio.h \
	%D%/zfs/sys/trace_zrlock.h \
	%D%/zfs/sys/zfs_bootenv_os.h \
	%D%/zfs/sys/zfs_context_os.h \
	%D%/zfs/sys/zfs_ctldir.h \
	%D%/zfs/sys/zfs_dir.h \
	%D%/zfs/sys/zfs_vfsops_os.h \
	%D%/zfs/sys/zfs_vnops_os.h \
	%D%/zfs/sys/zfs_znode_impl.h \
	%D%/zfs/sys/zpl.h

	kernel_spl_rpcdir = $(kerneldir)/spl/rpc
	kernel_spl_rpc_HEADERS = \
	+ %D%/spl/rpc/types.h \
	%D%/spl/rpc/xdr.h

	kernel_spl_sysdir = $(kerneldir)/spl/sys
	kernel_spl_sys_HEADERS = \
	%D%/spl/sys/acl.h \
	%D%/spl/sys/atomic.h \
	%D%/spl/sys/byteorder.h \
	%D%/spl/sys/callb.h \
	%D%/spl/sys/callo.h \
	%D%/spl/sys/cmn_err.h \
	%D%/spl/sys/condvar.h \
	%D%/spl/sys/cred.h \
	%D%/spl/sys/ctype.h \
	%D%/spl/sys/debug.h \
	%D%/spl/sys/disp.h \
	%D%/spl/sys/dkio.h \
	%D%/spl/sys/errno.h \
	%D%/spl/sys/fcntl.h \
	%D%/spl/sys/file.h \
	%D%/spl/sys/inttypes.h \
	%D%/spl/sys/isa_defs.h \
	%D%/spl/sys/kmem.h \
	%D%/spl/sys/kmem_cache.h \
	%D%/spl/sys/kstat.h \
	%D%/spl/sys/list.h \
	%D%/spl/sys/misc.h \
	%D%/spl/sys/mod_os.h \
	%D%/spl/sys/mutex.h \
	%D%/spl/sys/param.h \
	%D%/spl/sys/proc.h \
	%D%/spl/sys/processor.h \
	%D%/spl/sys/procfs_list.h \
	%D%/spl/sys/random.h \
	%D%/spl/sys/rwlock.h \
	%D%/spl/sys/shrinker.h \
	%D%/spl/sys/sid.h \
	%D%/spl/sys/signal.h \
	%D%/spl/sys/simd.h \
	%D%/spl/sys/stat.h \
	%D%/spl/sys/string.h \
	%D%/spl/sys/sunddi.h \
	%D%/spl/sys/sysmacros.h \
	%D%/spl/sys/systeminfo.h \
	%D%/spl/sys/taskq.h \
	%D%/spl/sys/thread.h \
	%D%/spl/sys/time.h \
	%D%/spl/sys/timer.h \
	%D%/spl/sys/trace.h \
	%D%/spl/sys/trace_spl.h \
	%D%/spl/sys/trace_taskq.h \
	%D%/spl/sys/tsd.h \
	%D%/spl/sys/types.h \
	%D%/spl/sys/types32.h \
	%D%/spl/sys/uio.h \
	%D%/spl/sys/user.h \
	%D%/spl/sys/vfs.h \
	%D%/spl/sys/vmem.h \
	%D%/spl/sys/vmsystm.h \
	%D%/spl/sys/vnode.h \
	%D%/spl/sys/wait.h \
	%D%/spl/sys/wmsum.h \
	%D%/spl/sys/zmod.h \
	%D%/spl/sys/zone.h

	kernel_spl_ia32dir = $(kernel_spl_sysdir)/ia32
	kernel_spl_ia32_HEADERS = \
	%D%/spl/sys/ia32/asm_linkage.h
	endif
	diff --git a/sys/contrib/openzfs/include/os/linux/kernel/linux/blkdev_compat.h b/sys/contrib/openzfs/include/os/linux/kernel/linux/blkdev_compat.h
	index f111e648ccf7..b0f398354e4f 100644
	--- a/sys/contrib/openzfs/include/os/linux/kernel/linux/blkdev_compat.h
	+++ b/sys/contrib/openzfs/include/os/linux/kernel/linux/blkdev_compat.h
	@@ -1,781 +1,783 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (C) 2011 Lawrence Livermore National Security, LLC.
	* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
	* Written by Brian Behlendorf <behlendorf1@llnl.gov>.
	* LLNL-CODE-403049.
	*/

	#ifndef _ZFS_BLKDEV_H
	#define _ZFS_BLKDEV_H

	#include <linux/blkdev.h>
	#include <linux/backing-dev.h>
	#include <linux/hdreg.h>
	#include <linux/major.h>
	#include <linux/msdos_fs.h> /* for SECTOR_* */
	#include <linux/bio.h>

	#ifdef HAVE_BLK_MQ
	#include <linux/blk-mq.h>
	#endif

	#ifndef HAVE_BLK_QUEUE_FLAG_SET
	static inline void
	blk_queue_flag_set(unsigned int flag, struct request_queue *q)
	{
	queue_flag_set(flag, q);
	}
	#endif

	#ifndef HAVE_BLK_QUEUE_FLAG_CLEAR
	static inline void
	blk_queue_flag_clear(unsigned int flag, struct request_queue *q)
	{
	queue_flag_clear(flag, q);
	}
	#endif

	/*
	* 4.7 API,
	* The blk_queue_write_cache() interface has replaced blk_queue_flush()
	* interface. However, the new interface is GPL-only thus we implement
	* our own trivial wrapper when the GPL-only version is detected.
	*
	* 2.6.36 - 4.6 API,
	* The blk_queue_flush() interface has replaced blk_queue_ordered()
	* interface. However, while the old interface was available to all the
	* new one is GPL-only. Thus if the GPL-only version is detected we
	* implement our own trivial helper.
	*/
	static inline void
	blk_queue_set_write_cache(struct request_queue *q, bool wc, bool fua)
	{
	#if defined(HAVE_BLK_QUEUE_WRITE_CACHE_GPL_ONLY)
	if (wc)
	blk_queue_flag_set(QUEUE_FLAG_WC, q);
	else
	blk_queue_flag_clear(QUEUE_FLAG_WC, q);
	if (fua)
	blk_queue_flag_set(QUEUE_FLAG_FUA, q);
	else
	blk_queue_flag_clear(QUEUE_FLAG_FUA, q);
	#elif defined(HAVE_BLK_QUEUE_WRITE_CACHE)
	blk_queue_write_cache(q, wc, fua);
	#elif defined(HAVE_BLK_QUEUE_FLUSH_GPL_ONLY)
	if (wc)
	q->flush_flags \|= REQ_FLUSH;
	if (fua)
	q->flush_flags \|= REQ_FUA;
	#elif defined(HAVE_BLK_QUEUE_FLUSH)
	blk_queue_flush(q, (wc ? REQ_FLUSH : 0) \| (fua ? REQ_FUA : 0));
	#else
	#error "Unsupported kernel"
	#endif
	}

	static inline void
	blk_queue_set_read_ahead(struct request_queue *q, unsigned long ra_pages)
	{
	#if !defined(HAVE_BLK_QUEUE_UPDATE_READAHEAD) && \
	!defined(HAVE_DISK_UPDATE_READAHEAD)
	#ifdef HAVE_BLK_QUEUE_BDI_DYNAMIC
	q->backing_dev_info->ra_pages = ra_pages;
	#else
	q->backing_dev_info.ra_pages = ra_pages;
	#endif
	#endif
	}

	#ifdef HAVE_BIO_BVEC_ITER
	#define BIO_BI_SECTOR(bio) (bio)->bi_iter.bi_sector
	#define BIO_BI_SIZE(bio) (bio)->bi_iter.bi_size
	#define BIO_BI_IDX(bio) (bio)->bi_iter.bi_idx
	#define BIO_BI_SKIP(bio) (bio)->bi_iter.bi_bvec_done
	#define bio_for_each_segment4(bv, bvp, b, i) \
	bio_for_each_segment((bv), (b), (i))
	typedef struct bvec_iter bvec_iterator_t;
	#else
	#define BIO_BI_SECTOR(bio) (bio)->bi_sector
	#define BIO_BI_SIZE(bio) (bio)->bi_size
	#define BIO_BI_IDX(bio) (bio)->bi_idx
	#define BIO_BI_SKIP(bio) (0)
	#define bio_for_each_segment4(bv, bvp, b, i) \
	bio_for_each_segment((bvp), (b), (i))
	typedef int bvec_iterator_t;
	#endif

	static inline void
	bio_set_flags_failfast(struct block_device bdev, int flags, bool dev,
	bool transport, bool driver)
	{
	#ifdef CONFIG_BUG
	/*
	* Disable FAILFAST for loopback devices because of the
	* following incorrect BUG_ON() in loop_make_request().
	* This support is also disabled for md devices because the
	* test suite layers md devices on top of loopback devices.
	* This may be removed when the loopback driver is fixed.
	*
	* BUG_ON(!lo \|\| (rw != READ && rw != WRITE));
	*/
	if ((MAJOR(bdev->bd_dev) == LOOP_MAJOR) \|\|
	(MAJOR(bdev->bd_dev) == MD_MAJOR))
	return;

	#ifdef BLOCK_EXT_MAJOR
	if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR)
	return;
	#endif /* BLOCK_EXT_MAJOR */
	#endif /* CONFIG_BUG */

	if (dev)
	*flags \|= REQ_FAILFAST_DEV;
	if (transport)
	*flags \|= REQ_FAILFAST_TRANSPORT;
	if (driver)
	*flags \|= REQ_FAILFAST_DRIVER;
	}

	/*
	* Maximum disk label length, it may be undefined for some kernels.
	*/
	#if !defined(DISK_NAME_LEN)
	#define DISK_NAME_LEN 32
	#endif /* DISK_NAME_LEN */

	#ifdef HAVE_BIO_BI_STATUS
	static inline int
	bi_status_to_errno(blk_status_t status)
	{
	switch (status) {
	case BLK_STS_OK:
	return (0);
	case BLK_STS_NOTSUPP:
	return (EOPNOTSUPP);
	case BLK_STS_TIMEOUT:
	return (ETIMEDOUT);
	case BLK_STS_NOSPC:
	return (ENOSPC);
	case BLK_STS_TRANSPORT:
	return (ENOLINK);
	case BLK_STS_TARGET:
	return (EREMOTEIO);
	#ifdef HAVE_BLK_STS_RESV_CONFLICT
	case BLK_STS_RESV_CONFLICT:
	#else
	case BLK_STS_NEXUS:
	#endif
	return (EBADE);
	case BLK_STS_MEDIUM:
	return (ENODATA);
	case BLK_STS_PROTECTION:
	return (EILSEQ);
	case BLK_STS_RESOURCE:
	return (ENOMEM);
	case BLK_STS_AGAIN:
	return (EAGAIN);
	case BLK_STS_IOERR:
	return (EIO);
	default:
	return (EIO);
	}
	}

	static inline blk_status_t
	errno_to_bi_status(int error)
	{
	switch (error) {
	case 0:
	return (BLK_STS_OK);
	case EOPNOTSUPP:
	return (BLK_STS_NOTSUPP);
	case ETIMEDOUT:
	return (BLK_STS_TIMEOUT);
	case ENOSPC:
	return (BLK_STS_NOSPC);
	case ENOLINK:
	return (BLK_STS_TRANSPORT);
	case EREMOTEIO:
	return (BLK_STS_TARGET);
	case EBADE:
	#ifdef HAVE_BLK_STS_RESV_CONFLICT
	return (BLK_STS_RESV_CONFLICT);
	#else
	return (BLK_STS_NEXUS);
	#endif
	case ENODATA:
	return (BLK_STS_MEDIUM);
	case EILSEQ:
	return (BLK_STS_PROTECTION);
	case ENOMEM:
	return (BLK_STS_RESOURCE);
	case EAGAIN:
	return (BLK_STS_AGAIN);
	case EIO:
	return (BLK_STS_IOERR);
	default:
	return (BLK_STS_IOERR);
	}
	}
	#endif /* HAVE_BIO_BI_STATUS */

	/*
	* 4.3 API change
	* The bio_endio() prototype changed slightly. These are helper
	* macro's to ensure the prototype and invocation are handled.
	*/
	#ifdef HAVE_1ARG_BIO_END_IO_T
	#ifdef HAVE_BIO_BI_STATUS
	#define BIO_END_IO_ERROR(bio) bi_status_to_errno(bio->bi_status)
	#define BIO_END_IO_PROTO(fn, x, z) static void fn(struct bio *x)
	#define BIO_END_IO(bio, error) bio_set_bi_status(bio, error)
	static inline void
	bio_set_bi_status(struct bio *bio, int error)
	{
	ASSERT3S(error, <=, 0);
	bio->bi_status = errno_to_bi_status(-error);
	bio_endio(bio);
	}
	#else
	#define BIO_END_IO_ERROR(bio) (-(bio->bi_error))
	#define BIO_END_IO_PROTO(fn, x, z) static void fn(struct bio *x)
	#define BIO_END_IO(bio, error) bio_set_bi_error(bio, error)
	static inline void
	bio_set_bi_error(struct bio *bio, int error)
	{
	ASSERT3S(error, <=, 0);
	bio->bi_error = error;
	bio_endio(bio);
	}
	#endif /* HAVE_BIO_BI_STATUS */

	#else
	#define BIO_END_IO_PROTO(fn, x, z) static void fn(struct bio *x, int z)
	#define BIO_END_IO(bio, error) bio_endio(bio, error);
	#endif /* HAVE_1ARG_BIO_END_IO_T */

	/*
	* 5.15 MACRO,
	* GD_DEAD
	*
	* 2.6.36 - 5.14 MACRO,
	* GENHD_FL_UP
	*
	* Check the disk status and return B_TRUE if alive
	* otherwise B_FALSE
	*/
	static inline boolean_t
	zfs_check_disk_status(struct block_device *bdev)
	{
	#if defined(GENHD_FL_UP)
	return (!!(bdev->bd_disk->flags & GENHD_FL_UP));
	#elif defined(GD_DEAD)
	return (!test_bit(GD_DEAD, &bdev->bd_disk->state));
	#else
	/*
	* This is encountered if neither GENHD_FL_UP nor GD_DEAD is available in
	* the kernel - likely due to an MACRO change that needs to be chased down.
	*/
	#error "Unsupported kernel: no usable disk status check"
	#endif
	}

	/*
	* 4.1 API,
	* 3.10.0 CentOS 7.x API,
	* blkdev_reread_part()
	*
	* For older kernels trigger a re-reading of the partition table by calling
	* check_disk_change() which calls flush_disk() to invalidate the device.
	*
	* For newer kernels (as of 5.10), bdev_check_media_change is used, in favor of
	* check_disk_change(), with the modification that invalidation is no longer
	* forced.
	*/
	#ifdef HAVE_CHECK_DISK_CHANGE
	#define zfs_check_media_change(bdev) check_disk_change(bdev)
	#ifdef HAVE_BLKDEV_REREAD_PART
	#define vdev_bdev_reread_part(bdev) blkdev_reread_part(bdev)
	#else
	#define vdev_bdev_reread_part(bdev) check_disk_change(bdev)
	#endif /* HAVE_BLKDEV_REREAD_PART */
	#else
	#ifdef HAVE_BDEV_CHECK_MEDIA_CHANGE
	static inline int
	zfs_check_media_change(struct block_device *bdev)
	{
	#ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
	struct gendisk *gd = bdev->bd_disk;
	const struct block_device_operations *bdo = gd->fops;
	#endif

	if (!bdev_check_media_change(bdev))
	return (0);

	#ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
	/*
	* Force revalidation, to mimic the old behavior of
	* check_disk_change()
	*/
	if (bdo->revalidate_disk)
	bdo->revalidate_disk(gd);
	#endif

	return (0);
	}
	#define vdev_bdev_reread_part(bdev) zfs_check_media_change(bdev)
	#elif defined(HAVE_DISK_CHECK_MEDIA_CHANGE)
	#define vdev_bdev_reread_part(bdev) disk_check_media_change(bdev->bd_disk)
	#define zfs_check_media_change(bdev) disk_check_media_change(bdev->bd_disk)
	#else
	/*
	* This is encountered if check_disk_change() and bdev_check_media_change()
	* are not available in the kernel - likely due to an API change that needs
	* to be chased down.
	*/
	#error "Unsupported kernel: no usable disk change check"
	#endif /* HAVE_BDEV_CHECK_MEDIA_CHANGE */
	#endif /* HAVE_CHECK_DISK_CHANGE */

	/*
	* 2.6.27 API change
	* The function was exported for use, prior to this it existed but the
	* symbol was not exported.
	*
	* 4.4.0-6.21 API change for Ubuntu
	* lookup_bdev() gained a second argument, FMODE_*, to check inode permissions.
	*
	* 5.11 API change
	* Changed to take a dev_t argument which is set on success and return a
	* non-zero error code on failure.
	*/
	static inline int
	vdev_lookup_bdev(const char path, dev_t dev)
	{
	#if defined(HAVE_DEVT_LOOKUP_BDEV)
	return (lookup_bdev(path, dev));
	#elif defined(HAVE_1ARG_LOOKUP_BDEV)
	struct block_device *bdev = lookup_bdev(path);
	if (IS_ERR(bdev))
	return (PTR_ERR(bdev));

	*dev = bdev->bd_dev;
	bdput(bdev);

	return (0);
	#elif defined(HAVE_MODE_LOOKUP_BDEV)
	struct block_device *bdev = lookup_bdev(path, FMODE_READ);
	if (IS_ERR(bdev))
	return (PTR_ERR(bdev));

	*dev = bdev->bd_dev;
	bdput(bdev);

	return (0);
	#else
	#error "Unsupported kernel"
	#endif
	}

	#if defined(HAVE_BLK_MODE_T)
	#define blk_mode_is_open_write(flag) ((flag) & BLK_OPEN_WRITE)
	#else
	#define blk_mode_is_open_write(flag) ((flag) & FMODE_WRITE)
	#endif

	/*
	* Kernels without bio_set_op_attrs use bi_rw for the bio flags.
	*/
	#if !defined(HAVE_BIO_SET_OP_ATTRS)
	static inline void
	bio_set_op_attrs(struct bio *bio, unsigned rw, unsigned flags)
	{
	#if defined(HAVE_BIO_BI_OPF)
	bio->bi_opf = rw \| flags;
	#else
	bio->bi_rw \|= rw \| flags;
	#endif /* HAVE_BIO_BI_OPF */
	}
	#endif

	/*
	* bio_set_flush - Set the appropriate flags in a bio to guarantee
	* data are on non-volatile media on completion.
	*
	* 2.6.37 - 4.8 API,
	* Introduce WRITE_FLUSH, WRITE_FUA, and WRITE_FLUSH_FUA flags as a
	* replacement for WRITE_BARRIER to allow expressing richer semantics
	* to the block layer. It's up to the block layer to implement the
	* semantics correctly. Use the WRITE_FLUSH_FUA flag combination.
	*
	* 4.8 - 4.9 API,
	* REQ_FLUSH was renamed to REQ_PREFLUSH. For consistency with previous
	* OpenZFS releases, prefer the WRITE_FLUSH_FUA flag set if it's available.
	*
	* 4.10 API,
	* The read/write flags and their modifiers, including WRITE_FLUSH,
	* WRITE_FUA and WRITE_FLUSH_FUA were removed from fs.h in
	* torvalds/linux@70fd7614 and replaced by direct flag modification
	* of the REQ_ flags in bio->bi_opf. Use REQ_PREFLUSH.
	*/
	static inline void
	bio_set_flush(struct bio *bio)
	{
	#if defined(HAVE_REQ_PREFLUSH) /* >= 4.10 */
	bio_set_op_attrs(bio, 0, REQ_PREFLUSH \| REQ_OP_WRITE);
	#elif defined(WRITE_FLUSH_FUA) /* >= 2.6.37 and <= 4.9 */
	bio_set_op_attrs(bio, 0, WRITE_FLUSH_FUA);
	#else
	#error "Allowing the build will cause bio_set_flush requests to be ignored."
	#endif
	}

	/*
	* 4.8 API,
	* REQ_OP_FLUSH
	*
	* 4.8-rc0 - 4.8-rc1,
	* REQ_PREFLUSH
	*
	* 2.6.36 - 4.7 API,
	* REQ_FLUSH
	*
	* in all cases but may have a performance impact for some kernels. It
	* has the advantage of minimizing kernel specific changes in the zvol code.
	*
	*/
	static inline boolean_t
	bio_is_flush(struct bio *bio)
	{
	#if defined(HAVE_REQ_OP_FLUSH) && defined(HAVE_BIO_BI_OPF)
	return ((bio_op(bio) == REQ_OP_FLUSH) \|\| (bio->bi_opf & REQ_PREFLUSH));
	#elif defined(HAVE_REQ_PREFLUSH) && defined(HAVE_BIO_BI_OPF)
	return (bio->bi_opf & REQ_PREFLUSH);
	#elif defined(HAVE_REQ_PREFLUSH) && !defined(HAVE_BIO_BI_OPF)
	return (bio->bi_rw & REQ_PREFLUSH);
	#elif defined(HAVE_REQ_FLUSH)
	return (bio->bi_rw & REQ_FLUSH);
	#else
	#error "Unsupported kernel"
	#endif
	}

	/*
	* 4.8 API,
	* REQ_FUA flag moved to bio->bi_opf
	*
	* 2.6.x - 4.7 API,
	* REQ_FUA
	*/
	static inline boolean_t
	bio_is_fua(struct bio *bio)
	{
	#if defined(HAVE_BIO_BI_OPF)
	return (bio->bi_opf & REQ_FUA);
	#elif defined(REQ_FUA)
	return (bio->bi_rw & REQ_FUA);
	#else
	#error "Allowing the build will cause fua requests to be ignored."
	#endif
	}

	/*
	* 4.8 API,
	* REQ_OP_DISCARD
	*
	* 2.6.36 - 4.7 API,
	* REQ_DISCARD
	*
	* In all cases the normal I/O path is used for discards. The only
	* difference is how the kernel tags individual I/Os as discards.
	*/
	static inline boolean_t
	bio_is_discard(struct bio *bio)
	{
	#if defined(HAVE_REQ_OP_DISCARD)
	return (bio_op(bio) == REQ_OP_DISCARD);
	#elif defined(HAVE_REQ_DISCARD)
	return (bio->bi_rw & REQ_DISCARD);
	#else
	#error "Unsupported kernel"
	#endif
	}

	/*
	* 4.8 API,
	* REQ_OP_SECURE_ERASE
	*
	* 2.6.36 - 4.7 API,
	* REQ_SECURE
	*/
	static inline boolean_t
	bio_is_secure_erase(struct bio *bio)
	{
	#if defined(HAVE_REQ_OP_SECURE_ERASE)
	return (bio_op(bio) == REQ_OP_SECURE_ERASE);
	#elif defined(REQ_SECURE)
	return (bio->bi_rw & REQ_SECURE);
	#else
	return (0);
	#endif
	}

	/*
	* 2.6.33 API change
	* Discard granularity and alignment restrictions may now be set. For
	* older kernels which do not support this it is safe to skip it.
	*/
	static inline void
	blk_queue_discard_granularity(struct request_queue *q, unsigned int dg)
	{
	q->limits.discard_granularity = dg;
	}

	/*
	* 5.19 API,
	* bdev_max_discard_sectors()
	*
	* 2.6.32 API,
	* blk_queue_discard()
	*/
	static inline boolean_t
	bdev_discard_supported(struct block_device *bdev)
	{
	#if defined(HAVE_BDEV_MAX_DISCARD_SECTORS)
	- return (!!bdev_max_discard_sectors(bdev));
	+ return (bdev_max_discard_sectors(bdev) > 0 &&
	+ bdev_discard_granularity(bdev) > 0);
	#elif defined(HAVE_BLK_QUEUE_DISCARD)
	- return (!!blk_queue_discard(bdev_get_queue(bdev)));
	+ return (blk_queue_discard(bdev_get_queue(bdev)) > 0 &&
	+ bdev_get_queue(bdev)->limits.discard_granularity > 0);
	#else
	#error "Unsupported kernel"
	#endif
	}

	/*
	* 5.19 API,
	* bdev_max_secure_erase_sectors()
	*
	* 4.8 API,
	* blk_queue_secure_erase()
	*
	* 2.6.36 - 4.7 API,
	* blk_queue_secdiscard()
	*/
	static inline boolean_t
	bdev_secure_discard_supported(struct block_device *bdev)
	{
	#if defined(HAVE_BDEV_MAX_SECURE_ERASE_SECTORS)
	return (!!bdev_max_secure_erase_sectors(bdev));
	#elif defined(HAVE_BLK_QUEUE_SECURE_ERASE)
	return (!!blk_queue_secure_erase(bdev_get_queue(bdev)));
	#elif defined(HAVE_BLK_QUEUE_SECDISCARD)
	return (!!blk_queue_secdiscard(bdev_get_queue(bdev)));
	#else
	#error "Unsupported kernel"
	#endif
	}

	/*
	* A common holder for vdev_bdev_open() is used to relax the exclusive open
	* semantics slightly. Internal vdev disk callers may pass VDEV_HOLDER to
	* allow them to open the device multiple times. Other kernel callers and
	* user space processes which don't pass this value will get EBUSY. This is
	* currently required for the correct operation of hot spares.
	*/
	#define VDEV_HOLDER ((void *)0x2401de7)

	static inline unsigned long
	blk_generic_start_io_acct(struct request_queue *q __attribute__((unused)),
	struct gendisk *disk __attribute__((unused)),
	int rw __attribute__((unused)), struct bio *bio)
	{
	#if defined(HAVE_BDEV_IO_ACCT_63)
	return (bdev_start_io_acct(bio->bi_bdev, bio_op(bio),
	jiffies));
	#elif defined(HAVE_BDEV_IO_ACCT_OLD)
	return (bdev_start_io_acct(bio->bi_bdev, bio_sectors(bio),
	bio_op(bio), jiffies));
	#elif defined(HAVE_DISK_IO_ACCT)
	return (disk_start_io_acct(disk, bio_sectors(bio), bio_op(bio)));
	#elif defined(HAVE_BIO_IO_ACCT)
	return (bio_start_io_acct(bio));
	#elif defined(HAVE_GENERIC_IO_ACCT_3ARG)
	unsigned long start_time = jiffies;
	generic_start_io_acct(rw, bio_sectors(bio), &disk->part0);
	return (start_time);
	#elif defined(HAVE_GENERIC_IO_ACCT_4ARG)
	unsigned long start_time = jiffies;
	generic_start_io_acct(q, rw, bio_sectors(bio), &disk->part0);
	return (start_time);
	#else
	/* Unsupported */
	return (0);
	#endif
	}

	static inline void
	blk_generic_end_io_acct(struct request_queue *q __attribute__((unused)),
	struct gendisk *disk __attribute__((unused)),
	int rw __attribute__((unused)), struct bio *bio, unsigned long start_time)
	{
	#if defined(HAVE_BDEV_IO_ACCT_63)
	bdev_end_io_acct(bio->bi_bdev, bio_op(bio), bio_sectors(bio),
	start_time);
	#elif defined(HAVE_BDEV_IO_ACCT_OLD)
	bdev_end_io_acct(bio->bi_bdev, bio_op(bio), start_time);
	#elif defined(HAVE_DISK_IO_ACCT)
	disk_end_io_acct(disk, bio_op(bio), start_time);
	#elif defined(HAVE_BIO_IO_ACCT)
	bio_end_io_acct(bio, start_time);
	#elif defined(HAVE_GENERIC_IO_ACCT_3ARG)
	generic_end_io_acct(rw, &disk->part0, start_time);
	#elif defined(HAVE_GENERIC_IO_ACCT_4ARG)
	generic_end_io_acct(q, rw, &disk->part0, start_time);
	#endif
	}

	#ifndef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
	static inline struct request_queue *
	blk_generic_alloc_queue(make_request_fn make_request, int node_id)
	{
	#if defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN)
	return (blk_alloc_queue(make_request, node_id));
	#elif defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN_RH)
	return (blk_alloc_queue_rh(make_request, node_id));
	#else
	struct request_queue *q = blk_alloc_queue(GFP_KERNEL);
	if (q != NULL)
	blk_queue_make_request(q, make_request);

	return (q);
	#endif
	}
	#endif /* !HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS */

	/*
	* All the io_*() helper functions below can operate on a bio, or a rq, but
	* not both. The older submit_bio() codepath will pass a bio, and the
	* newer blk-mq codepath will pass a rq.
	*/
	static inline int
	io_data_dir(struct bio bio, struct request rq)
	{
	#ifdef HAVE_BLK_MQ
	if (rq != NULL) {
	if (op_is_write(req_op(rq))) {
	return (WRITE);
	} else {
	return (READ);
	}
	}
	#else
	ASSERT3P(rq, ==, NULL);
	#endif
	return (bio_data_dir(bio));
	}

	static inline int
	io_is_flush(struct bio bio, struct request rq)
	{
	#ifdef HAVE_BLK_MQ
	if (rq != NULL)
	return (req_op(rq) == REQ_OP_FLUSH);
	#else
	ASSERT3P(rq, ==, NULL);
	#endif
	return (bio_is_flush(bio));
	}

	static inline int
	io_is_discard(struct bio bio, struct request rq)
	{
	#ifdef HAVE_BLK_MQ
	if (rq != NULL)
	return (req_op(rq) == REQ_OP_DISCARD);
	#else
	ASSERT3P(rq, ==, NULL);
	#endif
	return (bio_is_discard(bio));
	}

	static inline int
	io_is_secure_erase(struct bio bio, struct request rq)
	{
	#ifdef HAVE_BLK_MQ
	if (rq != NULL)
	return (req_op(rq) == REQ_OP_SECURE_ERASE);
	#else
	ASSERT3P(rq, ==, NULL);
	#endif
	return (bio_is_secure_erase(bio));
	}

	static inline int
	io_is_fua(struct bio bio, struct request rq)
	{
	#ifdef HAVE_BLK_MQ
	if (rq != NULL)
	return (rq->cmd_flags & REQ_FUA);
	#else
	ASSERT3P(rq, ==, NULL);
	#endif
	return (bio_is_fua(bio));
	}


	static inline uint64_t
	io_offset(struct bio bio, struct request rq)
	{
	#ifdef HAVE_BLK_MQ
	if (rq != NULL)
	return (blk_rq_pos(rq) << 9);
	#else
	ASSERT3P(rq, ==, NULL);
	#endif
	return (BIO_BI_SECTOR(bio) << 9);
	}

	static inline uint64_t
	io_size(struct bio bio, struct request rq)
	{
	#ifdef HAVE_BLK_MQ
	if (rq != NULL)
	return (blk_rq_bytes(rq));
	#else
	ASSERT3P(rq, ==, NULL);
	#endif
	return (BIO_BI_SIZE(bio));
	}

	static inline int
	io_has_data(struct bio bio, struct request rq)
	{
	#ifdef HAVE_BLK_MQ
	if (rq != NULL)
	return (bio_has_data(rq->bio));
	#else
	ASSERT3P(rq, ==, NULL);
	#endif
	return (bio_has_data(bio));
	}
	#endif /* _ZFS_BLKDEV_H */
	diff --git a/sys/contrib/openzfs/include/os/linux/kernel/linux/mm_compat.h b/sys/contrib/openzfs/include/os/linux/kernel/linux/mm_compat.h
	new file mode 100644
	index 000000000000..40056c68d6dd
	--- /dev/null
	+++ b/sys/contrib/openzfs/include/os/linux/kernel/linux/mm_compat.h
	@@ -0,0 +1,36 @@
	+/*
	+ * CDDL HEADER START
	+ *
	+ * The contents of this file are subject to the terms of the
	+ * Common Development and Distribution License (the "License").
	+ * You may not use this file except in compliance with the License.
	+ *
	+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	+ * or https://opensource.org/licenses/CDDL-1.0.
	+ * See the License for the specific language governing permissions
	+ * and limitations under the License.
	+ *
	+ * When distributing Covered Code, include this CDDL HEADER in each
	+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	+ * If applicable, add the following below this CDDL HEADER, with the
	+ * fields enclosed by brackets "[]" replaced with your own identifying
	+ * information: Portions Copyright [yyyy] [name of copyright owner]
	+ *
	+ * CDDL HEADER END
	+ */
	+
	+/*
	+ * Copyright (c) 2023, 2024, Klara Inc.
	+ */
	+
	+#ifndef _ZFS_MM_COMPAT_H
	+#define _ZFS_MM_COMPAT_H
	+
	+#include <linux/mm.h>
	+
	+/* 5.4 introduced page_size(). Older kernels can use a trivial macro instead */
	+#ifndef HAVE_MM_PAGE_SIZE
	+#define page_size(p) ((unsigned long)(PAGE_SIZE << compound_order(p)))
	+#endif
	+
	+#endif /* _ZFS_MM_COMPAT_H */
	diff --git a/sys/contrib/openzfs/include/os/linux/kernel/linux/mod_compat.h b/sys/contrib/openzfs/include/os/linux/kernel/linux/mod_compat.h
	index 8e20a9613539..039865b703ef 100644
	--- a/sys/contrib/openzfs/include/os/linux/kernel/linux/mod_compat.h
	+++ b/sys/contrib/openzfs/include/os/linux/kernel/linux/mod_compat.h
	@@ -1,204 +1,205 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (C) 2016 Gvozden Neskovic <neskovic@gmail.com>.
	* Copyright (c) 2020 by Delphix. All rights reserved.
	*/

	#ifndef _MOD_COMPAT_H
	#define _MOD_COMPAT_H

	#include <linux/module.h>
	#include <linux/moduleparam.h>

	/*
	* Despite constifying struct kernel_param_ops, some older kernels define a
	* `__check_old_set_param()` function in their headers that checks for a
	* non-constified `->set()`. This has long been fixed in Linux mainline, but
	* since we support older kernels, we workaround it by using a preprocessor
	* definition to disable it.
	*/
	#define __check_old_set_param(_) (0)

	typedef const struct kernel_param zfs_kernel_param_t;

	#define ZMOD_RW 0644
	#define ZMOD_RD 0444

	enum scope_prefix_types {
	zfs,
	zfs_arc,
	zfs_brt,
	zfs_condense,
	zfs_dbuf,
	zfs_dbuf_cache,
	zfs_deadman,
	zfs_dedup,
	zfs_l2arc,
	zfs_livelist,
	zfs_livelist_condense,
	zfs_lua,
	zfs_metaslab,
	zfs_mg,
	zfs_multihost,
	zfs_prefetch,
	zfs_reconstruct,
	zfs_recv,
	zfs_send,
	zfs_spa,
	zfs_trim,
	zfs_txg,
	zfs_vdev,
	+ zfs_vdev_disk,
	zfs_vdev_file,
	zfs_vdev_mirror,
	zfs_vnops,
	zfs_zevent,
	zfs_zio,
	zfs_zil
	};

	/*
	* While we define our own s64/u64 types, there is no reason to reimplement the
	* existing Linux kernel types, so we use the preprocessor to remap our
	* "custom" implementations to the kernel ones. This is done because the CPP
	* does not allow us to write conditional definitions. The fourth definition
	* exists because the CPP will not allow us to replace things like INT with int
	* before string concatenation.
	*/

	#define spl_param_set_int param_set_int
	#define spl_param_get_int param_get_int
	#define spl_param_ops_int param_ops_int
	#define spl_param_ops_INT param_ops_int

	#define spl_param_set_long param_set_long
	#define spl_param_get_long param_get_long
	#define spl_param_ops_long param_ops_long
	#define spl_param_ops_LONG param_ops_long

	#define spl_param_set_uint param_set_uint
	#define spl_param_get_uint param_get_uint
	#define spl_param_ops_uint param_ops_uint
	#define spl_param_ops_UINT param_ops_uint

	#define spl_param_set_ulong param_set_ulong
	#define spl_param_get_ulong param_get_ulong
	#define spl_param_ops_ulong param_ops_ulong
	#define spl_param_ops_ULONG param_ops_ulong

	#define spl_param_set_charp param_set_charp
	#define spl_param_get_charp param_get_charp
	#define spl_param_ops_charp param_ops_charp
	#define spl_param_ops_STRING param_ops_charp

	int spl_param_set_s64(const char val, zfs_kernel_param_t kp);
	extern int spl_param_get_s64(char buffer, zfs_kernel_param_t kp);
	extern const struct kernel_param_ops spl_param_ops_s64;
	#define spl_param_ops_S64 spl_param_ops_s64

	extern int spl_param_set_u64(const char val, zfs_kernel_param_t kp);
	extern int spl_param_get_u64(char buffer, zfs_kernel_param_t kp);
	extern const struct kernel_param_ops spl_param_ops_u64;
	#define spl_param_ops_U64 spl_param_ops_u64

	/*
	* Declare a module parameter / sysctl node
	*
	* "scope_prefix" the part of the sysctl / sysfs tree the node resides under
	* (currently a no-op on Linux)
	* "name_prefix" the part of the variable name that will be excluded from the
	* exported names on platforms with a hierarchical namespace
	* "name" the part of the variable that will be exposed on platforms with a
	* hierarchical namespace, or as name_prefix ## name on Linux
	* "type" the variable type
	* "perm" the permissions (read/write or read only)
	* "desc" a brief description of the option
	*
	* Examples:
	* ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, rotating_inc, UINT,
	* ZMOD_RW, "Rotating media load increment for non-seeking I/O's");
	* on FreeBSD:
	* vfs.zfs.vdev.mirror.rotating_inc
	* on Linux:
	* zfs_vdev_mirror_rotating_inc
	*
	* ZFS_MODULE_PARAM(zfs, , dmu_prefetch_max, UINT, ZMOD_RW,
	* "Limit one prefetch call to this size");
	* on FreeBSD:
	* vfs.zfs.dmu_prefetch_max
	* on Linux:
	* dmu_prefetch_max
	*/
	#define ZFS_MODULE_PARAM(scope_prefix, name_prefix, name, type, perm, desc) \
	_Static_assert( \
	sizeof (scope_prefix) == sizeof (enum scope_prefix_types), \
	"" #scope_prefix " size mismatch with enum scope_prefix_types"); \
	module_param_cb(name_prefix ## name, &spl_param_ops_ ## type, \
	&name_prefix ## name, perm); \
	MODULE_PARM_DESC(name_prefix ## name, desc)

	/*
	* Declare a module parameter / sysctl node
	*
	* "scope_prefix" the part of the the sysctl / sysfs tree the node resides under
	* (currently a no-op on Linux)
	* "name_prefix" the part of the variable name that will be excluded from the
	* exported names on platforms with a hierarchical namespace
	* "name" the part of the variable that will be exposed on platforms with a
	* hierarchical namespace, or as name_prefix ## name on Linux
	* "setfunc" setter function
	* "getfunc" getter function
	* "perm" the permissions (read/write or read only)
	* "desc" a brief description of the option
	*
	* Examples:
	* ZFS_MODULE_PARAM_CALL(zfs_spa, spa_, slop_shift, param_set_slop_shift,
	* param_get_int, ZMOD_RW, "Reserved free space in pool");
	* on FreeBSD:
	* vfs.zfs.spa_slop_shift
	* on Linux:
	* spa_slop_shift
	*/
	#define ZFS_MODULE_PARAM_CALL( \
	scope_prefix, name_prefix, name, setfunc, getfunc, perm, desc) \
	_Static_assert( \
	sizeof (scope_prefix) == sizeof (enum scope_prefix_types), \
	"" #scope_prefix " size mismatch with enum scope_prefix_types"); \
	module_param_call(name_prefix ## name, setfunc, getfunc, \
	&name_prefix ## name, perm); \
	MODULE_PARM_DESC(name_prefix ## name, desc)

	/*
	* As above, but there is no variable with the name name_prefix ## name,
	* so NULL is passed to module_param_call instead.
	*/
	#define ZFS_MODULE_VIRTUAL_PARAM_CALL( \
	scope_prefix, name_prefix, name, setfunc, getfunc, perm, desc) \
	_Static_assert( \
	sizeof (scope_prefix) == sizeof (enum scope_prefix_types), \
	"" #scope_prefix " size mismatch with enum scope_prefix_types"); \
	module_param_call(name_prefix ## name, setfunc, getfunc, NULL, perm); \
	MODULE_PARM_DESC(name_prefix ## name, desc)

	#define ZFS_MODULE_PARAM_ARGS const char buf, zfs_kernel_param_t kp

	#endif /* _MOD_COMPAT_H */
	diff --git a/sys/contrib/openzfs/include/os/linux/spl/rpc/types.h b/sys/contrib/openzfs/include/os/linux/spl/rpc/types.h
	new file mode 100644
	index 000000000000..5bbb4f2dec46
	--- /dev/null
	+++ b/sys/contrib/openzfs/include/os/linux/spl/rpc/types.h
	@@ -0,0 +1,30 @@
	+/*
	+ * Copyright (c) 2008 Sun Microsystems, Inc.
	+ * Written by Ricardo Correia <Ricardo.M.Correia@Sun.COM>
	+ *
	+ * This file is part of the SPL, Solaris Porting Layer.
	+ *
	+ * The SPL is free software; you can redistribute it and/or modify it
	+ * under the terms of the GNU General Public License as published by the
	+ * Free Software Foundation; either version 2 of the License, or (at your
	+ * option) any later version.
	+ *
	+ * The SPL is distributed in the hope that it will be useful, but WITHOUT
	+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	+ * for more details.
	+ *
	+ * You should have received a copy of the GNU General Public License along
	+ * with the SPL. If not, see <http://www.gnu.org/licenses/>.
	+ */
	+
	+#ifndef _SPL_RPC_TYPES_H
	+#define _SPL_RPC_TYPES_H
	+
	+#include <sys/types.h>
	+
	+/* Just enough to support rpc/xdr.h */
	+
	+typedef int bool_t;
	+
	+#endif /* SPL_RPC_TYPES_H */
	diff --git a/sys/contrib/openzfs/include/os/linux/spl/rpc/xdr.h b/sys/contrib/openzfs/include/os/linux/spl/rpc/xdr.h
	index b00f3542fcdf..5b621fa9c863 100644
	--- a/sys/contrib/openzfs/include/os/linux/spl/rpc/xdr.h
	+++ b/sys/contrib/openzfs/include/os/linux/spl/rpc/xdr.h
	@@ -1,153 +1,151 @@
	/*
	* Copyright (c) 2008 Sun Microsystems, Inc.
	* Written by Ricardo Correia <Ricardo.M.Correia@Sun.COM>
	*
	* This file is part of the SPL, Solaris Porting Layer.
	*
	* The SPL is free software; you can redistribute it and/or modify it
	* under the terms of the GNU General Public License as published by the
	* Free Software Foundation; either version 2 of the License, or (at your
	* option) any later version.
	*
	* The SPL is distributed in the hope that it will be useful, but WITHOUT
	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	* for more details.
	*
	* You should have received a copy of the GNU General Public License along
	* with the SPL. If not, see <http://www.gnu.org/licenses/>.
	*/

	#ifndef _SPL_RPC_XDR_H
	#define _SPL_RPC_XDR_H

	#include <sys/types.h>

	-typedef int bool_t;
	-
	/*
	* XDR enums and types.
	*/
	enum xdr_op {
	XDR_ENCODE,
	XDR_DECODE
	};

	struct xdr_ops;

	typedef struct {
	const struct xdr_ops *x_ops;
	/* Let caller know xdrmem_create() succeeds */
	caddr_t x_addr; /* Current buffer addr */
	caddr_t x_addr_end; /* End of the buffer */
	enum xdr_op x_op; /* Stream direction */
	} XDR;

	typedef bool_t (xdrproc_t)(XDR xdrs, void *ptr);

	struct xdr_ops {
	bool_t (xdr_control)(XDR , int, void *);

	bool_t (xdr_char)(XDR , char *);
	bool_t (xdr_u_short)(XDR , unsigned short *);
	bool_t (xdr_u_int)(XDR , unsigned *);
	bool_t (xdr_u_longlong_t)(XDR , u_longlong_t *);

	bool_t (xdr_opaque)(XDR , caddr_t, const uint_t);
	bool_t (xdr_string)(XDR , char **, const uint_t);
	bool_t (xdr_array)(XDR , caddr_t , uint_t , const uint_t,
	const uint_t, const xdrproc_t);
	};

	/*
	* XDR control operator.
	*/
	#define XDR_GET_BYTES_AVAIL 1

	struct xdr_bytesrec {
	bool_t xc_is_last_record;
	size_t xc_num_avail;
	};

	/*
	* XDR functions.
	*/
	void xdrmem_create(XDR *xdrs, const caddr_t addr, const uint_t size,
	const enum xdr_op op);

	#define xdr_control(xdrs, req, info) \
	(xdrs)->x_ops->xdr_control((xdrs), (req), (info))

	/*
	* For precaution, the following are defined as static inlines instead of macros
	* to get some amount of type safety.
	*
	* Also, macros wouldn't work in the case where typecasting is done, because it
	* must be possible to reference the functions' addresses by these names.
	*/
	static inline bool_t xdr_char(XDR xdrs, char cp)
	{
	return (xdrs->x_ops->xdr_char(xdrs, cp));
	}

	static inline bool_t xdr_u_short(XDR xdrs, unsigned short usp)
	{
	return (xdrs->x_ops->xdr_u_short(xdrs, usp));
	}

	static inline bool_t xdr_short(XDR xdrs, short sp)
	{
	BUILD_BUG_ON(sizeof (short) != 2);
	return (xdrs->x_ops->xdr_u_short(xdrs, (unsigned short *) sp));
	}

	static inline bool_t xdr_u_int(XDR xdrs, unsigned up)
	{
	return (xdrs->x_ops->xdr_u_int(xdrs, up));
	}

	static inline bool_t xdr_int(XDR xdrs, int ip)
	{
	BUILD_BUG_ON(sizeof (int) != 4);
	return (xdrs->x_ops->xdr_u_int(xdrs, (unsigned *)ip));
	}

	static inline bool_t xdr_u_longlong_t(XDR xdrs, u_longlong_t ullp)
	{
	return (xdrs->x_ops->xdr_u_longlong_t(xdrs, ullp));
	}

	static inline bool_t xdr_longlong_t(XDR xdrs, longlong_t llp)
	{
	BUILD_BUG_ON(sizeof (longlong_t) != 8);
	return (xdrs->x_ops->xdr_u_longlong_t(xdrs, (u_longlong_t *)llp));
	}

	/*
	* Fixed-length opaque data.
	*/
	static inline bool_t xdr_opaque(XDR *xdrs, caddr_t cp, const uint_t cnt)
	{
	return (xdrs->x_ops->xdr_opaque(xdrs, cp, cnt));
	}

	/*
	* Variable-length string.
	* The *sp buffer must have (maxsize + 1) bytes.
	*/
	static inline bool_t xdr_string(XDR xdrs, char *sp, const uint_t maxsize)
	{
	return (xdrs->x_ops->xdr_string(xdrs, sp, maxsize));
	}

	/*
	* Variable-length arrays.
	*/
	static inline bool_t xdr_array(XDR xdrs, caddr_t arrp, uint_t *sizep,
	const uint_t maxsize, const uint_t elsize, const xdrproc_t elproc)
	{
	return xdrs->x_ops->xdr_array(xdrs, arrp, sizep, maxsize, elsize,
	elproc);
	}

	#endif /* SPL_RPC_XDR_H */
	diff --git a/sys/contrib/openzfs/include/os/linux/spl/sys/debug.h b/sys/contrib/openzfs/include/os/linux/spl/sys/debug.h
	index 007238574fe1..288193ad21c5 100644
	--- a/sys/contrib/openzfs/include/os/linux/spl/sys/debug.h
	+++ b/sys/contrib/openzfs/include/os/linux/spl/sys/debug.h
	@@ -1,189 +1,201 @@
	/*
	* Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
	* Copyright (C) 2007 The Regents of the University of California.
	* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
	* Written by Brian Behlendorf <behlendorf1@llnl.gov>.
	* UCRL-CODE-235197
	*
	* This file is part of the SPL, Solaris Porting Layer.
	*
	* The SPL is free software; you can redistribute it and/or modify it
	* under the terms of the GNU General Public License as published by the
	* Free Software Foundation; either version 2 of the License, or (at your
	* option) any later version.
	*
	* The SPL is distributed in the hope that it will be useful, but WITHOUT
	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	* for more details.
	*
	* You should have received a copy of the GNU General Public License along
	* with the SPL. If not, see <http://www.gnu.org/licenses/>.
	*/

	/*
	* Available Solaris debug functions. All of the ASSERT() macros will be
	* compiled out when NDEBUG is defined, this is the default behavior for
	* the SPL. To enable assertions use the --enable-debug with configure.
	* The VERIFY() functions are never compiled out and cannot be disabled.
	*
	* PANIC() - Panic the node and print message.
	* ASSERT() - Assert X is true, if not panic.
	* ASSERT3B() - Assert boolean X OP Y is true, if not panic.
	* ASSERT3S() - Assert signed X OP Y is true, if not panic.
	* ASSERT3U() - Assert unsigned X OP Y is true, if not panic.
	* ASSERT3P() - Assert pointer X OP Y is true, if not panic.
	* ASSERT0() - Assert value is zero, if not panic.
	+ * ASSERT0P() - Assert pointer is null, if not panic.
	* VERIFY() - Verify X is true, if not panic.
	* VERIFY3B() - Verify boolean X OP Y is true, if not panic.
	* VERIFY3S() - Verify signed X OP Y is true, if not panic.
	* VERIFY3U() - Verify unsigned X OP Y is true, if not panic.
	* VERIFY3P() - Verify pointer X OP Y is true, if not panic.
	* VERIFY0() - Verify value is zero, if not panic.
	+ * VERIFY0P() - Verify pointer is null, if not panic.
	*/

	#ifndef _SPL_DEBUG_H
	#define _SPL_DEBUG_H

	/*
	* Common DEBUG functionality.
	*/
	#define __printflike(a, b) __printf(a, b)

	#ifndef __maybe_unused
	#define __maybe_unused __attribute__((unused))
	#endif

	/*
	* Without this, we see warnings from objtool during normal Linux builds when
	* the kernel is built with CONFIG_STACK_VALIDATION=y:
	*
	* warning: objtool: tsd_create() falls through to next function __list_add()
	* warning: objtool: .text: unexpected end of section
	*
	* Until the toolchain stops doing this, we must only define this attribute on
	* spl_panic() when doing static analysis.
	*/
	#if defined(__COVERITY__) \|\| defined(__clang_analyzer__)
	__attribute__((__noreturn__))
	#endif
	extern void spl_panic(const char file, const char func, int line,
	const char *fmt, ...);
	extern void spl_dumpstack(void);

	static inline int
	spl_assert(const char buf, const char file, const char *func, int line)
	{
	spl_panic(file, func, line, "%s", buf);
	return (0);
	}

	#define PANIC(fmt, a...) \
	spl_panic(__FILE__, __FUNCTION__, __LINE__, fmt, ## a)

	#define VERIFY(cond) \
	(void) (unlikely(!(cond)) && \
	spl_assert("VERIFY(" #cond ") failed\n", \
	__FILE__, __FUNCTION__, __LINE__))

	#define VERIFY3B(LEFT, OP, RIGHT) do { \
	const boolean_t _verify3_left = (boolean_t)(LEFT); \
	const boolean_t _verify3_right = (boolean_t)(RIGHT); \
	if (unlikely(!(_verify3_left OP _verify3_right))) \
	spl_panic(__FILE__, __FUNCTION__, __LINE__, \
	"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
	"failed (%d " #OP " %d)\n", \
	- (boolean_t)(_verify3_left), \
	- (boolean_t)(_verify3_right)); \
	+ (boolean_t)_verify3_left, \
	+ (boolean_t)_verify3_right); \
	} while (0)

	#define VERIFY3S(LEFT, OP, RIGHT) do { \
	const int64_t _verify3_left = (int64_t)(LEFT); \
	const int64_t _verify3_right = (int64_t)(RIGHT); \
	if (unlikely(!(_verify3_left OP _verify3_right))) \
	spl_panic(__FILE__, __FUNCTION__, __LINE__, \
	"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
	"failed (%lld " #OP " %lld)\n", \
	- (long long)(_verify3_left), \
	- (long long)(_verify3_right)); \
	+ (long long)_verify3_left, \
	+ (long long)_verify3_right); \
	} while (0)

	#define VERIFY3U(LEFT, OP, RIGHT) do { \
	const uint64_t _verify3_left = (uint64_t)(LEFT); \
	const uint64_t _verify3_right = (uint64_t)(RIGHT); \
	if (unlikely(!(_verify3_left OP _verify3_right))) \
	spl_panic(__FILE__, __FUNCTION__, __LINE__, \
	"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
	"failed (%llu " #OP " %llu)\n", \
	- (unsigned long long)(_verify3_left), \
	- (unsigned long long)(_verify3_right)); \
	+ (unsigned long long)_verify3_left, \
	+ (unsigned long long)_verify3_right); \
	} while (0)

	#define VERIFY3P(LEFT, OP, RIGHT) do { \
	const uintptr_t _verify3_left = (uintptr_t)(LEFT); \
	const uintptr_t _verify3_right = (uintptr_t)(RIGHT); \
	if (unlikely(!(_verify3_left OP _verify3_right))) \
	spl_panic(__FILE__, __FUNCTION__, __LINE__, \
	"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
	"failed (%px " #OP " %px)\n", \
	- (void *) (_verify3_left), \
	- (void *) (_verify3_right)); \
	+ (void *)_verify3_left, \
	+ (void *)_verify3_right); \
	} while (0)

	#define VERIFY0(RIGHT) do { \
	- const int64_t _verify3_left = (int64_t)(0); \
	- const int64_t _verify3_right = (int64_t)(RIGHT); \
	- if (unlikely(!(_verify3_left == _verify3_right))) \
	+ const int64_t _verify0_right = (int64_t)(RIGHT); \
	+ if (unlikely(!(0 == _verify0_right))) \
	spl_panic(__FILE__, __FUNCTION__, __LINE__, \
	- "VERIFY0(0 == " #RIGHT ") " \
	+ "VERIFY0(" #RIGHT ") " \
	"failed (0 == %lld)\n", \
	- (long long) (_verify3_right)); \
	+ (long long)_verify0_right); \
	+ } while (0)
	+
	+#define VERIFY0P(RIGHT) do { \
	+ const uintptr_t _verify0_right = (uintptr_t)(RIGHT); \
	+ if (unlikely(!(0 == _verify0_right))) \
	+ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
	+ "VERIFY0P(" #RIGHT ") " \
	+ "failed (NULL == %px)\n", \
	+ (void *)_verify0_right); \
	} while (0)

	#define VERIFY_IMPLY(A, B) \
	((void)(likely((!(A)) \|\| (B)) \|\| \
	spl_assert("(" #A ") implies (" #B ")", \
	__FILE__, __FUNCTION__, __LINE__)))

	#define VERIFY_EQUIV(A, B) \
	((void)(likely(!!(A) == !!(B)) \|\| \
	spl_assert("(" #A ") is equivalent to (" #B ")", \
	__FILE__, __FUNCTION__, __LINE__)))

	/*
	* Debugging disabled (--disable-debug)
	*/
	#ifdef NDEBUG

	#define ASSERT(x) ((void) sizeof ((uintptr_t)(x)))
	#define ASSERT3B(x, y, z) \
	((void) sizeof ((uintptr_t)(x)), (void) sizeof ((uintptr_t)(z)))
	#define ASSERT3S(x, y, z) \
	((void) sizeof ((uintptr_t)(x)), (void) sizeof ((uintptr_t)(z)))
	#define ASSERT3U(x, y, z) \
	((void) sizeof ((uintptr_t)(x)), (void) sizeof ((uintptr_t)(z)))
	#define ASSERT3P(x, y, z) \
	((void) sizeof ((uintptr_t)(x)), (void) sizeof ((uintptr_t)(z)))
	#define ASSERT0(x) ((void) sizeof ((uintptr_t)(x)))
	+#define ASSERT0P(x) ((void) sizeof ((uintptr_t)(x)))
	#define IMPLY(A, B) \
	((void) sizeof ((uintptr_t)(A)), (void) sizeof ((uintptr_t)(B)))
	#define EQUIV(A, B) \
	((void) sizeof ((uintptr_t)(A)), (void) sizeof ((uintptr_t)(B)))

	/*
	* Debugging enabled (--enable-debug)
	*/
	#else

	#define ASSERT3B VERIFY3B
	#define ASSERT3S VERIFY3S
	#define ASSERT3U VERIFY3U
	#define ASSERT3P VERIFY3P
	#define ASSERT0 VERIFY0
	+#define ASSERT0P VERIFY0P
	#define ASSERT VERIFY
	#define IMPLY VERIFY_IMPLY
	#define EQUIV VERIFY_EQUIV

	#endif /* NDEBUG */

	#endif /* SPL_DEBUG_H */
	diff --git a/sys/contrib/openzfs/include/os/linux/spl/sys/taskq.h b/sys/contrib/openzfs/include/os/linux/spl/sys/taskq.h
	index 6c1b4377a98a..6c0dbbefda7f 100644
	--- a/sys/contrib/openzfs/include/os/linux/spl/sys/taskq.h
	+++ b/sys/contrib/openzfs/include/os/linux/spl/sys/taskq.h
	@@ -1,170 +1,170 @@
	/*
	* Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
	* Copyright (C) 2007 The Regents of the University of California.
	* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
	* Written by Brian Behlendorf <behlendorf1@llnl.gov>.
	* UCRL-CODE-235197
	*
	* This file is part of the SPL, Solaris Porting Layer.
	*
	* The SPL is free software; you can redistribute it and/or modify it
	* under the terms of the GNU General Public License as published by the
	* Free Software Foundation; either version 2 of the License, or (at your
	* option) any later version.
	*
	* The SPL is distributed in the hope that it will be useful, but WITHOUT
	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	* for more details.
	*
	* You should have received a copy of the GNU General Public License along
	* with the SPL. If not, see <http://www.gnu.org/licenses/>.
	*/

	#ifndef _SPL_TASKQ_H
	#define _SPL_TASKQ_H

	#include <linux/module.h>
	#include <linux/gfp.h>
	#include <linux/slab.h>
	#include <linux/interrupt.h>
	#include <linux/kthread.h>
	#include <sys/types.h>
	#include <sys/thread.h>
	#include <sys/rwlock.h>
	#include <sys/wait.h>

	#define TASKQ_NAMELEN 31

	#define TASKQ_PREPOPULATE 0x00000001
	#define TASKQ_CPR_SAFE 0x00000002
	#define TASKQ_DYNAMIC 0x00000004
	#define TASKQ_THREADS_CPU_PCT 0x00000008
	#define TASKQ_DC_BATCH 0x00000010
	#define TASKQ_ACTIVE 0x80000000

	/*
	* Flags for taskq_dispatch. TQ_SLEEP/TQ_NOSLEEP should be same as
	* KM_SLEEP/KM_NOSLEEP. TQ_NOQUEUE/TQ_NOALLOC are set particularly
	* large so as not to conflict with already used GFP_* defines.
	*/
	#define TQ_SLEEP 0x00000000
	#define TQ_NOSLEEP 0x00000001
	#define TQ_PUSHPAGE 0x00000002
	#define TQ_NOQUEUE 0x01000000
	#define TQ_NOALLOC 0x02000000
	#define TQ_NEW 0x04000000
	#define TQ_FRONT 0x08000000

	/*
	* Reserved taskqid values.
	*/
	#define TASKQID_INVALID ((taskqid_t)0)
	#define TASKQID_INITIAL ((taskqid_t)1)

	/*
	* spin_lock(lock) and spin_lock_nested(lock,0) are equivalent,
	* so TQ_LOCK_DYNAMIC must not evaluate to 0
	*/
	typedef enum tq_lock_role {
	TQ_LOCK_GENERAL = 0,
	TQ_LOCK_DYNAMIC = 1,
	} tq_lock_role_t;

	typedef unsigned long taskqid_t;
	typedef void (task_func_t)(void *);

	typedef struct taskq {
	spinlock_t tq_lock; /* protects taskq_t */
	char tq_name; / taskq name */
	int tq_instance; /* instance of tq_name */
	struct list_head tq_thread_list; /* list of all threads */
	struct list_head tq_active_list; /* list of active threads */
	int tq_nactive; /* # of active threads */
	int tq_nthreads; /* # of existing threads */
	int tq_nspawn; /* # of threads being spawned */
	int tq_maxthreads; /* # of threads maximum */
	/* If PERCPU flag is set, percent of NCPUs to have as threads */
	int tq_cpu_pct;
	int tq_pri; /* priority */
	int tq_minalloc; /* min taskq_ent_t pool size */
	int tq_maxalloc; /* max taskq_ent_t pool size */
	int tq_nalloc; /* cur taskq_ent_t pool size */
	uint_t tq_flags; /* flags */
	taskqid_t tq_next_id; /* next pend/work id */
	taskqid_t tq_lowest_id; /* lowest pend/work id */
	struct list_head tq_free_list; /* free taskq_ent_t's */
	struct list_head tq_pend_list; /* pending taskq_ent_t's */
	struct list_head tq_prio_list; /* priority taskq_ent_t's */
	struct list_head tq_delay_list; /* delayed taskq_ent_t's */
	struct list_head tq_taskqs; /* all taskq_t's */
	spl_wait_queue_head_t tq_work_waitq; /* new work waitq */
	spl_wait_queue_head_t tq_wait_waitq; /* wait waitq */
	tq_lock_role_t tq_lock_class; /* class when taking tq_lock */
	/* list node for the cpu hotplug callback */
	struct hlist_node tq_hp_cb_node;
	boolean_t tq_hp_support;
	- unsigned long lastshouldstop; /* when to purge dynamic */
	+ unsigned long lastspawnstop; /* when to purge dynamic */
	} taskq_t;

	typedef struct taskq_ent {
	spinlock_t tqent_lock;
	spl_wait_queue_head_t tqent_waitq;
	struct timer_list tqent_timer;
	struct list_head tqent_list;
	taskqid_t tqent_id;
	task_func_t *tqent_func;
	void *tqent_arg;
	taskq_t *tqent_taskq;
	uintptr_t tqent_flags;
	unsigned long tqent_birth;
	} taskq_ent_t;

	#define TQENT_FLAG_PREALLOC 0x1
	#define TQENT_FLAG_CANCEL 0x2

	typedef struct taskq_thread {
	struct list_head tqt_thread_list;
	struct list_head tqt_active_list;
	struct task_struct *tqt_thread;
	taskq_t *tqt_tq;
	taskqid_t tqt_id;
	taskq_ent_t *tqt_task;
	uintptr_t tqt_flags;
	} taskq_thread_t;

	/* Global system-wide dynamic task queue available for all consumers */
	extern taskq_t *system_taskq;
	/* Global dynamic task queue for long delay */
	extern taskq_t *system_delay_taskq;

	/* List of all taskqs */
	extern struct list_head tq_list;
	extern struct rw_semaphore tq_list_sem;

	extern taskqid_t taskq_dispatch(taskq_t , task_func_t, void , uint_t);
	extern taskqid_t taskq_dispatch_delay(taskq_t , task_func_t, void ,
	uint_t, clock_t);
	extern void taskq_dispatch_ent(taskq_t , task_func_t, void , uint_t,
	taskq_ent_t *);
	extern int taskq_empty_ent(taskq_ent_t *);
	extern void taskq_init_ent(taskq_ent_t *);
	extern taskq_t taskq_create(const char , int, pri_t, int, int, uint_t);
	extern void taskq_destroy(taskq_t *);
	extern void taskq_wait_id(taskq_t *, taskqid_t);
	extern void taskq_wait_outstanding(taskq_t *, taskqid_t);
	extern void taskq_wait(taskq_t *);
	extern int taskq_cancel_id(taskq_t *, taskqid_t);
	extern int taskq_member(taskq_t , kthread_t );
	extern taskq_t *taskq_of_curthread(void);

	#define taskq_create_proc(name, nthreads, pri, min, max, proc, flags) \
	taskq_create(name, nthreads, pri, min, max, flags)
	#define taskq_create_sysdc(name, nthreads, min, max, proc, dc, flags) \
	((void) sizeof (dc), \
	taskq_create(name, nthreads, maxclsyspri, min, max, flags))

	int spl_taskq_init(void);
	void spl_taskq_fini(void);

	#endif /* _SPL_TASKQ_H */
	diff --git a/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_zil.h b/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_zil.h
	index afa1a274e43c..ae1caa3ac473 100644
	--- a/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_zil.h
	+++ b/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_zil.h
	@@ -1,267 +1,273 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	#if defined(_KERNEL)
	#if defined(HAVE_DECLARE_EVENT_CLASS)

	#undef TRACE_SYSTEM
	#define TRACE_SYSTEM zfs

	#undef TRACE_SYSTEM_VAR
	#define TRACE_SYSTEM_VAR zfs_zil

	#if !defined(_TRACE_ZIL_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	#define _TRACE_ZIL_H

	#include <linux/tracepoint.h>
	#include <sys/types.h>

	#define ZILOG_TP_STRUCT_ENTRY \
	__field(uint64_t, zl_lr_seq) \
	__field(uint64_t, zl_commit_lr_seq) \
	__field(uint64_t, zl_destroy_txg) \
	__field(uint64_t, zl_replaying_seq) \
	__field(uint32_t, zl_suspend) \
	__field(uint8_t, zl_suspending) \
	__field(uint8_t, zl_keep_first) \
	__field(uint8_t, zl_replay) \
	__field(uint8_t, zl_stop_sync) \
	__field(uint8_t, zl_logbias) \
	__field(uint8_t, zl_sync) \
	__field(int, zl_parse_error) \
	__field(uint64_t, zl_parse_blk_seq) \
	__field(uint64_t, zl_parse_lr_seq) \
	__field(uint64_t, zl_parse_blk_count) \
	__field(uint64_t, zl_parse_lr_count) \
	- __field(uint64_t, zl_cur_used) \
	+ __field(uint64_t, zl_cur_size) \
	+ __field(uint64_t, zl_cur_left) \
	+ __field(uint64_t, zl_cur_max) \
	__field(clock_t, zl_replay_time) \
	__field(uint64_t, zl_replay_blks)

	#define ZILOG_TP_FAST_ASSIGN \
	__entry->zl_lr_seq = zilog->zl_lr_seq; \
	__entry->zl_commit_lr_seq = zilog->zl_commit_lr_seq; \
	__entry->zl_destroy_txg = zilog->zl_destroy_txg; \
	__entry->zl_replaying_seq = zilog->zl_replaying_seq; \
	__entry->zl_suspend = zilog->zl_suspend; \
	__entry->zl_suspending = zilog->zl_suspending; \
	__entry->zl_keep_first = zilog->zl_keep_first; \
	__entry->zl_replay = zilog->zl_replay; \
	__entry->zl_stop_sync = zilog->zl_stop_sync; \
	__entry->zl_logbias = zilog->zl_logbias; \
	__entry->zl_sync = zilog->zl_sync; \
	__entry->zl_parse_error = zilog->zl_parse_error; \
	__entry->zl_parse_blk_seq = zilog->zl_parse_blk_seq; \
	__entry->zl_parse_lr_seq = zilog->zl_parse_lr_seq; \
	__entry->zl_parse_blk_count = zilog->zl_parse_blk_count;\
	__entry->zl_parse_lr_count = zilog->zl_parse_lr_count; \
	- __entry->zl_cur_used = zilog->zl_cur_used; \
	+ __entry->zl_cur_size = zilog->zl_cur_size; \
	+ __entry->zl_cur_left = zilog->zl_cur_left; \
	+ __entry->zl_cur_max = zilog->zl_cur_max; \
	__entry->zl_replay_time = zilog->zl_replay_time; \
	__entry->zl_replay_blks = zilog->zl_replay_blks;

	#define ZILOG_TP_PRINTK_FMT \
	"zl { lr_seq %llu commit_lr_seq %llu destroy_txg %llu " \
	"replaying_seq %llu suspend %u suspending %u keep_first %u " \
	"replay %u stop_sync %u logbias %u sync %u " \
	"parse_error %u parse_blk_seq %llu parse_lr_seq %llu " \
	"parse_blk_count %llu parse_lr_count %llu " \
	- "cur_used %llu replay_time %lu replay_blks %llu }"
	+ "cur_size %llu cur_left %llu cur_max %llu replay_time %lu " \
	+ "replay_blks %llu }"

	#define ZILOG_TP_PRINTK_ARGS \
	__entry->zl_lr_seq, __entry->zl_commit_lr_seq, \
	__entry->zl_destroy_txg, __entry->zl_replaying_seq, \
	__entry->zl_suspend, __entry->zl_suspending, \
	__entry->zl_keep_first, __entry->zl_replay, \
	__entry->zl_stop_sync, __entry->zl_logbias, __entry->zl_sync, \
	__entry->zl_parse_error, __entry->zl_parse_blk_seq, \
	__entry->zl_parse_lr_seq, __entry->zl_parse_blk_count, \
	- __entry->zl_parse_lr_count, __entry->zl_cur_used, \
	+ __entry->zl_parse_lr_count, __entry->zl_cur_size, \
	+ __entry->zl_cur_left, __entry->zl_cur_max, \
	__entry->zl_replay_time, __entry->zl_replay_blks

	#define ITX_TP_STRUCT_ENTRY \
	__field(itx_wr_state_t, itx_wr_state) \
	__field(uint8_t, itx_sync) \
	__field(zil_callback_t, itx_callback) \
	__field(void *, itx_callback_data) \
	__field(uint64_t, itx_oid) \
	\
	__field(uint64_t, lrc_txtype) \
	__field(uint64_t, lrc_reclen) \
	__field(uint64_t, lrc_txg) \
	__field(uint64_t, lrc_seq)

	#define ITX_TP_FAST_ASSIGN \
	__entry->itx_wr_state = itx->itx_wr_state; \
	__entry->itx_sync = itx->itx_sync; \
	__entry->itx_callback = itx->itx_callback; \
	__entry->itx_callback_data = itx->itx_callback_data; \
	__entry->itx_oid = itx->itx_oid; \
	\
	__entry->lrc_txtype = itx->itx_lr.lrc_txtype; \
	__entry->lrc_reclen = itx->itx_lr.lrc_reclen; \
	__entry->lrc_txg = itx->itx_lr.lrc_txg; \
	__entry->lrc_seq = itx->itx_lr.lrc_seq;

	#define ITX_TP_PRINTK_FMT \
	"itx { wr_state %u sync %u callback %p callback_data %p oid %llu" \
	" { txtype %llu reclen %llu txg %llu seq %llu } }"

	#define ITX_TP_PRINTK_ARGS \
	__entry->itx_wr_state, __entry->itx_sync, __entry->itx_callback,\
	__entry->itx_callback_data, __entry->itx_oid, \
	__entry->lrc_txtype, __entry->lrc_reclen, __entry->lrc_txg, \
	__entry->lrc_seq

	#define ZCW_TP_STRUCT_ENTRY \
	__field(lwb_t *, zcw_lwb) \
	__field(boolean_t, zcw_done) \
	__field(int, zcw_zio_error) \

	#define ZCW_TP_FAST_ASSIGN \
	__entry->zcw_lwb = zcw->zcw_lwb; \
	__entry->zcw_done = zcw->zcw_done; \
	__entry->zcw_zio_error = zcw->zcw_zio_error;

	#define ZCW_TP_PRINTK_FMT \
	"zcw { lwb %p done %u error %u }"

	#define ZCW_TP_PRINTK_ARGS \
	__entry->zcw_lwb, __entry->zcw_done, __entry->zcw_zio_error

	/*
	* Generic support for two argument tracepoints of the form:
	*
	* DTRACE_PROBE2(...,
	* zilog_t *, ...,
	* itx_t *, ...);
	*/

	#if defined(__clang__)
	#pragma clang diagnostic push
	#pragma clang diagnostic ignored "-Wordered-compare-function-pointers"
	#endif
	/* BEGIN CSTYLED */
	DECLARE_EVENT_CLASS(zfs_zil_process_itx_class,
	TP_PROTO(zilog_t zilog, itx_t itx),
	TP_ARGS(zilog, itx),
	TP_STRUCT__entry(
	ZILOG_TP_STRUCT_ENTRY
	ITX_TP_STRUCT_ENTRY
	),
	TP_fast_assign(
	ZILOG_TP_FAST_ASSIGN
	ITX_TP_FAST_ASSIGN
	),
	TP_printk(
	ZILOG_TP_PRINTK_FMT " " ITX_TP_PRINTK_FMT,
	ZILOG_TP_PRINTK_ARGS, ITX_TP_PRINTK_ARGS)
	);
	/* END CSTYLED */
	#if defined(__clang__)
	#pragma clang diagnostic pop
	#endif

	#define DEFINE_ZIL_PROCESS_ITX_EVENT(name) \
	DEFINE_EVENT(zfs_zil_process_itx_class, name, \
	TP_PROTO(zilog_t zilog, itx_t itx), \
	TP_ARGS(zilog, itx))
	DEFINE_ZIL_PROCESS_ITX_EVENT(zfs_zil__process__commit__itx);
	DEFINE_ZIL_PROCESS_ITX_EVENT(zfs_zil__process__normal__itx);

	/*
	* Generic support for two argument tracepoints of the form:
	*
	* DTRACE_PROBE2(...,
	* zilog_t *, ...,
	* zil_commit_waiter_t *, ...);
	*/
	/* BEGIN CSTYLED */
	DECLARE_EVENT_CLASS(zfs_zil_commit_io_error_class,
	TP_PROTO(zilog_t zilog, zil_commit_waiter_t zcw),
	TP_ARGS(zilog, zcw),
	TP_STRUCT__entry(
	ZILOG_TP_STRUCT_ENTRY
	ZCW_TP_STRUCT_ENTRY
	),
	TP_fast_assign(
	ZILOG_TP_FAST_ASSIGN
	ZCW_TP_FAST_ASSIGN
	),
	TP_printk(
	ZILOG_TP_PRINTK_FMT " " ZCW_TP_PRINTK_FMT,
	ZILOG_TP_PRINTK_ARGS, ZCW_TP_PRINTK_ARGS)
	);

	#define DEFINE_ZIL_COMMIT_IO_ERROR_EVENT(name) \
	DEFINE_EVENT(zfs_zil_commit_io_error_class, name, \
	TP_PROTO(zilog_t zilog, zil_commit_waiter_t zcw), \
	TP_ARGS(zilog, zcw))
	DEFINE_ZIL_COMMIT_IO_ERROR_EVENT(zfs_zil__commit__io__error);

	/*
	* Generic support for three argument tracepoints of the form:
	*
	* DTRACE_PROBE3(...,
	* zilog_t *, ...,
	* uint64_t, ...,
	* uint64_t, ...);
	*/
	/* BEGIN CSTYLED */
	DECLARE_EVENT_CLASS(zfs_zil_block_size_class,
	TP_PROTO(zilog_t *zilog, uint64_t res, uint64_t s1),
	TP_ARGS(zilog, res, s1),
	TP_STRUCT__entry(
	ZILOG_TP_STRUCT_ENTRY
	__field(uint64_t, res)
	__field(uint64_t, s1)
	),
	TP_fast_assign(
	ZILOG_TP_FAST_ASSIGN
	__entry->res = res;
	__entry->s1 = s1;
	),
	TP_printk(
	ZILOG_TP_PRINTK_FMT " res %llu s1 %llu",
	ZILOG_TP_PRINTK_ARGS, __entry->res, __entry->s1)
	);

	#define DEFINE_ZIL_BLOCK_SIZE_EVENT(name) \
	DEFINE_EVENT(zfs_zil_block_size_class, name, \
	TP_PROTO(zilog_t *zilog, uint64_t res, uint64_t s1), \
	TP_ARGS(zilog, res, s1))
	DEFINE_ZIL_BLOCK_SIZE_EVENT(zfs_zil__block__size);

	#endif /* _TRACE_ZIL_H */

	#undef TRACE_INCLUDE_PATH
	#undef TRACE_INCLUDE_FILE
	#define TRACE_INCLUDE_PATH sys
	#define TRACE_INCLUDE_FILE trace_zil
	#include <trace/define_trace.h>

	#else

	DEFINE_DTRACE_PROBE2(zil__process__commit__itx);
	DEFINE_DTRACE_PROBE2(zil__process__normal__itx);
	DEFINE_DTRACE_PROBE2(zil__commit__io__error);
	DEFINE_DTRACE_PROBE3(zil__block__size);

	#endif /* HAVE_DECLARE_EVENT_CLASS */
	#endif /* _KERNEL */
	diff --git a/sys/contrib/openzfs/include/sys/abd.h b/sys/contrib/openzfs/include/sys/abd.h
	index 750f9986c1da..bee38b831bc0 100644
	--- a/sys/contrib/openzfs/include/sys/abd.h
	+++ b/sys/contrib/openzfs/include/sys/abd.h
	@@ -1,226 +1,235 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2014 by Chunwei Chen. All rights reserved.
	* Copyright (c) 2016, 2019 by Delphix. All rights reserved.
	*/

	#ifndef _ABD_H
	#define _ABD_H

	#include <sys/isa_defs.h>
	#include <sys/debug.h>
	#include <sys/zfs_refcount.h>
	#include <sys/uio.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	typedef enum abd_flags {
	ABD_FLAG_LINEAR = 1 << 0, /* is buffer linear (or scattered)? */
	ABD_FLAG_OWNER = 1 << 1, /* does it own its data buffers? */
	ABD_FLAG_META = 1 << 2, /* does this represent FS metadata? */
	ABD_FLAG_MULTI_ZONE = 1 << 3, /* pages split over memory zones */
	ABD_FLAG_MULTI_CHUNK = 1 << 4, /* pages split over multiple chunks */
	ABD_FLAG_LINEAR_PAGE = 1 << 5, /* linear but allocd from page */
	ABD_FLAG_GANG = 1 << 6, /* mult ABDs chained together */
	ABD_FLAG_GANG_FREE = 1 << 7, /* gang ABD is responsible for mem */
	ABD_FLAG_ZEROS = 1 << 8, /* ABD for zero-filled buffer */
	ABD_FLAG_ALLOCD = 1 << 9, /* we allocated the abd_t */
	} abd_flags_t;

	typedef struct abd {
	abd_flags_t abd_flags;
	uint_t abd_size; /* excludes scattered abd_offset */
	list_node_t abd_gang_link;
	#ifdef ZFS_DEBUG
	struct abd *abd_parent;
	zfs_refcount_t abd_children;
	#endif
	kmutex_t abd_mtx;
	union {
	struct abd_scatter {
	uint_t abd_offset;
	#if defined(__FreeBSD__) && defined(_KERNEL)
	void abd_chunks[1]; / actually variable-length */
	#else
	uint_t abd_nents;
	struct scatterlist *abd_sgl;
	#endif
	} abd_scatter;
	struct abd_linear {
	void *abd_buf;
	struct scatterlist abd_sgl; / for LINEAR_PAGE */
	} abd_linear;
	struct abd_gang {
	list_t abd_gang_chain;
	} abd_gang;
	} abd_u;
	} abd_t;

	typedef int abd_iter_func_t(void buf, size_t len, void priv);
	typedef int abd_iter_func2_t(void bufa, void bufb, size_t len, void *priv);
	+#if defined(__linux__) && defined(_KERNEL)
	+typedef int abd_iter_page_func_t(struct page , size_t, size_t, void );
	+#endif

	extern int zfs_abd_scatter_enabled;

	/*
	* Allocations and deallocations
	*/

	__attribute__((malloc))
	abd_t *abd_alloc(size_t, boolean_t);
	__attribute__((malloc))
	abd_t *abd_alloc_linear(size_t, boolean_t);
	__attribute__((malloc))
	abd_t *abd_alloc_gang(void);
	__attribute__((malloc))
	abd_t *abd_alloc_for_io(size_t, boolean_t);
	__attribute__((malloc))
	abd_t abd_alloc_sametype(abd_t , size_t);
	boolean_t abd_size_alloc_linear(size_t);
	void abd_gang_add(abd_t , abd_t , boolean_t);
	void abd_free(abd_t *);
	abd_t abd_get_offset(abd_t , size_t);
	abd_t abd_get_offset_size(abd_t , size_t, size_t);
	abd_t abd_get_offset_struct(abd_t , abd_t *, size_t, size_t);
	abd_t *abd_get_zeros(size_t);
	abd_t abd_get_from_buf(void , size_t);
	void abd_cache_reap_now(void);

	/*
	* Conversion to and from a normal buffer
	*/

	void abd_to_buf(abd_t );
	void abd_borrow_buf(abd_t , size_t);
	void abd_borrow_buf_copy(abd_t , size_t);
	void abd_return_buf(abd_t , void , size_t);
	void abd_return_buf_copy(abd_t , void , size_t);
	void abd_take_ownership_of_buf(abd_t *, boolean_t);
	void abd_release_ownership_of_buf(abd_t *);

	/*
	* ABD operations
	*/

	int abd_iterate_func(abd_t , size_t, size_t, abd_iter_func_t , void *);
	int abd_iterate_func2(abd_t , abd_t , size_t, size_t, size_t,
	abd_iter_func2_t , void );
	+#if defined(__linux__) && defined(_KERNEL)
	+int abd_iterate_page_func(abd_t , size_t, size_t, abd_iter_page_func_t ,
	+ void *);
	+#endif
	void abd_copy_off(abd_t , abd_t , size_t, size_t, size_t);
	void abd_copy_from_buf_off(abd_t , const void , size_t, size_t);
	void abd_copy_to_buf_off(void , abd_t , size_t, size_t);
	int abd_cmp(abd_t , abd_t );
	int abd_cmp_buf_off(abd_t , const void , size_t, size_t);
	void abd_zero_off(abd_t *, size_t, size_t);
	void abd_verify(abd_t *);

	void abd_raidz_gen_iterate(abd_t *cabds, abd_t dabd,
	ssize_t csize, ssize_t dsize, const unsigned parity,
	void (func_raidz_gen)(void , const void , size_t, size_t));
	void abd_raidz_rec_iterate(abd_t cabds, abd_t tabds,
	ssize_t tsize, const unsigned parity,
	void (func_raidz_rec)(void t, const size_t tsize, void *c,
	const unsigned *mul),
	const unsigned *mul);

	/*
	* Wrappers for calls with offsets of 0
	*/

	static inline void
	abd_copy(abd_t dabd, abd_t sabd, size_t size)
	{
	abd_copy_off(dabd, sabd, 0, 0, size);
	}

	static inline void
	abd_copy_from_buf(abd_t abd, const void buf, size_t size)
	{
	abd_copy_from_buf_off(abd, buf, 0, size);
	}

	static inline void
	abd_copy_to_buf(void* buf, abd_t *abd, size_t size)
	{
	abd_copy_to_buf_off(buf, abd, 0, size);
	}

	static inline int
	abd_cmp_buf(abd_t abd, const void buf, size_t size)
	{
	return (abd_cmp_buf_off(abd, buf, 0, size));
	}

	static inline void
	abd_zero(abd_t *abd, size_t size)
	{
	abd_zero_off(abd, 0, size);
	}

	/*
	* ABD type check functions
	*/
	static inline boolean_t
	abd_is_linear(abd_t *abd)
	{
	return ((abd->abd_flags & ABD_FLAG_LINEAR) ? B_TRUE : B_FALSE);
	}

	static inline boolean_t
	abd_is_linear_page(abd_t *abd)
	{
	return ((abd->abd_flags & ABD_FLAG_LINEAR_PAGE) ? B_TRUE : B_FALSE);
	}

	static inline boolean_t
	abd_is_gang(abd_t *abd)
	{
	return ((abd->abd_flags & ABD_FLAG_GANG) ? B_TRUE : B_FALSE);
	}

	static inline uint_t
	abd_get_size(abd_t *abd)
	{
	return (abd->abd_size);
	}

	/*
	* Module lifecycle
	* Defined in each specific OS's abd_os.c
	*/

	void abd_init(void);
	void abd_fini(void);

	/*
	* Linux ABD bio functions
	+ * Note: these are only needed to support vdev_classic. See comment in
	+ * vdev_disk.c.
	*/
	#if defined(__linux__) && defined(_KERNEL)
	unsigned int abd_bio_map_off(struct bio , abd_t , unsigned int, size_t);
	unsigned long abd_nr_pages_off(abd_t *, unsigned int, size_t);
	#endif

	#ifdef __cplusplus
	}
	#endif

	#endif /* _ABD_H */
	diff --git a/sys/contrib/openzfs/include/sys/abd_impl.h b/sys/contrib/openzfs/include/sys/abd_impl.h
	index 40546d4af137..f88ea25e245d 100644
	--- a/sys/contrib/openzfs/include/sys/abd_impl.h
	+++ b/sys/contrib/openzfs/include/sys/abd_impl.h
	@@ -1,111 +1,131 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2014 by Chunwei Chen. All rights reserved.
	* Copyright (c) 2016, 2019 by Delphix. All rights reserved.
	+ * Copyright (c) 2023, 2024, Klara Inc.
	*/

	#ifndef _ABD_IMPL_H
	#define _ABD_IMPL_H

	#include <sys/abd.h>
	#include <sys/wmsum.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	typedef enum abd_stats_op {
	ABDSTAT_INCR, /* Increase abdstat values */
	ABDSTAT_DECR /* Decrease abdstat values */
	} abd_stats_op_t;

	-struct scatterlist; /* forward declaration */
	+/* forward declarations */
	+struct scatterlist;
	+struct page;

	struct abd_iter {
	/* public interface */
	- void iter_mapaddr; / addr corresponding to iter_pos */
	- size_t iter_mapsize; /* length of data valid at mapaddr */
	+ union {
	+ /* for abd_iter_map()/abd_iter_unmap() */
	+ struct {
	+ /* addr corresponding to iter_pos */
	+ void *iter_mapaddr;
	+ /* length of data valid at mapaddr */
	+ size_t iter_mapsize;
	+ };
	+ /* for abd_iter_page() */
	+ struct {
	+ /* current page */
	+ struct page *iter_page;
	+ /* offset of data in page */
	+ size_t iter_page_doff;
	+ /* size of data in page */
	+ size_t iter_page_dsize;
	+ };
	+ };

	/* private */
	abd_t iter_abd; / ABD being iterated through */
	size_t iter_pos;
	size_t iter_offset; /* offset in current sg/abd_buf, */
	/* abd_offset included */
	struct scatterlist iter_sg; / current sg */
	};

	extern abd_t *abd_zero_scatter;

	abd_t abd_gang_get_offset(abd_t , size_t *);
	abd_t *abd_alloc_struct(size_t);
	void abd_free_struct(abd_t *);

	/*
	* OS specific functions
	*/

	abd_t *abd_alloc_struct_impl(size_t);
	abd_t abd_get_offset_scatter(abd_t , abd_t *, size_t, size_t);
	void abd_free_struct_impl(abd_t *);
	void abd_alloc_chunks(abd_t *, size_t);
	void abd_free_chunks(abd_t *);
	void abd_update_scatter_stats(abd_t *, abd_stats_op_t);
	void abd_update_linear_stats(abd_t *, abd_stats_op_t);
	void abd_verify_scatter(abd_t *);
	void abd_free_linear_page(abd_t *);
	/* OS specific abd_iter functions */
	void abd_iter_init(struct abd_iter , abd_t );
	boolean_t abd_iter_at_end(struct abd_iter *);
	void abd_iter_advance(struct abd_iter *, size_t);
	void abd_iter_map(struct abd_iter *);
	void abd_iter_unmap(struct abd_iter *);
	+void abd_iter_page(struct abd_iter *);

	/*
	* Helper macros
	*/
	#define ABDSTAT_INCR(stat, val) \
	wmsum_add(&abd_sums.stat, (val))
	#define ABDSTAT_BUMP(stat) ABDSTAT_INCR(stat, 1)
	#define ABDSTAT_BUMPDOWN(stat) ABDSTAT_INCR(stat, -1)

	#define ABD_SCATTER(abd) (abd->abd_u.abd_scatter)
	#define ABD_LINEAR_BUF(abd) (abd->abd_u.abd_linear.abd_buf)
	#define ABD_GANG(abd) (abd->abd_u.abd_gang)

	#if defined(_KERNEL)
	#if defined(__FreeBSD__)
	#define abd_enter_critical(flags) critical_enter()
	#define abd_exit_critical(flags) critical_exit()
	#else
	#define abd_enter_critical(flags) local_irq_save(flags)
	#define abd_exit_critical(flags) local_irq_restore(flags)
	#endif
	#else /* !_KERNEL */
	#define abd_enter_critical(flags) ((void)0)
	#define abd_exit_critical(flags) ((void)0)
	#endif

	#ifdef __cplusplus
	}
	#endif

	#endif /* _ABD_IMPL_H */
	diff --git a/sys/contrib/openzfs/include/sys/dmu.h b/sys/contrib/openzfs/include/sys/dmu.h
	index 06b4dc27dfea..26b329b53f05 100644
	--- a/sys/contrib/openzfs/include/sys/dmu.h
	+++ b/sys/contrib/openzfs/include/sys/dmu.h
	@@ -1,1101 +1,1102 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
	* Copyright 2011 Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2012, Joyent, Inc. All rights reserved.
	* Copyright 2014 HybridCluster. All rights reserved.
	* Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
	* Copyright 2013 Saso Kiselkov. All rights reserved.
	* Copyright (c) 2017, Intel Corporation.
	* Copyright (c) 2022 Hewlett Packard Enterprise Development LP.
	*/

	/* Portions Copyright 2010 Robert Milkowski */

	#ifndef _SYS_DMU_H
	#define _SYS_DMU_H

	/*
	* This file describes the interface that the DMU provides for its
	* consumers.
	*
	* The DMU also interacts with the SPA. That interface is described in
	* dmu_spa.h.
	*/

	#include <sys/zfs_context.h>
	#include <sys/inttypes.h>
	#include <sys/cred.h>
	#include <sys/fs/zfs.h>
	#include <sys/zio_compress.h>
	#include <sys/zio_priority.h>
	#include <sys/uio.h>
	#include <sys/zfs_file.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	struct page;
	struct vnode;
	struct spa;
	struct zilog;
	struct zio;
	struct blkptr;
	struct zap_cursor;
	struct dsl_dataset;
	struct dsl_pool;
	struct dnode;
	struct drr_begin;
	struct drr_end;
	struct zbookmark_phys;
	struct spa;
	struct nvlist;
	struct arc_buf;
	struct zio_prop;
	struct sa_handle;
	struct dsl_crypto_params;
	struct locked_range;

	typedef struct objset objset_t;
	typedef struct dmu_tx dmu_tx_t;
	typedef struct dsl_dir dsl_dir_t;
	typedef struct dnode dnode_t;

	typedef enum dmu_object_byteswap {
	DMU_BSWAP_UINT8,
	DMU_BSWAP_UINT16,
	DMU_BSWAP_UINT32,
	DMU_BSWAP_UINT64,
	DMU_BSWAP_ZAP,
	DMU_BSWAP_DNODE,
	DMU_BSWAP_OBJSET,
	DMU_BSWAP_ZNODE,
	DMU_BSWAP_OLDACL,
	DMU_BSWAP_ACL,
	/*
	* Allocating a new byteswap type number makes the on-disk format
	* incompatible with any other format that uses the same number.
	*
	* Data can usually be structured to work with one of the
	* DMU_BSWAP_UINT* or DMU_BSWAP_ZAP types.
	*/
	DMU_BSWAP_NUMFUNCS
	} dmu_object_byteswap_t;

	#define DMU_OT_NEWTYPE 0x80
	#define DMU_OT_METADATA 0x40
	#define DMU_OT_ENCRYPTED 0x20
	#define DMU_OT_BYTESWAP_MASK 0x1f

	/*
	* Defines a uint8_t object type. Object types specify if the data
	* in the object is metadata (boolean) and how to byteswap the data
	* (dmu_object_byteswap_t). All of the types created by this method
	* are cached in the dbuf metadata cache.
	*/
	#define DMU_OT(byteswap, metadata, encrypted) \
	(DMU_OT_NEWTYPE \| \
	((metadata) ? DMU_OT_METADATA : 0) \| \
	((encrypted) ? DMU_OT_ENCRYPTED : 0) \| \
	((byteswap) & DMU_OT_BYTESWAP_MASK))

	#define DMU_OT_IS_VALID(ot) (((ot) & DMU_OT_NEWTYPE) ? \
	((ot) & DMU_OT_BYTESWAP_MASK) < DMU_BSWAP_NUMFUNCS : \
	(ot) < DMU_OT_NUMTYPES)

	#define DMU_OT_IS_METADATA_CACHED(ot) (((ot) & DMU_OT_NEWTYPE) ? \
	B_TRUE : dmu_ot[(ot)].ot_dbuf_metadata_cache)

	/*
	* MDB doesn't have dmu_ot; it defines these macros itself.
	*/
	#ifndef ZFS_MDB
	#define DMU_OT_IS_METADATA_IMPL(ot) (dmu_ot[ot].ot_metadata)
	#define DMU_OT_IS_ENCRYPTED_IMPL(ot) (dmu_ot[ot].ot_encrypt)
	#define DMU_OT_BYTESWAP_IMPL(ot) (dmu_ot[ot].ot_byteswap)
	#endif

	#define DMU_OT_IS_METADATA(ot) (((ot) & DMU_OT_NEWTYPE) ? \
	(((ot) & DMU_OT_METADATA) != 0) : \
	DMU_OT_IS_METADATA_IMPL(ot))

	#define DMU_OT_IS_DDT(ot) \
	((ot) == DMU_OT_DDT_ZAP)

	#define DMU_OT_IS_CRITICAL(ot) \
	(DMU_OT_IS_METADATA(ot) && \
	(ot) != DMU_OT_DNODE && \
	(ot) != DMU_OT_DIRECTORY_CONTENTS && \
	(ot) != DMU_OT_SA)

	/* Note: ztest uses DMU_OT_UINT64_OTHER as a proxy for file blocks */
	#define DMU_OT_IS_FILE(ot) \
	((ot) == DMU_OT_PLAIN_FILE_CONTENTS \|\| (ot) == DMU_OT_UINT64_OTHER)

	#define DMU_OT_IS_ENCRYPTED(ot) (((ot) & DMU_OT_NEWTYPE) ? \
	(((ot) & DMU_OT_ENCRYPTED) != 0) : \
	DMU_OT_IS_ENCRYPTED_IMPL(ot))

	/*
	* These object types use bp_fill != 1 for their L0 bp's. Therefore they can't
	* have their data embedded (i.e. use a BP_IS_EMBEDDED() bp), because bp_fill
	* is repurposed for embedded BPs.
	*/
	#define DMU_OT_HAS_FILL(ot) \
	((ot) == DMU_OT_DNODE \|\| (ot) == DMU_OT_OBJSET)

	#define DMU_OT_BYTESWAP(ot) (((ot) & DMU_OT_NEWTYPE) ? \
	((ot) & DMU_OT_BYTESWAP_MASK) : \
	DMU_OT_BYTESWAP_IMPL(ot))

	typedef enum dmu_object_type {
	DMU_OT_NONE,
	/* general: */
	DMU_OT_OBJECT_DIRECTORY, /* ZAP */
	DMU_OT_OBJECT_ARRAY, /* UINT64 */
	DMU_OT_PACKED_NVLIST, /* UINT8 (XDR by nvlist_pack/unpack) */
	DMU_OT_PACKED_NVLIST_SIZE, /* UINT64 */
	DMU_OT_BPOBJ, /* UINT64 */
	DMU_OT_BPOBJ_HDR, /* UINT64 */
	/* spa: */
	DMU_OT_SPACE_MAP_HEADER, /* UINT64 */
	DMU_OT_SPACE_MAP, /* UINT64 */
	/* zil: */
	DMU_OT_INTENT_LOG, /* UINT64 */
	/* dmu: */
	DMU_OT_DNODE, /* DNODE */
	DMU_OT_OBJSET, /* OBJSET */
	/* dsl: */
	DMU_OT_DSL_DIR, /* UINT64 */
	DMU_OT_DSL_DIR_CHILD_MAP, /* ZAP */
	DMU_OT_DSL_DS_SNAP_MAP, /* ZAP */
	DMU_OT_DSL_PROPS, /* ZAP */
	DMU_OT_DSL_DATASET, /* UINT64 */
	/* zpl: */
	DMU_OT_ZNODE, /* ZNODE */
	DMU_OT_OLDACL, /* Old ACL */
	DMU_OT_PLAIN_FILE_CONTENTS, /* UINT8 */
	DMU_OT_DIRECTORY_CONTENTS, /* ZAP */
	DMU_OT_MASTER_NODE, /* ZAP */
	DMU_OT_UNLINKED_SET, /* ZAP */
	/* zvol: */
	DMU_OT_ZVOL, /* UINT8 */
	DMU_OT_ZVOL_PROP, /* ZAP */
	/* other; for testing only! */
	DMU_OT_PLAIN_OTHER, /* UINT8 */
	DMU_OT_UINT64_OTHER, /* UINT64 */
	DMU_OT_ZAP_OTHER, /* ZAP */
	/* new object types: */
	DMU_OT_ERROR_LOG, /* ZAP */
	DMU_OT_SPA_HISTORY, /* UINT8 */
	DMU_OT_SPA_HISTORY_OFFSETS, /* spa_his_phys_t */
	DMU_OT_POOL_PROPS, /* ZAP */
	DMU_OT_DSL_PERMS, /* ZAP */
	DMU_OT_ACL, /* ACL */
	DMU_OT_SYSACL, /* SYSACL */
	DMU_OT_FUID, /* FUID table (Packed NVLIST UINT8) */
	DMU_OT_FUID_SIZE, /* FUID table size UINT64 */
	DMU_OT_NEXT_CLONES, /* ZAP */
	DMU_OT_SCAN_QUEUE, /* ZAP */
	DMU_OT_USERGROUP_USED, /* ZAP */
	DMU_OT_USERGROUP_QUOTA, /* ZAP */
	DMU_OT_USERREFS, /* ZAP */
	DMU_OT_DDT_ZAP, /* ZAP */
	DMU_OT_DDT_STATS, /* ZAP */
	DMU_OT_SA, /* System attr */
	DMU_OT_SA_MASTER_NODE, /* ZAP */
	DMU_OT_SA_ATTR_REGISTRATION, /* ZAP */
	DMU_OT_SA_ATTR_LAYOUTS, /* ZAP */
	DMU_OT_SCAN_XLATE, /* ZAP */
	DMU_OT_DEDUP, /* fake dedup BP from ddt_bp_create() */
	DMU_OT_DEADLIST, /* ZAP */
	DMU_OT_DEADLIST_HDR, /* UINT64 */
	DMU_OT_DSL_CLONES, /* ZAP */
	DMU_OT_BPOBJ_SUBOBJ, /* UINT64 */
	/*
	* Do not allocate new object types here. Doing so makes the on-disk
	* format incompatible with any other format that uses the same object
	* type number.
	*
	* When creating an object which does not have one of the above types
	* use the DMU_OTN_* type with the correct byteswap and metadata
	* values.
	*
	* The DMU_OTN_* types do not have entries in the dmu_ot table,
	* use the DMU_OT_IS_METADATA() and DMU_OT_BYTESWAP() macros instead
	* of indexing into dmu_ot directly (this works for both DMU_OT_* types
	* and DMU_OTN_* types).
	*/
	DMU_OT_NUMTYPES,

	/*
	* Names for valid types declared with DMU_OT().
	*/
	DMU_OTN_UINT8_DATA = DMU_OT(DMU_BSWAP_UINT8, B_FALSE, B_FALSE),
	DMU_OTN_UINT8_METADATA = DMU_OT(DMU_BSWAP_UINT8, B_TRUE, B_FALSE),
	DMU_OTN_UINT16_DATA = DMU_OT(DMU_BSWAP_UINT16, B_FALSE, B_FALSE),
	DMU_OTN_UINT16_METADATA = DMU_OT(DMU_BSWAP_UINT16, B_TRUE, B_FALSE),
	DMU_OTN_UINT32_DATA = DMU_OT(DMU_BSWAP_UINT32, B_FALSE, B_FALSE),
	DMU_OTN_UINT32_METADATA = DMU_OT(DMU_BSWAP_UINT32, B_TRUE, B_FALSE),
	DMU_OTN_UINT64_DATA = DMU_OT(DMU_BSWAP_UINT64, B_FALSE, B_FALSE),
	DMU_OTN_UINT64_METADATA = DMU_OT(DMU_BSWAP_UINT64, B_TRUE, B_FALSE),
	DMU_OTN_ZAP_DATA = DMU_OT(DMU_BSWAP_ZAP, B_FALSE, B_FALSE),
	DMU_OTN_ZAP_METADATA = DMU_OT(DMU_BSWAP_ZAP, B_TRUE, B_FALSE),

	DMU_OTN_UINT8_ENC_DATA = DMU_OT(DMU_BSWAP_UINT8, B_FALSE, B_TRUE),
	DMU_OTN_UINT8_ENC_METADATA = DMU_OT(DMU_BSWAP_UINT8, B_TRUE, B_TRUE),
	DMU_OTN_UINT16_ENC_DATA = DMU_OT(DMU_BSWAP_UINT16, B_FALSE, B_TRUE),
	DMU_OTN_UINT16_ENC_METADATA = DMU_OT(DMU_BSWAP_UINT16, B_TRUE, B_TRUE),
	DMU_OTN_UINT32_ENC_DATA = DMU_OT(DMU_BSWAP_UINT32, B_FALSE, B_TRUE),
	DMU_OTN_UINT32_ENC_METADATA = DMU_OT(DMU_BSWAP_UINT32, B_TRUE, B_TRUE),
	DMU_OTN_UINT64_ENC_DATA = DMU_OT(DMU_BSWAP_UINT64, B_FALSE, B_TRUE),
	DMU_OTN_UINT64_ENC_METADATA = DMU_OT(DMU_BSWAP_UINT64, B_TRUE, B_TRUE),
	DMU_OTN_ZAP_ENC_DATA = DMU_OT(DMU_BSWAP_ZAP, B_FALSE, B_TRUE),
	DMU_OTN_ZAP_ENC_METADATA = DMU_OT(DMU_BSWAP_ZAP, B_TRUE, B_TRUE),
	} dmu_object_type_t;

	/*
	* These flags are intended to be used to specify the "txg_how"
	* parameter when calling the dmu_tx_assign() function. See the comment
	* above dmu_tx_assign() for more details on the meaning of these flags.
	*/
	#define TXG_NOWAIT (0ULL)
	#define TXG_WAIT (1ULL<<0)
	#define TXG_NOTHROTTLE (1ULL<<1)

	void byteswap_uint64_array(void *buf, size_t size);
	void byteswap_uint32_array(void *buf, size_t size);
	void byteswap_uint16_array(void *buf, size_t size);
	void byteswap_uint8_array(void *buf, size_t size);
	void zap_byteswap(void *buf, size_t size);
	void zfs_oldacl_byteswap(void *buf, size_t size);
	void zfs_acl_byteswap(void *buf, size_t size);
	void zfs_znode_byteswap(void *buf, size_t size);

	#define DS_FIND_SNAPSHOTS (1<<0)
	#define DS_FIND_CHILDREN (1<<1)
	#define DS_FIND_SERIALIZE (1<<2)

	/*
	* The maximum number of bytes that can be accessed as part of one
	* operation, including metadata.
	*/
	#define DMU_MAX_ACCESS (64 * 1024 * 1024) /* 64MB */
	#define DMU_MAX_DELETEBLKCNT (20480) /* ~5MB of indirect blocks */

	#define DMU_USERUSED_OBJECT (-1ULL)
	#define DMU_GROUPUSED_OBJECT (-2ULL)
	#define DMU_PROJECTUSED_OBJECT (-3ULL)

	/*
	* Zap prefix for object accounting in DMU_{USER,GROUP,PROJECT}USED_OBJECT.
	*/
	#define DMU_OBJACCT_PREFIX "obj-"
	#define DMU_OBJACCT_PREFIX_LEN 4

	/*
	* artificial blkids for bonus buffer and spill blocks
	*/
	#define DMU_BONUS_BLKID (-1ULL)
	#define DMU_SPILL_BLKID (-2ULL)

	/*
	* Public routines to create, destroy, open, and close objsets.
	*/
	typedef void dmu_objset_create_sync_func_t(objset_t os, void arg,
	cred_t cr, dmu_tx_t tx);

	int dmu_objset_hold(const char name, const void tag, objset_t **osp);
	int dmu_objset_own(const char *name, dmu_objset_type_t type,
	boolean_t readonly, boolean_t key_required, const void *tag,
	objset_t **osp);
	void dmu_objset_rele(objset_t os, const void tag);
	void dmu_objset_disown(objset_t os, boolean_t key_required, const void tag);
	int dmu_objset_open_ds(struct dsl_dataset ds, objset_t *osp);

	void dmu_objset_evict_dbufs(objset_t *os);
	int dmu_objset_create(const char *name, dmu_objset_type_t type, uint64_t flags,
	struct dsl_crypto_params *dcp, dmu_objset_create_sync_func_t func,
	void *arg);
	int dmu_objset_clone(const char name, const char origin);
	int dsl_destroy_snapshots_nvl(struct nvlist *snaps, boolean_t defer,
	struct nvlist *errlist);
	int dmu_objset_snapshot_one(const char fsname, const char snapname);
	int dmu_objset_find(const char name, int func(const char , void ), void arg,
	int flags);
	void dmu_objset_byteswap(void *buf, size_t size);
	int dsl_dataset_rename_snapshot(const char *fsname,
	const char oldsnapname, const char newsnapname, boolean_t recursive);

	typedef struct dmu_buf {
	uint64_t db_object; /* object that this buffer is part of */
	uint64_t db_offset; /* byte offset in this object */
	uint64_t db_size; /* size of buffer in bytes */
	void db_data; / data in buffer */
	} dmu_buf_t;

	/*
	* The names of zap entries in the DIRECTORY_OBJECT of the MOS.
	*/
	#define DMU_POOL_DIRECTORY_OBJECT 1
	#define DMU_POOL_CONFIG "config"
	#define DMU_POOL_FEATURES_FOR_WRITE "features_for_write"
	#define DMU_POOL_FEATURES_FOR_READ "features_for_read"
	#define DMU_POOL_FEATURE_DESCRIPTIONS "feature_descriptions"
	#define DMU_POOL_FEATURE_ENABLED_TXG "feature_enabled_txg"
	#define DMU_POOL_ROOT_DATASET "root_dataset"
	#define DMU_POOL_SYNC_BPOBJ "sync_bplist"
	#define DMU_POOL_ERRLOG_SCRUB "errlog_scrub"
	#define DMU_POOL_ERRLOG_LAST "errlog_last"
	#define DMU_POOL_SPARES "spares"
	#define DMU_POOL_DEFLATE "deflate"
	#define DMU_POOL_HISTORY "history"
	#define DMU_POOL_PROPS "pool_props"
	#define DMU_POOL_L2CACHE "l2cache"
	#define DMU_POOL_TMP_USERREFS "tmp_userrefs"
	#define DMU_POOL_DDT "DDT-%s-%s-%s"
	#define DMU_POOL_DDT_STATS "DDT-statistics"
	#define DMU_POOL_CREATION_VERSION "creation_version"
	#define DMU_POOL_SCAN "scan"
	#define DMU_POOL_ERRORSCRUB "error_scrub"
	#define DMU_POOL_FREE_BPOBJ "free_bpobj"
	#define DMU_POOL_BPTREE_OBJ "bptree_obj"
	#define DMU_POOL_EMPTY_BPOBJ "empty_bpobj"
	#define DMU_POOL_CHECKSUM_SALT "org.illumos:checksum_salt"
	#define DMU_POOL_VDEV_ZAP_MAP "com.delphix:vdev_zap_map"
	#define DMU_POOL_REMOVING "com.delphix:removing"
	#define DMU_POOL_OBSOLETE_BPOBJ "com.delphix:obsolete_bpobj"
	#define DMU_POOL_CONDENSING_INDIRECT "com.delphix:condensing_indirect"
	#define DMU_POOL_ZPOOL_CHECKPOINT "com.delphix:zpool_checkpoint"
	#define DMU_POOL_LOG_SPACEMAP_ZAP "com.delphix:log_spacemap_zap"
	#define DMU_POOL_DELETED_CLONES "com.delphix:deleted_clones"

	/*
	* Allocate an object from this objset. The range of object numbers
	* available is (0, DN_MAX_OBJECT). Object 0 is the meta-dnode.
	*
	* The transaction must be assigned to a txg. The newly allocated
	* object will be "held" in the transaction (ie. you can modify the
	* newly allocated object in this transaction).
	*
	* dmu_object_alloc() chooses an object and returns it in *objectp.
	*
	* dmu_object_claim() allocates a specific object number. If that
	* number is already allocated, it fails and returns EEXIST.
	*
	* Return 0 on success, or ENOSPC or EEXIST as specified above.
	*/
	uint64_t dmu_object_alloc(objset_t *os, dmu_object_type_t ot,
	int blocksize, dmu_object_type_t bonus_type, int bonus_len, dmu_tx_t *tx);
	uint64_t dmu_object_alloc_ibs(objset_t *os, dmu_object_type_t ot, int blocksize,
	int indirect_blockshift,
	dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx);
	uint64_t dmu_object_alloc_dnsize(objset_t *os, dmu_object_type_t ot,
	int blocksize, dmu_object_type_t bonus_type, int bonus_len,
	int dnodesize, dmu_tx_t *tx);
	uint64_t dmu_object_alloc_hold(objset_t *os, dmu_object_type_t ot,
	int blocksize, int indirect_blockshift, dmu_object_type_t bonustype,
	int bonuslen, int dnodesize, dnode_t *allocated_dnode, const void tag,
	dmu_tx_t *tx);
	int dmu_object_claim(objset_t *os, uint64_t object, dmu_object_type_t ot,
	int blocksize, dmu_object_type_t bonus_type, int bonus_len, dmu_tx_t *tx);
	int dmu_object_claim_dnsize(objset_t *os, uint64_t object, dmu_object_type_t ot,
	int blocksize, dmu_object_type_t bonus_type, int bonus_len,
	int dnodesize, dmu_tx_t *tx);
	int dmu_object_reclaim(objset_t *os, uint64_t object, dmu_object_type_t ot,
	int blocksize, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *txp);
	int dmu_object_reclaim_dnsize(objset_t *os, uint64_t object,
	dmu_object_type_t ot, int blocksize, dmu_object_type_t bonustype,
	int bonuslen, int dnodesize, boolean_t keep_spill, dmu_tx_t *tx);
	int dmu_object_rm_spill(objset_t os, uint64_t object, dmu_tx_t tx);

	/*
	* Free an object from this objset.
	*
	* The object's data will be freed as well (ie. you don't need to call
	* dmu_free(object, 0, -1, tx)).
	*
	* The object need not be held in the transaction.
	*
	* If there are any holds on this object's buffers (via dmu_buf_hold()),
	* or tx holds on the object (via dmu_tx_hold_object()), you can not
	* free it; it fails and returns EBUSY.
	*
	* If the object is not allocated, it fails and returns ENOENT.
	*
	* Return 0 on success, or EBUSY or ENOENT as specified above.
	*/
	int dmu_object_free(objset_t os, uint64_t object, dmu_tx_t tx);

	/*
	* Find the next allocated or free object.
	*
	* The objectp parameter is in-out. It will be updated to be the next
	* object which is allocated. Ignore objects which have not been
	* modified since txg.
	*
	* XXX Can only be called on a objset with no dirty data.
	*
	* Returns 0 on success, or ENOENT if there are no more objects.
	*/
	int dmu_object_next(objset_t os, uint64_t objectp,
	boolean_t hole, uint64_t txg);

	/*
	* Set the number of levels on a dnode. nlevels must be greater than the
	* current number of levels or an EINVAL will be returned.
	*/
	int dmu_object_set_nlevels(objset_t *os, uint64_t object, int nlevels,
	dmu_tx_t *tx);

	/*
	* Set the data blocksize for an object.
	*
	* The object cannot have any blocks allocated beyond the first. If
	* the first block is allocated already, the new size must be greater
	* than the current block size. If these conditions are not met,
	* ENOTSUP will be returned.
	*
	* Returns 0 on success, or EBUSY if there are any holds on the object
	* contents, or ENOTSUP as described above.
	*/
	int dmu_object_set_blocksize(objset_t *os, uint64_t object, uint64_t size,
	int ibs, dmu_tx_t *tx);

	/*
	* Manually set the maxblkid on a dnode. This will adjust nlevels accordingly
	* to accommodate the change. When calling this function, the caller must
	* ensure that the object's nlevels can sufficiently support the new maxblkid.
	*/
	int dmu_object_set_maxblkid(objset_t *os, uint64_t object, uint64_t maxblkid,
	dmu_tx_t *tx);

	/*
	* Set the checksum property on a dnode. The new checksum algorithm will
	* apply to all newly written blocks; existing blocks will not be affected.
	*/
	void dmu_object_set_checksum(objset_t *os, uint64_t object, uint8_t checksum,
	dmu_tx_t *tx);

	/*
	* Set the compress property on a dnode. The new compression algorithm will
	* apply to all newly written blocks; existing blocks will not be affected.
	*/
	void dmu_object_set_compress(objset_t *os, uint64_t object, uint8_t compress,
	dmu_tx_t *tx);

	void dmu_write_embedded(objset_t *os, uint64_t object, uint64_t offset,
	void *data, uint8_t etype, uint8_t comp, int uncompressed_size,
	int compressed_size, int byteorder, dmu_tx_t *tx);
	void dmu_redact(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
	dmu_tx_t *tx);

	/*
	* Decide how to write a block: checksum, compression, number of copies, etc.
	*/
	#define WP_NOFILL 0x1
	#define WP_DMU_SYNC 0x2
	#define WP_SPILL 0x4

	void dmu_write_policy(objset_t os, dnode_t dn, int level, int wp,
	struct zio_prop *zp);

	/*
	* The bonus data is accessed more or less like a regular buffer.
	* You must dmu_bonus_hold() to get the buffer, which will give you a
	* dmu_buf_t with db_offset==-1ULL, and db_size = the size of the bonus
	* data. As with any normal buffer, you must call dmu_buf_will_dirty()
	* before modifying it, and the
	* object must be held in an assigned transaction before calling
	* dmu_buf_will_dirty. You may use dmu_buf_set_user() on the bonus
	* buffer as well. You must release what you hold with dmu_buf_rele().
	*
	* Returns ENOENT, EIO, or 0.
	*/
	int dmu_bonus_hold(objset_t os, uint64_t object, const void tag,
	dmu_buf_t **dbp);
	int dmu_bonus_hold_by_dnode(dnode_t dn, const void tag, dmu_buf_t **dbp,
	uint32_t flags);
	int dmu_bonus_max(void);
	int dmu_set_bonus(dmu_buf_t , int, dmu_tx_t );
	int dmu_set_bonustype(dmu_buf_t , dmu_object_type_t, dmu_tx_t );
	dmu_object_type_t dmu_get_bonustype(dmu_buf_t *);
	int dmu_rm_spill(objset_t , uint64_t, dmu_tx_t );

	/*
	* Special spill buffer support used by "SA" framework
	*/

	int dmu_spill_hold_by_bonus(dmu_buf_t bonus, uint32_t flags, const void tag,
	dmu_buf_t **dbp);
	int dmu_spill_hold_by_dnode(dnode_t *dn, uint32_t flags,
	const void tag, dmu_buf_t *dbp);
	int dmu_spill_hold_existing(dmu_buf_t bonus, const void tag, dmu_buf_t **dbp);

	/*
	* Obtain the DMU buffer from the specified object which contains the
	* specified offset. dmu_buf_hold() puts a "hold" on the buffer, so
	* that it will remain in memory. You must release the hold with
	* dmu_buf_rele(). You must not access the dmu_buf_t after releasing
	* what you hold. You must have a hold on any dmu_buf_t* you pass to the DMU.
	*
	* You must call dmu_buf_read, dmu_buf_will_dirty, or dmu_buf_will_fill
	* on the returned buffer before reading or writing the buffer's
	* db_data. The comments for those routines describe what particular
	* operations are valid after calling them.
	*
	* The object number must be a valid, allocated object number.
	*/
	int dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset,
	const void tag, dmu_buf_t *, int flags);
	int dmu_buf_hold_array(objset_t *os, uint64_t object, uint64_t offset,
	uint64_t length, int read, const void tag, int numbufsp,
	dmu_buf_t ***dbpp);
	int dmu_buf_hold_noread(objset_t *os, uint64_t object, uint64_t offset,
	const void tag, dmu_buf_t *dbp);
	int dmu_buf_hold_by_dnode(dnode_t *dn, uint64_t offset,
	const void tag, dmu_buf_t *dbp, int flags);
	int dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset,
	uint64_t length, boolean_t read, const void tag, int numbufsp,
	dmu_buf_t ***dbpp, uint32_t flags);
	int dmu_buf_hold_noread_by_dnode(dnode_t dn, uint64_t offset, const void tag,
	dmu_buf_t **dbp);
	/*
	* Add a reference to a dmu buffer that has already been held via
	* dmu_buf_hold() in the current context.
	*/
	void dmu_buf_add_ref(dmu_buf_t db, const void tag);

	/*
	* Attempt to add a reference to a dmu buffer that is in an unknown state,
	* using a pointer that may have been invalidated by eviction processing.
	* The request will succeed if the passed in dbuf still represents the
	* same os/object/blkid, is ineligible for eviction, and has at least
	* one hold by a user other than the syncer.
	*/
	boolean_t dmu_buf_try_add_ref(dmu_buf_t , objset_t os, uint64_t object,
	uint64_t blkid, const void *tag);

	void dmu_buf_rele(dmu_buf_t db, const void tag);
	uint64_t dmu_buf_refcount(dmu_buf_t *db);
	uint64_t dmu_buf_user_refcount(dmu_buf_t *db);

	/*
	* dmu_buf_hold_array holds the DMU buffers which contain all bytes in a
	* range of an object. A pointer to an array of dmu_buf_t*'s is
	* returned (in *dbpp).
	*
	* dmu_buf_rele_array releases the hold on an array of dmu_buf_t*'s, and
	* frees the array. The hold on the array of buffers MUST be released
	* with dmu_buf_rele_array. You can NOT release the hold on each buffer
	* individually with dmu_buf_rele.
	*/
	int dmu_buf_hold_array_by_bonus(dmu_buf_t *db, uint64_t offset,
	uint64_t length, boolean_t read, const void *tag,
	int numbufsp, dmu_buf_t **dbpp);
	void dmu_buf_rele_array(dmu_buf_t *, int numbufs, const void tag);

	typedef void dmu_buf_evict_func_t(void *user_ptr);

	/*
	* A DMU buffer user object may be associated with a dbuf for the
	* duration of its lifetime. This allows the user of a dbuf (client)
	* to attach private data to a dbuf (e.g. in-core only data such as a
	* dnode_children_t, zap_t, or zap_leaf_t) and be optionally notified
	* when that dbuf has been evicted. Clients typically respond to the
	* eviction notification by freeing their private data, thus ensuring
	* the same lifetime for both dbuf and private data.
	*
	* The mapping from a dmu_buf_user_t to any client private data is the
	* client's responsibility. All current consumers of the API with private
	* data embed a dmu_buf_user_t as the first member of the structure for
	* their private data. This allows conversions between the two types
	* with a simple cast. Since the DMU buf user API never needs access
	* to the private data, other strategies can be employed if necessary
	* or convenient for the client (e.g. using container_of() to do the
	* conversion for private data that cannot have the dmu_buf_user_t as
	* its first member).
	*
	* Eviction callbacks are executed without the dbuf mutex held or any
	* other type of mechanism to guarantee that the dbuf is still available.
	* For this reason, users must assume the dbuf has already been freed
	* and not reference the dbuf from the callback context.
	*
	* Users requesting "immediate eviction" are notified as soon as the dbuf
	* is only referenced by dirty records (dirties == holds). Otherwise the
	* notification occurs after eviction processing for the dbuf begins.
	*/
	typedef struct dmu_buf_user {
	/*
	* Asynchronous user eviction callback state.
	*/
	taskq_ent_t dbu_tqent;

	/*
	* This instance's eviction function pointers.
	*
	* dbu_evict_func_sync is called synchronously and then
	* dbu_evict_func_async is executed asynchronously on a taskq.
	*/
	dmu_buf_evict_func_t *dbu_evict_func_sync;
	dmu_buf_evict_func_t *dbu_evict_func_async;
	#ifdef ZFS_DEBUG
	/*
	* Pointer to user's dbuf pointer. NULL for clients that do
	* not associate a dbuf with their user data.
	*
	* The dbuf pointer is cleared upon eviction so as to catch
	* use-after-evict bugs in clients.
	*/
	dmu_buf_t **dbu_clear_on_evict_dbufp;
	#endif
	} dmu_buf_user_t;

	/*
	* Initialize the given dmu_buf_user_t instance with the eviction function
	* evict_func, to be called when the user is evicted.
	*
	* NOTE: This function should only be called once on a given dmu_buf_user_t.
	* To allow enforcement of this, dbu must already be zeroed on entry.
	*/
	static inline void
	dmu_buf_init_user(dmu_buf_user_t dbu, dmu_buf_evict_func_t evict_func_sync,
	dmu_buf_evict_func_t *evict_func_async,
	dmu_buf_t **clear_on_evict_dbufp __maybe_unused)
	{
	ASSERT(dbu->dbu_evict_func_sync == NULL);
	ASSERT(dbu->dbu_evict_func_async == NULL);

	/* must have at least one evict func */
	IMPLY(evict_func_sync == NULL, evict_func_async != NULL);
	dbu->dbu_evict_func_sync = evict_func_sync;
	dbu->dbu_evict_func_async = evict_func_async;
	taskq_init_ent(&dbu->dbu_tqent);
	#ifdef ZFS_DEBUG
	dbu->dbu_clear_on_evict_dbufp = clear_on_evict_dbufp;
	#endif
	}

	/*
	* Attach user data to a dbuf and mark it for normal (when the dbuf's
	* data is cleared or its reference count goes to zero) eviction processing.
	*
	* Returns NULL on success, or the existing user if another user currently
	* owns the buffer.
	*/
	void dmu_buf_set_user(dmu_buf_t db, dmu_buf_user_t *user);

	/*
	* Attach user data to a dbuf and mark it for immediate (its dirty and
	* reference counts are equal) eviction processing.
	*
	* Returns NULL on success, or the existing user if another user currently
	* owns the buffer.
	*/
	void dmu_buf_set_user_ie(dmu_buf_t db, dmu_buf_user_t *user);

	/*
	* Replace the current user of a dbuf.
	*
	* If given the current user of a dbuf, replaces the dbuf's user with
	* "new_user" and returns the user data pointer that was replaced.
	* Otherwise returns the current, and unmodified, dbuf user pointer.
	*/
	void dmu_buf_replace_user(dmu_buf_t db,
	dmu_buf_user_t old_user, dmu_buf_user_t new_user);

	/*
	* Remove the specified user data for a DMU buffer.
	*
	* Returns the user that was removed on success, or the current user if
	* another user currently owns the buffer.
	*/
	void dmu_buf_remove_user(dmu_buf_t db, dmu_buf_user_t *user);

	/*
	* Returns the user data (dmu_buf_user_t *) associated with this dbuf.
	*/
	void dmu_buf_get_user(dmu_buf_t db);

	objset_t dmu_buf_get_objset(dmu_buf_t db);
	-dnode_t dmu_buf_dnode_enter(dmu_buf_t db);
	-void dmu_buf_dnode_exit(dmu_buf_t *db);

	/* Block until any in-progress dmu buf user evictions complete. */
	void dmu_buf_user_evict_wait(void);

	/*
	* Returns the blkptr associated with this dbuf, or NULL if not set.
	*/
	struct blkptr dmu_buf_get_blkptr(dmu_buf_t db);

	/*
	* Indicate that you are going to modify the buffer's data (db_data).
	*
	* The transaction (tx) must be assigned to a txg (ie. you've called
	* dmu_tx_assign()). The buffer's object must be held in the tx
	* (ie. you've called dmu_tx_hold_object(tx, db->db_object)).
	*/
	void dmu_buf_will_dirty(dmu_buf_t db, dmu_tx_t tx);
	boolean_t dmu_buf_is_dirty(dmu_buf_t db, dmu_tx_t tx);
	void dmu_buf_set_crypt_params(dmu_buf_t *db_fake, boolean_t byteorder,
	const uint8_t salt, const uint8_t iv, const uint8_t mac, dmu_tx_t tx);

	/*
	* You must create a transaction, then hold the objects which you will
	* (or might) modify as part of this transaction. Then you must assign
	* the transaction to a transaction group. Once the transaction has
	* been assigned, you can modify buffers which belong to held objects as
	* part of this transaction. You can't modify buffers before the
	* transaction has been assigned; you can't modify buffers which don't
	* belong to objects which this transaction holds; you can't hold
	* objects once the transaction has been assigned. You may hold an
	* object which you are going to free (with dmu_object_free()), but you
	* don't have to.
	*
	* You can abort the transaction before it has been assigned.
	*
	* Note that you may hold buffers (with dmu_buf_hold) at any time,
	* regardless of transaction state.
	*/

	#define DMU_NEW_OBJECT (-1ULL)
	#define DMU_OBJECT_END (-1ULL)

	dmu_tx_t dmu_tx_create(objset_t os);
	void dmu_tx_hold_write(dmu_tx_t *tx, uint64_t object, uint64_t off, int len);
	void dmu_tx_hold_write_by_dnode(dmu_tx_t tx, dnode_t dn, uint64_t off,
	int len);
	void dmu_tx_hold_append(dmu_tx_t *tx, uint64_t object, uint64_t off, int len);
	void dmu_tx_hold_append_by_dnode(dmu_tx_t tx, dnode_t dn, uint64_t off,
	int len);
	void dmu_tx_hold_clone_by_dnode(dmu_tx_t tx, dnode_t dn, uint64_t off,
	int len);
	void dmu_tx_hold_free(dmu_tx_t *tx, uint64_t object, uint64_t off,
	uint64_t len);
	void dmu_tx_hold_free_by_dnode(dmu_tx_t tx, dnode_t dn, uint64_t off,
	uint64_t len);
	void dmu_tx_hold_zap(dmu_tx_t tx, uint64_t object, int add, const char name);
	void dmu_tx_hold_zap_by_dnode(dmu_tx_t tx, dnode_t dn, int add,
	const char *name);
	void dmu_tx_hold_bonus(dmu_tx_t *tx, uint64_t object);
	void dmu_tx_hold_bonus_by_dnode(dmu_tx_t tx, dnode_t dn);
	void dmu_tx_hold_spill(dmu_tx_t *tx, uint64_t object);
	void dmu_tx_hold_sa(dmu_tx_t tx, struct sa_handle hdl, boolean_t may_grow);
	void dmu_tx_hold_sa_create(dmu_tx_t *tx, int total_size);
	void dmu_tx_abort(dmu_tx_t *tx);
	int dmu_tx_assign(dmu_tx_t *tx, uint64_t txg_how);
	void dmu_tx_wait(dmu_tx_t *tx);
	void dmu_tx_commit(dmu_tx_t *tx);
	void dmu_tx_mark_netfree(dmu_tx_t *tx);

	/*
	* To register a commit callback, dmu_tx_callback_register() must be called.
	*
	* dcb_data is a pointer to caller private data that is passed on as a
	* callback parameter. The caller is responsible for properly allocating and
	* freeing it.
	*
	* When registering a callback, the transaction must be already created, but
	* it cannot be committed or aborted. It can be assigned to a txg or not.
	*
	* The callback will be called after the transaction has been safely written
	* to stable storage and will also be called if the dmu_tx is aborted.
	* If there is any error which prevents the transaction from being committed to
	* disk, the callback will be called with a value of error != 0.
	*
	* When multiple callbacks are registered to the transaction, the callbacks
	* will be called in reverse order to let Lustre, the only user of commit
	* callback currently, take the fast path of its commit callback handling.
	*/
	typedef void dmu_tx_callback_func_t(void *dcb_data, int error);

	void dmu_tx_callback_register(dmu_tx_t tx, dmu_tx_callback_func_t dcb_func,
	void *dcb_data);
	void dmu_tx_do_callbacks(list_t *cb_list, int error);

	/*
	* Free up the data blocks for a defined range of a file. If size is
	* -1, the range from offset to end-of-file is freed.
	*/
	int dmu_free_range(objset_t *os, uint64_t object, uint64_t offset,
	uint64_t size, dmu_tx_t *tx);
	int dmu_free_long_range(objset_t *os, uint64_t object, uint64_t offset,
	uint64_t size);
	int dmu_free_long_object(objset_t *os, uint64_t object);

	/*
	* Convenience functions.
	*
	* Canfail routines will return 0 on success, or an errno if there is a
	* nonrecoverable I/O error.
	*/
	#define DMU_READ_PREFETCH 0 /* prefetch */
	#define DMU_READ_NO_PREFETCH 1 /* don't prefetch */
	#define DMU_READ_NO_DECRYPT 2 /* don't decrypt */
	int dmu_read(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
	void *buf, uint32_t flags);
	int dmu_read_by_dnode(dnode_t dn, uint64_t offset, uint64_t size, void buf,
	uint32_t flags);
	void dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
	const void buf, dmu_tx_t tx);
	void dmu_write_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size,
	const void buf, dmu_tx_t tx);
	void dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
	dmu_tx_t *tx);
	#ifdef _KERNEL
	int dmu_read_uio(objset_t os, uint64_t object, zfs_uio_t uio, uint64_t size);
	int dmu_read_uio_dbuf(dmu_buf_t zdb, zfs_uio_t uio, uint64_t size);
	int dmu_read_uio_dnode(dnode_t dn, zfs_uio_t uio, uint64_t size);
	int dmu_write_uio(objset_t os, uint64_t object, zfs_uio_t uio, uint64_t size,
	dmu_tx_t *tx);
	int dmu_write_uio_dbuf(dmu_buf_t zdb, zfs_uio_t uio, uint64_t size,
	dmu_tx_t *tx);
	int dmu_write_uio_dnode(dnode_t dn, zfs_uio_t uio, uint64_t size,
	dmu_tx_t *tx);
	#endif
	struct arc_buf dmu_request_arcbuf(dmu_buf_t handle, int size);
	void dmu_return_arcbuf(struct arc_buf *buf);
	int dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset,
	struct arc_buf buf, dmu_tx_t tx);
	int dmu_assign_arcbuf_by_dbuf(dmu_buf_t *handle, uint64_t offset,
	struct arc_buf buf, dmu_tx_t tx);
	#define dmu_assign_arcbuf dmu_assign_arcbuf_by_dbuf
	extern uint_t zfs_max_recordsize;

	/*
	* Asynchronously try to read in the data.
	*/
	void dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset,
	uint64_t len, enum zio_priority pri);
	+void dmu_prefetch_by_dnode(dnode_t *dn, int64_t level, uint64_t offset,
	+ uint64_t len, enum zio_priority pri);
	+void dmu_prefetch_dnode(objset_t *os, uint64_t object, enum zio_priority pri);

	typedef struct dmu_object_info {
	/* All sizes are in bytes unless otherwise indicated. */
	uint32_t doi_data_block_size;
	uint32_t doi_metadata_block_size;
	dmu_object_type_t doi_type;
	dmu_object_type_t doi_bonus_type;
	uint64_t doi_bonus_size;
	uint8_t doi_indirection; /* 2 = dnode->indirect->data */
	uint8_t doi_checksum;
	uint8_t doi_compress;
	uint8_t doi_nblkptr;
	uint8_t doi_pad[4];
	uint64_t doi_dnodesize;
	uint64_t doi_physical_blocks_512; /* data + metadata, 512b blks */
	uint64_t doi_max_offset;
	uint64_t doi_fill_count; /* number of non-empty blocks */
	} dmu_object_info_t;

	typedef void (const arc_byteswap_func_t)(void buf, size_t size);

	typedef struct dmu_object_type_info {
	dmu_object_byteswap_t ot_byteswap;
	boolean_t ot_metadata;
	boolean_t ot_dbuf_metadata_cache;
	boolean_t ot_encrypt;
	const char *ot_name;
	} dmu_object_type_info_t;

	typedef const struct dmu_object_byteswap_info {
	arc_byteswap_func_t ob_func;
	const char *ob_name;
	} dmu_object_byteswap_info_t;

	extern const dmu_object_type_info_t dmu_ot[DMU_OT_NUMTYPES];
	extern dmu_object_byteswap_info_t dmu_ot_byteswap[DMU_BSWAP_NUMFUNCS];

	/*
	* Get information on a DMU object.
	*
	* Return 0 on success or ENOENT if object is not allocated.
	*
	* If doi is NULL, just indicates whether the object exists.
	*/
	int dmu_object_info(objset_t os, uint64_t object, dmu_object_info_t doi);
	void __dmu_object_info_from_dnode(struct dnode dn, dmu_object_info_t doi);
	/* Like dmu_object_info, but faster if you have a held dnode in hand. */
	void dmu_object_info_from_dnode(dnode_t dn, dmu_object_info_t doi);
	/* Like dmu_object_info, but faster if you have a held dbuf in hand. */
	void dmu_object_info_from_db(dmu_buf_t db, dmu_object_info_t doi);
	/*
	* Like dmu_object_info_from_db, but faster still when you only care about
	* the size.
	*/
	void dmu_object_size_from_db(dmu_buf_t db, uint32_t blksize,
	u_longlong_t *nblk512);

	void dmu_object_dnsize_from_db(dmu_buf_t db, int dnsize);

	typedef struct dmu_objset_stats {
	uint64_t dds_num_clones; /* number of clones of this */
	uint64_t dds_creation_txg;
	uint64_t dds_guid;
	dmu_objset_type_t dds_type;
	uint8_t dds_is_snapshot;
	uint8_t dds_inconsistent;
	uint8_t dds_redacted;
	char dds_origin[ZFS_MAX_DATASET_NAME_LEN];
	} dmu_objset_stats_t;

	/*
	* Get stats on a dataset.
	*/
	void dmu_objset_fast_stat(objset_t os, dmu_objset_stats_t stat);

	/*
	* Add entries to the nvlist for all the objset's properties. See
	* zfs_prop_table[] and zfs(1m) for details on the properties.
	*/
	void dmu_objset_stats(objset_t os, struct nvlist nv);

	/*
	* Get the space usage statistics for statvfs().
	*
	* refdbytes is the amount of space "referenced" by this objset.
	* availbytes is the amount of space available to this objset, taking
	* into account quotas & reservations, assuming that no other objsets
	* use the space first. These values correspond to the 'referenced' and
	* 'available' properties, described in the zfs(1m) manpage.
	*
	* usedobjs and availobjs are the number of objects currently allocated,
	* and available.
	*/
	void dmu_objset_space(objset_t os, uint64_t refdbytesp, uint64_t *availbytesp,
	uint64_t usedobjsp, uint64_t availobjsp);

	/*
	* The fsid_guid is a 56-bit ID that can change to avoid collisions.
	* (Contrast with the ds_guid which is a 64-bit ID that will never
	* change, so there is a small probability that it will collide.)
	*/
	uint64_t dmu_objset_fsid_guid(objset_t *os);

	/*
	* Get the [cm]time for an objset's snapshot dir
	*/
	inode_timespec_t dmu_objset_snap_cmtime(objset_t *os);

	int dmu_objset_is_snapshot(objset_t *os);

	extern struct spa dmu_objset_spa(objset_t os);
	extern struct zilog dmu_objset_zil(objset_t os);
	extern struct dsl_pool dmu_objset_pool(objset_t os);
	extern struct dsl_dataset dmu_objset_ds(objset_t os);
	extern void dmu_objset_name(objset_t os, char buf);
	extern dmu_objset_type_t dmu_objset_type(objset_t *os);
	extern uint64_t dmu_objset_id(objset_t *os);
	extern uint64_t dmu_objset_dnodesize(objset_t *os);
	extern zfs_sync_type_t dmu_objset_syncprop(objset_t *os);
	extern zfs_logbias_op_t dmu_objset_logbias(objset_t *os);
	extern int dmu_objset_blksize(objset_t *os);
	extern int dmu_snapshot_list_next(objset_t os, int namelen, char name,
	uint64_t id, uint64_t offp, boolean_t *case_conflict);
	extern int dmu_snapshot_lookup(objset_t os, const char name, uint64_t *val);
	extern int dmu_snapshot_realname(objset_t os, const char name, char *real,
	int maxlen, boolean_t *conflict);
	extern int dmu_dir_list_next(objset_t os, int namelen, char name,
	uint64_t idp, uint64_t offp);

	typedef struct zfs_file_info {
	uint64_t zfi_user;
	uint64_t zfi_group;
	uint64_t zfi_project;
	uint64_t zfi_generation;
	} zfs_file_info_t;

	typedef int file_info_cb_t(dmu_object_type_t bonustype, const void *data,
	struct zfs_file_info *zoi);
	extern void dmu_objset_register_type(dmu_objset_type_t ost,
	file_info_cb_t *cb);
	extern void dmu_objset_set_user(objset_t os, void user_ptr);
	extern void dmu_objset_get_user(objset_t os);

	/*
	* Return the txg number for the given assigned transaction.
	*/
	uint64_t dmu_tx_get_txg(dmu_tx_t *tx);

	/*
	* Synchronous write.
	* If a parent zio is provided this function initiates a write on the
	* provided buffer as a child of the parent zio.
	* In the absence of a parent zio, the write is completed synchronously.
	* At write completion, blk is filled with the bp of the written block.
	* Note that while the data covered by this function will be on stable
	* storage when the write completes this new data does not become a
	* permanent part of the file until the associated transaction commits.
	*/

	/*
	* {zfs,zvol,ztest}_get_done() args
	*/
	typedef struct zgd {
	struct lwb *zgd_lwb;
	struct blkptr *zgd_bp;
	dmu_buf_t *zgd_db;
	struct zfs_locked_range *zgd_lr;
	void *zgd_private;
	} zgd_t;

	typedef void dmu_sync_cb_t(zgd_t *arg, int error);
	int dmu_sync(struct zio zio, uint64_t txg, dmu_sync_cb_t done, zgd_t *zgd);

	/*
	* Find the next hole or data block in file starting at *off
	* Return found offset in *off. Return ESRCH for end of file.
	*/
	int dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole,
	uint64_t *off);

	int dmu_read_l0_bps(objset_t *os, uint64_t object, uint64_t offset,
	uint64_t length, struct blkptr bps, size_t nbpsp);
	int dmu_brt_clone(objset_t *os, uint64_t object, uint64_t offset,
	uint64_t length, dmu_tx_t tx, const struct blkptr bps, size_t nbps);

	/*
	* Initial setup and final teardown.
	*/
	extern void dmu_init(void);
	extern void dmu_fini(void);

	typedef void (dmu_traverse_cb_t)(objset_t os, void arg, struct blkptr bp,
	uint64_t object, uint64_t offset, int len);
	void dmu_traverse_objset(objset_t *os, uint64_t txg_start,
	dmu_traverse_cb_t cb, void *arg);

	int dmu_diff(const char tosnap_name, const char fromsnap_name,
	zfs_file_t fp, offset_t offp);

	/* CRC64 table */
	#define ZFS_CRC64_POLY 0xC96C5795D7870F42ULL /* ECMA-182, reflected form */
	extern uint64_t zfs_crc64_table[256];

	extern uint_t dmu_prefetch_max;

	#ifdef __cplusplus
	}
	#endif

	#endif /* _SYS_DMU_H */
	diff --git a/sys/contrib/openzfs/include/sys/dmu_objset.h b/sys/contrib/openzfs/include/sys/dmu_objset.h
	index 9f6e0fdd601b..a9123e862af7 100644
	--- a/sys/contrib/openzfs/include/sys/dmu_objset.h
	+++ b/sys/contrib/openzfs/include/sys/dmu_objset.h
	@@ -1,276 +1,277 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2012, 2018 by Delphix. All rights reserved.
	* Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
	* Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
	*/

	/* Portions Copyright 2010 Robert Milkowski */

	#ifndef _SYS_DMU_OBJSET_H
	#define _SYS_DMU_OBJSET_H

	#include <sys/spa.h>
	#include <sys/arc.h>
	#include <sys/txg.h>
	#include <sys/zfs_context.h>
	#include <sys/dnode.h>
	#include <sys/zio.h>
	#include <sys/zil.h>
	#include <sys/sa.h>
	#include <sys/zfs_ioctl.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	extern krwlock_t os_lock;

	struct dsl_pool;
	struct dsl_dataset;
	struct dmu_tx;

	#define OBJSET_PHYS_SIZE_V1 1024
	#define OBJSET_PHYS_SIZE_V2 2048
	#define OBJSET_PHYS_SIZE_V3 4096

	#define OBJSET_BUF_HAS_USERUSED(buf) \
	(arc_buf_size(buf) >= OBJSET_PHYS_SIZE_V2)
	#define OBJSET_BUF_HAS_PROJECTUSED(buf) \
	(arc_buf_size(buf) >= OBJSET_PHYS_SIZE_V3)

	#define OBJSET_FLAG_USERACCOUNTING_COMPLETE (1ULL << 0)
	#define OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE (1ULL << 1)
	#define OBJSET_FLAG_PROJECTQUOTA_COMPLETE (1ULL << 2)

	/*
	* This mask defines the set of flags which are "portable", meaning
	* that they can be preserved when doing a raw encrypted zfs send.
	* Flags included in this mask will be protected by os_portable_mac
	* when the block of dnodes is encrypted. No portable flags currently
	* exist.
	*/
	#define OBJSET_CRYPT_PORTABLE_FLAGS_MASK (0)

	#if defined(__clang__)
	#pragma clang diagnostic push
	#pragma clang diagnostic ignored "-Wgnu-variable-sized-type-not-at-end"
	#endif
	typedef struct objset_phys {
	dnode_phys_t os_meta_dnode;
	zil_header_t os_zil_header;
	uint64_t os_type;
	uint64_t os_flags;
	uint8_t os_portable_mac[ZIO_OBJSET_MAC_LEN];
	uint8_t os_local_mac[ZIO_OBJSET_MAC_LEN];
	char os_pad0[OBJSET_PHYS_SIZE_V2 - sizeof (dnode_phys_t)*3 -
	sizeof (zil_header_t) - sizeof (uint64_t)*2 -
	2*ZIO_OBJSET_MAC_LEN];
	dnode_phys_t os_userused_dnode;
	dnode_phys_t os_groupused_dnode;
	dnode_phys_t os_projectused_dnode;
	char os_pad1[OBJSET_PHYS_SIZE_V3 - OBJSET_PHYS_SIZE_V2 -
	sizeof (dnode_phys_t)];
	} objset_phys_t;
	#if defined(__clang__)
	#pragma clang diagnostic pop
	#endif

	typedef int (dmu_objset_upgrade_cb_t)(objset_t );

	#define OBJSET_PROP_UNINITIALIZED ((uint64_t)-1)
	struct objset {
	/* Immutable: */
	struct dsl_dataset *os_dsl_dataset;
	spa_t *os_spa;
	arc_buf_t *os_phys_buf;
	objset_phys_t *os_phys;
	boolean_t os_encrypted;

	/*
	* The following "special" dnodes have no parent, are exempt
	* from dnode_move(), and are not recorded in os_dnodes, but they
	* root their descendents in this objset using handles anyway, so
	* that all access to dnodes from dbufs consistently uses handles.
	*/
	dnode_handle_t os_meta_dnode;
	dnode_handle_t os_userused_dnode;
	dnode_handle_t os_groupused_dnode;
	dnode_handle_t os_projectused_dnode;
	zilog_t *os_zil;

	list_node_t os_evicting_node;

	/* can change, under dsl_dir's locks: */
	uint64_t os_dnodesize; /* default dnode size for new objects */
	enum zio_checksum os_checksum;
	enum zio_compress os_compress;
	uint8_t os_complevel;
	uint8_t os_copies;
	enum zio_checksum os_dedup_checksum;
	boolean_t os_dedup_verify;
	zfs_logbias_op_t os_logbias;
	zfs_cache_type_t os_primary_cache;
	zfs_cache_type_t os_secondary_cache;
	+ zfs_prefetch_type_t os_prefetch;
	zfs_sync_type_t os_sync;
	zfs_redundant_metadata_type_t os_redundant_metadata;
	uint64_t os_recordsize;
	/*
	* The next four values are used as a cache of whatever's on disk, and
	* are initialized the first time these properties are queried. Before
	* being initialized with their real values, their values are
	* OBJSET_PROP_UNINITIALIZED.
	*/
	uint64_t os_version;
	uint64_t os_normalization;
	uint64_t os_utf8only;
	uint64_t os_casesensitivity;
	/*
	* The largest zpl file block allowed in special class.
	* cached here instead of zfsvfs for easier access.
	*/
	int os_zpl_special_smallblock;

	/*
	* Pointer is constant; the blkptr it points to is protected by
	* os_dsl_dataset->ds_bp_rwlock
	*/
	blkptr_t *os_rootbp;

	/* no lock needed: */
	struct dmu_tx os_synctx; / XXX sketchy */
	zil_header_t os_zil_header;
	multilist_t os_synced_dnodes;
	uint64_t os_flags;
	uint64_t os_freed_dnodes;
	boolean_t os_rescan_dnodes;
	boolean_t os_raw_receive;

	/* os_phys_buf should be written raw next txg */
	boolean_t os_next_write_raw[TXG_SIZE];

	/* Protected by os_obj_lock */
	kmutex_t os_obj_lock;
	uint64_t os_obj_next_chunk;

	/* Per-CPU next object to allocate, protected by atomic ops. */
	uint64_t *os_obj_next_percpu;
	int os_obj_next_percpu_len;

	/* Protected by os_lock */
	kmutex_t os_lock;
	multilist_t os_dirty_dnodes[TXG_SIZE];
	list_t os_dnodes;
	list_t os_downgraded_dbufs;

	/* Protects changes to DMU_{USER,GROUP,PROJECT}USED_OBJECT */
	kmutex_t os_userused_lock;

	/* stuff we store for the user */
	kmutex_t os_user_ptr_lock;
	void *os_user_ptr;
	sa_os_t *os_sa;

	/* kernel thread to upgrade this dataset */
	kmutex_t os_upgrade_lock;
	taskqid_t os_upgrade_id;
	dmu_objset_upgrade_cb_t os_upgrade_cb;
	boolean_t os_upgrade_exit;
	int os_upgrade_status;
	};

	#define DMU_META_OBJSET 0
	#define DMU_META_DNODE_OBJECT 0
	#define DMU_OBJECT_IS_SPECIAL(obj) ((int64_t)(obj) <= 0)
	#define DMU_META_DNODE(os) ((os)->os_meta_dnode.dnh_dnode)
	#define DMU_USERUSED_DNODE(os) ((os)->os_userused_dnode.dnh_dnode)
	#define DMU_GROUPUSED_DNODE(os) ((os)->os_groupused_dnode.dnh_dnode)
	#define DMU_PROJECTUSED_DNODE(os) ((os)->os_projectused_dnode.dnh_dnode)

	/* called from zpl */
	int dmu_objset_hold(const char name, const void tag, objset_t **osp);
	int dmu_objset_hold_flags(const char name, boolean_t decrypt, const void tag,
	objset_t **osp);
	int dmu_objset_own(const char *name, dmu_objset_type_t type,
	boolean_t readonly, boolean_t decrypt, const void tag, objset_t *osp);
	int dmu_objset_own_obj(struct dsl_pool *dp, uint64_t obj,
	dmu_objset_type_t type, boolean_t readonly, boolean_t decrypt,
	const void tag, objset_t *osp);
	void dmu_objset_refresh_ownership(struct dsl_dataset *ds,
	struct dsl_dataset *newds, boolean_t decrypt, const void tag);
	void dmu_objset_rele(objset_t os, const void tag);
	void dmu_objset_rele_flags(objset_t os, boolean_t decrypt, const void tag);
	void dmu_objset_disown(objset_t os, boolean_t decrypt, const void tag);
	int dmu_objset_from_ds(struct dsl_dataset ds, objset_t *osp);

	void dmu_objset_stats(objset_t os, nvlist_t nv);
	void dmu_objset_fast_stat(objset_t os, dmu_objset_stats_t stat);
	void dmu_objset_space(objset_t os, uint64_t refdbytesp, uint64_t *availbytesp,
	uint64_t usedobjsp, uint64_t availobjsp);
	uint64_t dmu_objset_fsid_guid(objset_t *os);
	int dmu_objset_find_dp(struct dsl_pool *dp, uint64_t ddobj,
	int func(struct dsl_pool , struct dsl_dataset , void *),
	void *arg, int flags);
	void dmu_objset_evict_dbufs(objset_t *os);
	inode_timespec_t dmu_objset_snap_cmtime(objset_t *os);

	/* called from dsl */
	void dmu_objset_sync(objset_t os, zio_t zio, dmu_tx_t *tx);
	boolean_t dmu_objset_is_dirty(objset_t *os, uint64_t txg);
	objset_t dmu_objset_create_impl_dnstats(spa_t spa, struct dsl_dataset *ds,
	blkptr_t *bp, dmu_objset_type_t type, int levels, int blksz, int ibs,
	dmu_tx_t *tx);
	objset_t dmu_objset_create_impl(spa_t spa, struct dsl_dataset *ds,
	blkptr_t bp, dmu_objset_type_t type, dmu_tx_t tx);
	int dmu_objset_open_impl(spa_t spa, struct dsl_dataset ds, blkptr_t *bp,
	objset_t **osp);
	void dmu_objset_evict(objset_t *os);
	void dmu_objset_sync_done(objset_t os, dmu_tx_t tx);
	void dmu_objset_userquota_get_ids(dnode_t dn, boolean_t before, dmu_tx_t tx);
	boolean_t dmu_objset_userused_enabled(objset_t *os);
	void dmu_objset_userspace_upgrade(objset_t *os);
	boolean_t dmu_objset_userspace_present(objset_t *os);
	boolean_t dmu_objset_userobjused_enabled(objset_t *os);
	boolean_t dmu_objset_userobjspace_upgradable(objset_t *os);
	boolean_t dmu_objset_userobjspace_present(objset_t *os);
	boolean_t dmu_objset_incompatible_encryption_version(objset_t *os);
	boolean_t dmu_objset_projectquota_enabled(objset_t *os);
	boolean_t dmu_objset_projectquota_present(objset_t *os);
	boolean_t dmu_objset_projectquota_upgradable(objset_t *os);
	void dmu_objset_id_quota_upgrade(objset_t *os);
	int dmu_get_file_info(objset_t *os, dmu_object_type_t bonustype,
	const void data, zfs_file_info_t zfi);

	int dmu_fsname(const char snapname, char buf);

	void dmu_objset_evict_done(objset_t *os);
	void dmu_objset_willuse_space(objset_t os, int64_t space, dmu_tx_t tx);

	void dmu_objset_init(void);
	void dmu_objset_fini(void);

	#ifdef __cplusplus
	}
	#endif

	#endif /* _SYS_DMU_OBJSET_H */
	diff --git a/sys/contrib/openzfs/include/sys/dmu_zfetch.h b/sys/contrib/openzfs/include/sys/dmu_zfetch.h
	index f00e13cf03a6..322472fb1ae2 100644
	--- a/sys/contrib/openzfs/include/sys/dmu_zfetch.h
	+++ b/sys/contrib/openzfs/include/sys/dmu_zfetch.h
	@@ -1,86 +1,92 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
	* Use is subject to license terms.
	*/

	/*
	* Copyright (c) 2014, 2017 by Delphix. All rights reserved.
	*/

	#ifndef _DMU_ZFETCH_H
	#define _DMU_ZFETCH_H

	#include <sys/zfs_context.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	struct dnode; /* so we can reference dnode */

	typedef struct zfetch {
	kmutex_t zf_lock; /* protects zfetch structure */
	list_t zf_stream; /* list of zstream_t's */
	struct dnode zf_dnode; / dnode that owns this zfetch */
	int zf_numstreams; /* number of zstream_t's */
	} zfetch_t;

	+typedef struct zsrange {
	+ uint16_t start;
	+ uint16_t end;
	+} zsrange_t;
	+
	+#define ZFETCH_RANGES 9 /* Fits zstream_t into 128 bytes */
	+
	typedef struct zstream {
	+ list_node_t zs_node; /* link for zf_stream */
	uint64_t zs_blkid; /* expect next access at this blkid */
	+ uint_t zs_atime; /* time last prefetch issued */
	+ zsrange_t zs_ranges[ZFETCH_RANGES]; /* ranges from future */
	unsigned int zs_pf_dist; /* data prefetch distance in bytes */
	unsigned int zs_ipf_dist; /* L1 prefetch distance in bytes */
	uint64_t zs_pf_start; /* first data block to prefetch */
	uint64_t zs_pf_end; /* data block to prefetch up to */
	uint64_t zs_ipf_start; /* first data block to prefetch L1 */
	uint64_t zs_ipf_end; /* data block to prefetch L1 up to */
	-
	- list_node_t zs_node; /* link for zf_stream */
	- hrtime_t zs_atime; /* time last prefetch issued */
	- zfetch_t zs_fetch; / parent fetch */
	boolean_t zs_missed; /* stream saw cache misses */
	boolean_t zs_more; /* need more distant prefetch */
	zfs_refcount_t zs_callers; /* number of pending callers */
	/*
	* Number of stream references: dnode, callers and pending blocks.
	* The stream memory is freed when the number returns to zero.
	*/
	zfs_refcount_t zs_refs;
	} zstream_t;

	void zfetch_init(void);
	void zfetch_fini(void);

	void dmu_zfetch_init(zfetch_t , struct dnode );
	void dmu_zfetch_fini(zfetch_t *);
	zstream_t dmu_zfetch_prepare(zfetch_t , uint64_t, uint64_t, boolean_t,
	boolean_t);
	-void dmu_zfetch_run(zstream_t *, boolean_t, boolean_t);
	+void dmu_zfetch_run(zfetch_t , zstream_t , boolean_t, boolean_t);
	void dmu_zfetch(zfetch_t *, uint64_t, uint64_t, boolean_t, boolean_t,
	boolean_t);


	#ifdef __cplusplus
	}
	#endif

	#endif /* _DMU_ZFETCH_H */
	diff --git a/sys/contrib/openzfs/include/sys/fm/fs/zfs.h b/sys/contrib/openzfs/include/sys/fm/fs/zfs.h
	index fb9e8649221e..c746600cd2d5 100644
	--- a/sys/contrib/openzfs/include/sys/fm/fs/zfs.h
	+++ b/sys/contrib/openzfs/include/sys/fm/fs/zfs.h
	@@ -1,135 +1,137 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
	* Use is subject to license terms.
	*/

	/*
	* Copyright (c) 2020 by Delphix. All rights reserved.
	*/

	#ifndef _SYS_FM_FS_ZFS_H
	#define _SYS_FM_FS_ZFS_H

	#ifdef __cplusplus
	extern "C" {
	#endif

	#define ZFS_ERROR_CLASS "fs.zfs"

	#define FM_EREPORT_ZFS_CHECKSUM "checksum"
	#define FM_EREPORT_ZFS_AUTHENTICATION "authentication"
	#define FM_EREPORT_ZFS_IO "io"
	#define FM_EREPORT_ZFS_DATA "data"
	#define FM_EREPORT_ZFS_DELAY "delay"
	#define FM_EREPORT_ZFS_DEADMAN "deadman"
	#define FM_EREPORT_ZFS_POOL "zpool"
	#define FM_EREPORT_ZFS_DEVICE_UNKNOWN "vdev.unknown"
	#define FM_EREPORT_ZFS_DEVICE_OPEN_FAILED "vdev.open_failed"
	#define FM_EREPORT_ZFS_DEVICE_CORRUPT_DATA "vdev.corrupt_data"
	#define FM_EREPORT_ZFS_DEVICE_NO_REPLICAS "vdev.no_replicas"
	#define FM_EREPORT_ZFS_DEVICE_BAD_GUID_SUM "vdev.bad_guid_sum"
	#define FM_EREPORT_ZFS_DEVICE_TOO_SMALL "vdev.too_small"
	#define FM_EREPORT_ZFS_DEVICE_BAD_LABEL "vdev.bad_label"
	#define FM_EREPORT_ZFS_DEVICE_BAD_ASHIFT "vdev.bad_ashift"
	#define FM_EREPORT_ZFS_IO_FAILURE "io_failure"
	#define FM_EREPORT_ZFS_PROBE_FAILURE "probe_failure"
	#define FM_EREPORT_ZFS_LOG_REPLAY "log_replay"
	#define FM_EREPORT_ZFS_CONFIG_CACHE_WRITE "config_cache_write"

	#define FM_EREPORT_PAYLOAD_ZFS_POOL "pool"
	#define FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE "pool_failmode"
	#define FM_EREPORT_PAYLOAD_ZFS_POOL_GUID "pool_guid"
	#define FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT "pool_context"
	#define FM_EREPORT_PAYLOAD_ZFS_POOL_STATE "pool_state"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID "vdev_guid"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE "vdev_type"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_PATH "vdev_path"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_PHYSPATH "vdev_physpath"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_ENC_SYSFS_PATH "vdev_enc_sysfs_path"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_DEVID "vdev_devid"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_FRU "vdev_fru"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE "vdev_state"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_LASTSTATE "vdev_laststate"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_ASHIFT "vdev_ashift"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_COMP_TS "vdev_complete_ts"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_DELTA_TS "vdev_delta_ts"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_SPARE_PATHS "vdev_spare_paths"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_SPARE_GUIDS "vdev_spare_guids"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_READ_ERRORS "vdev_read_errors"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_WRITE_ERRORS "vdev_write_errors"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_ERRORS "vdev_cksum_errors"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_N "vdev_cksum_n"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_T "vdev_cksum_t"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_N "vdev_io_n"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_T "vdev_io_t"
	+#define FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_N "vdev_slow_io_n"
	+#define FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_T "vdev_slow_io_t"
	#define FM_EREPORT_PAYLOAD_ZFS_VDEV_DELAYS "vdev_delays"
	#define FM_EREPORT_PAYLOAD_ZFS_PARENT_GUID "parent_guid"
	#define FM_EREPORT_PAYLOAD_ZFS_PARENT_TYPE "parent_type"
	#define FM_EREPORT_PAYLOAD_ZFS_PARENT_PATH "parent_path"
	#define FM_EREPORT_PAYLOAD_ZFS_PARENT_DEVID "parent_devid"
	#define FM_EREPORT_PAYLOAD_ZFS_ZIO_OBJSET "zio_objset"
	#define FM_EREPORT_PAYLOAD_ZFS_ZIO_OBJECT "zio_object"
	#define FM_EREPORT_PAYLOAD_ZFS_ZIO_LEVEL "zio_level"
	#define FM_EREPORT_PAYLOAD_ZFS_ZIO_BLKID "zio_blkid"
	#define FM_EREPORT_PAYLOAD_ZFS_ZIO_ERR "zio_err"
	#define FM_EREPORT_PAYLOAD_ZFS_ZIO_OFFSET "zio_offset"
	#define FM_EREPORT_PAYLOAD_ZFS_ZIO_SIZE "zio_size"
	#define FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS "zio_flags"
	#define FM_EREPORT_PAYLOAD_ZFS_ZIO_STAGE "zio_stage"
	#define FM_EREPORT_PAYLOAD_ZFS_ZIO_PRIORITY "zio_priority"
	#define FM_EREPORT_PAYLOAD_ZFS_ZIO_PIPELINE "zio_pipeline"
	#define FM_EREPORT_PAYLOAD_ZFS_ZIO_DELAY "zio_delay"
	#define FM_EREPORT_PAYLOAD_ZFS_ZIO_TIMESTAMP "zio_timestamp"
	#define FM_EREPORT_PAYLOAD_ZFS_ZIO_DELTA "zio_delta"
	#define FM_EREPORT_PAYLOAD_ZFS_PREV_STATE "prev_state"
	#define FM_EREPORT_PAYLOAD_ZFS_CKSUM_ALGO "cksum_algorithm"
	#define FM_EREPORT_PAYLOAD_ZFS_CKSUM_BYTESWAP "cksum_byteswap"
	#define FM_EREPORT_PAYLOAD_ZFS_BAD_OFFSET_RANGES "bad_ranges"
	#define FM_EREPORT_PAYLOAD_ZFS_BAD_RANGE_MIN_GAP "bad_ranges_min_gap"
	#define FM_EREPORT_PAYLOAD_ZFS_BAD_RANGE_SETS "bad_range_sets"
	#define FM_EREPORT_PAYLOAD_ZFS_BAD_RANGE_CLEARS "bad_range_clears"
	#define FM_EREPORT_PAYLOAD_ZFS_BAD_SET_BITS "bad_set_bits"
	#define FM_EREPORT_PAYLOAD_ZFS_BAD_CLEARED_BITS "bad_cleared_bits"
	#define FM_EREPORT_PAYLOAD_ZFS_SNAPSHOT_NAME "snapshot_name"
	#define FM_EREPORT_PAYLOAD_ZFS_DEVICE_NAME "device_name"
	#define FM_EREPORT_PAYLOAD_ZFS_RAW_DEVICE_NAME "raw_name"
	#define FM_EREPORT_PAYLOAD_ZFS_VOLUME "volume"

	#define FM_EREPORT_FAILMODE_WAIT "wait"
	#define FM_EREPORT_FAILMODE_CONTINUE "continue"
	#define FM_EREPORT_FAILMODE_PANIC "panic"

	#define FM_RESOURCE_REMOVED "removed"
	#define FM_RESOURCE_AUTOREPLACE "autoreplace"
	#define FM_RESOURCE_STATECHANGE "statechange"

	#define FM_RESOURCE_ZFS_SNAPSHOT_MOUNT "snapshot_mount"
	#define FM_RESOURCE_ZFS_SNAPSHOT_UNMOUNT "snapshot_unmount"
	#define FM_RESOURCE_ZVOL_CREATE_SYMLINK "zvol_create"
	#define FM_RESOURCE_ZVOL_REMOVE_SYMLINK "zvol_remove"

	#ifdef __cplusplus
	}
	#endif

	#endif /* _SYS_FM_FS_ZFS_H */
	diff --git a/sys/contrib/openzfs/include/sys/fs/zfs.h b/sys/contrib/openzfs/include/sys/fs/zfs.h
	index bc940e8a7929..4329e4e86f2d 100644
	--- a/sys/contrib/openzfs/include/sys/fs/zfs.h
	+++ b/sys/contrib/openzfs/include/sys/fs/zfs.h
	@@ -1,1860 +1,1872 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	- * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
	+ * Copyright (c) 2011, 2024 by Delphix. All rights reserved.
	* Copyright 2011 Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2013, 2017 Joyent, Inc. All rights reserved.
	* Copyright (c) 2014 Integros [integros.com]
	* Copyright (c) 2017, Intel Corporation.
	* Copyright (c) 2019 Datto Inc.
	* Portions Copyright 2010 Robert Milkowski
	* Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
	* Copyright (c) 2022 Hewlett Packard Enterprise Development LP.
	*/

	#ifndef _SYS_FS_ZFS_H
	#define _SYS_FS_ZFS_H extern __attribute__((visibility("default")))

	#include <sys/time.h>
	#include <sys/zio_priority.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*
	* Types and constants shared between userland and the kernel.
	*/

	/*
	* Each dataset can be one of the following types. These constants can be
	* combined into masks that can be passed to various functions.
	*/
	typedef enum {
	ZFS_TYPE_INVALID = 0,
	ZFS_TYPE_FILESYSTEM = (1 << 0),
	ZFS_TYPE_SNAPSHOT = (1 << 1),
	ZFS_TYPE_VOLUME = (1 << 2),
	ZFS_TYPE_POOL = (1 << 3),
	ZFS_TYPE_BOOKMARK = (1 << 4),
	ZFS_TYPE_VDEV = (1 << 5),
	} zfs_type_t;

	/*
	* NB: lzc_dataset_type should be updated whenever a new objset type is added,
	* if it represents a real type of a dataset that can be created from userland.
	*/
	typedef enum dmu_objset_type {
	DMU_OST_NONE,
	DMU_OST_META,
	DMU_OST_ZFS,
	DMU_OST_ZVOL,
	DMU_OST_OTHER, /* For testing only! */
	DMU_OST_ANY, /* Be careful! */
	DMU_OST_NUMTYPES
	} dmu_objset_type_t;

	#define ZFS_TYPE_DATASET \
	(ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME \| ZFS_TYPE_SNAPSHOT)

	/*
	* All of these include the terminating NUL byte.
	*/
	#define ZAP_MAXNAMELEN 256
	#define ZAP_MAXVALUELEN (1024 * 8)
	#define ZAP_OLDMAXVALUELEN 1024
	#define ZFS_MAX_DATASET_NAME_LEN 256

	/*
	* Dataset properties are identified by these constants and must be added to
	* the end of this list to ensure that external consumers are not affected
	* by the change. If you make any changes to this list, be sure to update
	* the property table in module/zcommon/zfs_prop.c.
	*/
	typedef enum {
	ZPROP_CONT = -2,
	ZPROP_INVAL = -1,
	ZPROP_USERPROP = ZPROP_INVAL,
	ZFS_PROP_TYPE = 0,
	ZFS_PROP_CREATION,
	ZFS_PROP_USED,
	ZFS_PROP_AVAILABLE,
	ZFS_PROP_REFERENCED,
	ZFS_PROP_COMPRESSRATIO,
	ZFS_PROP_MOUNTED,
	ZFS_PROP_ORIGIN,
	ZFS_PROP_QUOTA,
	ZFS_PROP_RESERVATION,
	ZFS_PROP_VOLSIZE,
	ZFS_PROP_VOLBLOCKSIZE,
	ZFS_PROP_RECORDSIZE,
	ZFS_PROP_MOUNTPOINT,
	ZFS_PROP_SHARENFS,
	ZFS_PROP_CHECKSUM,
	ZFS_PROP_COMPRESSION,
	ZFS_PROP_ATIME,
	ZFS_PROP_DEVICES,
	ZFS_PROP_EXEC,
	ZFS_PROP_SETUID,
	ZFS_PROP_READONLY,
	ZFS_PROP_ZONED,
	ZFS_PROP_SNAPDIR,
	ZFS_PROP_ACLMODE,
	ZFS_PROP_ACLINHERIT,
	ZFS_PROP_CREATETXG,
	ZFS_PROP_NAME, /* not exposed to the user */
	ZFS_PROP_CANMOUNT,
	ZFS_PROP_ISCSIOPTIONS, /* not exposed to the user */
	ZFS_PROP_XATTR,
	ZFS_PROP_NUMCLONES, /* not exposed to the user */
	ZFS_PROP_COPIES,
	ZFS_PROP_VERSION,
	ZFS_PROP_UTF8ONLY,
	ZFS_PROP_NORMALIZE,
	ZFS_PROP_CASE,
	ZFS_PROP_VSCAN,
	ZFS_PROP_NBMAND,
	ZFS_PROP_SHARESMB,
	ZFS_PROP_REFQUOTA,
	ZFS_PROP_REFRESERVATION,
	ZFS_PROP_GUID,
	ZFS_PROP_PRIMARYCACHE,
	ZFS_PROP_SECONDARYCACHE,
	ZFS_PROP_USEDSNAP,
	ZFS_PROP_USEDDS,
	ZFS_PROP_USEDCHILD,
	ZFS_PROP_USEDREFRESERV,
	ZFS_PROP_USERACCOUNTING, /* not exposed to the user */
	ZFS_PROP_STMF_SHAREINFO, /* not exposed to the user */
	ZFS_PROP_DEFER_DESTROY,
	ZFS_PROP_USERREFS,
	ZFS_PROP_LOGBIAS,
	ZFS_PROP_UNIQUE, /* not exposed to the user */
	ZFS_PROP_OBJSETID,
	ZFS_PROP_DEDUP,
	ZFS_PROP_MLSLABEL,
	ZFS_PROP_SYNC,
	ZFS_PROP_DNODESIZE,
	ZFS_PROP_REFRATIO,
	ZFS_PROP_WRITTEN,
	ZFS_PROP_CLONES,
	ZFS_PROP_LOGICALUSED,
	ZFS_PROP_LOGICALREFERENCED,
	ZFS_PROP_INCONSISTENT, /* not exposed to the user */
	ZFS_PROP_VOLMODE,
	ZFS_PROP_FILESYSTEM_LIMIT,
	ZFS_PROP_SNAPSHOT_LIMIT,
	ZFS_PROP_FILESYSTEM_COUNT,
	ZFS_PROP_SNAPSHOT_COUNT,
	ZFS_PROP_SNAPDEV,
	ZFS_PROP_ACLTYPE,
	ZFS_PROP_SELINUX_CONTEXT,
	ZFS_PROP_SELINUX_FSCONTEXT,
	ZFS_PROP_SELINUX_DEFCONTEXT,
	ZFS_PROP_SELINUX_ROOTCONTEXT,
	ZFS_PROP_RELATIME,
	ZFS_PROP_REDUNDANT_METADATA,
	ZFS_PROP_OVERLAY,
	ZFS_PROP_PREV_SNAP,
	ZFS_PROP_RECEIVE_RESUME_TOKEN,
	ZFS_PROP_ENCRYPTION,
	ZFS_PROP_KEYLOCATION,
	ZFS_PROP_KEYFORMAT,
	ZFS_PROP_PBKDF2_SALT,
	ZFS_PROP_PBKDF2_ITERS,
	ZFS_PROP_ENCRYPTION_ROOT,
	ZFS_PROP_KEY_GUID,
	ZFS_PROP_KEYSTATUS,
	ZFS_PROP_REMAPTXG, /* obsolete - no longer used */
	ZFS_PROP_SPECIAL_SMALL_BLOCKS,
	ZFS_PROP_IVSET_GUID, /* not exposed to the user */
	ZFS_PROP_REDACTED,
	ZFS_PROP_REDACT_SNAPS,
	ZFS_PROP_SNAPSHOTS_CHANGED,
	+ ZFS_PROP_PREFETCH,
	ZFS_NUM_PROPS
	} zfs_prop_t;

	typedef enum {
	ZFS_PROP_USERUSED,
	ZFS_PROP_USERQUOTA,
	ZFS_PROP_GROUPUSED,
	ZFS_PROP_GROUPQUOTA,
	ZFS_PROP_USEROBJUSED,
	ZFS_PROP_USEROBJQUOTA,
	ZFS_PROP_GROUPOBJUSED,
	ZFS_PROP_GROUPOBJQUOTA,
	ZFS_PROP_PROJECTUSED,
	ZFS_PROP_PROJECTQUOTA,
	ZFS_PROP_PROJECTOBJUSED,
	ZFS_PROP_PROJECTOBJQUOTA,
	ZFS_NUM_USERQUOTA_PROPS
	} zfs_userquota_prop_t;

	_SYS_FS_ZFS_H const char *const zfs_userquota_prop_prefixes[
	ZFS_NUM_USERQUOTA_PROPS];

	/*
	* Pool properties are identified by these constants and must be added to the
	* end of this list to ensure that external consumers are not affected
	* by the change. Properties must be registered in zfs_prop_init().
	*/
	typedef enum {
	ZPOOL_PROP_INVAL = -1,
	ZPOOL_PROP_NAME,
	ZPOOL_PROP_SIZE,
	ZPOOL_PROP_CAPACITY,
	ZPOOL_PROP_ALTROOT,
	ZPOOL_PROP_HEALTH,
	ZPOOL_PROP_GUID,
	ZPOOL_PROP_VERSION,
	ZPOOL_PROP_BOOTFS,
	ZPOOL_PROP_DELEGATION,
	ZPOOL_PROP_AUTOREPLACE,
	ZPOOL_PROP_CACHEFILE,
	ZPOOL_PROP_FAILUREMODE,
	ZPOOL_PROP_LISTSNAPS,
	ZPOOL_PROP_AUTOEXPAND,
	ZPOOL_PROP_DEDUPDITTO,
	ZPOOL_PROP_DEDUPRATIO,
	ZPOOL_PROP_FREE,
	ZPOOL_PROP_ALLOCATED,
	ZPOOL_PROP_READONLY,
	ZPOOL_PROP_ASHIFT,
	ZPOOL_PROP_COMMENT,
	ZPOOL_PROP_EXPANDSZ,
	ZPOOL_PROP_FREEING,
	ZPOOL_PROP_FRAGMENTATION,
	ZPOOL_PROP_LEAKED,
	ZPOOL_PROP_MAXBLOCKSIZE,
	ZPOOL_PROP_TNAME,
	ZPOOL_PROP_MAXDNODESIZE,
	ZPOOL_PROP_MULTIHOST,
	ZPOOL_PROP_CHECKPOINT,
	ZPOOL_PROP_LOAD_GUID,
	ZPOOL_PROP_AUTOTRIM,
	ZPOOL_PROP_COMPATIBILITY,
	ZPOOL_PROP_BCLONEUSED,
	ZPOOL_PROP_BCLONESAVED,
	ZPOOL_PROP_BCLONERATIO,
	ZPOOL_NUM_PROPS
	} zpool_prop_t;

	/* Small enough to not hog a whole line of printout in zpool(8). */
	#define ZPROP_MAX_COMMENT 32
	#define ZPROP_BOOLEAN_NA 2

	#define ZPROP_VALUE "value"
	#define ZPROP_SOURCE "source"

	typedef enum {
	ZPROP_SRC_NONE = 0x1,
	ZPROP_SRC_DEFAULT = 0x2,
	ZPROP_SRC_TEMPORARY = 0x4,
	ZPROP_SRC_LOCAL = 0x8,
	ZPROP_SRC_INHERITED = 0x10,
	ZPROP_SRC_RECEIVED = 0x20
	} zprop_source_t;

	#define ZPROP_SRC_ALL 0x3f

	#define ZPROP_SOURCE_VAL_RECVD "$recvd"
	#define ZPROP_N_MORE_ERRORS "N_MORE_ERRORS"

	/*
	* Dataset flag implemented as a special entry in the props zap object
	* indicating that the dataset has received properties on or after
	* SPA_VERSION_RECVD_PROPS. The first such receive blows away local properties
	* just as it did in earlier versions, and thereafter, local properties are
	* preserved.
	*/
	#define ZPROP_HAS_RECVD "$hasrecvd"

	typedef enum {
	ZPROP_ERR_NOCLEAR = 0x1, /* failure to clear existing props */
	ZPROP_ERR_NORESTORE = 0x2 /* failure to restore props on error */
	} zprop_errflags_t;

	typedef int (zprop_func)(int, void );

	/*
	* Properties to be set on the root file system of a new pool
	* are stuffed into their own nvlist, which is then included in
	* the properties nvlist with the pool properties.
	*/
	#define ZPOOL_ROOTFS_PROPS "root-props-nvl"

	/*
	* Length of 'written@' and 'written#'
	*/
	#define ZFS_WRITTEN_PROP_PREFIX_LEN 8

	/*
	* VDEV properties are identified by these constants and must be added to the
	* end of this list to ensure that external consumers are not affected
	* by the change. If you make any changes to this list, be sure to update
	* the property table in usr/src/common/zfs/zpool_prop.c.
	*/
	typedef enum {
	VDEV_PROP_INVAL = -1,
	VDEV_PROP_USERPROP = VDEV_PROP_INVAL,
	VDEV_PROP_NAME,
	VDEV_PROP_CAPACITY,
	VDEV_PROP_STATE,
	VDEV_PROP_GUID,
	VDEV_PROP_ASIZE,
	VDEV_PROP_PSIZE,
	VDEV_PROP_ASHIFT,
	VDEV_PROP_SIZE,
	VDEV_PROP_FREE,
	VDEV_PROP_ALLOCATED,
	VDEV_PROP_COMMENT,
	VDEV_PROP_EXPANDSZ,
	VDEV_PROP_FRAGMENTATION,
	VDEV_PROP_BOOTSIZE,
	VDEV_PROP_PARITY,
	VDEV_PROP_PATH,
	VDEV_PROP_DEVID,
	VDEV_PROP_PHYS_PATH,
	VDEV_PROP_ENC_PATH,
	VDEV_PROP_FRU,
	VDEV_PROP_PARENT,
	VDEV_PROP_CHILDREN,
	VDEV_PROP_NUMCHILDREN,
	VDEV_PROP_READ_ERRORS,
	VDEV_PROP_WRITE_ERRORS,
	VDEV_PROP_CHECKSUM_ERRORS,
	VDEV_PROP_INITIALIZE_ERRORS,
	VDEV_PROP_OPS_NULL,
	VDEV_PROP_OPS_READ,
	VDEV_PROP_OPS_WRITE,
	VDEV_PROP_OPS_FREE,
	VDEV_PROP_OPS_CLAIM,
	VDEV_PROP_OPS_TRIM,
	VDEV_PROP_BYTES_NULL,
	VDEV_PROP_BYTES_READ,
	VDEV_PROP_BYTES_WRITE,
	VDEV_PROP_BYTES_FREE,
	VDEV_PROP_BYTES_CLAIM,
	VDEV_PROP_BYTES_TRIM,
	VDEV_PROP_REMOVING,
	VDEV_PROP_ALLOCATING,
	VDEV_PROP_FAILFAST,
	VDEV_PROP_CHECKSUM_N,
	VDEV_PROP_CHECKSUM_T,
	VDEV_PROP_IO_N,
	VDEV_PROP_IO_T,
	+ VDEV_PROP_RAIDZ_EXPANDING,
	+ VDEV_PROP_SLOW_IO_N,
	+ VDEV_PROP_SLOW_IO_T,
	VDEV_NUM_PROPS
	} vdev_prop_t;

	/*
	* Dataset property functions shared between libzfs and kernel.
	*/
	_SYS_FS_ZFS_H const char *zfs_prop_default_string(zfs_prop_t);
	_SYS_FS_ZFS_H uint64_t zfs_prop_default_numeric(zfs_prop_t);
	_SYS_FS_ZFS_H boolean_t zfs_prop_readonly(zfs_prop_t);
	_SYS_FS_ZFS_H boolean_t zfs_prop_visible(zfs_prop_t prop);
	_SYS_FS_ZFS_H boolean_t zfs_prop_inheritable(zfs_prop_t);
	_SYS_FS_ZFS_H boolean_t zfs_prop_setonce(zfs_prop_t);
	_SYS_FS_ZFS_H boolean_t zfs_prop_encryption_key_param(zfs_prop_t);
	_SYS_FS_ZFS_H boolean_t zfs_prop_valid_keylocation(const char *, boolean_t);
	_SYS_FS_ZFS_H const char *zfs_prop_to_name(zfs_prop_t);
	_SYS_FS_ZFS_H zfs_prop_t zfs_name_to_prop(const char *);
	_SYS_FS_ZFS_H boolean_t zfs_prop_user(const char *);
	_SYS_FS_ZFS_H boolean_t zfs_prop_userquota(const char *);
	_SYS_FS_ZFS_H boolean_t zfs_prop_written(const char *);
	_SYS_FS_ZFS_H int zfs_prop_index_to_string(zfs_prop_t, uint64_t, const char **);
	_SYS_FS_ZFS_H int zfs_prop_string_to_index(zfs_prop_t, const char *,
	uint64_t *);
	_SYS_FS_ZFS_H uint64_t zfs_prop_random_value(zfs_prop_t, uint64_t seed);
	_SYS_FS_ZFS_H boolean_t zfs_prop_valid_for_type(int, zfs_type_t, boolean_t);

	/*
	* Pool property functions shared between libzfs and kernel.
	*/
	_SYS_FS_ZFS_H zpool_prop_t zpool_name_to_prop(const char *);
	_SYS_FS_ZFS_H const char *zpool_prop_to_name(zpool_prop_t);
	_SYS_FS_ZFS_H const char *zpool_prop_default_string(zpool_prop_t);
	_SYS_FS_ZFS_H uint64_t zpool_prop_default_numeric(zpool_prop_t);
	_SYS_FS_ZFS_H boolean_t zpool_prop_readonly(zpool_prop_t);
	_SYS_FS_ZFS_H boolean_t zpool_prop_setonce(zpool_prop_t);
	_SYS_FS_ZFS_H boolean_t zpool_prop_feature(const char *);
	_SYS_FS_ZFS_H boolean_t zpool_prop_unsupported(const char *);
	_SYS_FS_ZFS_H int zpool_prop_index_to_string(zpool_prop_t, uint64_t,
	const char **);
	_SYS_FS_ZFS_H int zpool_prop_string_to_index(zpool_prop_t, const char *,
	uint64_t *);
	_SYS_FS_ZFS_H uint64_t zpool_prop_random_value(zpool_prop_t, uint64_t seed);

	/*
	* VDEV property functions shared between libzfs and kernel.
	*/
	_SYS_FS_ZFS_H vdev_prop_t vdev_name_to_prop(const char *);
	_SYS_FS_ZFS_H boolean_t vdev_prop_user(const char *name);
	_SYS_FS_ZFS_H const char *vdev_prop_to_name(vdev_prop_t);
	_SYS_FS_ZFS_H const char *vdev_prop_default_string(vdev_prop_t);
	_SYS_FS_ZFS_H uint64_t vdev_prop_default_numeric(vdev_prop_t);
	_SYS_FS_ZFS_H boolean_t vdev_prop_readonly(vdev_prop_t prop);
	_SYS_FS_ZFS_H int vdev_prop_index_to_string(vdev_prop_t, uint64_t,
	const char **);
	_SYS_FS_ZFS_H int vdev_prop_string_to_index(vdev_prop_t, const char *,
	uint64_t *);
	_SYS_FS_ZFS_H boolean_t zpool_prop_vdev(const char *name);
	_SYS_FS_ZFS_H uint64_t vdev_prop_random_value(vdev_prop_t prop, uint64_t seed);

	/*
	* Definitions for the Delegation.
	*/
	typedef enum {
	ZFS_DELEG_WHO_UNKNOWN = 0,
	ZFS_DELEG_USER = 'u',
	ZFS_DELEG_USER_SETS = 'U',
	ZFS_DELEG_GROUP = 'g',
	ZFS_DELEG_GROUP_SETS = 'G',
	ZFS_DELEG_EVERYONE = 'e',
	ZFS_DELEG_EVERYONE_SETS = 'E',
	ZFS_DELEG_CREATE = 'c',
	ZFS_DELEG_CREATE_SETS = 'C',
	ZFS_DELEG_NAMED_SET = 's',
	ZFS_DELEG_NAMED_SET_SETS = 'S'
	} zfs_deleg_who_type_t;

	typedef enum {
	ZFS_DELEG_NONE = 0,
	ZFS_DELEG_PERM_LOCAL = 1,
	ZFS_DELEG_PERM_DESCENDENT = 2,
	ZFS_DELEG_PERM_LOCALDESCENDENT = 3,
	ZFS_DELEG_PERM_CREATE = 4
	} zfs_deleg_inherit_t;

	#define ZFS_DELEG_PERM_UID "uid"
	#define ZFS_DELEG_PERM_GID "gid"
	#define ZFS_DELEG_PERM_GROUPS "groups"

	#define ZFS_MLSLABEL_DEFAULT "none"

	#define ZFS_SMB_ACL_SRC "src"
	#define ZFS_SMB_ACL_TARGET "target"

	typedef enum {
	ZFS_CANMOUNT_OFF = 0,
	ZFS_CANMOUNT_ON = 1,
	ZFS_CANMOUNT_NOAUTO = 2
	} zfs_canmount_type_t;

	typedef enum {
	ZFS_LOGBIAS_LATENCY = 0,
	ZFS_LOGBIAS_THROUGHPUT = 1
	} zfs_logbias_op_t;

	typedef enum zfs_share_op {
	ZFS_SHARE_NFS = 0,
	ZFS_UNSHARE_NFS = 1,
	ZFS_SHARE_SMB = 2,
	ZFS_UNSHARE_SMB = 3
	} zfs_share_op_t;

	typedef enum zfs_smb_acl_op {
	ZFS_SMB_ACL_ADD,
	ZFS_SMB_ACL_REMOVE,
	ZFS_SMB_ACL_RENAME,
	ZFS_SMB_ACL_PURGE
	} zfs_smb_acl_op_t;

	typedef enum zfs_cache_type {
	ZFS_CACHE_NONE = 0,
	ZFS_CACHE_METADATA = 1,
	ZFS_CACHE_ALL = 2
	} zfs_cache_type_t;

	typedef enum {
	ZFS_SYNC_STANDARD = 0,
	ZFS_SYNC_ALWAYS = 1,
	ZFS_SYNC_DISABLED = 2
	} zfs_sync_type_t;

	typedef enum {
	ZFS_XATTR_OFF = 0,
	ZFS_XATTR_DIR = 1,
	ZFS_XATTR_SA = 2
	} zfs_xattr_type_t;

	typedef enum {
	ZFS_DNSIZE_LEGACY = 0,
	ZFS_DNSIZE_AUTO = 1,
	ZFS_DNSIZE_1K = 1024,
	ZFS_DNSIZE_2K = 2048,
	ZFS_DNSIZE_4K = 4096,
	ZFS_DNSIZE_8K = 8192,
	ZFS_DNSIZE_16K = 16384
	} zfs_dnsize_type_t;

	typedef enum {
	ZFS_REDUNDANT_METADATA_ALL,
	ZFS_REDUNDANT_METADATA_MOST,
	ZFS_REDUNDANT_METADATA_SOME,
	ZFS_REDUNDANT_METADATA_NONE
	} zfs_redundant_metadata_type_t;

	typedef enum {
	ZFS_VOLMODE_DEFAULT = 0,
	ZFS_VOLMODE_GEOM = 1,
	ZFS_VOLMODE_DEV = 2,
	ZFS_VOLMODE_NONE = 3
	} zfs_volmode_t;

	typedef enum zfs_keystatus {
	ZFS_KEYSTATUS_NONE = 0,
	ZFS_KEYSTATUS_UNAVAILABLE,
	ZFS_KEYSTATUS_AVAILABLE,
	} zfs_keystatus_t;

	typedef enum zfs_keyformat {
	ZFS_KEYFORMAT_NONE = 0,
	ZFS_KEYFORMAT_RAW,
	ZFS_KEYFORMAT_HEX,
	ZFS_KEYFORMAT_PASSPHRASE,
	ZFS_KEYFORMAT_FORMATS
	} zfs_keyformat_t;

	typedef enum zfs_key_location {
	ZFS_KEYLOCATION_NONE = 0,
	ZFS_KEYLOCATION_PROMPT,
	ZFS_KEYLOCATION_URI,
	ZFS_KEYLOCATION_LOCATIONS
	} zfs_keylocation_t;

	+typedef enum {
	+ ZFS_PREFETCH_NONE = 0,
	+ ZFS_PREFETCH_METADATA = 1,
	+ ZFS_PREFETCH_ALL = 2
	+} zfs_prefetch_type_t;
	+
	#define DEFAULT_PBKDF2_ITERATIONS 350000
	#define MIN_PBKDF2_ITERATIONS 100000

	/*
	* On-disk version number.
	*/
	#define SPA_VERSION_1 1ULL
	#define SPA_VERSION_2 2ULL
	#define SPA_VERSION_3 3ULL
	#define SPA_VERSION_4 4ULL
	#define SPA_VERSION_5 5ULL
	#define SPA_VERSION_6 6ULL
	#define SPA_VERSION_7 7ULL
	#define SPA_VERSION_8 8ULL
	#define SPA_VERSION_9 9ULL
	#define SPA_VERSION_10 10ULL
	#define SPA_VERSION_11 11ULL
	#define SPA_VERSION_12 12ULL
	#define SPA_VERSION_13 13ULL
	#define SPA_VERSION_14 14ULL
	#define SPA_VERSION_15 15ULL
	#define SPA_VERSION_16 16ULL
	#define SPA_VERSION_17 17ULL
	#define SPA_VERSION_18 18ULL
	#define SPA_VERSION_19 19ULL
	#define SPA_VERSION_20 20ULL
	#define SPA_VERSION_21 21ULL
	#define SPA_VERSION_22 22ULL
	#define SPA_VERSION_23 23ULL
	#define SPA_VERSION_24 24ULL
	#define SPA_VERSION_25 25ULL
	#define SPA_VERSION_26 26ULL
	#define SPA_VERSION_27 27ULL
	#define SPA_VERSION_28 28ULL
	#define SPA_VERSION_5000 5000ULL

	/*
	* The incrementing pool version number has been replaced by pool feature
	* flags. For more details, see zfeature.c.
	*/
	#define SPA_VERSION SPA_VERSION_5000
	#define SPA_VERSION_STRING "5000"

	/*
	* Symbolic names for the changes that caused a SPA_VERSION switch.
	* Used in the code when checking for presence or absence of a feature.
	* Feel free to define multiple symbolic names for each version if there
	* were multiple changes to on-disk structures during that version.
	*
	* NOTE: When checking the current SPA_VERSION in your code, be sure
	* to use spa_version() since it reports the version of the
	* last synced uberblock. Checking the in-flight version can
	* be dangerous in some cases.
	*/
	#define SPA_VERSION_INITIAL SPA_VERSION_1
	#define SPA_VERSION_DITTO_BLOCKS SPA_VERSION_2
	#define SPA_VERSION_SPARES SPA_VERSION_3
	#define SPA_VERSION_RAIDZ2 SPA_VERSION_3
	#define SPA_VERSION_BPOBJ_ACCOUNT SPA_VERSION_3
	#define SPA_VERSION_RAIDZ_DEFLATE SPA_VERSION_3
	#define SPA_VERSION_DNODE_BYTES SPA_VERSION_3
	#define SPA_VERSION_ZPOOL_HISTORY SPA_VERSION_4
	#define SPA_VERSION_GZIP_COMPRESSION SPA_VERSION_5
	#define SPA_VERSION_BOOTFS SPA_VERSION_6
	#define SPA_VERSION_SLOGS SPA_VERSION_7
	#define SPA_VERSION_DELEGATED_PERMS SPA_VERSION_8
	#define SPA_VERSION_FUID SPA_VERSION_9
	#define SPA_VERSION_REFRESERVATION SPA_VERSION_9
	#define SPA_VERSION_REFQUOTA SPA_VERSION_9
	#define SPA_VERSION_UNIQUE_ACCURATE SPA_VERSION_9
	#define SPA_VERSION_L2CACHE SPA_VERSION_10
	#define SPA_VERSION_NEXT_CLONES SPA_VERSION_11
	#define SPA_VERSION_ORIGIN SPA_VERSION_11
	#define SPA_VERSION_DSL_SCRUB SPA_VERSION_11
	#define SPA_VERSION_SNAP_PROPS SPA_VERSION_12
	#define SPA_VERSION_USED_BREAKDOWN SPA_VERSION_13
	#define SPA_VERSION_PASSTHROUGH_X SPA_VERSION_14
	#define SPA_VERSION_USERSPACE SPA_VERSION_15
	#define SPA_VERSION_STMF_PROP SPA_VERSION_16
	#define SPA_VERSION_RAIDZ3 SPA_VERSION_17
	#define SPA_VERSION_USERREFS SPA_VERSION_18
	#define SPA_VERSION_HOLES SPA_VERSION_19
	#define SPA_VERSION_ZLE_COMPRESSION SPA_VERSION_20
	#define SPA_VERSION_DEDUP SPA_VERSION_21
	#define SPA_VERSION_RECVD_PROPS SPA_VERSION_22
	#define SPA_VERSION_SLIM_ZIL SPA_VERSION_23
	#define SPA_VERSION_SA SPA_VERSION_24
	#define SPA_VERSION_SCAN SPA_VERSION_25
	#define SPA_VERSION_DIR_CLONES SPA_VERSION_26
	#define SPA_VERSION_DEADLISTS SPA_VERSION_26
	#define SPA_VERSION_FAST_SNAP SPA_VERSION_27
	#define SPA_VERSION_MULTI_REPLACE SPA_VERSION_28
	#define SPA_VERSION_BEFORE_FEATURES SPA_VERSION_28
	#define SPA_VERSION_FEATURES SPA_VERSION_5000

	#define SPA_VERSION_IS_SUPPORTED(v) \
	(((v) >= SPA_VERSION_INITIAL && (v) <= SPA_VERSION_BEFORE_FEATURES) \|\| \
	((v) >= SPA_VERSION_FEATURES && (v) <= SPA_VERSION))

	/*
	* ZPL version - rev'd whenever an incompatible on-disk format change
	* occurs. This is independent of SPA/DMU/ZAP versioning. You must
	* also update the version_table[] and help message in zfs_prop.c.
	*/
	#define ZPL_VERSION_1 1ULL
	#define ZPL_VERSION_2 2ULL
	#define ZPL_VERSION_3 3ULL
	#define ZPL_VERSION_4 4ULL
	#define ZPL_VERSION_5 5ULL
	#define ZPL_VERSION ZPL_VERSION_5
	#define ZPL_VERSION_STRING "5"

	#define ZPL_VERSION_INITIAL ZPL_VERSION_1
	#define ZPL_VERSION_DIRENT_TYPE ZPL_VERSION_2
	#define ZPL_VERSION_FUID ZPL_VERSION_3
	#define ZPL_VERSION_NORMALIZATION ZPL_VERSION_3
	#define ZPL_VERSION_SYSATTR ZPL_VERSION_3
	#define ZPL_VERSION_USERSPACE ZPL_VERSION_4
	#define ZPL_VERSION_SA ZPL_VERSION_5

	/* Persistent L2ARC version */
	#define L2ARC_PERSISTENT_VERSION_1 1ULL
	#define L2ARC_PERSISTENT_VERSION L2ARC_PERSISTENT_VERSION_1
	#define L2ARC_PERSISTENT_VERSION_STRING "1"

	/* Rewind policy information */
	#define ZPOOL_NO_REWIND 1 /* No policy - default behavior */
	#define ZPOOL_NEVER_REWIND 2 /* Do not search for best txg or rewind */
	#define ZPOOL_TRY_REWIND 4 /* Search for best txg, but do not rewind */
	#define ZPOOL_DO_REWIND 8 /* Rewind to best txg w/in deferred frees */
	#define ZPOOL_EXTREME_REWIND 16 /* Allow extreme measures to find best txg */
	#define ZPOOL_REWIND_MASK 28 /* All the possible rewind bits */
	#define ZPOOL_REWIND_POLICIES 31 /* All the possible policy bits */

	typedef struct zpool_load_policy {
	uint32_t zlp_rewind; /* rewind policy requested */
	uint64_t zlp_maxmeta; /* max acceptable meta-data errors */
	uint64_t zlp_maxdata; /* max acceptable data errors */
	uint64_t zlp_txg; /* specific txg to load */
	} zpool_load_policy_t;

	/*
	* The following are configuration names used in the nvlist describing a pool's
	* configuration. New on-disk names should be prefixed with "<reversed-DNS>:"
	* (e.g. "org.openzfs:") to avoid conflicting names being developed
	* independently.
	*/
	#define ZPOOL_CONFIG_VERSION "version"
	#define ZPOOL_CONFIG_POOL_NAME "name"
	#define ZPOOL_CONFIG_POOL_STATE "state"
	#define ZPOOL_CONFIG_POOL_TXG "txg"
	#define ZPOOL_CONFIG_POOL_GUID "pool_guid"
	#define ZPOOL_CONFIG_CREATE_TXG "create_txg"
	#define ZPOOL_CONFIG_TOP_GUID "top_guid"
	#define ZPOOL_CONFIG_VDEV_TREE "vdev_tree"
	#define ZPOOL_CONFIG_TYPE "type"
	#define ZPOOL_CONFIG_CHILDREN "children"
	#define ZPOOL_CONFIG_ID "id"
	#define ZPOOL_CONFIG_GUID "guid"
	#define ZPOOL_CONFIG_INDIRECT_OBJECT "com.delphix:indirect_object"
	#define ZPOOL_CONFIG_INDIRECT_BIRTHS "com.delphix:indirect_births"
	#define ZPOOL_CONFIG_PREV_INDIRECT_VDEV "com.delphix:prev_indirect_vdev"
	#define ZPOOL_CONFIG_PATH "path"
	#define ZPOOL_CONFIG_DEVID "devid"
	#define ZPOOL_CONFIG_SPARE_ID "spareid"
	#define ZPOOL_CONFIG_METASLAB_ARRAY "metaslab_array"
	#define ZPOOL_CONFIG_METASLAB_SHIFT "metaslab_shift"
	#define ZPOOL_CONFIG_ASHIFT "ashift"
	#define ZPOOL_CONFIG_ASIZE "asize"
	#define ZPOOL_CONFIG_DTL "DTL"
	#define ZPOOL_CONFIG_SCAN_STATS "scan_stats" /* not stored on disk */
	#define ZPOOL_CONFIG_REMOVAL_STATS "removal_stats" /* not stored on disk */
	#define ZPOOL_CONFIG_CHECKPOINT_STATS "checkpoint_stats" /* not on disk */
	#define ZPOOL_CONFIG_VDEV_STATS "vdev_stats" /* not stored on disk */
	#define ZPOOL_CONFIG_INDIRECT_SIZE "indirect_size" /* not stored on disk */

	/* container nvlist of extended stats */
	#define ZPOOL_CONFIG_VDEV_STATS_EX "vdev_stats_ex"

	/* Active queue read/write stats */
	#define ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE "vdev_sync_r_active_queue"
	#define ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE "vdev_sync_w_active_queue"
	#define ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE "vdev_async_r_active_queue"
	#define ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE "vdev_async_w_active_queue"
	#define ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE "vdev_async_scrub_active_queue"
	#define ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE "vdev_async_trim_active_queue"
	#define ZPOOL_CONFIG_VDEV_REBUILD_ACTIVE_QUEUE "vdev_rebuild_active_queue"

	/* Queue sizes */
	#define ZPOOL_CONFIG_VDEV_SYNC_R_PEND_QUEUE "vdev_sync_r_pend_queue"
	#define ZPOOL_CONFIG_VDEV_SYNC_W_PEND_QUEUE "vdev_sync_w_pend_queue"
	#define ZPOOL_CONFIG_VDEV_ASYNC_R_PEND_QUEUE "vdev_async_r_pend_queue"
	#define ZPOOL_CONFIG_VDEV_ASYNC_W_PEND_QUEUE "vdev_async_w_pend_queue"
	#define ZPOOL_CONFIG_VDEV_SCRUB_PEND_QUEUE "vdev_async_scrub_pend_queue"
	#define ZPOOL_CONFIG_VDEV_TRIM_PEND_QUEUE "vdev_async_trim_pend_queue"
	#define ZPOOL_CONFIG_VDEV_REBUILD_PEND_QUEUE "vdev_rebuild_pend_queue"

	/* Latency read/write histogram stats */
	#define ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO "vdev_tot_r_lat_histo"
	#define ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO "vdev_tot_w_lat_histo"
	#define ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO "vdev_disk_r_lat_histo"
	#define ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO "vdev_disk_w_lat_histo"
	#define ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO "vdev_sync_r_lat_histo"
	#define ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO "vdev_sync_w_lat_histo"
	#define ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO "vdev_async_r_lat_histo"
	#define ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO "vdev_async_w_lat_histo"
	#define ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO "vdev_scrub_histo"
	#define ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO "vdev_trim_histo"
	#define ZPOOL_CONFIG_VDEV_REBUILD_LAT_HISTO "vdev_rebuild_histo"

	/* Request size histograms */
	#define ZPOOL_CONFIG_VDEV_SYNC_IND_R_HISTO "vdev_sync_ind_r_histo"
	#define ZPOOL_CONFIG_VDEV_SYNC_IND_W_HISTO "vdev_sync_ind_w_histo"
	#define ZPOOL_CONFIG_VDEV_ASYNC_IND_R_HISTO "vdev_async_ind_r_histo"
	#define ZPOOL_CONFIG_VDEV_ASYNC_IND_W_HISTO "vdev_async_ind_w_histo"
	#define ZPOOL_CONFIG_VDEV_IND_SCRUB_HISTO "vdev_ind_scrub_histo"
	#define ZPOOL_CONFIG_VDEV_IND_TRIM_HISTO "vdev_ind_trim_histo"
	#define ZPOOL_CONFIG_VDEV_IND_REBUILD_HISTO "vdev_ind_rebuild_histo"
	#define ZPOOL_CONFIG_VDEV_SYNC_AGG_R_HISTO "vdev_sync_agg_r_histo"
	#define ZPOOL_CONFIG_VDEV_SYNC_AGG_W_HISTO "vdev_sync_agg_w_histo"
	#define ZPOOL_CONFIG_VDEV_ASYNC_AGG_R_HISTO "vdev_async_agg_r_histo"
	#define ZPOOL_CONFIG_VDEV_ASYNC_AGG_W_HISTO "vdev_async_agg_w_histo"
	#define ZPOOL_CONFIG_VDEV_AGG_SCRUB_HISTO "vdev_agg_scrub_histo"
	#define ZPOOL_CONFIG_VDEV_AGG_TRIM_HISTO "vdev_agg_trim_histo"
	#define ZPOOL_CONFIG_VDEV_AGG_REBUILD_HISTO "vdev_agg_rebuild_histo"

	/* Number of slow IOs */
	#define ZPOOL_CONFIG_VDEV_SLOW_IOS "vdev_slow_ios"

	/* vdev enclosure sysfs path */
	#define ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH "vdev_enc_sysfs_path"

	#define ZPOOL_CONFIG_WHOLE_DISK "whole_disk"
	#define ZPOOL_CONFIG_ERRCOUNT "error_count"
	#define ZPOOL_CONFIG_NOT_PRESENT "not_present"
	#define ZPOOL_CONFIG_SPARES "spares"
	#define ZPOOL_CONFIG_IS_SPARE "is_spare"
	#define ZPOOL_CONFIG_NPARITY "nparity"
	#define ZPOOL_CONFIG_HOSTID "hostid"
	#define ZPOOL_CONFIG_HOSTNAME "hostname"
	#define ZPOOL_CONFIG_LOADED_TIME "initial_load_time"
	#define ZPOOL_CONFIG_UNSPARE "unspare"
	#define ZPOOL_CONFIG_PHYS_PATH "phys_path"
	#define ZPOOL_CONFIG_IS_LOG "is_log"
	#define ZPOOL_CONFIG_L2CACHE "l2cache"
	#define ZPOOL_CONFIG_HOLE_ARRAY "hole_array"
	#define ZPOOL_CONFIG_VDEV_CHILDREN "vdev_children"
	#define ZPOOL_CONFIG_IS_HOLE "is_hole"
	#define ZPOOL_CONFIG_DDT_HISTOGRAM "ddt_histogram"
	#define ZPOOL_CONFIG_DDT_OBJ_STATS "ddt_object_stats"
	#define ZPOOL_CONFIG_DDT_STATS "ddt_stats"
	#define ZPOOL_CONFIG_SPLIT "splitcfg"
	#define ZPOOL_CONFIG_ORIG_GUID "orig_guid"
	#define ZPOOL_CONFIG_SPLIT_GUID "split_guid"
	#define ZPOOL_CONFIG_SPLIT_LIST "guid_list"
	#define ZPOOL_CONFIG_NONALLOCATING "non_allocating"
	#define ZPOOL_CONFIG_REMOVING "removing"
	#define ZPOOL_CONFIG_RESILVER_TXG "resilver_txg"
	#define ZPOOL_CONFIG_REBUILD_TXG "rebuild_txg"
	#define ZPOOL_CONFIG_COMMENT "comment"
	#define ZPOOL_CONFIG_SUSPENDED "suspended" /* not stored on disk */
	#define ZPOOL_CONFIG_SUSPENDED_REASON "suspended_reason" /* not stored */
	#define ZPOOL_CONFIG_TIMESTAMP "timestamp" /* not stored on disk */
	#define ZPOOL_CONFIG_BOOTFS "bootfs" /* not stored on disk */
	#define ZPOOL_CONFIG_MISSING_DEVICES "missing_vdevs" /* not stored on disk */
	#define ZPOOL_CONFIG_LOAD_INFO "load_info" /* not stored on disk */
	#define ZPOOL_CONFIG_REWIND_INFO "rewind_info" /* not stored on disk */
	#define ZPOOL_CONFIG_UNSUP_FEAT "unsup_feat" /* not stored on disk */
	#define ZPOOL_CONFIG_ENABLED_FEAT "enabled_feat" /* not stored on disk */
	#define ZPOOL_CONFIG_CAN_RDONLY "can_rdonly" /* not stored on disk */
	#define ZPOOL_CONFIG_FEATURES_FOR_READ "features_for_read"
	#define ZPOOL_CONFIG_FEATURE_STATS "feature_stats" /* not stored on disk */
	#define ZPOOL_CONFIG_ERRATA "errata" /* not stored on disk */
	#define ZPOOL_CONFIG_VDEV_ROOT_ZAP "com.klarasystems:vdev_zap_root"
	#define ZPOOL_CONFIG_VDEV_TOP_ZAP "com.delphix:vdev_zap_top"
	#define ZPOOL_CONFIG_VDEV_LEAF_ZAP "com.delphix:vdev_zap_leaf"
	#define ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS "com.delphix:has_per_vdev_zaps"
	#define ZPOOL_CONFIG_RESILVER_DEFER "com.datto:resilver_defer"
	#define ZPOOL_CONFIG_CACHEFILE "cachefile" /* not stored on disk */
	#define ZPOOL_CONFIG_MMP_STATE "mmp_state" /* not stored on disk */
	#define ZPOOL_CONFIG_MMP_TXG "mmp_txg" /* not stored on disk */
	#define ZPOOL_CONFIG_MMP_SEQ "mmp_seq" /* not stored on disk */
	#define ZPOOL_CONFIG_MMP_HOSTNAME "mmp_hostname" /* not stored on disk */
	#define ZPOOL_CONFIG_MMP_HOSTID "mmp_hostid" /* not stored on disk */
	#define ZPOOL_CONFIG_ALLOCATION_BIAS "alloc_bias" /* not stored on disk */
	#define ZPOOL_CONFIG_EXPANSION_TIME "expansion_time" /* not stored */
	#define ZPOOL_CONFIG_REBUILD_STATS "org.openzfs:rebuild_stats"
	#define ZPOOL_CONFIG_COMPATIBILITY "compatibility"

	/*
	* The persistent vdev state is stored as separate values rather than a single
	* 'vdev_state' entry. This is because a device can be in multiple states, such
	* as offline and degraded.
	*/
	#define ZPOOL_CONFIG_OFFLINE "offline"
	#define ZPOOL_CONFIG_FAULTED "faulted"
	#define ZPOOL_CONFIG_DEGRADED "degraded"
	#define ZPOOL_CONFIG_REMOVED "removed"
	#define ZPOOL_CONFIG_FRU "fru"
	#define ZPOOL_CONFIG_AUX_STATE "aux_state"

	/* Pool load policy parameters */
	#define ZPOOL_LOAD_POLICY "load-policy"
	#define ZPOOL_LOAD_REWIND_POLICY "load-rewind-policy"
	#define ZPOOL_LOAD_REQUEST_TXG "load-request-txg"
	#define ZPOOL_LOAD_META_THRESH "load-meta-thresh"
	#define ZPOOL_LOAD_DATA_THRESH "load-data-thresh"

	/* Rewind data discovered */
	#define ZPOOL_CONFIG_LOAD_TIME "rewind_txg_ts"
	#define ZPOOL_CONFIG_LOAD_META_ERRORS "verify_meta_errors"
	#define ZPOOL_CONFIG_LOAD_DATA_ERRORS "verify_data_errors"
	#define ZPOOL_CONFIG_REWIND_TIME "seconds_of_rewind"

	/* dRAID configuration */
	#define ZPOOL_CONFIG_DRAID_NDATA "draid_ndata"
	#define ZPOOL_CONFIG_DRAID_NSPARES "draid_nspares"
	#define ZPOOL_CONFIG_DRAID_NGROUPS "draid_ngroups"

	#define VDEV_TYPE_ROOT "root"
	#define VDEV_TYPE_MIRROR "mirror"
	#define VDEV_TYPE_REPLACING "replacing"
	#define VDEV_TYPE_RAIDZ "raidz"
	#define VDEV_TYPE_DRAID "draid"
	#define VDEV_TYPE_DRAID_SPARE "dspare"
	#define VDEV_TYPE_DISK "disk"
	#define VDEV_TYPE_FILE "file"
	#define VDEV_TYPE_MISSING "missing"
	#define VDEV_TYPE_HOLE "hole"
	#define VDEV_TYPE_SPARE "spare"
	#define VDEV_TYPE_LOG "log"
	#define VDEV_TYPE_L2CACHE "l2cache"
	#define VDEV_TYPE_INDIRECT "indirect"

	#define VDEV_RAIDZ_MAXPARITY 3

	#define VDEV_DRAID_MAXPARITY 3
	#define VDEV_DRAID_MIN_CHILDREN 2
	#define VDEV_DRAID_MAX_CHILDREN UINT8_MAX

	/* VDEV_TOP_ZAP_* are used in top-level vdev ZAP objects. */
	#define VDEV_TOP_ZAP_INDIRECT_OBSOLETE_SM \
	"com.delphix:indirect_obsolete_sm"
	#define VDEV_TOP_ZAP_OBSOLETE_COUNTS_ARE_PRECISE \
	"com.delphix:obsolete_counts_are_precise"
	#define VDEV_TOP_ZAP_POOL_CHECKPOINT_SM \
	"com.delphix:pool_checkpoint_sm"
	#define VDEV_TOP_ZAP_MS_UNFLUSHED_PHYS_TXGS \
	"com.delphix:ms_unflushed_phys_txgs"

	#define VDEV_TOP_ZAP_VDEV_REBUILD_PHYS \
	"org.openzfs:vdev_rebuild"

	#define VDEV_TOP_ZAP_ALLOCATION_BIAS \
	"org.zfsonlinux:allocation_bias"

	/* vdev metaslab allocation bias */
	#define VDEV_ALLOC_BIAS_LOG "log"
	#define VDEV_ALLOC_BIAS_SPECIAL "special"
	#define VDEV_ALLOC_BIAS_DEDUP "dedup"

	/* vdev initialize state */
	#define VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET \
	"com.delphix:next_offset_to_initialize"
	#define VDEV_LEAF_ZAP_INITIALIZE_STATE \
	"com.delphix:vdev_initialize_state"
	#define VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME \
	"com.delphix:vdev_initialize_action_time"

	/* vdev TRIM state */
	#define VDEV_LEAF_ZAP_TRIM_LAST_OFFSET \
	"org.zfsonlinux:next_offset_to_trim"
	#define VDEV_LEAF_ZAP_TRIM_STATE \
	"org.zfsonlinux:vdev_trim_state"
	#define VDEV_LEAF_ZAP_TRIM_ACTION_TIME \
	"org.zfsonlinux:vdev_trim_action_time"
	#define VDEV_LEAF_ZAP_TRIM_RATE \
	"org.zfsonlinux:vdev_trim_rate"
	#define VDEV_LEAF_ZAP_TRIM_PARTIAL \
	"org.zfsonlinux:vdev_trim_partial"
	#define VDEV_LEAF_ZAP_TRIM_SECURE \
	"org.zfsonlinux:vdev_trim_secure"

	/*
	* This is needed in userland to report the minimum necessary device size.
	*/
	#define SPA_MINDEVSIZE (64ULL << 20)

	/*
	* Set if the fragmentation has not yet been calculated. This can happen
	* because the space maps have not been upgraded or the histogram feature
	* is not enabled.
	*/
	#define ZFS_FRAG_INVALID UINT64_MAX

	/*
	* The location of the pool configuration repository, shared between kernel and
	* userland.
	*/
	#define ZPOOL_CACHE_BOOT "/boot/zfs/zpool.cache"
	#define ZPOOL_CACHE "/etc/zfs/zpool.cache"
	/*
	* Settings for zpool compatibility features files
	*/
	#define ZPOOL_SYSCONF_COMPAT_D SYSCONFDIR "/zfs/compatibility.d"
	#define ZPOOL_DATA_COMPAT_D PKGDATADIR "/compatibility.d"
	#define ZPOOL_COMPAT_MAXSIZE 16384

	/*
	* Hard-wired compatibility settings
	*/
	#define ZPOOL_COMPAT_LEGACY "legacy"
	#define ZPOOL_COMPAT_OFF "off"

	/*
	* vdev states are ordered from least to most healthy.
	* A vdev that's CANT_OPEN or below is considered unusable.
	*/
	typedef enum vdev_state {
	VDEV_STATE_UNKNOWN = 0, /* Uninitialized vdev */
	VDEV_STATE_CLOSED, /* Not currently open */
	VDEV_STATE_OFFLINE, /* Not allowed to open */
	VDEV_STATE_REMOVED, /* Explicitly removed from system */
	VDEV_STATE_CANT_OPEN, /* Tried to open, but failed */
	VDEV_STATE_FAULTED, /* External request to fault device */
	VDEV_STATE_DEGRADED, /* Replicated vdev with unhealthy kids */
	VDEV_STATE_HEALTHY /* Presumed good */
	} vdev_state_t;

	#define VDEV_STATE_ONLINE VDEV_STATE_HEALTHY

	/*
	* vdev aux states. When a vdev is in the CANT_OPEN state, the aux field
	* of the vdev stats structure uses these constants to distinguish why.
	*/
	typedef enum vdev_aux {
	VDEV_AUX_NONE, /* no error */
	VDEV_AUX_OPEN_FAILED, /* ldi_open_() or vn_open() failed /
	VDEV_AUX_CORRUPT_DATA, /* bad label or disk contents */
	VDEV_AUX_NO_REPLICAS, /* insufficient number of replicas */
	VDEV_AUX_BAD_GUID_SUM, /* vdev guid sum doesn't match */
	VDEV_AUX_TOO_SMALL, /* vdev size is too small */
	VDEV_AUX_BAD_LABEL, /* the label is OK but invalid */
	VDEV_AUX_VERSION_NEWER, /* on-disk version is too new */
	VDEV_AUX_VERSION_OLDER, /* on-disk version is too old */
	VDEV_AUX_UNSUP_FEAT, /* unsupported features */
	VDEV_AUX_SPARED, /* hot spare used in another pool */
	VDEV_AUX_ERR_EXCEEDED, /* too many errors */
	VDEV_AUX_IO_FAILURE, /* experienced I/O failure */
	VDEV_AUX_BAD_LOG, /* cannot read log chain(s) */
	VDEV_AUX_EXTERNAL, /* external diagnosis or forced fault */
	VDEV_AUX_SPLIT_POOL, /* vdev was split off into another pool */
	VDEV_AUX_BAD_ASHIFT, /* vdev ashift is invalid */
	VDEV_AUX_EXTERNAL_PERSIST, /* persistent forced fault */
	VDEV_AUX_ACTIVE, /* vdev active on a different host */
	VDEV_AUX_CHILDREN_OFFLINE, /* all children are offline */
	VDEV_AUX_ASHIFT_TOO_BIG, /* vdev's min block size is too large */
	} vdev_aux_t;

	/*
	* pool state. The following states are written to disk as part of the normal
	* SPA lifecycle: ACTIVE, EXPORTED, DESTROYED, SPARE, L2CACHE. The remaining
	* states are software abstractions used at various levels to communicate
	* pool state.
	*/
	typedef enum pool_state {
	POOL_STATE_ACTIVE = 0, /* In active use */
	POOL_STATE_EXPORTED, /* Explicitly exported */
	POOL_STATE_DESTROYED, /* Explicitly destroyed */
	POOL_STATE_SPARE, /* Reserved for hot spare use */
	POOL_STATE_L2CACHE, /* Level 2 ARC device */
	POOL_STATE_UNINITIALIZED, /* Internal spa_t state */
	POOL_STATE_UNAVAIL, /* Internal libzfs state */
	POOL_STATE_POTENTIALLY_ACTIVE /* Internal libzfs state */
	} pool_state_t;

	/*
	* mmp state. The following states provide additional detail describing
	* why a pool couldn't be safely imported.
	*/
	typedef enum mmp_state {
	MMP_STATE_ACTIVE = 0, /* In active use */
	MMP_STATE_INACTIVE, /* Inactive and safe to import */
	MMP_STATE_NO_HOSTID /* System hostid is not set */
	} mmp_state_t;

	/*
	* Scan Functions.
	*/
	typedef enum pool_scan_func {
	POOL_SCAN_NONE,
	POOL_SCAN_SCRUB,
	POOL_SCAN_RESILVER,
	POOL_SCAN_ERRORSCRUB,
	POOL_SCAN_FUNCS
	} pool_scan_func_t;

	/*
	* Used to control scrub pause and resume.
	*/
	typedef enum pool_scrub_cmd {
	POOL_SCRUB_NORMAL = 0,
	POOL_SCRUB_PAUSE,
	POOL_SCRUB_FLAGS_END
	} pool_scrub_cmd_t;

	typedef enum {
	CS_NONE,
	CS_CHECKPOINT_EXISTS,
	CS_CHECKPOINT_DISCARDING,
	CS_NUM_STATES
	} checkpoint_state_t;

	typedef struct pool_checkpoint_stat {
	uint64_t pcs_state; /* checkpoint_state_t */
	uint64_t pcs_start_time; /* time checkpoint/discard started */
	uint64_t pcs_space; /* checkpointed space */
	} pool_checkpoint_stat_t;

	/*
	* ZIO types. Needed to interpret vdev statistics below.
	*/
	typedef enum zio_type {
	ZIO_TYPE_NULL = 0,
	ZIO_TYPE_READ,
	ZIO_TYPE_WRITE,
	ZIO_TYPE_FREE,
	ZIO_TYPE_CLAIM,
	ZIO_TYPE_IOCTL,
	ZIO_TYPE_TRIM,
	ZIO_TYPES
	} zio_type_t;

	/*
	* Pool statistics. Note: all fields should be 64-bit because this
	* is passed between kernel and userland as an nvlist uint64 array.
	*/
	typedef struct pool_scan_stat {
	/* values stored on disk */
	uint64_t pss_func; /* pool_scan_func_t */
	uint64_t pss_state; /* dsl_scan_state_t */
	uint64_t pss_start_time; /* scan start time */
	uint64_t pss_end_time; /* scan end time */
	uint64_t pss_to_examine; /* total bytes to scan */
	uint64_t pss_examined; /* total bytes located by scanner */
	uint64_t pss_skipped; /* total bytes skipped by scanner */
	uint64_t pss_processed; /* total processed bytes */
	uint64_t pss_errors; /* scan errors */

	/* values not stored on disk */
	uint64_t pss_pass_exam; /* examined bytes per scan pass */
	uint64_t pss_pass_start; /* start time of a scan pass */
	uint64_t pss_pass_scrub_pause; /* pause time of a scrub pass */
	/* cumulative time scrub spent paused, needed for rate calculation */
	uint64_t pss_pass_scrub_spent_paused;
	uint64_t pss_pass_issued; /* issued bytes per scan pass */
	uint64_t pss_issued; /* total bytes checked by scanner */

	/* error scrub values stored on disk */
	uint64_t pss_error_scrub_func; /* pool_scan_func_t */
	uint64_t pss_error_scrub_state; /* dsl_scan_state_t */
	uint64_t pss_error_scrub_start; /* error scrub start time */
	uint64_t pss_error_scrub_end; /* error scrub end time */
	uint64_t pss_error_scrub_examined; /* error blocks issued I/O */
	/* error blocks to be issued I/O */
	uint64_t pss_error_scrub_to_be_examined;

	/* error scrub values not stored on disk */
	/* error scrub pause time in milliseconds */
	uint64_t pss_pass_error_scrub_pause;

	} pool_scan_stat_t;

	typedef struct pool_removal_stat {
	uint64_t prs_state; /* dsl_scan_state_t */
	uint64_t prs_removing_vdev;
	uint64_t prs_start_time;
	uint64_t prs_end_time;
	uint64_t prs_to_copy; /* bytes that need to be copied */
	uint64_t prs_copied; /* bytes copied so far */
	/*
	* bytes of memory used for indirect mappings.
	* This includes all removed vdevs.
	*/
	uint64_t prs_mapping_memory;
	} pool_removal_stat_t;

	typedef enum dsl_scan_state {
	DSS_NONE,
	DSS_SCANNING,
	DSS_FINISHED,
	DSS_CANCELED,
	DSS_ERRORSCRUBBING,
	DSS_NUM_STATES
	} dsl_scan_state_t;

	typedef struct vdev_rebuild_stat {
	uint64_t vrs_state; /* vdev_rebuild_state_t */
	uint64_t vrs_start_time; /* time_t */
	uint64_t vrs_end_time; /* time_t */
	uint64_t vrs_scan_time_ms; /* total run time (millisecs) */
	uint64_t vrs_bytes_scanned; /* allocated bytes scanned */
	uint64_t vrs_bytes_issued; /* read bytes issued */
	uint64_t vrs_bytes_rebuilt; /* rebuilt bytes */
	uint64_t vrs_bytes_est; /* total bytes to scan */
	uint64_t vrs_errors; /* scanning errors */
	uint64_t vrs_pass_time_ms; /* pass run time (millisecs) */
	uint64_t vrs_pass_bytes_scanned; /* bytes scanned since start/resume */
	uint64_t vrs_pass_bytes_issued; /* bytes rebuilt since start/resume */
	uint64_t vrs_pass_bytes_skipped; /* bytes skipped since start/resume */
	} vdev_rebuild_stat_t;

	/*
	* Errata described by https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-ER.
	* The ordering of this enum must be maintained to ensure the errata identifiers
	* map to the correct documentation. New errata may only be appended to the
	* list and must contain corresponding documentation at the above link.
	*/
	typedef enum zpool_errata {
	ZPOOL_ERRATA_NONE,
	ZPOOL_ERRATA_ZOL_2094_SCRUB,
	ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY,
	ZPOOL_ERRATA_ZOL_6845_ENCRYPTION,
	ZPOOL_ERRATA_ZOL_8308_ENCRYPTION,
	} zpool_errata_t;

	/*
	* Vdev statistics. Note: all fields should be 64-bit because this
	* is passed between kernel and user land as an nvlist uint64 array.
	*
	* The vs_ops[] and vs_bytes[] arrays must always be an array size of 6 in
	* order to keep subsequent members at their known fixed offsets. When
	* adding a new field it must be added to the end the structure.
	*/
	#define VS_ZIO_TYPES 6

	typedef struct vdev_stat {
	hrtime_t vs_timestamp; /* time since vdev load */
	uint64_t vs_state; /* vdev state */
	uint64_t vs_aux; /* see vdev_aux_t */
	uint64_t vs_alloc; /* space allocated */
	uint64_t vs_space; /* total capacity */
	uint64_t vs_dspace; /* deflated capacity */
	uint64_t vs_rsize; /* replaceable dev size */
	uint64_t vs_esize; /* expandable dev size */
	uint64_t vs_ops[VS_ZIO_TYPES]; /* operation count */
	uint64_t vs_bytes[VS_ZIO_TYPES]; /* bytes read/written */
	uint64_t vs_read_errors; /* read errors */
	uint64_t vs_write_errors; /* write errors */
	uint64_t vs_checksum_errors; /* checksum errors */
	uint64_t vs_initialize_errors; /* initializing errors */
	uint64_t vs_self_healed; /* self-healed bytes */
	uint64_t vs_scan_removing; /* removing? */
	uint64_t vs_scan_processed; /* scan processed bytes */
	uint64_t vs_fragmentation; /* device fragmentation */
	uint64_t vs_initialize_bytes_done; /* bytes initialized */
	uint64_t vs_initialize_bytes_est; /* total bytes to initialize */
	uint64_t vs_initialize_state; /* vdev_initializing_state_t */
	uint64_t vs_initialize_action_time; /* time_t */
	uint64_t vs_checkpoint_space; /* checkpoint-consumed space */
	uint64_t vs_resilver_deferred; /* resilver deferred */
	uint64_t vs_slow_ios; /* slow IOs */
	uint64_t vs_trim_errors; /* trimming errors */
	uint64_t vs_trim_notsup; /* supported by device */
	uint64_t vs_trim_bytes_done; /* bytes trimmed */
	uint64_t vs_trim_bytes_est; /* total bytes to trim */
	uint64_t vs_trim_state; /* vdev_trim_state_t */
	uint64_t vs_trim_action_time; /* time_t */
	uint64_t vs_rebuild_processed; /* bytes rebuilt */
	uint64_t vs_configured_ashift; /* TLV vdev_ashift */
	uint64_t vs_logical_ashift; /* vdev_logical_ashift */
	uint64_t vs_physical_ashift; /* vdev_physical_ashift */
	uint64_t vs_noalloc; /* allocations halted? */
	uint64_t vs_pspace; /* physical capacity */
	} vdev_stat_t;

	#define VDEV_STAT_VALID(field, uint64_t_field_count) \
	((uint64_t_field_count * sizeof (uint64_t)) >= \
	(offsetof(vdev_stat_t, field) + sizeof (((vdev_stat_t *)NULL)->field)))

	/*
	* Extended stats
	*
	* These are stats which aren't included in the original iostat output. For
	* convenience, they are grouped together in vdev_stat_ex, although each stat
	* is individually exported as an nvlist.
	*/
	typedef struct vdev_stat_ex {
	/* Number of ZIOs issued to disk and waiting to finish */
	uint64_t vsx_active_queue[ZIO_PRIORITY_NUM_QUEUEABLE];

	/* Number of ZIOs pending to be issued to disk */
	uint64_t vsx_pend_queue[ZIO_PRIORITY_NUM_QUEUEABLE];

	/*
	* Below are the histograms for various latencies. Buckets are in
	* units of nanoseconds.
	*/

	/*
	* 2^37 nanoseconds = 134s. Timeouts will probably start kicking in
	* before this.
	*/
	#define VDEV_L_HISTO_BUCKETS 37 /* Latency histo buckets */
	#define VDEV_RQ_HISTO_BUCKETS 25 /* Request size histo buckets */

	/* Amount of time in ZIO queue (ns) */
	uint64_t vsx_queue_histo[ZIO_PRIORITY_NUM_QUEUEABLE]
	[VDEV_L_HISTO_BUCKETS];

	/* Total ZIO latency (ns). Includes queuing and disk access time */
	uint64_t vsx_total_histo[ZIO_TYPES][VDEV_L_HISTO_BUCKETS];

	/* Amount of time to read/write the disk (ns) */
	uint64_t vsx_disk_histo[ZIO_TYPES][VDEV_L_HISTO_BUCKETS];

	/* "lookup the bucket for a value" histogram macros */
	#define HISTO(val, buckets) (val != 0 ? MIN(highbit64(val) - 1, \
	buckets - 1) : 0)
	#define L_HISTO(a) HISTO(a, VDEV_L_HISTO_BUCKETS)
	#define RQ_HISTO(a) HISTO(a, VDEV_RQ_HISTO_BUCKETS)

	/* Physical IO histogram */
	uint64_t vsx_ind_histo[ZIO_PRIORITY_NUM_QUEUEABLE]
	[VDEV_RQ_HISTO_BUCKETS];

	/* Delegated (aggregated) physical IO histogram */
	uint64_t vsx_agg_histo[ZIO_PRIORITY_NUM_QUEUEABLE]
	[VDEV_RQ_HISTO_BUCKETS];

	} vdev_stat_ex_t;

	/*
	* Initialize functions.
	*/
	typedef enum pool_initialize_func {
	POOL_INITIALIZE_START,
	POOL_INITIALIZE_CANCEL,
	POOL_INITIALIZE_SUSPEND,
	POOL_INITIALIZE_UNINIT,
	POOL_INITIALIZE_FUNCS
	} pool_initialize_func_t;

	/*
	* TRIM functions.
	*/
	typedef enum pool_trim_func {
	POOL_TRIM_START,
	POOL_TRIM_CANCEL,
	POOL_TRIM_SUSPEND,
	POOL_TRIM_FUNCS
	} pool_trim_func_t;

	/*
	* DDT statistics. Note: all fields should be 64-bit because this
	* is passed between kernel and userland as an nvlist uint64 array.
	*/
	typedef struct ddt_object {
	uint64_t ddo_count; /* number of elements in ddt */
	uint64_t ddo_dspace; /* size of ddt on disk */
	uint64_t ddo_mspace; /* size of ddt in-core */
	} ddt_object_t;

	typedef struct ddt_stat {
	uint64_t dds_blocks; /* blocks */
	uint64_t dds_lsize; /* logical size */
	uint64_t dds_psize; /* physical size */
	uint64_t dds_dsize; /* deflated allocated size */
	uint64_t dds_ref_blocks; /* referenced blocks */
	uint64_t dds_ref_lsize; /* referenced lsize * refcnt */
	uint64_t dds_ref_psize; /* referenced psize * refcnt */
	uint64_t dds_ref_dsize; /* referenced dsize * refcnt */
	} ddt_stat_t;

	typedef struct ddt_histogram {
	ddt_stat_t ddh_stat[64]; /* power-of-two histogram buckets */
	} ddt_histogram_t;

	#define ZVOL_DRIVER "zvol"
	#define ZFS_DRIVER "zfs"
	#define ZFS_DEV "/dev/zfs"
	#define ZFS_DEVDIR "/dev"

	#define ZFS_SUPER_MAGIC 0x2fc12fc1

	/* general zvol path */
	#define ZVOL_DIR "/dev/zvol/"

	#define ZVOL_MAJOR 230
	#define ZVOL_MINOR_BITS 4
	#define ZVOL_MINOR_MASK ((1U << ZVOL_MINOR_BITS) - 1)
	#define ZVOL_MINORS (1 << 4)
	#define ZVOL_DEV_NAME "zd"

	#define ZVOL_PROP_NAME "name"
	#define ZVOL_DEFAULT_BLOCKSIZE 16384

	typedef enum {
	VDEV_INITIALIZE_NONE,
	VDEV_INITIALIZE_ACTIVE,
	VDEV_INITIALIZE_CANCELED,
	VDEV_INITIALIZE_SUSPENDED,
	VDEV_INITIALIZE_COMPLETE
	} vdev_initializing_state_t;

	typedef enum {
	VDEV_TRIM_NONE,
	VDEV_TRIM_ACTIVE,
	VDEV_TRIM_CANCELED,
	VDEV_TRIM_SUSPENDED,
	VDEV_TRIM_COMPLETE,
	} vdev_trim_state_t;

	typedef enum {
	VDEV_REBUILD_NONE,
	VDEV_REBUILD_ACTIVE,
	VDEV_REBUILD_CANCELED,
	VDEV_REBUILD_COMPLETE,
	} vdev_rebuild_state_t;

	/*
	* nvlist name constants. Facilitate restricting snapshot iteration range for
	* the "list next snapshot" ioctl
	*/
	#define SNAP_ITER_MIN_TXG "snap_iter_min_txg"
	#define SNAP_ITER_MAX_TXG "snap_iter_max_txg"

	/*
	* /dev/zfs ioctl numbers.
	*
	* These numbers cannot change over time. New ioctl numbers must be appended.
	*/
	typedef enum zfs_ioc {
	/*
	* Core features - 88/128 numbers reserved.
	*/
	#ifdef __FreeBSD__
	ZFS_IOC_FIRST = 0,
	#else
	ZFS_IOC_FIRST = ('Z' << 8),
	#endif
	ZFS_IOC = ZFS_IOC_FIRST,
	ZFS_IOC_POOL_CREATE = ZFS_IOC_FIRST, /* 0x5a00 */
	ZFS_IOC_POOL_DESTROY, /* 0x5a01 */
	ZFS_IOC_POOL_IMPORT, /* 0x5a02 */
	ZFS_IOC_POOL_EXPORT, /* 0x5a03 */
	ZFS_IOC_POOL_CONFIGS, /* 0x5a04 */
	ZFS_IOC_POOL_STATS, /* 0x5a05 */
	ZFS_IOC_POOL_TRYIMPORT, /* 0x5a06 */
	ZFS_IOC_POOL_SCAN, /* 0x5a07 */
	ZFS_IOC_POOL_FREEZE, /* 0x5a08 */
	ZFS_IOC_POOL_UPGRADE, /* 0x5a09 */
	ZFS_IOC_POOL_GET_HISTORY, /* 0x5a0a */
	ZFS_IOC_VDEV_ADD, /* 0x5a0b */
	ZFS_IOC_VDEV_REMOVE, /* 0x5a0c */
	ZFS_IOC_VDEV_SET_STATE, /* 0x5a0d */
	ZFS_IOC_VDEV_ATTACH, /* 0x5a0e */
	ZFS_IOC_VDEV_DETACH, /* 0x5a0f */
	ZFS_IOC_VDEV_SETPATH, /* 0x5a10 */
	ZFS_IOC_VDEV_SETFRU, /* 0x5a11 */
	ZFS_IOC_OBJSET_STATS, /* 0x5a12 */
	ZFS_IOC_OBJSET_ZPLPROPS, /* 0x5a13 */
	ZFS_IOC_DATASET_LIST_NEXT, /* 0x5a14 */
	ZFS_IOC_SNAPSHOT_LIST_NEXT, /* 0x5a15 */
	ZFS_IOC_SET_PROP, /* 0x5a16 */
	ZFS_IOC_CREATE, /* 0x5a17 */
	ZFS_IOC_DESTROY, /* 0x5a18 */
	ZFS_IOC_ROLLBACK, /* 0x5a19 */
	ZFS_IOC_RENAME, /* 0x5a1a */
	ZFS_IOC_RECV, /* 0x5a1b */
	ZFS_IOC_SEND, /* 0x5a1c */
	ZFS_IOC_INJECT_FAULT, /* 0x5a1d */
	ZFS_IOC_CLEAR_FAULT, /* 0x5a1e */
	ZFS_IOC_INJECT_LIST_NEXT, /* 0x5a1f */
	ZFS_IOC_ERROR_LOG, /* 0x5a20 */
	ZFS_IOC_CLEAR, /* 0x5a21 */
	ZFS_IOC_PROMOTE, /* 0x5a22 */
	ZFS_IOC_SNAPSHOT, /* 0x5a23 */
	ZFS_IOC_DSOBJ_TO_DSNAME, /* 0x5a24 */
	ZFS_IOC_OBJ_TO_PATH, /* 0x5a25 */
	ZFS_IOC_POOL_SET_PROPS, /* 0x5a26 */
	ZFS_IOC_POOL_GET_PROPS, /* 0x5a27 */
	ZFS_IOC_SET_FSACL, /* 0x5a28 */
	ZFS_IOC_GET_FSACL, /* 0x5a29 */
	ZFS_IOC_SHARE, /* 0x5a2a */
	ZFS_IOC_INHERIT_PROP, /* 0x5a2b */
	ZFS_IOC_SMB_ACL, /* 0x5a2c */
	ZFS_IOC_USERSPACE_ONE, /* 0x5a2d */
	ZFS_IOC_USERSPACE_MANY, /* 0x5a2e */
	ZFS_IOC_USERSPACE_UPGRADE, /* 0x5a2f */
	ZFS_IOC_HOLD, /* 0x5a30 */
	ZFS_IOC_RELEASE, /* 0x5a31 */
	ZFS_IOC_GET_HOLDS, /* 0x5a32 */
	ZFS_IOC_OBJSET_RECVD_PROPS, /* 0x5a33 */
	ZFS_IOC_VDEV_SPLIT, /* 0x5a34 */
	ZFS_IOC_NEXT_OBJ, /* 0x5a35 */
	ZFS_IOC_DIFF, /* 0x5a36 */
	ZFS_IOC_TMP_SNAPSHOT, /* 0x5a37 */
	ZFS_IOC_OBJ_TO_STATS, /* 0x5a38 */
	ZFS_IOC_SPACE_WRITTEN, /* 0x5a39 */
	ZFS_IOC_SPACE_SNAPS, /* 0x5a3a */
	ZFS_IOC_DESTROY_SNAPS, /* 0x5a3b */
	ZFS_IOC_POOL_REGUID, /* 0x5a3c */
	ZFS_IOC_POOL_REOPEN, /* 0x5a3d */
	ZFS_IOC_SEND_PROGRESS, /* 0x5a3e */
	ZFS_IOC_LOG_HISTORY, /* 0x5a3f */
	ZFS_IOC_SEND_NEW, /* 0x5a40 */
	ZFS_IOC_SEND_SPACE, /* 0x5a41 */
	ZFS_IOC_CLONE, /* 0x5a42 */
	ZFS_IOC_BOOKMARK, /* 0x5a43 */
	ZFS_IOC_GET_BOOKMARKS, /* 0x5a44 */
	ZFS_IOC_DESTROY_BOOKMARKS, /* 0x5a45 */
	ZFS_IOC_RECV_NEW, /* 0x5a46 */
	ZFS_IOC_POOL_SYNC, /* 0x5a47 */
	ZFS_IOC_CHANNEL_PROGRAM, /* 0x5a48 */
	ZFS_IOC_LOAD_KEY, /* 0x5a49 */
	ZFS_IOC_UNLOAD_KEY, /* 0x5a4a */
	ZFS_IOC_CHANGE_KEY, /* 0x5a4b */
	ZFS_IOC_REMAP, /* 0x5a4c */
	ZFS_IOC_POOL_CHECKPOINT, /* 0x5a4d */
	ZFS_IOC_POOL_DISCARD_CHECKPOINT, /* 0x5a4e */
	ZFS_IOC_POOL_INITIALIZE, /* 0x5a4f */
	ZFS_IOC_POOL_TRIM, /* 0x5a50 */
	ZFS_IOC_REDACT, /* 0x5a51 */
	ZFS_IOC_GET_BOOKMARK_PROPS, /* 0x5a52 */
	ZFS_IOC_WAIT, /* 0x5a53 */
	ZFS_IOC_WAIT_FS, /* 0x5a54 */
	ZFS_IOC_VDEV_GET_PROPS, /* 0x5a55 */
	ZFS_IOC_VDEV_SET_PROPS, /* 0x5a56 */
	ZFS_IOC_POOL_SCRUB, /* 0x5a57 */

	/*
	* Per-platform (Optional) - 8/128 numbers reserved.
	*/
	ZFS_IOC_PLATFORM = ZFS_IOC_FIRST + 0x80,
	ZFS_IOC_EVENTS_NEXT, /* 0x81 (Linux) */
	ZFS_IOC_EVENTS_CLEAR, /* 0x82 (Linux) */
	ZFS_IOC_EVENTS_SEEK, /* 0x83 (Linux) */
	ZFS_IOC_NEXTBOOT, /* 0x84 (FreeBSD) */
	ZFS_IOC_JAIL, /* 0x85 (FreeBSD) */
	ZFS_IOC_USERNS_ATTACH = ZFS_IOC_JAIL, /* 0x85 (Linux) */
	ZFS_IOC_UNJAIL, /* 0x86 (FreeBSD) */
	ZFS_IOC_USERNS_DETACH = ZFS_IOC_UNJAIL, /* 0x86 (Linux) */
	ZFS_IOC_SET_BOOTENV, /* 0x87 */
	ZFS_IOC_GET_BOOTENV, /* 0x88 */
	ZFS_IOC_LAST
	} zfs_ioc_t;

	/*
	* zvol ioctl to get dataset name
	*/
	#define BLKZNAME _IOR(0x12, 125, char[ZFS_MAX_DATASET_NAME_LEN])

	#ifdef __linux__

	/*
	* IOCTLs to update and retrieve additional file level attributes on
	* Linux.
	*/
	#define ZFS_IOC_GETDOSFLAGS _IOR(0x83, 1, uint64_t)
	#define ZFS_IOC_SETDOSFLAGS _IOW(0x83, 2, uint64_t)

	/*
	* Additional file level attributes, that are stored
	* in the upper half of z_pflags
	*/
	#define ZFS_READONLY 0x0000000100000000ull
	#define ZFS_HIDDEN 0x0000000200000000ull
	#define ZFS_SYSTEM 0x0000000400000000ull
	#define ZFS_ARCHIVE 0x0000000800000000ull
	#define ZFS_IMMUTABLE 0x0000001000000000ull
	#define ZFS_NOUNLINK 0x0000002000000000ull
	#define ZFS_APPENDONLY 0x0000004000000000ull
	#define ZFS_NODUMP 0x0000008000000000ull
	#define ZFS_OPAQUE 0x0000010000000000ull
	#define ZFS_AV_QUARANTINED 0x0000020000000000ull
	#define ZFS_AV_MODIFIED 0x0000040000000000ull
	#define ZFS_REPARSE 0x0000080000000000ull
	#define ZFS_OFFLINE 0x0000100000000000ull
	#define ZFS_SPARSE 0x0000200000000000ull

	#define ZFS_DOS_FL_USER_VISIBLE (ZFS_IMMUTABLE \| ZFS_APPENDONLY \| \
	ZFS_NOUNLINK \| ZFS_ARCHIVE \| ZFS_NODUMP \| ZFS_SYSTEM \| \
	ZFS_HIDDEN \| ZFS_READONLY \| ZFS_REPARSE \| ZFS_OFFLINE \| \
	ZFS_SPARSE)

	#endif

	/*
	* ZFS-specific error codes used for returning descriptive errors
	* to the userland through zfs ioctls.
	*
	* The enum implicitly includes all the error codes from errno.h.
	* New code should use and extend this enum for errors that are
	* not described precisely by generic errno codes.
	*
	* These numbers should not change over time. New entries should be appended.
	*
	* (Keep in sync with contrib/pyzfs/libzfs_core/_constants.py)
	*/
	typedef enum {
	ZFS_ERR_CHECKPOINT_EXISTS = 1024,
	ZFS_ERR_DISCARDING_CHECKPOINT,
	ZFS_ERR_NO_CHECKPOINT,
	ZFS_ERR_DEVRM_IN_PROGRESS,
	ZFS_ERR_VDEV_TOO_BIG,
	ZFS_ERR_IOC_CMD_UNAVAIL,
	ZFS_ERR_IOC_ARG_UNAVAIL,
	ZFS_ERR_IOC_ARG_REQUIRED,
	ZFS_ERR_IOC_ARG_BADTYPE,
	ZFS_ERR_WRONG_PARENT,
	ZFS_ERR_FROM_IVSET_GUID_MISSING,
	ZFS_ERR_FROM_IVSET_GUID_MISMATCH,
	ZFS_ERR_SPILL_BLOCK_FLAG_MISSING,
	ZFS_ERR_UNKNOWN_SEND_STREAM_FEATURE,
	ZFS_ERR_EXPORT_IN_PROGRESS,
	ZFS_ERR_BOOKMARK_SOURCE_NOT_ANCESTOR,
	ZFS_ERR_STREAM_TRUNCATED,
	ZFS_ERR_STREAM_LARGE_BLOCK_MISMATCH,
	ZFS_ERR_RESILVER_IN_PROGRESS,
	ZFS_ERR_REBUILD_IN_PROGRESS,
	ZFS_ERR_BADPROP,
	ZFS_ERR_VDEV_NOTSUP,
	ZFS_ERR_NOT_USER_NAMESPACE,
	ZFS_ERR_RESUME_EXISTS,
	ZFS_ERR_CRYPTO_NOTSUP,
	+ ZFS_ERR_RAIDZ_EXPAND_IN_PROGRESS,
	+ ZFS_ERR_ASHIFT_MISMATCH,
	} zfs_errno_t;

	/*
	* Internal SPA load state. Used by FMA diagnosis engine.
	*/
	typedef enum {
	SPA_LOAD_NONE, /* no load in progress */
	SPA_LOAD_OPEN, /* normal open */
	SPA_LOAD_IMPORT, /* import in progress */
	SPA_LOAD_TRYIMPORT, /* tryimport in progress */
	SPA_LOAD_RECOVER, /* recovery requested */
	SPA_LOAD_ERROR, /* load failed */
	SPA_LOAD_CREATE /* creation in progress */
	} spa_load_state_t;

	typedef enum {
	ZPOOL_WAIT_CKPT_DISCARD,
	ZPOOL_WAIT_FREE,
	ZPOOL_WAIT_INITIALIZE,
	ZPOOL_WAIT_REPLACE,
	ZPOOL_WAIT_REMOVE,
	ZPOOL_WAIT_RESILVER,
	ZPOOL_WAIT_SCRUB,
	ZPOOL_WAIT_TRIM,
	ZPOOL_WAIT_NUM_ACTIVITIES
	} zpool_wait_activity_t;

	typedef enum {
	ZFS_WAIT_DELETEQ,
	ZFS_WAIT_NUM_ACTIVITIES
	} zfs_wait_activity_t;

	/*
	* Bookmark name values.
	*/
	#define ZPOOL_ERR_LIST "error list"
	#define ZPOOL_ERR_DATASET "dataset"
	#define ZPOOL_ERR_OBJECT "object"

	#define HIS_MAX_RECORD_LEN (MAXPATHLEN + MAXPATHLEN + 1)

	/*
	* The following are names used in the nvlist describing
	* the pool's history log.
	*/
	#define ZPOOL_HIST_RECORD "history record"
	#define ZPOOL_HIST_TIME "history time"
	#define ZPOOL_HIST_CMD "history command"
	#define ZPOOL_HIST_WHO "history who"
	#define ZPOOL_HIST_ZONE "history zone"
	#define ZPOOL_HIST_HOST "history hostname"
	#define ZPOOL_HIST_TXG "history txg"
	#define ZPOOL_HIST_INT_EVENT "history internal event"
	#define ZPOOL_HIST_INT_STR "history internal str"
	#define ZPOOL_HIST_INT_NAME "internal_name"
	#define ZPOOL_HIST_IOCTL "ioctl"
	#define ZPOOL_HIST_INPUT_NVL "in_nvl"
	#define ZPOOL_HIST_OUTPUT_NVL "out_nvl"
	#define ZPOOL_HIST_OUTPUT_SIZE "out_size"
	#define ZPOOL_HIST_DSNAME "dsname"
	#define ZPOOL_HIST_DSID "dsid"
	#define ZPOOL_HIST_ERRNO "errno"
	#define ZPOOL_HIST_ELAPSED_NS "elapsed_ns"

	/*
	* Special nvlist name that will not have its args recorded in the pool's
	* history log.
	*/
	#define ZPOOL_HIDDEN_ARGS "hidden_args"

	/*
	* The following are names used when invoking ZFS_IOC_POOL_INITIALIZE.
	*/
	#define ZPOOL_INITIALIZE_COMMAND "initialize_command"
	#define ZPOOL_INITIALIZE_VDEVS "initialize_vdevs"

	/*
	* The following are names used when invoking ZFS_IOC_POOL_TRIM.
	*/
	#define ZPOOL_TRIM_COMMAND "trim_command"
	#define ZPOOL_TRIM_VDEVS "trim_vdevs"
	#define ZPOOL_TRIM_RATE "trim_rate"
	#define ZPOOL_TRIM_SECURE "trim_secure"

	/*
	* The following are names used when invoking ZFS_IOC_POOL_WAIT.
	*/
	#define ZPOOL_WAIT_ACTIVITY "wait_activity"
	#define ZPOOL_WAIT_TAG "wait_tag"
	#define ZPOOL_WAIT_WAITED "wait_waited"

	/*
	* The following are names used when invoking ZFS_IOC_VDEV_GET_PROP.
	*/
	#define ZPOOL_VDEV_PROPS_GET_VDEV "vdevprops_get_vdev"
	#define ZPOOL_VDEV_PROPS_GET_PROPS "vdevprops_get_props"

	/*
	* The following are names used when invoking ZFS_IOC_VDEV_SET_PROP.
	*/
	#define ZPOOL_VDEV_PROPS_SET_VDEV "vdevprops_set_vdev"
	#define ZPOOL_VDEV_PROPS_SET_PROPS "vdevprops_set_props"

	/*
	* The following are names used when invoking ZFS_IOC_WAIT_FS.
	*/
	#define ZFS_WAIT_ACTIVITY "wait_activity"
	#define ZFS_WAIT_WAITED "wait_waited"

	/*
	* Flags for ZFS_IOC_VDEV_SET_STATE
	*/
	#define ZFS_ONLINE_CHECKREMOVE 0x1
	#define ZFS_ONLINE_UNSPARE 0x2
	#define ZFS_ONLINE_FORCEFAULT 0x4
	#define ZFS_ONLINE_EXPAND 0x8
	#define ZFS_ONLINE_SPARE 0x10
	#define ZFS_OFFLINE_TEMPORARY 0x1

	/*
	* Flags for ZFS_IOC_POOL_IMPORT
	*/
	#define ZFS_IMPORT_NORMAL 0x0
	#define ZFS_IMPORT_VERBATIM 0x1
	#define ZFS_IMPORT_ANY_HOST 0x2
	#define ZFS_IMPORT_MISSING_LOG 0x4
	#define ZFS_IMPORT_ONLY 0x8
	#define ZFS_IMPORT_TEMP_NAME 0x10
	#define ZFS_IMPORT_SKIP_MMP 0x20
	#define ZFS_IMPORT_LOAD_KEYS 0x40
	#define ZFS_IMPORT_CHECKPOINT 0x80

	/*
	* Channel program argument/return nvlist keys and defaults.
	*/
	#define ZCP_ARG_PROGRAM "program"
	#define ZCP_ARG_ARGLIST "arg"
	#define ZCP_ARG_SYNC "sync"
	#define ZCP_ARG_INSTRLIMIT "instrlimit"
	#define ZCP_ARG_MEMLIMIT "memlimit"

	#define ZCP_ARG_CLIARGV "argv"

	#define ZCP_RET_ERROR "error"
	#define ZCP_RET_RETURN "return"

	#define ZCP_DEFAULT_INSTRLIMIT (10 * 1000 * 1000)
	#define ZCP_MAX_INSTRLIMIT (10 * ZCP_DEFAULT_INSTRLIMIT)
	#define ZCP_DEFAULT_MEMLIMIT (10 * 1024 * 1024)
	#define ZCP_MAX_MEMLIMIT (10 * ZCP_DEFAULT_MEMLIMIT)

	/*
	* Sysevent payload members. ZFS will generate the following sysevents with the
	* given payloads:
	*
	* ESC_ZFS_RESILVER_START
	* ESC_ZFS_RESILVER_FINISH
	*
	* ZFS_EV_POOL_NAME DATA_TYPE_STRING
	* ZFS_EV_POOL_GUID DATA_TYPE_UINT64
	* ZFS_EV_RESILVER_TYPE DATA_TYPE_STRING
	*
	* ESC_ZFS_POOL_DESTROY
	* ESC_ZFS_POOL_REGUID
	*
	* ZFS_EV_POOL_NAME DATA_TYPE_STRING
	* ZFS_EV_POOL_GUID DATA_TYPE_UINT64
	*
	* ESC_ZFS_VDEV_REMOVE
	* ESC_ZFS_VDEV_CLEAR
	* ESC_ZFS_VDEV_CHECK
	*
	* ZFS_EV_POOL_NAME DATA_TYPE_STRING
	* ZFS_EV_POOL_GUID DATA_TYPE_UINT64
	* ZFS_EV_VDEV_PATH DATA_TYPE_STRING (optional)
	* ZFS_EV_VDEV_GUID DATA_TYPE_UINT64
	*
	* ESC_ZFS_HISTORY_EVENT
	*
	* ZFS_EV_POOL_NAME DATA_TYPE_STRING
	* ZFS_EV_POOL_GUID DATA_TYPE_UINT64
	* ZFS_EV_HIST_TIME DATA_TYPE_UINT64 (optional)
	* ZFS_EV_HIST_CMD DATA_TYPE_STRING (optional)
	* ZFS_EV_HIST_WHO DATA_TYPE_UINT64 (optional)
	* ZFS_EV_HIST_ZONE DATA_TYPE_STRING (optional)
	* ZFS_EV_HIST_HOST DATA_TYPE_STRING (optional)
	* ZFS_EV_HIST_TXG DATA_TYPE_UINT64 (optional)
	* ZFS_EV_HIST_INT_EVENT DATA_TYPE_UINT64 (optional)
	* ZFS_EV_HIST_INT_STR DATA_TYPE_STRING (optional)
	* ZFS_EV_HIST_INT_NAME DATA_TYPE_STRING (optional)
	* ZFS_EV_HIST_IOCTL DATA_TYPE_STRING (optional)
	* ZFS_EV_HIST_DSNAME DATA_TYPE_STRING (optional)
	* ZFS_EV_HIST_DSID DATA_TYPE_UINT64 (optional)
	*
	* The ZFS_EV_HIST_* members will correspond to the ZPOOL_HIST_* members in the
	* history log nvlist. The keynames will be free of any spaces or other
	* characters that could be potentially unexpected to consumers of the
	* sysevents.
	*/
	#define ZFS_EV_POOL_NAME "pool_name"
	#define ZFS_EV_POOL_GUID "pool_guid"
	#define ZFS_EV_VDEV_PATH "vdev_path"
	#define ZFS_EV_VDEV_GUID "vdev_guid"
	#define ZFS_EV_HIST_TIME "history_time"
	#define ZFS_EV_HIST_CMD "history_command"
	#define ZFS_EV_HIST_WHO "history_who"
	#define ZFS_EV_HIST_ZONE "history_zone"
	#define ZFS_EV_HIST_HOST "history_hostname"
	#define ZFS_EV_HIST_TXG "history_txg"
	#define ZFS_EV_HIST_INT_EVENT "history_internal_event"
	#define ZFS_EV_HIST_INT_STR "history_internal_str"
	#define ZFS_EV_HIST_INT_NAME "history_internal_name"
	#define ZFS_EV_HIST_IOCTL "history_ioctl"
	#define ZFS_EV_HIST_DSNAME "history_dsname"
	#define ZFS_EV_HIST_DSID "history_dsid"
	#define ZFS_EV_RESILVER_TYPE "resilver_type"

	/*
	* We currently support block sizes from 512 bytes to 16MB.
	* The benefits of larger blocks, and thus larger IO, need to be weighed
	* against the cost of COWing a giant block to modify one byte, and the
	* large latency of reading or writing a large block.
	*
	* The recordsize property can not be set larger than zfs_max_recordsize
	* (default 16MB on 64-bit and 1MB on 32-bit). See the comment near
	* zfs_max_recordsize in dsl_dataset.c for details.
	*
	* Note that although the LSIZE field of the blkptr_t can store sizes up
	* to 32MB, the dnode's dn_datablkszsec can only store sizes up to
	* 32MB - 512 bytes. Therefore, we limit SPA_MAXBLOCKSIZE to 16MB.
	*/
	#define SPA_MINBLOCKSHIFT 9
	#define SPA_OLD_MAXBLOCKSHIFT 17
	#define SPA_MAXBLOCKSHIFT 24
	#define SPA_MINBLOCKSIZE (1ULL << SPA_MINBLOCKSHIFT)
	#define SPA_OLD_MAXBLOCKSIZE (1ULL << SPA_OLD_MAXBLOCKSHIFT)
	#define SPA_MAXBLOCKSIZE (1ULL << SPA_MAXBLOCKSHIFT)

	/* supported encryption algorithms */
	enum zio_encrypt {
	ZIO_CRYPT_INHERIT = 0,
	ZIO_CRYPT_ON,
	ZIO_CRYPT_OFF,
	ZIO_CRYPT_AES_128_CCM,
	ZIO_CRYPT_AES_192_CCM,
	ZIO_CRYPT_AES_256_CCM,
	ZIO_CRYPT_AES_128_GCM,
	ZIO_CRYPT_AES_192_GCM,
	ZIO_CRYPT_AES_256_GCM,
	ZIO_CRYPT_FUNCTIONS
	};

	#define ZIO_CRYPT_ON_VALUE ZIO_CRYPT_AES_256_GCM
	#define ZIO_CRYPT_DEFAULT ZIO_CRYPT_OFF

	/*
	* xattr namespace prefixes. These are forbidden in xattr names.
	*
	* For cross-platform compatibility, xattrs in the user namespace should not be
	* prefixed with the namespace name, but for backwards compatibility with older
	* ZFS on Linux versions we do prefix the namespace.
	*/
	#define ZFS_XA_NS_FREEBSD_PREFIX "freebsd:"
	#define ZFS_XA_NS_FREEBSD_PREFIX_LEN strlen("freebsd:")
	#define ZFS_XA_NS_LINUX_SECURITY_PREFIX "security."
	#define ZFS_XA_NS_LINUX_SECURITY_PREFIX_LEN strlen("security.")
	#define ZFS_XA_NS_LINUX_SYSTEM_PREFIX "system."
	#define ZFS_XA_NS_LINUX_SYSTEM_PREFIX_LEN strlen("system.")
	#define ZFS_XA_NS_LINUX_TRUSTED_PREFIX "trusted."
	#define ZFS_XA_NS_LINUX_TRUSTED_PREFIX_LEN strlen("trusted.")
	#define ZFS_XA_NS_LINUX_USER_PREFIX "user."
	#define ZFS_XA_NS_LINUX_USER_PREFIX_LEN strlen("user.")

	#define ZFS_XA_NS_PREFIX_MATCH(ns, name) \
	(strncmp(name, ZFS_XA_NS_##ns##_PREFIX, \
	ZFS_XA_NS_##ns##_PREFIX_LEN) == 0)

	#define ZFS_XA_NS_PREFIX_FORBIDDEN(name) \
	(ZFS_XA_NS_PREFIX_MATCH(FREEBSD, name) \|\| \
	ZFS_XA_NS_PREFIX_MATCH(LINUX_SECURITY, name) \|\| \
	ZFS_XA_NS_PREFIX_MATCH(LINUX_SYSTEM, name) \|\| \
	ZFS_XA_NS_PREFIX_MATCH(LINUX_TRUSTED, name) \|\| \
	ZFS_XA_NS_PREFIX_MATCH(LINUX_USER, name))

	#ifdef __cplusplus
	}
	#endif

	#endif /* _SYS_FS_ZFS_H */
	diff --git a/sys/contrib/openzfs/include/sys/multilist.h b/sys/contrib/openzfs/include/sys/multilist.h
	index 26f37c37ab38..e7de86f2379b 100644
	--- a/sys/contrib/openzfs/include/sys/multilist.h
	+++ b/sys/contrib/openzfs/include/sys/multilist.h
	@@ -1,108 +1,111 @@
	/*
	* CDDL HEADER START
	*
	* This file and its contents are supplied under the terms of the
	* Common Development and Distribution License ("CDDL"), version 1.0.
	* You may only use this file in accordance with the terms of version
	* 1.0 of the CDDL.
	*
	* A full copy of the text of the CDDL should have accompanied this
	* source. A copy of the CDDL is also available via the Internet at
	* http://www.illumos.org/license/CDDL.
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2013, 2017 by Delphix. All rights reserved.
	*/

	#ifndef _SYS_MULTILIST_H
	#define _SYS_MULTILIST_H

	#include <sys/zfs_context.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	typedef list_node_t multilist_node_t;
	typedef struct multilist multilist_t;
	typedef struct multilist_sublist multilist_sublist_t;
	typedef unsigned int multilist_sublist_index_func_t(multilist_t , void );

	struct multilist_sublist {
	/*
	* The mutex used internally to implement thread safe insertions
	* and removals to this individual sublist. It can also be locked
	* by a consumer using multilist_sublist_{lock,unlock}, which is
	* useful if a consumer needs to traverse the list in a thread
	* safe manner.
	*/
	kmutex_t mls_lock;
	/*
	* The actual list object containing all objects in this sublist.
	*/
	list_t mls_list;
	/*
	* Pad to cache line, in an effort to try and prevent cache line
	* contention.
	*/
	} ____cacheline_aligned;

	struct multilist {
	/*
	* This is used to get to the multilist_node_t structure given
	* the void *object contained on the list.
	*/
	size_t ml_offset;
	/*
	* The number of sublists used internally by this multilist.
	*/
	uint64_t ml_num_sublists;
	/*
	* The array of pointers to the actual sublists.
	*/
	multilist_sublist_t *ml_sublists;
	/*
	* Pointer to function which determines the sublist to use
	* when inserting and removing objects from this multilist.
	* Please see the comment above multilist_create for details.
	*/
	multilist_sublist_index_func_t *ml_index_func;
	};

	void multilist_create(multilist_t *, size_t, size_t,
	multilist_sublist_index_func_t *);
	void multilist_destroy(multilist_t *);

	void multilist_insert(multilist_t , void );
	void multilist_remove(multilist_t , void );
	int multilist_is_empty(multilist_t *);

	unsigned int multilist_get_num_sublists(multilist_t *);
	unsigned int multilist_get_random_index(multilist_t *);

	-multilist_sublist_t multilist_sublist_lock(multilist_t , unsigned int);
	+void multilist_sublist_lock(multilist_sublist_t *);
	+multilist_sublist_t multilist_sublist_lock_idx(multilist_t , unsigned int);
	multilist_sublist_t multilist_sublist_lock_obj(multilist_t , void *);
	void multilist_sublist_unlock(multilist_sublist_t *);

	void multilist_sublist_insert_head(multilist_sublist_t , void );
	void multilist_sublist_insert_tail(multilist_sublist_t , void );
	+void multilist_sublist_insert_after(multilist_sublist_t , void , void *);
	+void multilist_sublist_insert_before(multilist_sublist_t , void , void *);
	void multilist_sublist_move_forward(multilist_sublist_t mls, void obj);
	void multilist_sublist_remove(multilist_sublist_t , void );
	int multilist_sublist_is_empty(multilist_sublist_t *);
	int multilist_sublist_is_empty_idx(multilist_t *, unsigned int);

	void multilist_sublist_head(multilist_sublist_t );
	void multilist_sublist_tail(multilist_sublist_t );
	void multilist_sublist_next(multilist_sublist_t , void *);
	void multilist_sublist_prev(multilist_sublist_t , void *);

	void multilist_link_init(multilist_node_t *);
	int multilist_link_active(multilist_node_t *);

	#ifdef __cplusplus
	}
	#endif

	#endif /* _SYS_MULTILIST_H */
	diff --git a/sys/contrib/openzfs/include/sys/spa.h b/sys/contrib/openzfs/include/sys/spa.h
	index 87ddbd90e170..6611141b9569 100644
	--- a/sys/contrib/openzfs/include/sys/spa.h
	+++ b/sys/contrib/openzfs/include/sys/spa.h
	@@ -1,1237 +1,1243 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	- * Copyright (c) 2011, 2021 by Delphix. All rights reserved.
	+ * Copyright (c) 2011, 2024 by Delphix. All rights reserved.
	* Copyright 2011 Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
	* Copyright 2013 Saso Kiselkov. All rights reserved.
	* Copyright (c) 2014 Integros [integros.com]
	* Copyright 2017 Joyent, Inc.
	* Copyright (c) 2017, Intel Corporation.
	* Copyright (c) 2019, Allan Jude
	* Copyright (c) 2019, Klara Inc.
	* Copyright (c) 2019, Datto Inc.
	*/

	#ifndef _SYS_SPA_H
	#define _SYS_SPA_H

	#include <sys/avl.h>
	#include <sys/zfs_context.h>
	#include <sys/kstat.h>
	#include <sys/nvpair.h>
	#include <sys/sysmacros.h>
	#include <sys/types.h>
	#include <sys/fs/zfs.h>
	#include <sys/spa_checksum.h>
	#include <sys/dmu.h>
	#include <sys/space_map.h>
	#include <sys/bitops.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*
	* Forward references that lots of things need.
	*/
	typedef struct spa spa_t;
	typedef struct vdev vdev_t;
	typedef struct metaslab metaslab_t;
	typedef struct metaslab_group metaslab_group_t;
	typedef struct metaslab_class metaslab_class_t;
	typedef struct zio zio_t;
	typedef struct zilog zilog_t;
	typedef struct spa_aux_vdev spa_aux_vdev_t;
	typedef struct ddt ddt_t;
	typedef struct ddt_entry ddt_entry_t;
	typedef struct zbookmark_phys zbookmark_phys_t;
	typedef struct zbookmark_err_phys zbookmark_err_phys_t;

	struct bpobj;
	struct bplist;
	struct dsl_pool;
	struct dsl_dataset;
	struct dsl_crypto_params;

	/*
	* Alignment Shift (ashift) is an immutable, internal top-level vdev property
	* which can only be set at vdev creation time. Physical writes are always done
	* according to it, which makes 2^ashift the smallest possible IO on a vdev.
	*
	* We currently allow values ranging from 512 bytes (2^9 = 512) to 64 KiB
	* (2^16 = 65,536).
	*/
	#define ASHIFT_MIN 9
	#define ASHIFT_MAX 16

	/*
	* Size of block to hold the configuration data (a packed nvlist)
	*/
	#define SPA_CONFIG_BLOCKSIZE (1ULL << 14)

	/*
	* The DVA size encodings for LSIZE and PSIZE support blocks up to 32MB.
	* The ASIZE encoding should be at least 64 times larger (6 more bits)
	* to support up to 4-way RAID-Z mirror mode with worst-case gang block
	* overhead, three DVAs per bp, plus one more bit in case we do anything
	* else that expands the ASIZE.
	*/
	#define SPA_LSIZEBITS 16 /* LSIZE up to 32M (2^16 * 512) */
	#define SPA_PSIZEBITS 16 /* PSIZE up to 32M (2^16 * 512) */
	#define SPA_ASIZEBITS 24 /* ASIZE up to 64 times larger */

	#define SPA_COMPRESSBITS 7
	#define SPA_VDEVBITS 24
	#define SPA_COMPRESSMASK ((1U << SPA_COMPRESSBITS) - 1)

	/*
	* All SPA data is represented by 128-bit data virtual addresses (DVAs).
	* The members of the dva_t should be considered opaque outside the SPA.
	*/
	typedef struct dva {
	uint64_t dva_word[2];
	} dva_t;


	/*
	* Some checksums/hashes need a 256-bit initialization salt. This salt is kept
	* secret and is suitable for use in MAC algorithms as the key.
	*/
	typedef struct zio_cksum_salt {
	uint8_t zcs_bytes[32];
	} zio_cksum_salt_t;

	/*
	* Each block is described by its DVAs, time of birth, checksum, etc.
	* The word-by-word, bit-by-bit layout of the blkptr is as follows:
	*
	* 64 56 48 40 32 24 16 8 0
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 0 \| pad \| vdev1 \| GRID \| ASIZE \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 1 \|G\| offset1 \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 2 \| pad \| vdev2 \| GRID \| ASIZE \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 3 \|G\| offset2 \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 4 \| pad \| vdev3 \| GRID \| ASIZE \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 5 \|G\| offset3 \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 6 \|BDX\|lvl\| type \| cksum \|E\| comp\| PSIZE \| LSIZE \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 7 \| padding \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 8 \| padding \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 9 \| physical birth txg \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* a \| logical birth txg \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* b \| fill count \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* c \| checksum[0] \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* d \| checksum[1] \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* e \| checksum[2] \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* f \| checksum[3] \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	*
	* Legend:
	*
	* vdev virtual device ID
	* offset offset into virtual device
	* LSIZE logical size
	* PSIZE physical size (after compression)
	* ASIZE allocated size (including RAID-Z parity and gang block headers)
	* GRID RAID-Z layout information (reserved for future use)
	* cksum checksum function
	* comp compression function
	* G gang block indicator
	* B byteorder (endianness)
	* D dedup
	* X encryption
	* E blkptr_t contains embedded data (see below)
	* lvl level of indirection
	* type DMU object type
	* phys birth txg when dva[0] was written; zero if same as logical birth txg
	* note that typically all the dva's would be written in this
	* txg, but they could be different if they were moved by
	* device removal.
	* log. birth transaction group in which the block was logically born
	* fill count number of non-zero blocks under this bp
	* checksum[4] 256-bit checksum of the data this bp describes
	*/

	/*
	* The blkptr_t's of encrypted blocks also need to store the encryption
	* parameters so that the block can be decrypted. This layout is as follows:
	*
	* 64 56 48 40 32 24 16 8 0
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 0 \| vdev1 \| GRID \| ASIZE \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 1 \|G\| offset1 \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 2 \| vdev2 \| GRID \| ASIZE \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 3 \|G\| offset2 \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 4 \| salt \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 5 \| IV1 \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 6 \|BDX\|lvl\| type \| cksum \|E\| comp\| PSIZE \| LSIZE \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 7 \| padding \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 8 \| padding \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 9 \| physical birth txg \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* a \| logical birth txg \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* b \| IV2 \| fill count \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* c \| checksum[0] \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* d \| checksum[1] \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* e \| MAC[0] \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* f \| MAC[1] \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	*
	* Legend:
	*
	* salt Salt for generating encryption keys
	* IV1 First 64 bits of encryption IV
	* X Block requires encryption handling (set to 1)
	* E blkptr_t contains embedded data (set to 0, see below)
	* fill count number of non-zero blocks under this bp (truncated to 32 bits)
	* IV2 Last 32 bits of encryption IV
	* checksum[2] 128-bit checksum of the data this bp describes
	* MAC[2] 128-bit message authentication code for this data
	*
	* The X bit being set indicates that this block is one of 3 types. If this is
	* a level 0 block with an encrypted object type, the block is encrypted
	* (see BP_IS_ENCRYPTED()). If this is a level 0 block with an unencrypted
	* object type, this block is authenticated with an HMAC (see
	* BP_IS_AUTHENTICATED()). Otherwise (if level > 0), this bp will use the MAC
	* words to store a checksum-of-MACs from the level below (see
	* BP_HAS_INDIRECT_MAC_CKSUM()). For convenience in the code, BP_IS_PROTECTED()
	* refers to both encrypted and authenticated blocks and BP_USES_CRYPT()
	* refers to any of these 3 kinds of blocks.
	*
	* The additional encryption parameters are the salt, IV, and MAC which are
	* explained in greater detail in the block comment at the top of zio_crypt.c.
	* The MAC occupies half of the checksum space since it serves a very similar
	* purpose: to prevent data corruption on disk. The only functional difference
	* is that the checksum is used to detect on-disk corruption whether or not the
	* encryption key is loaded and the MAC provides additional protection against
	* malicious disk tampering. We use the 3rd DVA to store the salt and first
	* 64 bits of the IV. As a result encrypted blocks can only have 2 copies
	* maximum instead of the normal 3. The last 32 bits of the IV are stored in
	* the upper bits of what is usually the fill count. Note that only blocks at
	* level 0 or -2 are ever encrypted, which allows us to guarantee that these
	* 32 bits are not trampled over by other code (see zio_crypt.c for details).
	* The salt and IV are not used for authenticated bps or bps with an indirect
	* MAC checksum, so these blocks can utilize all 3 DVAs and the full 64 bits
	* for the fill count.
	*/

	/*
	* "Embedded" blkptr_t's don't actually point to a block, instead they
	* have a data payload embedded in the blkptr_t itself. See the comment
	* in blkptr.c for more details.
	*
	* The blkptr_t is laid out as follows:
	*
	* 64 56 48 40 32 24 16 8 0
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 0 \| payload \|
	* 1 \| payload \|
	* 2 \| payload \|
	* 3 \| payload \|
	* 4 \| payload \|
	* 5 \| payload \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 6 \|BDX\|lvl\| type \| etype \|E\| comp\| PSIZE\| LSIZE \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 7 \| payload \|
	* 8 \| payload \|
	* 9 \| payload \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* a \| logical birth txg \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* b \| payload \|
	* c \| payload \|
	* d \| payload \|
	* e \| payload \|
	* f \| payload \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	*
	* Legend:
	*
	* payload contains the embedded data
	* B (byteorder) byteorder (endianness)
	* D (dedup) padding (set to zero)
	* X encryption (set to zero)
	* E (embedded) set to one
	* lvl indirection level
	* type DMU object type
	* etype how to interpret embedded data (BP_EMBEDDED_TYPE_*)
	* comp compression function of payload
	* PSIZE size of payload after compression, in bytes
	* LSIZE logical size of payload, in bytes
	* note that 25 bits is enough to store the largest
	* "normal" BP's LSIZE (2^16 * 2^9) in bytes
	* log. birth transaction group in which the block was logically born
	*
	* Note that LSIZE and PSIZE are stored in bytes, whereas for non-embedded
	* bp's they are stored in units of SPA_MINBLOCKSHIFT.
	* Generally, the generic BP_GET_*() macros can be used on embedded BP's.
	* The B, D, X, lvl, type, and comp fields are stored the same as with normal
	* BP's so the BP_SET_* macros can be used with them. etype, PSIZE, LSIZE must
	* be set with the BPE_SET_* macros. BP_SET_EMBEDDED() should be called before
	* other macros, as they assert that they are only used on BP's of the correct
	* "embedded-ness". Encrypted blkptr_t's cannot be embedded because they use
	* the payload space for encryption parameters (see the comment above on
	* how encryption parameters are stored).
	*/

	#define BPE_GET_ETYPE(bp) \
	(ASSERT(BP_IS_EMBEDDED(bp)), \
	BF64_GET((bp)->blk_prop, 40, 8))
	#define BPE_SET_ETYPE(bp, t) do { \
	ASSERT(BP_IS_EMBEDDED(bp)); \
	BF64_SET((bp)->blk_prop, 40, 8, t); \
	} while (0)

	#define BPE_GET_LSIZE(bp) \
	(ASSERT(BP_IS_EMBEDDED(bp)), \
	BF64_GET_SB((bp)->blk_prop, 0, 25, 0, 1))
	#define BPE_SET_LSIZE(bp, x) do { \
	ASSERT(BP_IS_EMBEDDED(bp)); \
	BF64_SET_SB((bp)->blk_prop, 0, 25, 0, 1, x); \
	} while (0)

	#define BPE_GET_PSIZE(bp) \
	(ASSERT(BP_IS_EMBEDDED(bp)), \
	BF64_GET_SB((bp)->blk_prop, 25, 7, 0, 1))
	#define BPE_SET_PSIZE(bp, x) do { \
	ASSERT(BP_IS_EMBEDDED(bp)); \
	BF64_SET_SB((bp)->blk_prop, 25, 7, 0, 1, x); \
	} while (0)

	typedef enum bp_embedded_type {
	BP_EMBEDDED_TYPE_DATA,
	BP_EMBEDDED_TYPE_RESERVED, /* Reserved for Delphix byteswap feature. */
	BP_EMBEDDED_TYPE_REDACTED,
	NUM_BP_EMBEDDED_TYPES
	} bp_embedded_type_t;

	#define BPE_NUM_WORDS 14
	#define BPE_PAYLOAD_SIZE (BPE_NUM_WORDS * sizeof (uint64_t))
	#define BPE_IS_PAYLOADWORD(bp, wp) \
	((wp) != &(bp)->blk_prop && (wp) != &(bp)->blk_birth)

	#define SPA_BLKPTRSHIFT 7 /* blkptr_t is 128 bytes */
	#define SPA_DVAS_PER_BP 3 /* Number of DVAs in a bp */
	#define SPA_SYNC_MIN_VDEVS 3 /* min vdevs to update during sync */

	/*
	* A block is a hole when it has either 1) never been written to, or
	* 2) is zero-filled. In both cases, ZFS can return all zeroes for all reads
	* without physically allocating disk space. Holes are represented in the
	* blkptr_t structure by zeroed blk_dva. Correct checking for holes is
	* done through the BP_IS_HOLE macro. For holes, the logical size, level,
	* DMU object type, and birth times are all also stored for holes that
	* were written to at some point (i.e. were punched after having been filled).
	*/
	typedef struct blkptr {
	dva_t blk_dva[SPA_DVAS_PER_BP]; /* Data Virtual Addresses */
	uint64_t blk_prop; /* size, compression, type, etc */
	uint64_t blk_pad[2]; /* Extra space for the future */
	uint64_t blk_phys_birth; /* txg when block was allocated */
	uint64_t blk_birth; /* transaction group at birth */
	uint64_t blk_fill; /* fill count */
	zio_cksum_t blk_cksum; /* 256-bit checksum */
	} blkptr_t;

	/*
	* Macros to get and set fields in a bp or DVA.
	*/

	/*
	* Note, for gang blocks, DVA_GET_ASIZE() is the total space allocated for
	* this gang DVA including its children BP's. The space allocated at this
	* DVA's vdev/offset is vdev_gang_header_asize(vdev).
	*/
	#define DVA_GET_ASIZE(dva) \
	BF64_GET_SB((dva)->dva_word[0], 0, SPA_ASIZEBITS, SPA_MINBLOCKSHIFT, 0)
	#define DVA_SET_ASIZE(dva, x) \
	BF64_SET_SB((dva)->dva_word[0], 0, SPA_ASIZEBITS, \
	SPA_MINBLOCKSHIFT, 0, x)

	#define DVA_GET_GRID(dva) BF64_GET((dva)->dva_word[0], 24, 8)
	#define DVA_SET_GRID(dva, x) BF64_SET((dva)->dva_word[0], 24, 8, x)

	#define DVA_GET_VDEV(dva) BF64_GET((dva)->dva_word[0], 32, SPA_VDEVBITS)
	#define DVA_SET_VDEV(dva, x) \
	BF64_SET((dva)->dva_word[0], 32, SPA_VDEVBITS, x)

	#define DVA_GET_OFFSET(dva) \
	BF64_GET_SB((dva)->dva_word[1], 0, 63, SPA_MINBLOCKSHIFT, 0)
	#define DVA_SET_OFFSET(dva, x) \
	BF64_SET_SB((dva)->dva_word[1], 0, 63, SPA_MINBLOCKSHIFT, 0, x)

	#define DVA_GET_GANG(dva) BF64_GET((dva)->dva_word[1], 63, 1)
	#define DVA_SET_GANG(dva, x) BF64_SET((dva)->dva_word[1], 63, 1, x)

	#define BP_GET_LSIZE(bp) \
	(BP_IS_EMBEDDED(bp) ? \
	(BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA ? BPE_GET_LSIZE(bp) : 0): \
	BF64_GET_SB((bp)->blk_prop, 0, SPA_LSIZEBITS, SPA_MINBLOCKSHIFT, 1))
	#define BP_SET_LSIZE(bp, x) do { \
	ASSERT(!BP_IS_EMBEDDED(bp)); \
	BF64_SET_SB((bp)->blk_prop, \
	0, SPA_LSIZEBITS, SPA_MINBLOCKSHIFT, 1, x); \
	} while (0)

	#define BP_GET_PSIZE(bp) \
	(BP_IS_EMBEDDED(bp) ? 0 : \
	BF64_GET_SB((bp)->blk_prop, 16, SPA_PSIZEBITS, SPA_MINBLOCKSHIFT, 1))
	#define BP_SET_PSIZE(bp, x) do { \
	ASSERT(!BP_IS_EMBEDDED(bp)); \
	BF64_SET_SB((bp)->blk_prop, \
	16, SPA_PSIZEBITS, SPA_MINBLOCKSHIFT, 1, x); \
	} while (0)

	#define BP_GET_COMPRESS(bp) \
	BF64_GET((bp)->blk_prop, 32, SPA_COMPRESSBITS)
	#define BP_SET_COMPRESS(bp, x) \
	BF64_SET((bp)->blk_prop, 32, SPA_COMPRESSBITS, x)

	#define BP_IS_EMBEDDED(bp) BF64_GET((bp)->blk_prop, 39, 1)
	#define BP_SET_EMBEDDED(bp, x) BF64_SET((bp)->blk_prop, 39, 1, x)

	#define BP_GET_CHECKSUM(bp) \
	(BP_IS_EMBEDDED(bp) ? ZIO_CHECKSUM_OFF : \
	BF64_GET((bp)->blk_prop, 40, 8))
	#define BP_SET_CHECKSUM(bp, x) do { \
	ASSERT(!BP_IS_EMBEDDED(bp)); \
	BF64_SET((bp)->blk_prop, 40, 8, x); \
	} while (0)

	#define BP_GET_TYPE(bp) BF64_GET((bp)->blk_prop, 48, 8)
	#define BP_SET_TYPE(bp, x) BF64_SET((bp)->blk_prop, 48, 8, x)

	#define BP_GET_LEVEL(bp) BF64_GET((bp)->blk_prop, 56, 5)
	#define BP_SET_LEVEL(bp, x) BF64_SET((bp)->blk_prop, 56, 5, x)

	/* encrypted, authenticated, and MAC cksum bps use the same bit */
	#define BP_USES_CRYPT(bp) BF64_GET((bp)->blk_prop, 61, 1)
	#define BP_SET_CRYPT(bp, x) BF64_SET((bp)->blk_prop, 61, 1, x)

	#define BP_IS_ENCRYPTED(bp) \
	(BP_USES_CRYPT(bp) && \
	BP_GET_LEVEL(bp) <= 0 && \
	DMU_OT_IS_ENCRYPTED(BP_GET_TYPE(bp)))

	#define BP_IS_AUTHENTICATED(bp) \
	(BP_USES_CRYPT(bp) && \
	BP_GET_LEVEL(bp) <= 0 && \
	!DMU_OT_IS_ENCRYPTED(BP_GET_TYPE(bp)))

	#define BP_HAS_INDIRECT_MAC_CKSUM(bp) \
	(BP_USES_CRYPT(bp) && BP_GET_LEVEL(bp) > 0)

	#define BP_IS_PROTECTED(bp) \
	(BP_IS_ENCRYPTED(bp) \|\| BP_IS_AUTHENTICATED(bp))

	#define BP_GET_DEDUP(bp) BF64_GET((bp)->blk_prop, 62, 1)
	#define BP_SET_DEDUP(bp, x) BF64_SET((bp)->blk_prop, 62, 1, x)

	#define BP_GET_BYTEORDER(bp) BF64_GET((bp)->blk_prop, 63, 1)
	#define BP_SET_BYTEORDER(bp, x) BF64_SET((bp)->blk_prop, 63, 1, x)

	#define BP_GET_FREE(bp) BF64_GET((bp)->blk_fill, 0, 1)
	#define BP_SET_FREE(bp, x) BF64_SET((bp)->blk_fill, 0, 1, x)

	#define BP_PHYSICAL_BIRTH(bp) \
	(BP_IS_EMBEDDED(bp) ? 0 : \
	(bp)->blk_phys_birth ? (bp)->blk_phys_birth : (bp)->blk_birth)

	#define BP_SET_BIRTH(bp, logical, physical) \
	{ \
	ASSERT(!BP_IS_EMBEDDED(bp)); \
	(bp)->blk_birth = (logical); \
	(bp)->blk_phys_birth = ((logical) == (physical) ? 0 : (physical)); \
	}

	#define BP_GET_FILL(bp) \
	((BP_IS_ENCRYPTED(bp)) ? BF64_GET((bp)->blk_fill, 0, 32) : \
	((BP_IS_EMBEDDED(bp)) ? 1 : (bp)->blk_fill))

	#define BP_SET_FILL(bp, fill) \
	{ \
	if (BP_IS_ENCRYPTED(bp)) \
	BF64_SET((bp)->blk_fill, 0, 32, fill); \
	else \
	(bp)->blk_fill = fill; \
	}

	#define BP_GET_IV2(bp) \
	(ASSERT(BP_IS_ENCRYPTED(bp)), \
	BF64_GET((bp)->blk_fill, 32, 32))
	#define BP_SET_IV2(bp, iv2) \
	{ \
	ASSERT(BP_IS_ENCRYPTED(bp)); \
	BF64_SET((bp)->blk_fill, 32, 32, iv2); \
	}

	#define BP_IS_METADATA(bp) \
	(BP_GET_LEVEL(bp) > 0 \|\| DMU_OT_IS_METADATA(BP_GET_TYPE(bp)))

	#define BP_GET_ASIZE(bp) \
	(BP_IS_EMBEDDED(bp) ? 0 : \
	DVA_GET_ASIZE(&(bp)->blk_dva[0]) + \
	DVA_GET_ASIZE(&(bp)->blk_dva[1]) + \
	(DVA_GET_ASIZE(&(bp)->blk_dva[2]) * !BP_IS_ENCRYPTED(bp)))

	#define BP_GET_UCSIZE(bp) \
	(BP_IS_METADATA(bp) ? BP_GET_PSIZE(bp) : BP_GET_LSIZE(bp))

	#define BP_GET_NDVAS(bp) \
	(BP_IS_EMBEDDED(bp) ? 0 : \
	!!DVA_GET_ASIZE(&(bp)->blk_dva[0]) + \
	!!DVA_GET_ASIZE(&(bp)->blk_dva[1]) + \
	(!!DVA_GET_ASIZE(&(bp)->blk_dva[2]) * !BP_IS_ENCRYPTED(bp)))

	#define BP_COUNT_GANG(bp) \
	(BP_IS_EMBEDDED(bp) ? 0 : \
	(DVA_GET_GANG(&(bp)->blk_dva[0]) + \
	DVA_GET_GANG(&(bp)->blk_dva[1]) + \
	(DVA_GET_GANG(&(bp)->blk_dva[2]) * !BP_IS_ENCRYPTED(bp))))

	#define DVA_EQUAL(dva1, dva2) \
	((dva1)->dva_word[1] == (dva2)->dva_word[1] && \
	(dva1)->dva_word[0] == (dva2)->dva_word[0])

	#define BP_EQUAL(bp1, bp2) \
	(BP_PHYSICAL_BIRTH(bp1) == BP_PHYSICAL_BIRTH(bp2) && \
	(bp1)->blk_birth == (bp2)->blk_birth && \
	DVA_EQUAL(&(bp1)->blk_dva[0], &(bp2)->blk_dva[0]) && \
	DVA_EQUAL(&(bp1)->blk_dva[1], &(bp2)->blk_dva[1]) && \
	DVA_EQUAL(&(bp1)->blk_dva[2], &(bp2)->blk_dva[2]))


	#define DVA_IS_VALID(dva) (DVA_GET_ASIZE(dva) != 0)

	#define BP_IDENTITY(bp) (ASSERT(!BP_IS_EMBEDDED(bp)), &(bp)->blk_dva[0])
	#define BP_IS_GANG(bp) \
	(BP_IS_EMBEDDED(bp) ? B_FALSE : DVA_GET_GANG(BP_IDENTITY(bp)))
	#define DVA_IS_EMPTY(dva) ((dva)->dva_word[0] == 0ULL && \
	(dva)->dva_word[1] == 0ULL)
	#define BP_IS_HOLE(bp) \
	(!BP_IS_EMBEDDED(bp) && DVA_IS_EMPTY(BP_IDENTITY(bp)))

	#define BP_SET_REDACTED(bp) \
	{ \
	BP_SET_EMBEDDED(bp, B_TRUE); \
	BPE_SET_ETYPE(bp, BP_EMBEDDED_TYPE_REDACTED); \
	}
	#define BP_IS_REDACTED(bp) \
	(BP_IS_EMBEDDED(bp) && BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_REDACTED)

	/* BP_IS_RAIDZ(bp) assumes no block compression */
	#define BP_IS_RAIDZ(bp) (DVA_GET_ASIZE(&(bp)->blk_dva[0]) > \
	BP_GET_PSIZE(bp))

	#define BP_ZERO(bp) \
	{ \
	(bp)->blk_dva[0].dva_word[0] = 0; \
	(bp)->blk_dva[0].dva_word[1] = 0; \
	(bp)->blk_dva[1].dva_word[0] = 0; \
	(bp)->blk_dva[1].dva_word[1] = 0; \
	(bp)->blk_dva[2].dva_word[0] = 0; \
	(bp)->blk_dva[2].dva_word[1] = 0; \
	(bp)->blk_prop = 0; \
	(bp)->blk_pad[0] = 0; \
	(bp)->blk_pad[1] = 0; \
	(bp)->blk_phys_birth = 0; \
	(bp)->blk_birth = 0; \
	(bp)->blk_fill = 0; \
	ZIO_SET_CHECKSUM(&(bp)->blk_cksum, 0, 0, 0, 0); \
	}

	#ifdef _ZFS_BIG_ENDIAN
	#define ZFS_HOST_BYTEORDER (0ULL)
	#else
	#define ZFS_HOST_BYTEORDER (1ULL)
	#endif

	#define BP_SHOULD_BYTESWAP(bp) (BP_GET_BYTEORDER(bp) != ZFS_HOST_BYTEORDER)

	#define BP_SPRINTF_LEN 400

	/*
	* This macro allows code sharing between zfs, libzpool, and mdb.
	* 'func' is either kmem_scnprintf() or mdb_snprintf().
	* 'ws' (whitespace) can be ' ' for single-line format, '\n' for multi-line.
	*/

	#define SNPRINTF_BLKPTR(func, ws, buf, size, bp, type, checksum, compress) \
	{ \
	static const char *const copyname[] = \
	{ "zero", "single", "double", "triple" }; \
	int len = 0; \
	int copies = 0; \
	const char *crypt_type; \
	if (bp != NULL) { \
	if (BP_IS_ENCRYPTED(bp)) { \
	crypt_type = "encrypted"; \
	/* LINTED E_SUSPICIOUS_COMPARISON */ \
	} else if (BP_IS_AUTHENTICATED(bp)) { \
	crypt_type = "authenticated"; \
	} else if (BP_HAS_INDIRECT_MAC_CKSUM(bp)) { \
	crypt_type = "indirect-MAC"; \
	} else { \
	crypt_type = "unencrypted"; \
	} \
	} \
	if (bp == NULL) { \
	len += func(buf + len, size - len, "<NULL>"); \
	} else if (BP_IS_HOLE(bp)) { \
	len += func(buf + len, size - len, \
	"HOLE [L%llu %s] " \
	"size=%llxL birth=%lluL", \
	(u_longlong_t)BP_GET_LEVEL(bp), \
	type, \
	(u_longlong_t)BP_GET_LSIZE(bp), \
	(u_longlong_t)bp->blk_birth); \
	} else if (BP_IS_EMBEDDED(bp)) { \
	len = func(buf + len, size - len, \
	"EMBEDDED [L%llu %s] et=%u %s " \
	"size=%llxL/%llxP birth=%lluL", \
	(u_longlong_t)BP_GET_LEVEL(bp), \
	type, \
	(int)BPE_GET_ETYPE(bp), \
	compress, \
	(u_longlong_t)BPE_GET_LSIZE(bp), \
	(u_longlong_t)BPE_GET_PSIZE(bp), \
	(u_longlong_t)bp->blk_birth); \
	} else if (BP_IS_REDACTED(bp)) { \
	len += func(buf + len, size - len, \
	"REDACTED [L%llu %s] size=%llxL birth=%lluL", \
	(u_longlong_t)BP_GET_LEVEL(bp), \
	type, \
	(u_longlong_t)BP_GET_LSIZE(bp), \
	(u_longlong_t)bp->blk_birth); \
	} else { \
	for (int d = 0; d < BP_GET_NDVAS(bp); d++) { \
	const dva_t *dva = &bp->blk_dva[d]; \
	if (DVA_IS_VALID(dva)) \
	copies++; \
	len += func(buf + len, size - len, \
	"DVA[%d]=<%llu:%llx:%llx>%c", d, \
	(u_longlong_t)DVA_GET_VDEV(dva), \
	(u_longlong_t)DVA_GET_OFFSET(dva), \
	(u_longlong_t)DVA_GET_ASIZE(dva), \
	ws); \
	} \
	ASSERT3S(copies, >, 0); \
	if (BP_IS_ENCRYPTED(bp)) { \
	len += func(buf + len, size - len, \
	"salt=%llx iv=%llx:%llx%c", \
	(u_longlong_t)bp->blk_dva[2].dva_word[0], \
	(u_longlong_t)bp->blk_dva[2].dva_word[1], \
	(u_longlong_t)BP_GET_IV2(bp), \
	ws); \
	} \
	if (BP_IS_GANG(bp) && \
	DVA_GET_ASIZE(&bp->blk_dva[2]) <= \
	DVA_GET_ASIZE(&bp->blk_dva[1]) / 2) \
	copies--; \
	len += func(buf + len, size - len, \
	"[L%llu %s] %s %s %s %s %s %s %s%c" \
	"size=%llxL/%llxP birth=%lluL/%lluP fill=%llu%c" \
	"cksum=%016llx:%016llx:%016llx:%016llx", \
	(u_longlong_t)BP_GET_LEVEL(bp), \
	type, \
	checksum, \
	compress, \
	crypt_type, \
	BP_GET_BYTEORDER(bp) == 0 ? "BE" : "LE", \
	BP_IS_GANG(bp) ? "gang" : "contiguous", \
	BP_GET_DEDUP(bp) ? "dedup" : "unique", \
	copyname[copies], \
	ws, \
	(u_longlong_t)BP_GET_LSIZE(bp), \
	(u_longlong_t)BP_GET_PSIZE(bp), \
	(u_longlong_t)bp->blk_birth, \
	(u_longlong_t)BP_PHYSICAL_BIRTH(bp), \
	(u_longlong_t)BP_GET_FILL(bp), \
	ws, \
	(u_longlong_t)bp->blk_cksum.zc_word[0], \
	(u_longlong_t)bp->blk_cksum.zc_word[1], \
	(u_longlong_t)bp->blk_cksum.zc_word[2], \
	(u_longlong_t)bp->blk_cksum.zc_word[3]); \
	} \
	ASSERT(len < size); \
	}

	#define BP_GET_BUFC_TYPE(bp) \
	(BP_IS_METADATA(bp) ? ARC_BUFC_METADATA : ARC_BUFC_DATA)

	typedef enum spa_import_type {
	SPA_IMPORT_EXISTING,
	SPA_IMPORT_ASSEMBLE
	} spa_import_type_t;

	typedef enum spa_mode {
	SPA_MODE_UNINIT = 0,
	SPA_MODE_READ = 1,
	SPA_MODE_WRITE = 2,
	} spa_mode_t;

	/*
	* Send TRIM commands in-line during normal pool operation while deleting.
	* OFF: no
	* ON: yes
	*/
	typedef enum {
	SPA_AUTOTRIM_OFF = 0, /* default */
	SPA_AUTOTRIM_ON,
	} spa_autotrim_t;

	/*
	* Reason TRIM command was issued, used internally for accounting purposes.
	*/
	typedef enum trim_type {
	TRIM_TYPE_MANUAL = 0,
	TRIM_TYPE_AUTO = 1,
	TRIM_TYPE_SIMPLE = 2
	} trim_type_t;

	/* state manipulation functions */
	extern int spa_open(const char pool, spa_t , const void tag);
	extern int spa_open_rewind(const char pool, spa_t , const void tag,
	nvlist_t policy, nvlist_t *config);
	extern int spa_get_stats(const char pool, nvlist_t config, char altroot,
	size_t buflen);
	extern int spa_create(const char pool, nvlist_t nvroot, nvlist_t *props,
	nvlist_t zplprops, struct dsl_crypto_params dcp);
	extern int spa_import(char pool, nvlist_t config, nvlist_t *props,
	uint64_t flags);
	extern nvlist_t spa_tryimport(nvlist_t tryconfig);
	extern int spa_destroy(const char *pool);
	extern int spa_checkpoint(const char *pool);
	extern int spa_checkpoint_discard(const char *pool);
	extern int spa_export(const char pool, nvlist_t *oldconfig, boolean_t force,
	boolean_t hardforce);
	extern int spa_reset(const char *pool);
	extern void spa_async_request(spa_t *spa, int flag);
	extern void spa_async_unrequest(spa_t *spa, int flag);
	extern void spa_async_suspend(spa_t *spa);
	extern void spa_async_resume(spa_t *spa);
	extern int spa_async_tasks(spa_t *spa);
	extern spa_t spa_inject_addref(char pool);
	extern void spa_inject_delref(spa_t *spa);
	extern void spa_scan_stat_init(spa_t *spa);
	extern int spa_scan_get_stats(spa_t spa, pool_scan_stat_t ps);
	extern int bpobj_enqueue_alloc_cb(void arg, const blkptr_t bp, dmu_tx_t *tx);
	extern int bpobj_enqueue_free_cb(void arg, const blkptr_t bp, dmu_tx_t *tx);

	#define SPA_ASYNC_CONFIG_UPDATE 0x01
	#define SPA_ASYNC_REMOVE 0x02
	-#define SPA_ASYNC_PROBE 0x04
	+#define SPA_ASYNC_FAULT_VDEV 0x04
	#define SPA_ASYNC_RESILVER_DONE 0x08
	#define SPA_ASYNC_RESILVER 0x10
	#define SPA_ASYNC_AUTOEXPAND 0x20
	#define SPA_ASYNC_REMOVE_DONE 0x40
	#define SPA_ASYNC_REMOVE_STOP 0x80
	#define SPA_ASYNC_INITIALIZE_RESTART 0x100
	#define SPA_ASYNC_TRIM_RESTART 0x200
	#define SPA_ASYNC_AUTOTRIM_RESTART 0x400
	#define SPA_ASYNC_L2CACHE_REBUILD 0x800
	#define SPA_ASYNC_L2CACHE_TRIM 0x1000
	#define SPA_ASYNC_REBUILD_DONE 0x2000
	#define SPA_ASYNC_DETACH_SPARE 0x4000

	/* device manipulation */
	-extern int spa_vdev_add(spa_t spa, nvlist_t nvroot);
	+extern int spa_vdev_add(spa_t spa, nvlist_t nvroot, boolean_t ashift_check);
	extern int spa_vdev_attach(spa_t spa, uint64_t guid, nvlist_t nvroot,
	int replacing, int rebuild);
	extern int spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid,
	int replace_done);
	extern int spa_vdev_alloc(spa_t *spa, uint64_t guid);
	extern int spa_vdev_noalloc(spa_t *spa, uint64_t guid);
	extern boolean_t spa_vdev_remove_active(spa_t *spa);
	extern int spa_vdev_initialize(spa_t spa, nvlist_t nv, uint64_t cmd_type,
	nvlist_t *vdev_errlist);
	extern int spa_vdev_trim(spa_t spa, nvlist_t nv, uint64_t cmd_type,
	uint64_t rate, boolean_t partial, boolean_t secure, nvlist_t *vdev_errlist);
	extern int spa_vdev_setpath(spa_t spa, uint64_t guid, const char newpath);
	extern int spa_vdev_setfru(spa_t spa, uint64_t guid, const char newfru);
	extern int spa_vdev_split_mirror(spa_t spa, const char newname,
	nvlist_t config, nvlist_t props, boolean_t exp);

	/* spare state (which is global across all pools) */
	extern void spa_spare_add(vdev_t *vd);
	extern void spa_spare_remove(vdev_t *vd);
	extern boolean_t spa_spare_exists(uint64_t guid, uint64_t pool, int refcnt);
	extern void spa_spare_activate(vdev_t *vd);

	/* L2ARC state (which is global across all pools) */
	extern void spa_l2cache_add(vdev_t *vd);
	extern void spa_l2cache_remove(vdev_t *vd);
	extern boolean_t spa_l2cache_exists(uint64_t guid, uint64_t *pool);
	extern void spa_l2cache_activate(vdev_t *vd);
	extern void spa_l2cache_drop(spa_t *spa);

	/* scanning */
	extern int spa_scan(spa_t *spa, pool_scan_func_t func);
	extern int spa_scan_stop(spa_t *spa);
	extern int spa_scrub_pause_resume(spa_t *spa, pool_scrub_cmd_t flag);

	/* spa syncing */
	extern void spa_sync(spa_t spa, uint64_t txg); / only for DMU use */
	extern void spa_sync_allpools(void);

	extern uint_t zfs_sync_pass_deferred_free;

	/* spa namespace global mutex */
	extern kmutex_t spa_namespace_lock;

	/*
	* SPA configuration functions in spa_config.c
	*/

	#define SPA_CONFIG_UPDATE_POOL 0
	#define SPA_CONFIG_UPDATE_VDEVS 1

	extern void spa_write_cachefile(spa_t *, boolean_t, boolean_t, boolean_t);
	extern void spa_config_load(void);
	extern int spa_all_configs(uint64_t generation, nvlist_t *pools);
	extern void spa_config_set(spa_t spa, nvlist_t config);
	extern nvlist_t spa_config_generate(spa_t spa, vdev_t *vd, uint64_t txg,
	int getstats);
	extern void spa_config_update(spa_t *spa, int what);
	extern int spa_config_parse(spa_t spa, vdev_t vdp, nvlist_t nv,
	vdev_t *parent, uint_t id, int atype);


	/*
	* Miscellaneous SPA routines in spa_misc.c
	*/

	/* Namespace manipulation */
	extern spa_t spa_lookup(const char name);
	extern spa_t spa_add(const char name, nvlist_t config, const char altroot);
	extern void spa_remove(spa_t *spa);
	extern spa_t spa_next(spa_t prev);

	/* Refcount functions */
	extern void spa_open_ref(spa_t spa, const void tag);
	extern void spa_close(spa_t spa, const void tag);
	extern void spa_async_close(spa_t spa, const void tag);
	extern boolean_t spa_refcount_zero(spa_t *spa);

	#define SCL_NONE 0x00
	#define SCL_CONFIG 0x01
	#define SCL_STATE 0x02
	#define SCL_L2ARC 0x04 /* hack until L2ARC 2.0 */
	#define SCL_ALLOC 0x08
	#define SCL_ZIO 0x10
	#define SCL_FREE 0x20
	#define SCL_VDEV 0x40
	#define SCL_LOCKS 7
	#define SCL_ALL ((1 << SCL_LOCKS) - 1)
	#define SCL_STATE_ALL (SCL_STATE \| SCL_L2ARC \| SCL_ZIO)

	/* Historical pool statistics */
	typedef struct spa_history_kstat {
	kmutex_t lock;
	uint64_t count;
	uint64_t size;
	kstat_t *kstat;
	void *priv;
	list_t list;
	} spa_history_kstat_t;

	typedef struct spa_history_list {
	uint64_t size;
	procfs_list_t procfs_list;
	} spa_history_list_t;

	typedef struct spa_stats {
	spa_history_list_t read_history;
	spa_history_list_t txg_history;
	spa_history_kstat_t tx_assign_histogram;
	spa_history_list_t mmp_history;
	spa_history_kstat_t state; /* pool state */
	spa_history_kstat_t guid; /* pool guid */
	spa_history_kstat_t iostats;
	} spa_stats_t;

	typedef enum txg_state {
	TXG_STATE_BIRTH = 0,
	TXG_STATE_OPEN = 1,
	TXG_STATE_QUIESCED = 2,
	TXG_STATE_WAIT_FOR_SYNC = 3,
	TXG_STATE_SYNCED = 4,
	TXG_STATE_COMMITTED = 5,
	} txg_state_t;

	typedef struct txg_stat {
	vdev_stat_t vs1;
	vdev_stat_t vs2;
	uint64_t txg;
	uint64_t ndirty;
	} txg_stat_t;

	/* Assorted pool IO kstats */
	typedef struct spa_iostats {
	kstat_named_t trim_extents_written;
	kstat_named_t trim_bytes_written;
	kstat_named_t trim_extents_skipped;
	kstat_named_t trim_bytes_skipped;
	kstat_named_t trim_extents_failed;
	kstat_named_t trim_bytes_failed;
	kstat_named_t autotrim_extents_written;
	kstat_named_t autotrim_bytes_written;
	kstat_named_t autotrim_extents_skipped;
	kstat_named_t autotrim_bytes_skipped;
	kstat_named_t autotrim_extents_failed;
	kstat_named_t autotrim_bytes_failed;
	kstat_named_t simple_trim_extents_written;
	kstat_named_t simple_trim_bytes_written;
	kstat_named_t simple_trim_extents_skipped;
	kstat_named_t simple_trim_bytes_skipped;
	kstat_named_t simple_trim_extents_failed;
	kstat_named_t simple_trim_bytes_failed;
	} spa_iostats_t;

	extern void spa_stats_init(spa_t *spa);
	extern void spa_stats_destroy(spa_t *spa);
	extern void spa_read_history_add(spa_t spa, const zbookmark_phys_t zb,
	uint32_t aflags);
	extern void spa_txg_history_add(spa_t *spa, uint64_t txg, hrtime_t birth_time);
	extern int spa_txg_history_set(spa_t *spa, uint64_t txg,
	txg_state_t completed_state, hrtime_t completed_time);
	extern txg_stat_t spa_txg_history_init_io(spa_t , uint64_t,
	struct dsl_pool *);
	extern void spa_txg_history_fini_io(spa_t , txg_stat_t );
	extern void spa_tx_assign_add_nsecs(spa_t *spa, uint64_t nsecs);
	extern int spa_mmp_history_set_skip(spa_t *spa, uint64_t mmp_kstat_id);
	extern int spa_mmp_history_set(spa_t *spa, uint64_t mmp_kstat_id, int io_error,
	hrtime_t duration);
	extern void spa_mmp_history_add(spa_t *spa, uint64_t txg, uint64_t timestamp,
	uint64_t mmp_delay, vdev_t *vd, int label, uint64_t mmp_kstat_id,
	int error);
	extern void spa_iostats_trim_add(spa_t *spa, trim_type_t type,
	uint64_t extents_written, uint64_t bytes_written,
	uint64_t extents_skipped, uint64_t bytes_skipped,
	uint64_t extents_failed, uint64_t bytes_failed);
	extern void spa_import_progress_add(spa_t *spa);
	extern void spa_import_progress_remove(uint64_t spa_guid);
	extern int spa_import_progress_set_mmp_check(uint64_t pool_guid,
	uint64_t mmp_sec_remaining);
	extern int spa_import_progress_set_max_txg(uint64_t pool_guid,
	uint64_t max_txg);
	extern int spa_import_progress_set_state(uint64_t pool_guid,
	spa_load_state_t spa_load_state);
	+extern void spa_import_progress_set_notes(spa_t *spa,
	+ const char *fmt, ...) __printflike(2, 3);
	+extern void spa_import_progress_set_notes_nolog(spa_t *spa,
	+ const char *fmt, ...) __printflike(2, 3);

	/* Pool configuration locks */
	extern int spa_config_tryenter(spa_t spa, int locks, const void tag,
	krw_t rw);
	extern void spa_config_enter(spa_t spa, int locks, const void tag, krw_t rw);
	extern void spa_config_enter_mmp(spa_t spa, int locks, const void tag,
	krw_t rw);
	extern void spa_config_exit(spa_t spa, int locks, const void tag);
	extern int spa_config_held(spa_t *spa, int locks, krw_t rw);

	/* Pool vdev add/remove lock */
	extern uint64_t spa_vdev_enter(spa_t *spa);
	extern uint64_t spa_vdev_detach_enter(spa_t *spa, uint64_t guid);
	extern uint64_t spa_vdev_config_enter(spa_t *spa);
	extern void spa_vdev_config_exit(spa_t spa, vdev_t vd, uint64_t txg,
	int error, const char *tag);
	extern int spa_vdev_exit(spa_t spa, vdev_t vd, uint64_t txg, int error);

	/* Pool vdev state change lock */
	extern void spa_vdev_state_enter(spa_t *spa, int oplock);
	extern int spa_vdev_state_exit(spa_t spa, vdev_t vd, int error);

	/* Log state */
	typedef enum spa_log_state {
	SPA_LOG_UNKNOWN = 0, /* unknown log state */
	SPA_LOG_MISSING, /* missing log(s) */
	SPA_LOG_CLEAR, /* clear the log(s) */
	SPA_LOG_GOOD, /* log(s) are good */
	} spa_log_state_t;

	extern spa_log_state_t spa_get_log_state(spa_t *spa);
	extern void spa_set_log_state(spa_t *spa, spa_log_state_t state);
	extern int spa_reset_logs(spa_t *spa);

	/* Log claim callback */
	extern void spa_claim_notify(zio_t *zio);
	extern void spa_deadman(void *);

	/* Accessor functions */
	extern boolean_t spa_shutting_down(spa_t *spa);
	extern struct dsl_pool spa_get_dsl(spa_t spa);
	extern boolean_t spa_is_initializing(spa_t *spa);
	extern boolean_t spa_indirect_vdevs_loaded(spa_t *spa);
	extern blkptr_t spa_get_rootblkptr(spa_t spa);
	extern void spa_set_rootblkptr(spa_t spa, const blkptr_t bp);
	extern void spa_altroot(spa_t , char , size_t);
	extern uint32_t spa_sync_pass(spa_t *spa);
	extern char spa_name(spa_t spa);
	extern uint64_t spa_guid(spa_t *spa);
	extern uint64_t spa_load_guid(spa_t *spa);
	extern uint64_t spa_last_synced_txg(spa_t *spa);
	extern uint64_t spa_first_txg(spa_t *spa);
	extern uint64_t spa_syncing_txg(spa_t *spa);
	extern uint64_t spa_final_dirty_txg(spa_t *spa);
	extern uint64_t spa_version(spa_t *spa);
	extern pool_state_t spa_state(spa_t *spa);
	extern spa_load_state_t spa_load_state(spa_t *spa);
	extern uint64_t spa_freeze_txg(spa_t *spa);
	extern uint64_t spa_get_worst_case_asize(spa_t *spa, uint64_t lsize);
	extern uint64_t spa_get_dspace(spa_t *spa);
	extern uint64_t spa_get_checkpoint_space(spa_t *spa);
	extern uint64_t spa_get_slop_space(spa_t *spa);
	extern void spa_update_dspace(spa_t *spa);
	extern uint64_t spa_version(spa_t *spa);
	extern boolean_t spa_deflate(spa_t *spa);
	extern metaslab_class_t spa_normal_class(spa_t spa);
	extern metaslab_class_t spa_log_class(spa_t spa);
	extern metaslab_class_t spa_embedded_log_class(spa_t spa);
	extern metaslab_class_t spa_special_class(spa_t spa);
	extern metaslab_class_t spa_dedup_class(spa_t spa);
	extern metaslab_class_t spa_preferred_class(spa_t spa, uint64_t size,
	dmu_object_type_t objtype, uint_t level, uint_t special_smallblk);

	extern void spa_evicting_os_register(spa_t , objset_t os);
	extern void spa_evicting_os_deregister(spa_t , objset_t os);
	extern void spa_evicting_os_wait(spa_t *spa);
	extern int spa_max_replication(spa_t *spa);
	extern int spa_prev_software_version(spa_t *spa);
	extern uint64_t spa_get_failmode(spa_t *spa);
	extern uint64_t spa_get_deadman_failmode(spa_t *spa);
	extern void spa_set_deadman_failmode(spa_t spa, const char failmode);
	extern boolean_t spa_suspended(spa_t *spa);
	extern uint64_t spa_bootfs(spa_t *spa);
	extern uint64_t spa_delegation(spa_t *spa);
	extern objset_t spa_meta_objset(spa_t spa);
	extern space_map_t spa_syncing_log_sm(spa_t spa);
	extern uint64_t spa_deadman_synctime(spa_t *spa);
	extern uint64_t spa_deadman_ziotime(spa_t *spa);
	extern uint64_t spa_dirty_data(spa_t *spa);
	extern spa_autotrim_t spa_get_autotrim(spa_t *spa);

	/* Miscellaneous support routines */
	extern void spa_load_failed(spa_t spa, const char fmt, ...)
	__attribute__((format(printf, 2, 3)));
	extern void spa_load_note(spa_t spa, const char fmt, ...)
	__attribute__((format(printf, 2, 3)));
	extern void spa_activate_mos_feature(spa_t spa, const char feature,
	dmu_tx_t *tx);
	extern void spa_deactivate_mos_feature(spa_t spa, const char feature);
	extern spa_t *spa_by_guid(uint64_t pool_guid, uint64_t device_guid);
	extern boolean_t spa_guid_exists(uint64_t pool_guid, uint64_t device_guid);
	extern char spa_strdup(const char );
	extern void spa_strfree(char *);
	extern uint64_t spa_generate_guid(spa_t *spa);
	extern void snprintf_blkptr(char buf, size_t buflen, const blkptr_t bp);
	extern void spa_freeze(spa_t *spa);
	extern int spa_change_guid(spa_t *spa);
	extern void spa_upgrade(spa_t *spa, uint64_t version);
	extern void spa_evict_all(void);
	extern vdev_t spa_lookup_by_guid(spa_t spa, uint64_t guid,
	boolean_t l2cache);
	extern boolean_t spa_has_l2cache(spa_t *, uint64_t guid);
	extern boolean_t spa_has_spare(spa_t *, uint64_t guid);
	extern uint64_t dva_get_dsize_sync(spa_t spa, const dva_t dva);
	extern uint64_t bp_get_dsize_sync(spa_t spa, const blkptr_t bp);
	extern uint64_t bp_get_dsize(spa_t spa, const blkptr_t bp);
	extern boolean_t spa_has_slogs(spa_t *spa);
	extern boolean_t spa_is_root(spa_t *spa);
	extern boolean_t spa_writeable(spa_t *spa);
	extern boolean_t spa_has_pending_synctask(spa_t *spa);
	extern int spa_maxblocksize(spa_t *spa);
	extern int spa_maxdnodesize(spa_t *spa);
	extern boolean_t spa_has_checkpoint(spa_t *spa);
	extern boolean_t spa_importing_readonly_checkpoint(spa_t *spa);
	extern boolean_t spa_suspend_async_destroy(spa_t *spa);
	extern uint64_t spa_min_claim_txg(spa_t *spa);
	extern boolean_t zfs_dva_valid(spa_t spa, const dva_t dva,
	const blkptr_t *bp);
	typedef void (*spa_remap_cb_t)(uint64_t vdev, uint64_t offset, uint64_t size,
	void *arg);
	extern boolean_t spa_remap_blkptr(spa_t spa, blkptr_t bp,
	spa_remap_cb_t callback, void *arg);
	extern uint64_t spa_get_last_removal_txg(spa_t *spa);
	extern boolean_t spa_trust_config(spa_t *spa);
	extern uint64_t spa_missing_tvds_allowed(spa_t *spa);
	extern void spa_set_missing_tvds(spa_t *spa, uint64_t missing);
	extern boolean_t spa_top_vdevs_spacemap_addressable(spa_t *spa);
	extern uint64_t spa_total_metaslabs(spa_t *spa);
	extern boolean_t spa_multihost(spa_t *spa);
	extern uint32_t spa_get_hostid(spa_t *spa);
	extern void spa_activate_allocation_classes(spa_t , dmu_tx_t );
	extern boolean_t spa_livelist_delete_check(spa_t *spa);

	+extern boolean_t spa_mmp_remote_host_activity(spa_t *spa);
	+
	extern spa_mode_t spa_mode(spa_t *spa);
	extern uint64_t zfs_strtonum(const char str, char *nptr);

	extern char *spa_his_ievent_table[];

	extern void spa_history_create_obj(spa_t spa, dmu_tx_t tx);
	extern int spa_history_get(spa_t spa, uint64_t offset, uint64_t *len_read,
	char *his_buf);
	extern int spa_history_log(spa_t spa, const char his_buf);
	extern int spa_history_log_nvl(spa_t spa, nvlist_t nvl);
	extern void spa_history_log_version(spa_t spa, const char operation,
	dmu_tx_t *tx);
	extern void spa_history_log_internal(spa_t spa, const char operation,
	dmu_tx_t tx, const char fmt, ...) __printflike(4, 5);
	extern void spa_history_log_internal_ds(struct dsl_dataset ds, const char op,
	dmu_tx_t tx, const char fmt, ...) __printflike(4, 5);
	extern void spa_history_log_internal_dd(dsl_dir_t dd, const char operation,
	dmu_tx_t tx, const char fmt, ...) __printflike(4, 5);

	extern const char spa_state_to_name(spa_t spa);

	/* error handling */
	struct zbookmark_phys;
	extern void spa_log_error(spa_t spa, const zbookmark_phys_t zb,
	const uint64_t *birth);
	extern void spa_remove_error(spa_t spa, zbookmark_phys_t zb,
	const uint64_t *birth);
	extern int zfs_ereport_post(const char clazz, spa_t spa, vdev_t *vd,
	const zbookmark_phys_t zb, zio_t zio, uint64_t state);
	extern boolean_t zfs_ereport_is_valid(const char clazz, spa_t spa, vdev_t *vd,
	zio_t *zio);
	extern void zfs_ereport_taskq_fini(void);
	extern void zfs_ereport_clear(spa_t spa, vdev_t vd);
	extern nvlist_t zfs_event_create(spa_t spa, vdev_t vd, const char type,
	const char name, nvlist_t aux);
	extern void zfs_post_remove(spa_t spa, vdev_t vd);
	extern void zfs_post_state_change(spa_t spa, vdev_t vd, uint64_t laststate);
	extern void zfs_post_autoreplace(spa_t spa, vdev_t vd);
	extern uint64_t spa_approx_errlog_size(spa_t *spa);
	extern int spa_get_errlog(spa_t spa, void uaddr, uint64_t *count);
	extern uint64_t spa_get_last_errlog_size(spa_t *spa);
	extern void spa_errlog_rotate(spa_t *spa);
	extern void spa_errlog_drain(spa_t *spa);
	extern void spa_errlog_sync(spa_t *spa, uint64_t txg);
	extern void spa_get_errlists(spa_t spa, avl_tree_t last, avl_tree_t *scrub);
	extern void spa_delete_dataset_errlog(spa_t spa, uint64_t ds, dmu_tx_t tx);
	extern void spa_swap_errlog(spa_t *spa, uint64_t new_head_ds,
	uint64_t old_head_ds, dmu_tx_t *tx);
	extern void sync_error_list(spa_t spa, avl_tree_t t, uint64_t *obj,
	dmu_tx_t *tx);
	extern void spa_upgrade_errlog(spa_t spa, dmu_tx_t tx);
	extern int find_top_affected_fs(spa_t *spa, uint64_t head_ds,
	zbookmark_err_phys_t zep, uint64_t top_affected_fs);
	extern int find_birth_txg(struct dsl_dataset ds, zbookmark_err_phys_t zep,
	uint64_t *birth_txg);
	extern void zep_to_zb(uint64_t dataset, zbookmark_err_phys_t *zep,
	zbookmark_phys_t *zb);
	extern void name_to_errphys(char buf, zbookmark_err_phys_t zep);

	/* vdev mirror */
	extern void vdev_mirror_stat_init(void);
	extern void vdev_mirror_stat_fini(void);

	/* Initialization and termination */
	extern void spa_init(spa_mode_t mode);
	extern void spa_fini(void);
	extern void spa_boot_init(void);

	/* properties */
	extern int spa_prop_set(spa_t spa, nvlist_t nvp);
	extern int spa_prop_get(spa_t spa, nvlist_t *nvp);
	extern void spa_prop_clear_bootfs(spa_t spa, uint64_t obj, dmu_tx_t tx);
	extern void spa_configfile_set(spa_t , nvlist_t , boolean_t);

	/* asynchronous event notification */
	extern void spa_event_notify(spa_t spa, vdev_t vdev, nvlist_t *hist_nvl,
	const char *name);
	extern void zfs_ereport_zvol_post(const char subclass, const char name,
	const char device_name, const char raw_name);

	/* waiting for pool activities to complete */
	extern int spa_wait(const char *pool, zpool_wait_activity_t activity,
	boolean_t *waited);
	extern int spa_wait_tag(const char *name, zpool_wait_activity_t activity,
	uint64_t tag, boolean_t *waited);
	extern void spa_notify_waiters(spa_t *spa);
	extern void spa_wake_waiters(spa_t *spa);

	extern void spa_import_os(spa_t *spa);
	extern void spa_export_os(spa_t *spa);
	extern void spa_activate_os(spa_t *spa);
	extern void spa_deactivate_os(spa_t *spa);

	/* module param call functions */
	int param_set_deadman_ziotime(ZFS_MODULE_PARAM_ARGS);
	int param_set_deadman_synctime(ZFS_MODULE_PARAM_ARGS);
	int param_set_slop_shift(ZFS_MODULE_PARAM_ARGS);
	int param_set_deadman_failmode(ZFS_MODULE_PARAM_ARGS);

	#ifdef ZFS_DEBUG
	#define dprintf_bp(bp, fmt, ...) do { \
	if (zfs_flags & ZFS_DEBUG_DPRINTF) { \
	char *__blkbuf = kmem_alloc(BP_SPRINTF_LEN, KM_SLEEP); \
	snprintf_blkptr(__blkbuf, BP_SPRINTF_LEN, (bp)); \
	dprintf(fmt " %s\n", __VA_ARGS__, __blkbuf); \
	kmem_free(__blkbuf, BP_SPRINTF_LEN); \
	} \
	} while (0)
	#else
	#define dprintf_bp(bp, fmt, ...)
	#endif

	extern spa_mode_t spa_mode_global;
	extern int zfs_deadman_enabled;
	extern uint64_t zfs_deadman_synctime_ms;
	extern uint64_t zfs_deadman_ziotime_ms;
	extern uint64_t zfs_deadman_checktime_ms;

	extern kmem_cache_t *zio_buf_cache[];
	extern kmem_cache_t *zio_data_buf_cache[];

	#ifdef __cplusplus
	}
	#endif

	#endif /* _SYS_SPA_H */
	diff --git a/sys/contrib/openzfs/include/sys/uberblock_impl.h b/sys/contrib/openzfs/include/sys/uberblock_impl.h
	index 03bcfa8f4dd1..13fce9c29e2d 100644
	--- a/sys/contrib/openzfs/include/sys/uberblock_impl.h
	+++ b/sys/contrib/openzfs/include/sys/uberblock_impl.h
	@@ -1,145 +1,145 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2016, 2017 by Delphix. All rights reserved.
	*/

	#ifndef _SYS_UBERBLOCK_IMPL_H
	#define _SYS_UBERBLOCK_IMPL_H

	#include <sys/uberblock.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*
	* The uberblock version is incremented whenever an incompatible on-disk
	* format change is made to the SPA, DMU, or ZAP.
	*
	* Note: the first two fields should never be moved. When a storage pool
	* is opened, the uberblock must be read off the disk before the version
	* can be checked. If the ub_version field is moved, we may not detect
	* version mismatch. If the ub_magic field is moved, applications that
	* expect the magic number in the first word won't work.
	*/
	#define UBERBLOCK_MAGIC 0x00bab10c /* oo-ba-bloc! */
	#define UBERBLOCK_SHIFT 10 /* up to 1K */
	#define MMP_MAGIC 0xa11cea11 /* all-see-all */

	#define MMP_INTERVAL_VALID_BIT 0x01
	#define MMP_SEQ_VALID_BIT 0x02
	#define MMP_FAIL_INT_VALID_BIT 0x04

	-#define MMP_VALID(ubp) (ubp->ub_magic == UBERBLOCK_MAGIC && \
	- ubp->ub_mmp_magic == MMP_MAGIC)
	-#define MMP_INTERVAL_VALID(ubp) (MMP_VALID(ubp) && (ubp->ub_mmp_config & \
	+#define MMP_VALID(ubp) ((ubp)->ub_magic == UBERBLOCK_MAGIC && \
	+ (ubp)->ub_mmp_magic == MMP_MAGIC)
	+#define MMP_INTERVAL_VALID(ubp) (MMP_VALID(ubp) && ((ubp)->ub_mmp_config & \
	MMP_INTERVAL_VALID_BIT))
	-#define MMP_SEQ_VALID(ubp) (MMP_VALID(ubp) && (ubp->ub_mmp_config & \
	+#define MMP_SEQ_VALID(ubp) (MMP_VALID(ubp) && ((ubp)->ub_mmp_config & \
	MMP_SEQ_VALID_BIT))
	-#define MMP_FAIL_INT_VALID(ubp) (MMP_VALID(ubp) && (ubp->ub_mmp_config & \
	+#define MMP_FAIL_INT_VALID(ubp) (MMP_VALID(ubp) && ((ubp)->ub_mmp_config & \
	MMP_FAIL_INT_VALID_BIT))

	-#define MMP_INTERVAL(ubp) ((ubp->ub_mmp_config & 0x00000000FFFFFF00) \
	+#define MMP_INTERVAL(ubp) (((ubp)->ub_mmp_config & 0x00000000FFFFFF00) \
	>> 8)
	-#define MMP_SEQ(ubp) ((ubp->ub_mmp_config & 0x0000FFFF00000000) \
	+#define MMP_SEQ(ubp) (((ubp)->ub_mmp_config & 0x0000FFFF00000000) \
	>> 32)
	-#define MMP_FAIL_INT(ubp) ((ubp->ub_mmp_config & 0xFFFF000000000000) \
	+#define MMP_FAIL_INT(ubp) (((ubp)->ub_mmp_config & 0xFFFF000000000000) \
	>> 48)

	#define MMP_INTERVAL_SET(write) \
	(((uint64_t)(write & 0xFFFFFF) << 8) \| MMP_INTERVAL_VALID_BIT)

	#define MMP_SEQ_SET(seq) \
	(((uint64_t)(seq & 0xFFFF) << 32) \| MMP_SEQ_VALID_BIT)

	#define MMP_FAIL_INT_SET(fail) \
	(((uint64_t)(fail & 0xFFFF) << 48) \| MMP_FAIL_INT_VALID_BIT)

	struct uberblock {
	uint64_t ub_magic; /* UBERBLOCK_MAGIC */
	uint64_t ub_version; /* SPA_VERSION */
	uint64_t ub_txg; /* txg of last sync */
	uint64_t ub_guid_sum; /* sum of all vdev guids */
	uint64_t ub_timestamp; /* UTC time of last sync */
	blkptr_t ub_rootbp; /* MOS objset_phys_t */

	/* highest SPA_VERSION supported by software that wrote this txg */
	uint64_t ub_software_version;

	/* Maybe missing in uberblocks we read, but always written */
	uint64_t ub_mmp_magic; /* MMP_MAGIC */
	/*
	* If ub_mmp_delay == 0 and ub_mmp_magic is valid, MMP is off.
	* Otherwise, nanosec since last MMP write.
	*/
	uint64_t ub_mmp_delay;

	/*
	* The ub_mmp_config contains the multihost write interval, multihost
	* fail intervals, sequence number for sub-second granularity, and
	* valid bit mask. This layout is as follows:
	*
	* 64 56 48 40 32 24 16 8 0
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	* 0 \| Fail Intervals\| Seq \| Write Interval (ms) \| VALID \|
	* +-------+-------+-------+-------+-------+-------+-------+-------+
	*
	* This allows a write_interval of (2^24/1000)s, over 4.5 hours
	*
	* VALID Bits:
	* - 0x01 - Write Interval (ms)
	* - 0x02 - Sequence number exists
	* - 0x04 - Fail Intervals
	* - 0xf8 - Reserved
	*/
	uint64_t ub_mmp_config;

	/*
	* ub_checkpoint_txg indicates two things about the current uberblock:
	*
	* 1] If it is not zero then this uberblock is a checkpoint. If it is
	* zero, then this uberblock is not a checkpoint.
	*
	* 2] On checkpointed uberblocks, the value of ub_checkpoint_txg is
	* the ub_txg that the uberblock had at the time we moved it to
	* the MOS config.
	*
	* The field is set when we checkpoint the uberblock and continues to
	* hold that value even after we've rewound (unlike the ub_txg that
	* is reset to a higher value).
	*
	* Besides checks used to determine whether we are reopening the
	* pool from a checkpointed uberblock [see spa_ld_select_uberblock()],
	* the value of the field is used to determine which ZIL blocks have
	* been allocated according to the ms_sm when we are rewinding to a
	* checkpoint. Specifically, if blk_birth > ub_checkpoint_txg, then
	* the ZIL block is not allocated [see uses of spa_min_claim_txg()].
	*/
	uint64_t ub_checkpoint_txg;
	};

	#ifdef __cplusplus
	}
	#endif

	#endif /* _SYS_UBERBLOCK_IMPL_H */
	diff --git a/sys/contrib/openzfs/include/sys/vdev_impl.h b/sys/contrib/openzfs/include/sys/vdev_impl.h
	index 3f2312c23438..8c6ab316fa18 100644
	--- a/sys/contrib/openzfs/include/sys/vdev_impl.h
	+++ b/sys/contrib/openzfs/include/sys/vdev_impl.h
	@@ -1,652 +1,655 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
	* Copyright (c) 2017, Intel Corporation.
	+ * Copyright (c) 2023, Klara Inc.
	*/

	#ifndef _SYS_VDEV_IMPL_H
	#define _SYS_VDEV_IMPL_H

	#include <sys/avl.h>
	#include <sys/bpobj.h>
	#include <sys/dmu.h>
	#include <sys/metaslab.h>
	#include <sys/nvpair.h>
	#include <sys/space_map.h>
	#include <sys/vdev.h>
	#include <sys/dkio.h>
	#include <sys/uberblock_impl.h>
	#include <sys/vdev_indirect_mapping.h>
	#include <sys/vdev_indirect_births.h>
	#include <sys/vdev_rebuild.h>
	#include <sys/vdev_removal.h>
	#include <sys/zfs_ratelimit.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*
	* Virtual device descriptors.
	*
	* All storage pool operations go through the virtual device framework,
	* which provides data replication and I/O scheduling.
	*/

	/*
	* Forward declarations that lots of things need.
	*/
	typedef struct vdev_queue vdev_queue_t;
	struct abd;

	extern uint_t zfs_vdev_queue_depth_pct;
	extern uint_t zfs_vdev_def_queue_depth;
	extern uint_t zfs_vdev_async_write_max_active;

	/*
	* Virtual device operations
	*/
	typedef int vdev_init_func_t(spa_t spa, nvlist_t nv, void **tsd);
	typedef void vdev_kobj_post_evt_func_t(vdev_t *vd);
	typedef void vdev_fini_func_t(vdev_t *vd);
	typedef int vdev_open_func_t(vdev_t vd, uint64_t size, uint64_t *max_size,
	uint64_t ashift, uint64_t pshift);
	typedef void vdev_close_func_t(vdev_t *vd);
	typedef uint64_t vdev_asize_func_t(vdev_t *vd, uint64_t psize);
	typedef uint64_t vdev_min_asize_func_t(vdev_t *vd);
	typedef uint64_t vdev_min_alloc_func_t(vdev_t *vd);
	typedef void vdev_io_start_func_t(zio_t *zio);
	typedef void vdev_io_done_func_t(zio_t *zio);
	typedef void vdev_state_change_func_t(vdev_t *vd, int, int);
	typedef boolean_t vdev_need_resilver_func_t(vdev_t vd, const dva_t dva,
	size_t psize, uint64_t phys_birth);
	typedef void vdev_hold_func_t(vdev_t *vd);
	typedef void vdev_rele_func_t(vdev_t *vd);

	typedef void vdev_remap_cb_t(uint64_t inner_offset, vdev_t *vd,
	uint64_t offset, uint64_t size, void *arg);
	typedef void vdev_remap_func_t(vdev_t *vd, uint64_t offset, uint64_t size,
	vdev_remap_cb_t callback, void *arg);
	/*
	* Given a target vdev, translates the logical range "in" to the physical
	* range "res"
	*/
	typedef void vdev_xlation_func_t(vdev_t cvd, const range_seg64_t logical,
	range_seg64_t physical, range_seg64_t remain);
	typedef uint64_t vdev_rebuild_asize_func_t(vdev_t *vd, uint64_t start,
	uint64_t size, uint64_t max_segment);
	typedef void vdev_metaslab_init_func_t(vdev_t vd, uint64_t startp,
	uint64_t *sizep);
	typedef void vdev_config_generate_func_t(vdev_t vd, nvlist_t nv);
	typedef uint64_t vdev_nparity_func_t(vdev_t *vd);
	typedef uint64_t vdev_ndisks_func_t(vdev_t *vd);

	typedef const struct vdev_ops {
	vdev_init_func_t *vdev_op_init;
	vdev_fini_func_t *vdev_op_fini;
	vdev_open_func_t *vdev_op_open;
	vdev_close_func_t *vdev_op_close;
	vdev_asize_func_t *vdev_op_asize;
	vdev_min_asize_func_t *vdev_op_min_asize;
	vdev_min_alloc_func_t *vdev_op_min_alloc;
	vdev_io_start_func_t *vdev_op_io_start;
	vdev_io_done_func_t *vdev_op_io_done;
	vdev_state_change_func_t *vdev_op_state_change;
	vdev_need_resilver_func_t *vdev_op_need_resilver;
	vdev_hold_func_t *vdev_op_hold;
	vdev_rele_func_t *vdev_op_rele;
	vdev_remap_func_t *vdev_op_remap;
	vdev_xlation_func_t *vdev_op_xlate;
	vdev_rebuild_asize_func_t *vdev_op_rebuild_asize;
	vdev_metaslab_init_func_t *vdev_op_metaslab_init;
	vdev_config_generate_func_t *vdev_op_config_generate;
	vdev_nparity_func_t *vdev_op_nparity;
	vdev_ndisks_func_t *vdev_op_ndisks;
	vdev_kobj_post_evt_func_t *vdev_op_kobj_evt_post;
	char vdev_op_type[16];
	boolean_t vdev_op_leaf;
	} vdev_ops_t;

	/*
	* Virtual device properties
	*/
	typedef union vdev_queue_class {
	struct {
	ulong_t vqc_list_numnodes;
	list_t vqc_list;
	};
	avl_tree_t vqc_tree;
	} vdev_queue_class_t;

	struct vdev_queue {
	vdev_t *vq_vdev;
	vdev_queue_class_t vq_class[ZIO_PRIORITY_NUM_QUEUEABLE];
	avl_tree_t vq_read_offset_tree;
	avl_tree_t vq_write_offset_tree;
	uint64_t vq_last_offset;
	zio_priority_t vq_last_prio; /* Last sent I/O priority. */
	uint32_t vq_cqueued; /* Classes with queued I/Os. */
	uint32_t vq_cactive[ZIO_PRIORITY_NUM_QUEUEABLE];
	uint32_t vq_active; /* Number of active I/Os. */
	uint32_t vq_ia_active; /* Active interactive I/Os. */
	uint32_t vq_nia_credit; /* Non-interactive I/Os credit. */
	list_t vq_active_list; /* List of active I/Os. */
	hrtime_t vq_io_complete_ts; /* time last i/o completed */
	hrtime_t vq_io_delta_ts;
	zio_t vq_io_search; /* used as local for stack reduction */
	kmutex_t vq_lock;
	};

	typedef enum vdev_alloc_bias {
	VDEV_BIAS_NONE,
	VDEV_BIAS_LOG, /* dedicated to ZIL data (SLOG) */
	VDEV_BIAS_SPECIAL, /* dedicated to ddt, metadata, and small blks */
	VDEV_BIAS_DEDUP /* dedicated to dedup metadata */
	} vdev_alloc_bias_t;


	/*
	* On-disk indirect vdev state.
	*
	* An indirect vdev is described exclusively in the MOS config of a pool.
	* The config for an indirect vdev includes several fields, which are
	* accessed in memory by a vdev_indirect_config_t.
	*/
	typedef struct vdev_indirect_config {
	/*
	* Object (in MOS) which contains the indirect mapping. This object
	* contains an array of vdev_indirect_mapping_entry_phys_t ordered by
	* vimep_src. The bonus buffer for this object is a
	* vdev_indirect_mapping_phys_t. This object is allocated when a vdev
	* removal is initiated.
	*
	* Note that this object can be empty if none of the data on the vdev
	* has been copied yet.
	*/
	uint64_t vic_mapping_object;

	/*
	* Object (in MOS) which contains the birth times for the mapping
	* entries. This object contains an array of
	* vdev_indirect_birth_entry_phys_t sorted by vibe_offset. The bonus
	* buffer for this object is a vdev_indirect_birth_phys_t. This object
	* is allocated when a vdev removal is initiated.
	*
	* Note that this object can be empty if none of the vdev has yet been
	* copied.
	*/
	uint64_t vic_births_object;

	/*
	* This is the vdev ID which was removed previous to this vdev, or
	* UINT64_MAX if there are no previously removed vdevs.
	*/
	uint64_t vic_prev_indirect_vdev;
	} vdev_indirect_config_t;

	/*
	* Virtual device descriptor
	*/
	struct vdev {
	/*
	* Common to all vdev types.
	*/
	uint64_t vdev_id; /* child number in vdev parent */
	uint64_t vdev_guid; /* unique ID for this vdev */
	uint64_t vdev_guid_sum; /* self guid + all child guids */
	uint64_t vdev_orig_guid; /* orig. guid prior to remove */
	uint64_t vdev_asize; /* allocatable device capacity */
	uint64_t vdev_min_asize; /* min acceptable asize */
	uint64_t vdev_max_asize; /* max acceptable asize */
	uint64_t vdev_ashift; /* block alignment shift */

	/*
	* Logical block alignment shift
	*
	* The smallest sized/aligned I/O supported by the device.
	*/
	uint64_t vdev_logical_ashift;
	/*
	* Physical block alignment shift
	*
	* The device supports logical I/Os with vdev_logical_ashift
	* size/alignment, but optimum performance will be achieved by
	* aligning/sizing requests to vdev_physical_ashift. Smaller
	* requests may be inflated or incur device level read-modify-write
	* operations.
	*
	* May be 0 to indicate no preference (i.e. use vdev_logical_ashift).
	*/
	uint64_t vdev_physical_ashift;
	uint64_t vdev_state; /* see VDEV_STATE_* #defines */
	uint64_t vdev_prevstate; /* used when reopening a vdev */
	vdev_ops_t vdev_ops; / vdev operations */
	spa_t vdev_spa; / spa for this vdev */
	void vdev_tsd; / type-specific data */
	vdev_t vdev_top; / top-level vdev */
	vdev_t vdev_parent; / parent vdev */
	vdev_t *vdev_child; / array of children */
	uint64_t vdev_children; /* number of children */
	vdev_stat_t vdev_stat; /* virtual device statistics */
	vdev_stat_ex_t vdev_stat_ex; /* extended statistics */
	boolean_t vdev_expanding; /* expand the vdev? */
	boolean_t vdev_reopening; /* reopen in progress? */
	boolean_t vdev_nonrot; /* true if solid state */
	int vdev_load_error; /* error on last load */
	int vdev_open_error; /* error on last open */
	int vdev_validate_error; /* error on last validate */
	kthread_t vdev_open_thread; / thread opening children */
	kthread_t vdev_validate_thread; / thread validating children */
	uint64_t vdev_crtxg; /* txg when top-level was added */
	uint64_t vdev_root_zap;

	/*
	* Top-level vdev state.
	*/
	uint64_t vdev_ms_array; /* metaslab array object */
	uint64_t vdev_ms_shift; /* metaslab size shift */
	uint64_t vdev_ms_count; /* number of metaslabs */
	metaslab_group_t vdev_mg; / metaslab group */
	metaslab_group_t vdev_log_mg; / embedded slog metaslab group */
	metaslab_t *vdev_ms; / metaslab array */
	txg_list_t vdev_ms_list; /* per-txg dirty metaslab lists */
	txg_list_t vdev_dtl_list; /* per-txg dirty DTL lists */
	txg_node_t vdev_txg_node; /* per-txg dirty vdev linkage */
	boolean_t vdev_remove_wanted; /* async remove wanted? */
	- boolean_t vdev_probe_wanted; /* async probe wanted? */
	+ boolean_t vdev_fault_wanted; /* async faulted wanted? */
	list_node_t vdev_config_dirty_node; /* config dirty list */
	list_node_t vdev_state_dirty_node; /* state dirty list */
	uint64_t vdev_deflate_ratio; /* deflation ratio (x512) */
	uint64_t vdev_islog; /* is an intent log device */
	uint64_t vdev_noalloc; /* device is passivated? */
	uint64_t vdev_removing; /* device is being removed? */
	uint64_t vdev_failfast; /* device failfast setting */
	boolean_t vdev_ishole; /* is a hole in the namespace */
	uint64_t vdev_top_zap;
	vdev_alloc_bias_t vdev_alloc_bias; /* metaslab allocation bias */

	/* pool checkpoint related */
	space_map_t vdev_checkpoint_sm; / contains reserved blocks */

	/* Initialize related */
	boolean_t vdev_initialize_exit_wanted;
	vdev_initializing_state_t vdev_initialize_state;
	list_node_t vdev_initialize_node;
	kthread_t *vdev_initialize_thread;
	/* Protects vdev_initialize_thread and vdev_initialize_state. */
	kmutex_t vdev_initialize_lock;
	kcondvar_t vdev_initialize_cv;
	uint64_t vdev_initialize_offset[TXG_SIZE];
	uint64_t vdev_initialize_last_offset;
	range_tree_t vdev_initialize_tree; / valid while initializing */
	uint64_t vdev_initialize_bytes_est;
	uint64_t vdev_initialize_bytes_done;
	uint64_t vdev_initialize_action_time; /* start and end time */

	/* TRIM related */
	boolean_t vdev_trim_exit_wanted;
	boolean_t vdev_autotrim_exit_wanted;
	vdev_trim_state_t vdev_trim_state;
	list_node_t vdev_trim_node;
	kmutex_t vdev_autotrim_lock;
	kcondvar_t vdev_autotrim_cv;
	kcondvar_t vdev_autotrim_kick_cv;
	kthread_t *vdev_autotrim_thread;
	/* Protects vdev_trim_thread and vdev_trim_state. */
	kmutex_t vdev_trim_lock;
	kcondvar_t vdev_trim_cv;
	kthread_t *vdev_trim_thread;
	uint64_t vdev_trim_offset[TXG_SIZE];
	uint64_t vdev_trim_last_offset;
	uint64_t vdev_trim_bytes_est;
	uint64_t vdev_trim_bytes_done;
	uint64_t vdev_trim_rate; /* requested rate (bytes/sec) */
	uint64_t vdev_trim_partial; /* requested partial TRIM */
	uint64_t vdev_trim_secure; /* requested secure TRIM */
	uint64_t vdev_trim_action_time; /* start and end time */

	/* Rebuild related */
	boolean_t vdev_rebuilding;
	boolean_t vdev_rebuild_exit_wanted;
	boolean_t vdev_rebuild_cancel_wanted;
	boolean_t vdev_rebuild_reset_wanted;
	kmutex_t vdev_rebuild_lock;
	kcondvar_t vdev_rebuild_cv;
	kthread_t *vdev_rebuild_thread;
	vdev_rebuild_t vdev_rebuild_config;

	/* For limiting outstanding I/Os (initialize, TRIM) */
	kmutex_t vdev_initialize_io_lock;
	kcondvar_t vdev_initialize_io_cv;
	uint64_t vdev_initialize_inflight;
	kmutex_t vdev_trim_io_lock;
	kcondvar_t vdev_trim_io_cv;
	uint64_t vdev_trim_inflight[3];

	/*
	* Values stored in the config for an indirect or removing vdev.
	*/
	vdev_indirect_config_t vdev_indirect_config;

	/*
	* The vdev_indirect_rwlock protects the vdev_indirect_mapping
	* pointer from changing on indirect vdevs (when it is condensed).
	* Note that removing (not yet indirect) vdevs have different
	* access patterns (the mapping is not accessed from open context,
	* e.g. from zio_read) and locking strategy (e.g. svr_lock).
	*/
	krwlock_t vdev_indirect_rwlock;
	vdev_indirect_mapping_t *vdev_indirect_mapping;
	vdev_indirect_births_t *vdev_indirect_births;

	/*
	* In memory data structures used to manage the obsolete sm, for
	* indirect or removing vdevs.
	*
	* The vdev_obsolete_segments is the in-core record of the segments
	* that are no longer referenced anywhere in the pool (due to
	* being freed or remapped and not referenced by any snapshots).
	* During a sync, segments are added to vdev_obsolete_segments
	* via vdev_indirect_mark_obsolete(); at the end of each sync
	* pass, this is appended to vdev_obsolete_sm via
	* vdev_indirect_sync_obsolete(). The vdev_obsolete_lock
	* protects against concurrent modifications of vdev_obsolete_segments
	* from multiple zio threads.
	*/
	kmutex_t vdev_obsolete_lock;
	range_tree_t *vdev_obsolete_segments;
	space_map_t *vdev_obsolete_sm;

	/*
	* Protects the vdev_scan_io_queue field itself as well as the
	* structure's contents (when present).
	*/
	kmutex_t vdev_scan_io_queue_lock;
	struct dsl_scan_io_queue *vdev_scan_io_queue;

	/*
	* Leaf vdev state.
	*/
	range_tree_t vdev_dtl[DTL_TYPES]; / dirty time logs */
	space_map_t vdev_dtl_sm; / dirty time log space map */
	txg_node_t vdev_dtl_node; /* per-txg dirty DTL linkage */
	uint64_t vdev_dtl_object; /* DTL object */
	uint64_t vdev_psize; /* physical device capacity */
	uint64_t vdev_wholedisk; /* true if this is a whole disk */
	uint64_t vdev_offline; /* persistent offline state */
	uint64_t vdev_faulted; /* persistent faulted state */
	uint64_t vdev_degraded; /* persistent degraded state */
	uint64_t vdev_removed; /* persistent removed state */
	uint64_t vdev_resilver_txg; /* persistent resilvering state */
	uint64_t vdev_rebuild_txg; /* persistent rebuilding state */
	char vdev_path; / vdev path (if any) */
	char vdev_devid; / vdev devid (if any) */
	char vdev_physpath; / vdev device path (if any) */
	char vdev_enc_sysfs_path; / enclosure sysfs path */
	char vdev_fru; / physical FRU location */
	uint64_t vdev_not_present; /* not present during import */
	uint64_t vdev_unspare; /* unspare when resilvering done */
	boolean_t vdev_nowritecache; /* true if flushwritecache failed */
	boolean_t vdev_has_trim; /* TRIM is supported */
	boolean_t vdev_has_securetrim; /* secure TRIM is supported */
	boolean_t vdev_checkremove; /* temporary online test */
	boolean_t vdev_forcefault; /* force online fault */
	boolean_t vdev_splitting; /* split or repair in progress */
	boolean_t vdev_delayed_close; /* delayed device close? */
	boolean_t vdev_tmpoffline; /* device taken offline temporarily? */
	boolean_t vdev_detached; /* device detached? */
	boolean_t vdev_cant_read; /* vdev is failing all reads */
	boolean_t vdev_cant_write; /* vdev is failing all writes */
	boolean_t vdev_isspare; /* was a hot spare */
	boolean_t vdev_isl2cache; /* was a l2cache device */
	boolean_t vdev_copy_uberblocks; /* post expand copy uberblocks */
	boolean_t vdev_resilver_deferred; /* resilver deferred */
	boolean_t vdev_kobj_flag; /* kobj event record */
	boolean_t vdev_attaching; /* vdev attach ashift handling */
	vdev_queue_t vdev_queue; /* I/O deadline schedule queue */
	spa_aux_vdev_t vdev_aux; / for l2cache and spares vdevs */
	zio_t vdev_probe_zio; / root of current probe */
	vdev_aux_t vdev_label_aux; /* on-disk aux state */
	uint64_t vdev_leaf_zap;
	hrtime_t vdev_mmp_pending; /* 0 if write finished */
	uint64_t vdev_mmp_kstat_id; /* to find kstat entry */
	uint64_t vdev_expansion_time; /* vdev's last expansion time */
	list_node_t vdev_leaf_node; /* leaf vdev list */

	/*
	* For DTrace to work in userland (libzpool) context, these fields must
	* remain at the end of the structure. DTrace will use the kernel's
	* CTF definition for 'struct vdev', and since the size of a kmutex_t is
	* larger in userland, the offsets for the rest of the fields would be
	* incorrect.
	*/
	kmutex_t vdev_dtl_lock; /* vdev_dtl_{map,resilver} */
	kmutex_t vdev_stat_lock; /* vdev_stat */
	kmutex_t vdev_probe_lock; /* protects vdev_probe_zio */

	/*
	* We rate limit ZIO delay, deadman, and checksum events, since they
	* can flood ZED with tons of events when a drive is acting up.
	*/
	zfs_ratelimit_t vdev_delay_rl;
	zfs_ratelimit_t vdev_deadman_rl;
	zfs_ratelimit_t vdev_checksum_rl;

	/*
	- * Checksum and IO thresholds for tuning ZED
	+ * Vdev properties for tuning ZED or zfsd
	*/
	uint64_t vdev_checksum_n;
	uint64_t vdev_checksum_t;
	uint64_t vdev_io_n;
	uint64_t vdev_io_t;
	+ uint64_t vdev_slow_io_n;
	+ uint64_t vdev_slow_io_t;
	};

	#define VDEV_PAD_SIZE (8 << 10)
	/* 2 padding areas (vl_pad1 and vl_be) to skip */
	#define VDEV_SKIP_SIZE VDEV_PAD_SIZE * 2
	#define VDEV_PHYS_SIZE (112 << 10)
	#define VDEV_UBERBLOCK_RING (128 << 10)

	/*
	* MMP blocks occupy the last MMP_BLOCKS_PER_LABEL slots in the uberblock
	* ring when MMP is enabled.
	*/
	#define MMP_BLOCKS_PER_LABEL 1

	/* The largest uberblock we support is 8k. */
	#define MAX_UBERBLOCK_SHIFT (13)
	#define VDEV_UBERBLOCK_SHIFT(vd) \
	MIN(MAX((vd)->vdev_top->vdev_ashift, UBERBLOCK_SHIFT), \
	MAX_UBERBLOCK_SHIFT)
	#define VDEV_UBERBLOCK_COUNT(vd) \
	(VDEV_UBERBLOCK_RING >> VDEV_UBERBLOCK_SHIFT(vd))
	#define VDEV_UBERBLOCK_OFFSET(vd, n) \
	offsetof(vdev_label_t, vl_uberblock[(n) << VDEV_UBERBLOCK_SHIFT(vd)])
	#define VDEV_UBERBLOCK_SIZE(vd) (1ULL << VDEV_UBERBLOCK_SHIFT(vd))

	typedef struct vdev_phys {
	char vp_nvlist[VDEV_PHYS_SIZE - sizeof (zio_eck_t)];
	zio_eck_t vp_zbt;
	} vdev_phys_t;

	typedef enum vbe_vers {
	/*
	* The bootenv file is stored as ascii text in the envblock.
	* It is used by the GRUB bootloader used on Linux to store the
	* contents of the grubenv file. The file is stored as raw ASCII,
	* and is protected by an embedded checksum. By default, GRUB will
	* check if the boot filesystem supports storing the environment data
	* in a special location, and if so, will invoke filesystem specific
	* logic to retrieve it. This can be overridden by a variable, should
	* the user so desire.
	*/
	VB_RAW = 0,

	/*
	* The bootenv file is converted to an nvlist and then packed into the
	* envblock.
	*/
	VB_NVLIST = 1
	} vbe_vers_t;

	typedef struct vdev_boot_envblock {
	uint64_t vbe_version;
	char vbe_bootenv[VDEV_PAD_SIZE - sizeof (uint64_t) -
	sizeof (zio_eck_t)];
	zio_eck_t vbe_zbt;
	} vdev_boot_envblock_t;
	_Static_assert(sizeof (vdev_boot_envblock_t) == VDEV_PAD_SIZE,
	"vdev_boot_envblock_t wrong size");

	typedef struct vdev_label {
	char vl_pad1[VDEV_PAD_SIZE]; /* 8K */
	vdev_boot_envblock_t vl_be; /* 8K */
	vdev_phys_t vl_vdev_phys; /* 112K */
	char vl_uberblock[VDEV_UBERBLOCK_RING]; /* 128K */
	} vdev_label_t; /* 256K total */

	/*
	* vdev_dirty() flags
	*/
	#define VDD_METASLAB 0x01
	#define VDD_DTL 0x02

	/* Offset of embedded boot loader region on each label */
	#define VDEV_BOOT_OFFSET (2 * sizeof (vdev_label_t))
	/*
	* Size of embedded boot loader region on each label.
	* The total size of the first two labels plus the boot area is 4MB.
	*/
	#define VDEV_BOOT_SIZE (7ULL << 19) /* 3.5M */

	/*
	* Size of label regions at the start and end of each leaf device.
	*/
	#define VDEV_LABEL_START_SIZE (2 * sizeof (vdev_label_t) + VDEV_BOOT_SIZE)
	#define VDEV_LABEL_END_SIZE (2 * sizeof (vdev_label_t))
	#define VDEV_LABELS 4
	#define VDEV_BEST_LABEL VDEV_LABELS
	#define VDEV_OFFSET_IS_LABEL(vd, off) \
	(((off) < VDEV_LABEL_START_SIZE) \|\| \
	((off) >= ((vd)->vdev_psize - VDEV_LABEL_END_SIZE)))

	#define VDEV_ALLOC_LOAD 0
	#define VDEV_ALLOC_ADD 1
	#define VDEV_ALLOC_SPARE 2
	#define VDEV_ALLOC_L2CACHE 3
	#define VDEV_ALLOC_ROOTPOOL 4
	#define VDEV_ALLOC_SPLIT 5
	#define VDEV_ALLOC_ATTACH 6

	/*
	* Allocate or free a vdev
	*/
	extern vdev_t vdev_alloc_common(spa_t spa, uint_t id, uint64_t guid,
	vdev_ops_t *ops);
	extern int vdev_alloc(spa_t spa, vdev_t vdp, nvlist_t config,
	vdev_t *parent, uint_t id, int alloctype);
	extern void vdev_free(vdev_t *vd);

	/*
	* Add or remove children and parents
	*/
	extern void vdev_add_child(vdev_t pvd, vdev_t cvd);
	extern void vdev_remove_child(vdev_t pvd, vdev_t cvd);
	extern void vdev_compact_children(vdev_t *pvd);
	extern vdev_t vdev_add_parent(vdev_t cvd, vdev_ops_t *ops);
	extern void vdev_remove_parent(vdev_t *cvd);

	/*
	* vdev sync load and sync
	*/
	extern boolean_t vdev_log_state_valid(vdev_t *vd);
	extern int vdev_load(vdev_t *vd);
	extern int vdev_dtl_load(vdev_t *vd);
	extern void vdev_sync(vdev_t *vd, uint64_t txg);
	extern void vdev_sync_done(vdev_t *vd, uint64_t txg);
	extern void vdev_dirty(vdev_t vd, int flags, void arg, uint64_t txg);
	extern void vdev_dirty_leaves(vdev_t *vd, int flags, uint64_t txg);

	/*
	* Available vdev types.
	*/
	extern vdev_ops_t vdev_root_ops;
	extern vdev_ops_t vdev_mirror_ops;
	extern vdev_ops_t vdev_replacing_ops;
	extern vdev_ops_t vdev_raidz_ops;
	extern vdev_ops_t vdev_draid_ops;
	extern vdev_ops_t vdev_draid_spare_ops;
	extern vdev_ops_t vdev_disk_ops;
	extern vdev_ops_t vdev_file_ops;
	extern vdev_ops_t vdev_missing_ops;
	extern vdev_ops_t vdev_hole_ops;
	extern vdev_ops_t vdev_spare_ops;
	extern vdev_ops_t vdev_indirect_ops;

	/*
	* Common size functions
	*/
	extern void vdev_default_xlate(vdev_t vd, const range_seg64_t logical_rs,
	range_seg64_t physical_rs, range_seg64_t remain_rs);
	extern uint64_t vdev_default_asize(vdev_t *vd, uint64_t psize);
	extern uint64_t vdev_default_min_asize(vdev_t *vd);
	extern uint64_t vdev_get_min_asize(vdev_t *vd);
	extern void vdev_set_min_asize(vdev_t *vd);
	extern uint64_t vdev_get_min_alloc(vdev_t *vd);
	extern uint64_t vdev_get_nparity(vdev_t *vd);
	extern uint64_t vdev_get_ndisks(vdev_t *vd);

	/*
	* Global variables
	*/
	extern int zfs_vdev_standard_sm_blksz;

	/*
	* Functions from vdev_indirect.c
	*/
	extern void vdev_indirect_sync_obsolete(vdev_t vd, dmu_tx_t tx);
	extern boolean_t vdev_indirect_should_condense(vdev_t *vd);
	extern void spa_condense_indirect_start_sync(vdev_t vd, dmu_tx_t tx);
	extern int vdev_obsolete_sm_object(vdev_t vd, uint64_t sm_obj);
	extern int vdev_obsolete_counts_are_precise(vdev_t vd, boolean_t are_precise);

	/*
	* Other miscellaneous functions
	*/
	int vdev_checkpoint_sm_object(vdev_t vd, uint64_t sm_obj);
	void vdev_metaslab_group_create(vdev_t *vd);
	uint64_t vdev_best_ashift(uint64_t logical, uint64_t a, uint64_t b);

	/*
	* Vdev ashift optimization tunables
	*/
	extern uint_t zfs_vdev_min_auto_ashift;
	extern uint_t zfs_vdev_max_auto_ashift;
	int param_set_min_auto_ashift(ZFS_MODULE_PARAM_ARGS);
	int param_set_max_auto_ashift(ZFS_MODULE_PARAM_ARGS);

	#ifdef __cplusplus
	}
	#endif

	#endif /* _SYS_VDEV_IMPL_H */
	diff --git a/sys/contrib/openzfs/include/sys/zap.h b/sys/contrib/openzfs/include/sys/zap.h
	index 308a7c7284d7..96ddcc324b65 100644
	--- a/sys/contrib/openzfs/include/sys/zap.h
	+++ b/sys/contrib/openzfs/include/sys/zap.h
	@@ -1,515 +1,523 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2012, 2018 by Delphix. All rights reserved.
	* Copyright 2017 Nexenta Systems, Inc.
	*/

	#ifndef _SYS_ZAP_H
	#define _SYS_ZAP_H

	/*
	* ZAP - ZFS Attribute Processor
	*
	* The ZAP is a module which sits on top of the DMU (Data Management
	* Unit) and implements a higher-level storage primitive using DMU
	* objects. Its primary consumer is the ZPL (ZFS Posix Layer).
	*
	* A "zapobj" is a DMU object which the ZAP uses to stores attributes.
	* Users should use only zap routines to access a zapobj - they should
	* not access the DMU object directly using DMU routines.
	*
	* The attributes stored in a zapobj are name-value pairs. The name is
	* a zero-terminated string of up to ZAP_MAXNAMELEN bytes (including
	* terminating NULL). The value is an array of integers, which may be
	* 1, 2, 4, or 8 bytes long. The total space used by the array (number
	* of integers * integer length) can be up to ZAP_MAXVALUELEN bytes.
	* Note that an 8-byte integer value can be used to store the location
	* (object number) of another dmu object (which may be itself a zapobj).
	* Note that you can use a zero-length attribute to store a single bit
	* of information - the attribute is present or not.
	*
	* The ZAP routines are thread-safe. However, you must observe the
	* DMU's restriction that a transaction may not be operated on
	* concurrently.
	*
	* Any of the routines that return an int may return an I/O error (EIO
	* or ECHECKSUM).
	*
	*
	* Implementation / Performance Notes:
	*
	* The ZAP is intended to operate most efficiently on attributes with
	* short (49 bytes or less) names and single 8-byte values, for which
	* the microzap will be used. The ZAP should be efficient enough so
	* that the user does not need to cache these attributes.
	*
	* The ZAP's locking scheme makes its routines thread-safe. Operations
	* on different zapobjs will be processed concurrently. Operations on
	* the same zapobj which only read data will be processed concurrently.
	* Operations on the same zapobj which modify data will be processed
	* concurrently when there are many attributes in the zapobj (because
	* the ZAP uses per-block locking - more than 128 * (number of cpus)
	* small attributes will suffice).
	*/

	/*
	* We're using zero-terminated byte strings (ie. ASCII or UTF-8 C
	* strings) for the names of attributes, rather than a byte string
	* bounded by an explicit length. If some day we want to support names
	* in character sets which have embedded zeros (eg. UTF-16, UTF-32),
	* we'll have to add routines for using length-bounded strings.
	*/

	#include <sys/dmu.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*
	* Specifies matching criteria for ZAP lookups.
	* MT_NORMALIZE Use ZAP normalization flags, which can include both
	* unicode normalization and case-insensitivity.
	* MT_MATCH_CASE Do case-sensitive lookups even if MT_NORMALIZE is
	* specified and ZAP normalization flags include
	* U8_TEXTPREP_TOUPPER.
	*/
	typedef enum matchtype {
	MT_NORMALIZE = 1 << 0,
	MT_MATCH_CASE = 1 << 1,
	} matchtype_t;

	typedef enum zap_flags {
	/* Use 64-bit hash value (serialized cursors will always use 64-bits) */
	ZAP_FLAG_HASH64 = 1 << 0,
	/* Key is binary, not string (zap_add_uint64() can be used) */
	ZAP_FLAG_UINT64_KEY = 1 << 1,
	/*
	* First word of key (which must be an array of uint64) is
	* already randomly distributed.
	*/
	ZAP_FLAG_PRE_HASHED_KEY = 1 << 2,
	#if defined(__linux__) && defined(_KERNEL)
	} zfs_zap_flags_t;
	#define zap_flags_t zfs_zap_flags_t
	#else
	} zap_flags_t;
	#endif

	/*
	* Create a new zapobj with no attributes and return its object number.
	*/
	uint64_t zap_create(objset_t *ds, dmu_object_type_t ot,
	dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx);
	uint64_t zap_create_dnsize(objset_t *ds, dmu_object_type_t ot,
	dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx);
	uint64_t zap_create_norm(objset_t *ds, int normflags, dmu_object_type_t ot,
	dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx);
	uint64_t zap_create_norm_dnsize(objset_t *ds, int normflags,
	dmu_object_type_t ot, dmu_object_type_t bonustype, int bonuslen,
	int dnodesize, dmu_tx_t *tx);
	uint64_t zap_create_flags(objset_t *os, int normflags, zap_flags_t flags,
	dmu_object_type_t ot, int leaf_blockshift, int indirect_blockshift,
	dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx);
	uint64_t zap_create_flags_dnsize(objset_t *os, int normflags,
	zap_flags_t flags, dmu_object_type_t ot, int leaf_blockshift,
	int indirect_blockshift, dmu_object_type_t bonustype, int bonuslen,
	int dnodesize, dmu_tx_t *tx);
	uint64_t zap_create_hold(objset_t *os, int normflags, zap_flags_t flags,
	dmu_object_type_t ot, int leaf_blockshift, int indirect_blockshift,
	dmu_object_type_t bonustype, int bonuslen, int dnodesize,
	dnode_t *allocated_dnode, const void tag, dmu_tx_t *tx);

	uint64_t zap_create_link(objset_t *os, dmu_object_type_t ot,
	uint64_t parent_obj, const char name, dmu_tx_t tx);
	uint64_t zap_create_link_dnsize(objset_t *os, dmu_object_type_t ot,
	uint64_t parent_obj, const char name, int dnodesize, dmu_tx_t tx);

	/*
	* Initialize an already-allocated object.
	*/
	void mzap_create_impl(dnode_t *dn, int normflags, zap_flags_t flags,
	dmu_tx_t *tx);

	/*
	* Create a new zapobj with no attributes from the given (unallocated)
	* object number.
	*/
	int zap_create_claim(objset_t *ds, uint64_t obj, dmu_object_type_t ot,
	dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx);
	int zap_create_claim_dnsize(objset_t *ds, uint64_t obj, dmu_object_type_t ot,
	dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx);
	int zap_create_claim_norm(objset_t *ds, uint64_t obj,
	int normflags, dmu_object_type_t ot,
	dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx);
	int zap_create_claim_norm_dnsize(objset_t *ds, uint64_t obj,
	int normflags, dmu_object_type_t ot,
	dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx);

	/*
	* The zapobj passed in must be a valid ZAP object for all of the
	* following routines.
	*/

	/*
	* Destroy this zapobj and all its attributes.
	*
	* Frees the object number using dmu_object_free.
	*/
	int zap_destroy(objset_t ds, uint64_t zapobj, dmu_tx_t tx);

	/*
	* Manipulate attributes.
	*
	* 'integer_size' is in bytes, and must be 1, 2, 4, or 8.
	*/

	/*
	* Retrieve the contents of the attribute with the given name.
	*
	* If the requested attribute does not exist, the call will fail and
	* return ENOENT.
	*
	* If 'integer_size' is smaller than the attribute's integer size, the
	* call will fail and return EINVAL.
	*
	* If 'integer_size' is equal to or larger than the attribute's integer
	* size, the call will succeed and return 0.
	*
	* When converting to a larger integer size, the integers will be treated as
	* unsigned (ie. no sign-extension will be performed).
	*
	* 'num_integers' is the length (in integers) of 'buf'.
	*
	* If the attribute is longer than the buffer, as many integers as will
	* fit will be transferred to 'buf'. If the entire attribute was not
	* transferred, the call will return EOVERFLOW.
	*/
	int zap_lookup(objset_t ds, uint64_t zapobj, const char name,
	uint64_t integer_size, uint64_t num_integers, void *buf);

	/*
	* If rn_len is nonzero, realname will be set to the name of the found
	* entry (which may be different from the requested name if matchtype is
	* not MT_EXACT).
	*
	* If normalization_conflictp is not NULL, it will be set if there is
	* another name with the same case/unicode normalized form.
	*/
	int zap_lookup_norm(objset_t ds, uint64_t zapobj, const char name,
	uint64_t integer_size, uint64_t num_integers, void *buf,
	matchtype_t mt, char *realname, int rn_len,
	boolean_t *normalization_conflictp);
	int zap_lookup_uint64(objset_t os, uint64_t zapobj, const uint64_t key,
	int key_numints, uint64_t integer_size, uint64_t num_integers, void *buf);
	int zap_contains(objset_t ds, uint64_t zapobj, const char name);
	int zap_prefetch(objset_t os, uint64_t zapobj, const char name);
	int zap_prefetch_uint64(objset_t os, uint64_t zapobj, const uint64_t key,
	int key_numints);

	int zap_lookup_by_dnode(dnode_t dn, const char name,
	uint64_t integer_size, uint64_t num_integers, void *buf);
	int zap_lookup_norm_by_dnode(dnode_t dn, const char name,
	uint64_t integer_size, uint64_t num_integers, void *buf,
	matchtype_t mt, char *realname, int rn_len,
	boolean_t *ncp);

	int zap_count_write_by_dnode(dnode_t dn, const char name,
	int add, zfs_refcount_t towrite, zfs_refcount_t tooverwrite);

	/*
	* Create an attribute with the given name and value.
	*
	* If an attribute with the given name already exists, the call will
	* fail and return EEXIST.
	*/
	int zap_add(objset_t ds, uint64_t zapobj, const char key,
	int integer_size, uint64_t num_integers,
	const void val, dmu_tx_t tx);
	int zap_add_by_dnode(dnode_t dn, const char key,
	int integer_size, uint64_t num_integers,
	const void val, dmu_tx_t tx);
	int zap_add_uint64(objset_t ds, uint64_t zapobj, const uint64_t key,
	int key_numints, int integer_size, uint64_t num_integers,
	const void val, dmu_tx_t tx);
	+int zap_add_uint64_by_dnode(dnode_t dn, const uint64_t key,
	+ int key_numints, int integer_size, uint64_t num_integers,
	+ const void val, dmu_tx_t tx);

	/*
	* Set the attribute with the given name to the given value. If an
	* attribute with the given name does not exist, it will be created. If
	* an attribute with the given name already exists, the previous value
	* will be overwritten. The integer_size may be different from the
	* existing attribute's integer size, in which case the attribute's
	* integer size will be updated to the new value.
	*/
	int zap_update(objset_t ds, uint64_t zapobj, const char name,
	int integer_size, uint64_t num_integers, const void val, dmu_tx_t tx);
	int zap_update_uint64(objset_t os, uint64_t zapobj, const uint64_t key,
	int key_numints,
	int integer_size, uint64_t num_integers, const void val, dmu_tx_t tx);
	+int zap_update_uint64_by_dnode(dnode_t dn, const uint64_t key,
	+ int key_numints,
	+ int integer_size, uint64_t num_integers, const void val, dmu_tx_t tx);

	/*
	* Get the length (in integers) and the integer size of the specified
	* attribute.
	*
	* If the requested attribute does not exist, the call will fail and
	* return ENOENT.
	*/
	int zap_length(objset_t ds, uint64_t zapobj, const char name,
	uint64_t integer_size, uint64_t num_integers);
	int zap_length_uint64(objset_t os, uint64_t zapobj, const uint64_t key,
	int key_numints, uint64_t integer_size, uint64_t num_integers);

	/*
	* Remove the specified attribute.
	*
	* If the specified attribute does not exist, the call will fail and
	* return ENOENT.
	*/
	int zap_remove(objset_t ds, uint64_t zapobj, const char name, dmu_tx_t *tx);
	int zap_remove_norm(objset_t ds, uint64_t zapobj, const char name,
	matchtype_t mt, dmu_tx_t *tx);
	int zap_remove_by_dnode(dnode_t dn, const char name, dmu_tx_t *tx);
	int zap_remove_uint64(objset_t os, uint64_t zapobj, const uint64_t key,
	int key_numints, dmu_tx_t *tx);
	+int zap_remove_uint64_by_dnode(dnode_t dn, const uint64_t key,
	+ int key_numints, dmu_tx_t *tx);

	/*
	* Returns (in *count) the number of attributes in the specified zap
	* object.
	*/
	int zap_count(objset_t ds, uint64_t zapobj, uint64_t count);

	/*
	* Returns (in name) the name of the entry whose (value & mask)
	* (za_first_integer) is value, or ENOENT if not found. The string
	* pointed to by name must be at least 256 bytes long. If mask==0, the
	* match must be exact (ie, same as mask=-1ULL).
	*/
	int zap_value_search(objset_t *os, uint64_t zapobj,
	uint64_t value, uint64_t mask, char *name);

	/*
	* Transfer all the entries from fromobj into intoobj. Only works on
	* int_size=8 num_integers=1 values. Fails if there are any duplicated
	* entries.
	*/
	int zap_join(objset_t os, uint64_t fromobj, uint64_t intoobj, dmu_tx_t tx);

	/* Same as zap_join, but set the values to 'value'. */
	int zap_join_key(objset_t *os, uint64_t fromobj, uint64_t intoobj,
	uint64_t value, dmu_tx_t *tx);

	/* Same as zap_join, but add together any duplicated entries. */
	int zap_join_increment(objset_t *os, uint64_t fromobj, uint64_t intoobj,
	dmu_tx_t *tx);

	/*
	* Manipulate entries where the name + value are the "same" (the name is
	* a stringified version of the value).
	*/
	int zap_add_int(objset_t os, uint64_t obj, uint64_t value, dmu_tx_t tx);
	int zap_remove_int(objset_t os, uint64_t obj, uint64_t value, dmu_tx_t tx);
	int zap_lookup_int(objset_t *os, uint64_t obj, uint64_t value);
	int zap_increment_int(objset_t *os, uint64_t obj, uint64_t key, int64_t delta,
	dmu_tx_t *tx);

	/* Here the key is an int and the value is a different int. */
	int zap_add_int_key(objset_t *os, uint64_t obj,
	uint64_t key, uint64_t value, dmu_tx_t *tx);
	int zap_update_int_key(objset_t *os, uint64_t obj,
	uint64_t key, uint64_t value, dmu_tx_t *tx);
	int zap_lookup_int_key(objset_t *os, uint64_t obj,
	uint64_t key, uint64_t *valuep);

	int zap_increment(objset_t os, uint64_t obj, const char name, int64_t delta,
	dmu_tx_t *tx);

	struct zap;
	struct zap_leaf;
	typedef struct zap_cursor {
	/* This structure is opaque! */
	objset_t *zc_objset;
	struct zap *zc_zap;
	struct zap_leaf *zc_leaf;
	uint64_t zc_zapobj;
	uint64_t zc_serialized;
	uint64_t zc_hash;
	uint32_t zc_cd;
	boolean_t zc_prefetch;
	} zap_cursor_t;

	typedef struct {
	int za_integer_length;
	/*
	* za_normalization_conflict will be set if there are additional
	* entries with this normalized form (eg, "foo" and "Foo").
	*/
	boolean_t za_normalization_conflict;
	uint64_t za_num_integers;
	uint64_t za_first_integer; /* no sign extension for <8byte ints */
	char za_name[ZAP_MAXNAMELEN];
	} zap_attribute_t;

	/*
	* The interface for listing all the attributes of a zapobj can be
	* thought of as cursor moving down a list of the attributes one by
	* one. The cookie returned by the zap_cursor_serialize routine is
	* persistent across system calls (and across reboot, even).
	*/

	/*
	* Initialize a zap cursor, pointing to the "first" attribute of the
	* zapobj. You must _fini the cursor when you are done with it.
	*/
	void zap_cursor_init(zap_cursor_t zc, objset_t os, uint64_t zapobj);
	void zap_cursor_init_noprefetch(zap_cursor_t zc, objset_t os,
	uint64_t zapobj);
	void zap_cursor_fini(zap_cursor_t *zc);

	/*
	* Get the attribute currently pointed to by the cursor. Returns
	* ENOENT if at the end of the attributes.
	*/
	int zap_cursor_retrieve(zap_cursor_t zc, zap_attribute_t za);

	/*
	* Advance the cursor to the next attribute.
	*/
	void zap_cursor_advance(zap_cursor_t *zc);

	/*
	* Get a persistent cookie pointing to the current position of the zap
	* cursor. The low 4 bits in the cookie are always zero, and thus can
	* be used as to differentiate a serialized cookie from a different type
	* of value. The cookie will be less than 2^32 as long as there are
	* fewer than 2^22 (4.2 million) entries in the zap object.
	*/
	uint64_t zap_cursor_serialize(zap_cursor_t *zc);

	/*
	* Initialize a zap cursor pointing to the position recorded by
	* zap_cursor_serialize (in the "serialized" argument). You can also
	* use a "serialized" argument of 0 to start at the beginning of the
	* zapobj (ie. zap_cursor_init_serialized(..., 0) is equivalent to
	* zap_cursor_init(...).)
	*/
	void zap_cursor_init_serialized(zap_cursor_t zc, objset_t ds,
	uint64_t zapobj, uint64_t serialized);


	#define ZAP_HISTOGRAM_SIZE 10

	typedef struct zap_stats {
	/*
	* Size of the pointer table (in number of entries).
	* This is always a power of 2, or zero if it's a microzap.
	* In general, it should be considerably greater than zs_num_leafs.
	*/
	uint64_t zs_ptrtbl_len;

	uint64_t zs_blocksize; /* size of zap blocks */

	/*
	* The number of blocks used. Note that some blocks may be
	* wasted because old ptrtbl's and large name/value blocks are
	* not reused. (Although their space is reclaimed, we don't
	* reuse those offsets in the object.)
	*/
	uint64_t zs_num_blocks;

	/*
	* Pointer table values from zap_ptrtbl in the zap_phys_t
	*/
	uint64_t zs_ptrtbl_nextblk; /* next (larger) copy start block */
	uint64_t zs_ptrtbl_blks_copied; /* number source blocks copied */
	uint64_t zs_ptrtbl_zt_blk; /* starting block number */
	uint64_t zs_ptrtbl_zt_numblks; /* number of blocks */
	uint64_t zs_ptrtbl_zt_shift; /* bits to index it */

	/*
	* Values of the other members of the zap_phys_t
	*/
	uint64_t zs_block_type; /* ZBT_HEADER */
	uint64_t zs_magic; /* ZAP_MAGIC */
	uint64_t zs_num_leafs; /* The number of leaf blocks */
	uint64_t zs_num_entries; /* The number of zap entries */
	uint64_t zs_salt; /* salt to stir into hash function */

	/*
	* Histograms. For all histograms, the last index
	* (ZAP_HISTOGRAM_SIZE-1) includes any values which are greater
	* than what can be represented. For example
	* zs_leafs_with_n5_entries[ZAP_HISTOGRAM_SIZE-1] is the number
	* of leafs with more than 45 entries.
	*/

	/*
	* zs_leafs_with_n_pointers[n] is the number of leafs with
	* 2^n pointers to it.
	*/
	uint64_t zs_leafs_with_2n_pointers[ZAP_HISTOGRAM_SIZE];

	/*
	* zs_leafs_with_n_entries[n] is the number of leafs with
	* [n5, (n+1)5) entries. In the current implementation, there
	* can be at most 55 entries in any block, but there may be
	* fewer if the name or value is large, or the block is not
	* completely full.
	*/
	uint64_t zs_blocks_with_n5_entries[ZAP_HISTOGRAM_SIZE];

	/*
	* zs_leafs_n_tenths_full[n] is the number of leafs whose
	* fullness is in the range [n/10, (n+1)/10).
	*/
	uint64_t zs_blocks_n_tenths_full[ZAP_HISTOGRAM_SIZE];

	/*
	* zs_entries_using_n_chunks[n] is the number of entries which
	* consume n 24-byte chunks. (Note, large names/values only use
	* one chunk, but contribute to zs_num_blocks_large.)
	*/
	uint64_t zs_entries_using_n_chunks[ZAP_HISTOGRAM_SIZE];

	/*
	* zs_buckets_with_n_entries[n] is the number of buckets (each
	* leaf has 64 buckets) with n entries.
	* zs_buckets_with_n_entries[1] should be very close to
	* zs_num_entries.
	*/
	uint64_t zs_buckets_with_n_entries[ZAP_HISTOGRAM_SIZE];
	} zap_stats_t;

	/*
	* Get statistics about a ZAP object. Note: you need to be aware of the
	* internal implementation of the ZAP to correctly interpret some of the
	* statistics. This interface shouldn't be relied on unless you really
	* know what you're doing.
	*/
	int zap_get_stats(objset_t ds, uint64_t zapobj, zap_stats_t zs);

	#ifdef __cplusplus
	}
	#endif

	#endif /* _SYS_ZAP_H */
	diff --git a/sys/contrib/openzfs/include/sys/zap_impl.h b/sys/contrib/openzfs/include/sys/zap_impl.h
	index 74853f5faceb..2959aa9b2ca4 100644
	--- a/sys/contrib/openzfs/include/sys/zap_impl.h
	+++ b/sys/contrib/openzfs/include/sys/zap_impl.h
	@@ -1,240 +1,241 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
	* Copyright (c) 2013, 2016 by Delphix. All rights reserved.
	* Copyright 2017 Nexenta Systems, Inc.
	*/

	#ifndef _SYS_ZAP_IMPL_H
	#define _SYS_ZAP_IMPL_H

	#include <sys/zap.h>
	#include <sys/zfs_context.h>
	#include <sys/avl.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	extern int fzap_default_block_shift;

	#define ZAP_MAGIC 0x2F52AB2ABULL

	#define FZAP_BLOCK_SHIFT(zap) ((zap)->zap_f.zap_block_shift)

	#define MZAP_ENT_LEN 64
	#define MZAP_NAME_LEN (MZAP_ENT_LEN - 8 - 4 - 2)
	#define MZAP_MAX_BLKSZ SPA_OLD_MAXBLOCKSIZE

	#define ZAP_NEED_CD (-1U)

	typedef struct mzap_ent_phys {
	uint64_t mze_value;
	uint32_t mze_cd;
	uint16_t mze_pad; /* in case we want to chain them someday */
	char mze_name[MZAP_NAME_LEN];
	} mzap_ent_phys_t;

	typedef struct mzap_phys {
	uint64_t mz_block_type; /* ZBT_MICRO */
	uint64_t mz_salt;
	uint64_t mz_normflags;
	uint64_t mz_pad[5];
	mzap_ent_phys_t mz_chunk[1];
	/* actually variable size depending on block size */
	} mzap_phys_t;

	typedef struct mzap_ent {
	uint32_t mze_hash;
	uint16_t mze_cd; /* copy from mze_phys->mze_cd */
	uint16_t mze_chunkid;
	} mzap_ent_t;

	#define MZE_PHYS(zap, mze) \
	(&zap_m_phys(zap)->mz_chunk[(mze)->mze_chunkid])

	/*
	* The (fat) zap is stored in one object. It is an array of
	* 1<<FZAP_BLOCK_SHIFT byte blocks. The layout looks like one of:
	*
	* ptrtbl fits in first block:
	* [zap_phys_t zap_ptrtbl_shift < 6] [zap_leaf_t] ...
	*
	* ptrtbl too big for first block:
	* [zap_phys_t zap_ptrtbl_shift >= 6] [zap_leaf_t] [ptrtbl] ...
	*
	*/

	struct dmu_buf;
	struct zap_leaf;

	#define ZBT_LEAF ((1ULL << 63) + 0)
	#define ZBT_HEADER ((1ULL << 63) + 1)
	#define ZBT_MICRO ((1ULL << 63) + 3)
	/* any other values are ptrtbl blocks */

	/*
	* the embedded pointer table takes up half a block:
	* block size / entry size (2^3) / 2
	*/
	#define ZAP_EMBEDDED_PTRTBL_SHIFT(zap) (FZAP_BLOCK_SHIFT(zap) - 3 - 1)

	/*
	* The embedded pointer table starts half-way through the block. Since
	* the pointer table itself is half the block, it starts at (64-bit)
	* word number (1<<ZAP_EMBEDDED_PTRTBL_SHIFT(zap)).
	*/
	#define ZAP_EMBEDDED_PTRTBL_ENT(zap, idx) \
	((uint64_t *)zap_f_phys(zap)) \
	[(idx) + (1<<ZAP_EMBEDDED_PTRTBL_SHIFT(zap))]

	/*
	* TAKE NOTE:
	* If zap_phys_t is modified, zap_byteswap() must be modified.
	*/
	typedef struct zap_phys {
	uint64_t zap_block_type; /* ZBT_HEADER */
	uint64_t zap_magic; /* ZAP_MAGIC */

	struct zap_table_phys {
	uint64_t zt_blk; /* starting block number */
	uint64_t zt_numblks; /* number of blocks */
	uint64_t zt_shift; /* bits to index it */
	uint64_t zt_nextblk; /* next (larger) copy start block */
	uint64_t zt_blks_copied; /* number source blocks copied */
	} zap_ptrtbl;

	uint64_t zap_freeblk; /* the next free block */
	uint64_t zap_num_leafs; /* number of leafs */
	uint64_t zap_num_entries; /* number of entries */
	uint64_t zap_salt; /* salt to stir into hash function */
	uint64_t zap_normflags; /* flags for u8_textprep_str() */
	uint64_t zap_flags; /* zap_flags_t */
	/*
	* This structure is followed by padding, and then the embedded
	* pointer table. The embedded pointer table takes up second
	* half of the block. It is accessed using the
	* ZAP_EMBEDDED_PTRTBL_ENT() macro.
	*/
	} zap_phys_t;

	typedef struct zap_table_phys zap_table_phys_t;

	typedef struct zap {
	dmu_buf_user_t zap_dbu;
	objset_t *zap_objset;
	uint64_t zap_object;
	+ dnode_t *zap_dnode;
	struct dmu_buf *zap_dbuf;
	krwlock_t zap_rwlock;
	boolean_t zap_ismicro;
	int zap_normflags;
	uint64_t zap_salt;
	union {
	struct {
	/*
	* zap_num_entries_mtx protects
	* zap_num_entries
	*/
	kmutex_t zap_num_entries_mtx;
	int zap_block_shift;
	} zap_fat;
	struct {
	int16_t zap_num_entries;
	int16_t zap_num_chunks;
	int16_t zap_alloc_next;
	zfs_btree_t zap_tree;
	} zap_micro;
	} zap_u;
	} zap_t;

	static inline zap_phys_t *
	zap_f_phys(zap_t *zap)
	{
	return (zap->zap_dbuf->db_data);
	}

	static inline mzap_phys_t *
	zap_m_phys(zap_t *zap)
	{
	return (zap->zap_dbuf->db_data);
	}

	typedef struct zap_name {
	zap_t *zn_zap;
	int zn_key_intlen;
	const void *zn_key_orig;
	int zn_key_orig_numints;
	const void *zn_key_norm;
	int zn_key_norm_numints;
	uint64_t zn_hash;
	matchtype_t zn_matchtype;
	int zn_normflags;
	char zn_normbuf[ZAP_MAXNAMELEN];
	} zap_name_t;

	#define zap_f zap_u.zap_fat
	#define zap_m zap_u.zap_micro

	boolean_t zap_match(zap_name_t zn, const char matchname);
	int zap_lockdir(objset_t os, uint64_t obj, dmu_tx_t tx,
	krw_t lti, boolean_t fatreader, boolean_t adding, const void *tag,
	zap_t **zapp);
	void zap_unlockdir(zap_t zap, const void tag);
	void zap_evict_sync(void *dbu);
	zap_name_t zap_name_alloc_str(zap_t zap, const char *key, matchtype_t mt);
	void zap_name_free(zap_name_t *zn);
	int zap_hashbits(zap_t *zap);
	uint32_t zap_maxcd(zap_t *zap);
	uint64_t zap_getflags(zap_t *zap);

	#define ZAP_HASH_IDX(hash, n) (((n) == 0) ? 0 : ((hash) >> (64 - (n))))

	void fzap_byteswap(void *buf, size_t size);
	int fzap_count(zap_t zap, uint64_t count);
	int fzap_lookup(zap_name_t *zn,
	uint64_t integer_size, uint64_t num_integers, void *buf,
	char realname, int rn_len, boolean_t normalization_conflictp);
	void fzap_prefetch(zap_name_t *zn);
	int fzap_add(zap_name_t *zn, uint64_t integer_size, uint64_t num_integers,
	const void val, const void tag, dmu_tx_t *tx);
	int fzap_update(zap_name_t *zn,
	int integer_size, uint64_t num_integers, const void *val,
	const void tag, dmu_tx_t tx);
	int fzap_length(zap_name_t *zn,
	uint64_t integer_size, uint64_t num_integers);
	int fzap_remove(zap_name_t zn, dmu_tx_t tx);
	int fzap_cursor_retrieve(zap_t zap, zap_cursor_t zc, zap_attribute_t *za);
	void fzap_get_stats(zap_t zap, zap_stats_t zs);
	void zap_put_leaf(struct zap_leaf *l);

	int fzap_add_cd(zap_name_t *zn,
	uint64_t integer_size, uint64_t num_integers,
	const void val, uint32_t cd, const void tag, dmu_tx_t *tx);
	void fzap_upgrade(zap_t zap, dmu_tx_t tx, zap_flags_t flags);

	#ifdef __cplusplus
	}
	#endif

	#endif /* _SYS_ZAP_IMPL_H */
	diff --git a/sys/contrib/openzfs/include/sys/zap_leaf.h b/sys/contrib/openzfs/include/sys/zap_leaf.h
	index ebc67c2bf465..e54456d3472b 100644
	--- a/sys/contrib/openzfs/include/sys/zap_leaf.h
	+++ b/sys/contrib/openzfs/include/sys/zap_leaf.h
	@@ -1,255 +1,255 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
	*/

	#ifndef _SYS_ZAP_LEAF_H
	#define _SYS_ZAP_LEAF_H

	#include <sys/zap.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	struct zap;
	struct zap_name;
	struct zap_stats;

	#define ZAP_LEAF_MAGIC 0x2AB1EAF

	/* chunk size = 24 bytes */
	#define ZAP_LEAF_CHUNKSIZE 24

	/*
	* The amount of space available for chunks is:
	* block size (1<<l->l_bs) - hash entry size (2) * number of hash
	* entries - header space (2*chunksize)
	*/
	#define ZAP_LEAF_NUMCHUNKS_BS(bs) \
	- (((1<<(bs)) - 2*ZAP_LEAF_HASH_NUMENTRIES_BS(bs)) / \
	+ (((1U << (bs)) - 2 * ZAP_LEAF_HASH_NUMENTRIES_BS(bs)) / \
	ZAP_LEAF_CHUNKSIZE - 2)

	#define ZAP_LEAF_NUMCHUNKS(l) (ZAP_LEAF_NUMCHUNKS_BS(((l)->l_bs)))

	#define ZAP_LEAF_NUMCHUNKS_DEF \
	(ZAP_LEAF_NUMCHUNKS_BS(fzap_default_block_shift))

	/*
	* The amount of space within the chunk available for the array is:
	* chunk size - space for type (1) - space for next pointer (2)
	*/
	#define ZAP_LEAF_ARRAY_BYTES (ZAP_LEAF_CHUNKSIZE - 3)

	#define ZAP_LEAF_ARRAY_NCHUNKS(bytes) \
	(((bytes)+ZAP_LEAF_ARRAY_BYTES-1)/ZAP_LEAF_ARRAY_BYTES)

	/*
	* Low water mark: when there are only this many chunks free, start
	* growing the ptrtbl. Ideally, this should be larger than a
	* "reasonably-sized" entry. 20 chunks is more than enough for the
	* largest directory entry (MAXNAMELEN (256) byte name, 8-byte value),
	* while still being only around 3% for 16k blocks.
	*/
	#define ZAP_LEAF_LOW_WATER (20)

	/*
	* The leaf hash table has block size / 2^5 (32) number of entries,
	* which should be more than enough for the maximum number of entries,
	* which is less than block size / CHUNKSIZE (24) / minimum number of
	* chunks per entry (3).
	*/
	#define ZAP_LEAF_HASH_SHIFT_BS(bs) ((bs) - 5)
	-#define ZAP_LEAF_HASH_NUMENTRIES_BS(bs) (1 << ZAP_LEAF_HASH_SHIFT_BS(bs))
	+#define ZAP_LEAF_HASH_NUMENTRIES_BS(bs) (1U << ZAP_LEAF_HASH_SHIFT_BS(bs))
	#define ZAP_LEAF_HASH_SHIFT(l) (ZAP_LEAF_HASH_SHIFT_BS(((l)->l_bs)))
	#define ZAP_LEAF_HASH_NUMENTRIES(l) (ZAP_LEAF_HASH_NUMENTRIES_BS(((l)->l_bs)))

	/*
	* The chunks start immediately after the hash table. The end of the
	* hash table is at l_hash + HASH_NUMENTRIES, which we simply cast to a
	* chunk_t.
	*/
	#define ZAP_LEAF_CHUNK(l, idx) \
	((zap_leaf_chunk_t *) \
	(zap_leaf_phys(l)->l_hash + ZAP_LEAF_HASH_NUMENTRIES(l)))[idx]
	#define ZAP_LEAF_ENTRY(l, idx) (&ZAP_LEAF_CHUNK(l, idx).l_entry)

	typedef enum zap_chunk_type {
	ZAP_CHUNK_FREE = 253,
	ZAP_CHUNK_ENTRY = 252,
	ZAP_CHUNK_ARRAY = 251,
	ZAP_CHUNK_TYPE_MAX = 250
	} zap_chunk_type_t;

	#define ZLF_ENTRIES_CDSORTED (1<<0)

	/*
	* TAKE NOTE:
	* If zap_leaf_phys_t is modified, zap_leaf_byteswap() must be modified.
	*/
	typedef struct zap_leaf_phys {
	struct zap_leaf_header {
	/* Public to ZAP */
	uint64_t lh_block_type; /* ZBT_LEAF */
	uint64_t lh_pad1;
	uint64_t lh_prefix; /* hash prefix of this leaf */
	uint32_t lh_magic; /* ZAP_LEAF_MAGIC */
	uint16_t lh_nfree; /* number free chunks */
	uint16_t lh_nentries; /* number of entries */
	uint16_t lh_prefix_len; /* num bits used to id this */

	/* Private to zap_leaf */
	uint16_t lh_freelist; /* chunk head of free list */
	uint8_t lh_flags; /* ZLF_* flags */
	uint8_t lh_pad2[11];
	} l_hdr; /* 2 24-byte chunks */

	/*
	* The header is followed by a hash table with
	* ZAP_LEAF_HASH_NUMENTRIES(zap) entries. The hash table is
	* followed by an array of ZAP_LEAF_NUMCHUNKS(zap)
	* zap_leaf_chunk structures. These structures are accessed
	* with the ZAP_LEAF_CHUNK() macro.
	*/

	- uint16_t l_hash[1];
	+ uint16_t l_hash[];
	} zap_leaf_phys_t;

	typedef union zap_leaf_chunk {
	struct zap_leaf_entry {
	uint8_t le_type; /* always ZAP_CHUNK_ENTRY */
	uint8_t le_value_intlen; /* size of value's ints */
	uint16_t le_next; /* next entry in hash chain */
	uint16_t le_name_chunk; /* first chunk of the name */
	uint16_t le_name_numints; /* ints in name (incl null) */
	uint16_t le_value_chunk; /* first chunk of the value */
	uint16_t le_value_numints; /* value length in ints */
	uint32_t le_cd; /* collision differentiator */
	uint64_t le_hash; /* hash value of the name */
	} l_entry;
	struct zap_leaf_array {
	uint8_t la_type; /* always ZAP_CHUNK_ARRAY */
	uint8_t la_array[ZAP_LEAF_ARRAY_BYTES];
	uint16_t la_next; /* next blk or CHAIN_END */
	} l_array;
	struct zap_leaf_free {
	uint8_t lf_type; /* always ZAP_CHUNK_FREE */
	uint8_t lf_pad[ZAP_LEAF_ARRAY_BYTES];
	uint16_t lf_next; /* next in free list, or CHAIN_END */
	} l_free;
	} zap_leaf_chunk_t;

	typedef struct zap_leaf {
	dmu_buf_user_t l_dbu;
	krwlock_t l_rwlock;
	uint64_t l_blkid; /* 1<<ZAP_BLOCK_SHIFT byte block off */
	- int l_bs; /* block size shift */
	+ uint_t l_bs; /* block size shift */
	dmu_buf_t *l_dbuf;
	} zap_leaf_t;

	static inline zap_leaf_phys_t *
	zap_leaf_phys(zap_leaf_t *l)
	{
	return (l->l_dbuf->db_data);
	}

	typedef struct zap_entry_handle {
	/* Set by zap_leaf and public to ZAP */
	uint64_t zeh_num_integers;
	uint64_t zeh_hash;
	uint32_t zeh_cd;
	uint8_t zeh_integer_size;

	/* Private to zap_leaf */
	uint16_t zeh_fakechunk;
	uint16_t *zeh_chunkp;
	zap_leaf_t *zeh_leaf;
	} zap_entry_handle_t;

	/*
	* Return a handle to the named entry, or ENOENT if not found. The hash
	* value must equal zap_hash(name).
	*/
	extern int zap_leaf_lookup(zap_leaf_t *l,
	struct zap_name zn, zap_entry_handle_t zeh);

	/*
	* Return a handle to the entry with this hash+cd, or the entry with the
	* next closest hash+cd.
	*/
	extern int zap_leaf_lookup_closest(zap_leaf_t *l,
	uint64_t hash, uint32_t cd, zap_entry_handle_t *zeh);

	/*
	* Read the first num_integers in the attribute. Integer size
	* conversion will be done without sign extension. Return EINVAL if
	* integer_size is too small. Return EOVERFLOW if there are more than
	* num_integers in the attribute.
	*/
	extern int zap_entry_read(const zap_entry_handle_t *zeh,
	uint8_t integer_size, uint64_t num_integers, void *buf);

	extern int zap_entry_read_name(struct zap zap, const zap_entry_handle_t zeh,
	uint16_t buflen, char *buf);

	/*
	* Replace the value of an existing entry.
	*
	* May fail if it runs out of space (ENOSPC).
	*/
	extern int zap_entry_update(zap_entry_handle_t *zeh,
	uint8_t integer_size, uint64_t num_integers, const void *buf);

	/*
	* Remove an entry.
	*/
	extern void zap_entry_remove(zap_entry_handle_t *zeh);

	/*
	* Create an entry. An equal entry must not exist, and this entry must
	* belong in this leaf (according to its hash value). Fills in the
	* entry handle on success. Returns 0 on success or ENOSPC on failure.
	*/
	extern int zap_entry_create(zap_leaf_t l, struct zap_name zn, uint32_t cd,
	uint8_t integer_size, uint64_t num_integers, const void *buf,
	zap_entry_handle_t *zeh);

	/* Determine whether there is another entry with the same normalized form. */
	extern boolean_t zap_entry_normalization_conflict(zap_entry_handle_t *zeh,
	struct zap_name zn, const char name, struct zap *zap);

	/*
	* Other stuff.
	*/

	extern void zap_leaf_init(zap_leaf_t *l, boolean_t sort);
	-extern void zap_leaf_byteswap(zap_leaf_phys_t *buf, int len);
	+extern void zap_leaf_byteswap(zap_leaf_phys_t *buf, size_t len);
	extern void zap_leaf_split(zap_leaf_t l, zap_leaf_t nl, boolean_t sort);
	extern void zap_leaf_stats(struct zap zap, zap_leaf_t l,
	struct zap_stats *zs);

	#ifdef __cplusplus
	}
	#endif

	#endif /* _SYS_ZAP_LEAF_H */
	diff --git a/sys/contrib/openzfs/include/sys/zil_impl.h b/sys/contrib/openzfs/include/sys/zil_impl.h
	index f780ad3d61bc..9a34bafc1c77 100644
	--- a/sys/contrib/openzfs/include/sys/zil_impl.h
	+++ b/sys/contrib/openzfs/include/sys/zil_impl.h
	@@ -1,256 +1,260 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2012, 2018 by Delphix. All rights reserved.
	*/

	/* Portions Copyright 2010 Robert Milkowski */

	#ifndef _SYS_ZIL_IMPL_H
	#define _SYS_ZIL_IMPL_H

	#include <sys/zil.h>
	#include <sys/dmu_objset.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*
	* Possible states for a given lwb structure.
	*
	* An lwb will start out in the "new" state, and transition to the "opened"
	* state via a call to zil_lwb_write_open() on first itx assignment. When
	* transitioning from "new" to "opened" the zilog's "zl_issuer_lock" must be
	* held.
	*
	* After the lwb is "opened", it can be assigned number of itxs and transition
	* into the "closed" state via zil_lwb_write_close() when full or on timeout.
	* When transitioning from "opened" to "closed" the zilog's "zl_issuer_lock"
	* must be held. New lwb allocation also takes "zl_lock" to protect the list.
	*
	* After the lwb is "closed", it can transition into the "ready" state via
	* zil_lwb_write_issue(). "zl_lock" must be held when making this transition.
	* Since it is done by the same thread, "zl_issuer_lock" is not needed.
	*
	* When lwb in "ready" state receives its block pointer, it can transition to
	* "issued". "zl_lock" must be held when making this transition.
	*
	* After the lwb's write zio completes, it transitions into the "write
	* done" state via zil_lwb_write_done(); and then into the "flush done"
	* state via zil_lwb_flush_vdevs_done(). When transitioning from
	* "issued" to "write done", and then from "write done" to "flush done",
	* the zilog's "zl_lock" must be held, not the "zl_issuer_lock".
	*
	* The zilog's "zl_issuer_lock" can become heavily contended in certain
	* workloads, so we specifically avoid acquiring that lock when
	* transitioning an lwb from "issued" to "done". This allows us to avoid
	* having to acquire the "zl_issuer_lock" for each lwb ZIO completion,
	* which would have added more lock contention on an already heavily
	* contended lock.
	*
	* Additionally, correctness when reading an lwb's state is often
	* achieved by exploiting the fact that these state transitions occur in
	* this specific order; i.e. "new" to "opened" to "closed" to "ready" to
	* "issued" to "write_done" and finally "flush_done".
	*
	* Thus, if an lwb is in the "new" or "opened" state, holding the
	* "zl_issuer_lock" will prevent a concurrent thread from transitioning
	* that lwb to the "closed" state. Likewise, if an lwb is already in the
	* "ready" state, holding the "zl_lock" will prevent a concurrent thread
	* from transitioning that lwb to the "issued" state.
	*/
	typedef enum {
	LWB_STATE_NEW,
	LWB_STATE_OPENED,
	LWB_STATE_CLOSED,
	LWB_STATE_READY,
	LWB_STATE_ISSUED,
	LWB_STATE_WRITE_DONE,
	LWB_STATE_FLUSH_DONE,
	LWB_NUM_STATES
	} lwb_state_t;

	/*
	* Log write block (lwb)
	*
	* Prior to an lwb being issued to disk via zil_lwb_write_issue(), it
	* will be protected by the zilog's "zl_issuer_lock". Basically, prior
	* to it being issued, it will only be accessed by the thread that's
	* holding the "zl_issuer_lock". After the lwb is issued, the zilog's
	* "zl_lock" is used to protect the lwb against concurrent access.
	*/
	typedef struct lwb {
	zilog_t lwb_zilog; / back pointer to log struct */
	blkptr_t lwb_blk; /* on disk address of this log blk */
	boolean_t lwb_slim; /* log block has slim format */
	boolean_t lwb_slog; /* lwb_blk is on SLOG device */
	int lwb_error; /* log block allocation error */
	int lwb_nmax; /* max bytes in the buffer */
	int lwb_nused; /* # used bytes in buffer */
	int lwb_nfilled; /* # filled bytes in buffer */
	int lwb_sz; /* size of block and buffer */
	lwb_state_t lwb_state; /* the state of this lwb */
	char lwb_buf; / log write buffer */
	zio_t lwb_child_zio; / parent zio for children */
	zio_t lwb_write_zio; / zio for the lwb buffer */
	zio_t lwb_root_zio; / root zio for lwb write and flushes */
	hrtime_t lwb_issued_timestamp; /* when was the lwb issued? */
	uint64_t lwb_issued_txg; /* the txg when the write is issued */
	uint64_t lwb_alloc_txg; /* the txg when lwb_blk is allocated */
	uint64_t lwb_max_txg; /* highest txg in this lwb */
	list_node_t lwb_node; /* zilog->zl_lwb_list linkage */
	list_node_t lwb_issue_node; /* linkage of lwbs ready for issue */
	list_t lwb_itxs; /* list of itx's */
	list_t lwb_waiters; /* list of zil_commit_waiter's */
	avl_tree_t lwb_vdev_tree; /* vdevs to flush after lwb write */
	kmutex_t lwb_vdev_lock; /* protects lwb_vdev_tree */
	} lwb_t;

	/*
	* ZIL commit waiter.
	*
	* This structure is allocated each time zil_commit() is called, and is
	* used by zil_commit() to communicate with other parts of the ZIL, such
	* that zil_commit() can know when it safe for it return. For more
	* details, see the comment above zil_commit().
	*
	* The "zcw_lock" field is used to protect the commit waiter against
	* concurrent access. This lock is often acquired while already holding
	* the zilog's "zl_issuer_lock" or "zl_lock"; see the functions
	* zil_process_commit_list() and zil_lwb_flush_vdevs_done() as examples
	* of this. Thus, one must be careful not to acquire the
	* "zl_issuer_lock" or "zl_lock" when already holding the "zcw_lock";
	* e.g. see the zil_commit_waiter_timeout() function.
	*/
	typedef struct zil_commit_waiter {
	kcondvar_t zcw_cv; /* signalled when "done" */
	kmutex_t zcw_lock; /* protects fields of this struct */
	list_node_t zcw_node; /* linkage in lwb_t:lwb_waiter list */
	lwb_t zcw_lwb; / back pointer to lwb when linked */
	boolean_t zcw_done; /* B_TRUE when "done", else B_FALSE */
	int zcw_zio_error; /* contains the zio io_error value */
	} zil_commit_waiter_t;

	/*
	* Intent log transaction lists
	*/
	typedef struct itxs {
	list_t i_sync_list; /* list of synchronous itxs */
	avl_tree_t i_async_tree; /* tree of foids for async itxs */
	} itxs_t;

	typedef struct itxg {
	kmutex_t itxg_lock; /* lock for this structure */
	uint64_t itxg_txg; /* txg for this chain */
	itxs_t itxg_itxs; / sync and async itxs */
	} itxg_t;

	/* for async nodes we build up an AVL tree of lists of async itxs per file */
	typedef struct itx_async_node {
	uint64_t ia_foid; /* file object id */
	list_t ia_list; /* list of async itxs for this foid */
	avl_node_t ia_node; /* AVL tree linkage */
	} itx_async_node_t;

	/*
	* Vdev flushing: during a zil_commit(), we build up an AVL tree of the vdevs
	* we've touched so we know which ones need a write cache flush at the end.
	*/
	typedef struct zil_vdev_node {
	uint64_t zv_vdev; /* vdev to be flushed */
	avl_node_t zv_node; /* AVL tree linkage */
	} zil_vdev_node_t;

	-#define ZIL_PREV_BLKS 16
	+#define ZIL_BURSTS 8

	/*
	* Stable storage intent log management structure. One per dataset.
	*/
	struct zilog {
	kmutex_t zl_lock; /* protects most zilog_t fields */
	struct dsl_pool zl_dmu_pool; / DSL pool */
	spa_t zl_spa; / handle for read/write log */
	const zil_header_t zl_header; / log header buffer */
	objset_t zl_os; / object set we're logging */
	zil_get_data_t zl_get_data; / callback to get object content */
	lwb_t zl_last_lwb_opened; / most recent lwb opened */
	hrtime_t zl_last_lwb_latency; /* zio latency of last lwb done */
	uint64_t zl_lr_seq; /* on-disk log record sequence number */
	uint64_t zl_commit_lr_seq; /* last committed on-disk lr seq */
	uint64_t zl_destroy_txg; /* txg of last zil_destroy() */
	uint64_t zl_replayed_seq[TXG_SIZE]; /* last replayed rec seq */
	uint64_t zl_replaying_seq; /* current replay seq number */
	uint32_t zl_suspend; /* log suspend count */
	kcondvar_t zl_cv_suspend; /* log suspend completion */
	uint8_t zl_suspending; /* log is currently suspending */
	uint8_t zl_keep_first; /* keep first log block in destroy */
	uint8_t zl_replay; /* replaying records while set */
	uint8_t zl_stop_sync; /* for debugging */
	kmutex_t zl_issuer_lock; /* single writer, per ZIL, at a time */
	uint8_t zl_logbias; /* latency or throughput */
	uint8_t zl_sync; /* synchronous or asynchronous */
	int zl_parse_error; /* last zil_parse() error */
	uint64_t zl_parse_blk_seq; /* highest blk seq on last parse */
	uint64_t zl_parse_lr_seq; /* highest lr seq on last parse */
	uint64_t zl_parse_blk_count; /* number of blocks parsed */
	uint64_t zl_parse_lr_count; /* number of log records parsed */
	itxg_t zl_itxg[TXG_SIZE]; /* intent log txg chains */
	list_t zl_itx_commit_list; /* itx list to be committed */
	- uint64_t zl_cur_used; /* current commit log size used */
	+ uint64_t zl_cur_size; /* current burst full size */
	+ uint64_t zl_cur_left; /* current burst remaining size */
	+ uint64_t zl_cur_max; /* biggest record in current burst */
	list_t zl_lwb_list; /* in-flight log write list */
	avl_tree_t zl_bp_tree; /* track bps during log parse */
	clock_t zl_replay_time; /* lbolt of when replay started */
	uint64_t zl_replay_blks; /* number of log blocks replayed */
	zil_header_t zl_old_header; /* debugging aid */
	- uint_t zl_prev_blks[ZIL_PREV_BLKS]; /* size - sector rounded */
	+ uint_t zl_parallel; /* workload is multi-threaded */
	uint_t zl_prev_rotor; /* rotor for zl_prev[] */
	+ uint_t zl_prev_opt[ZIL_BURSTS]; /* optimal block size */
	+ uint_t zl_prev_min[ZIL_BURSTS]; /* minimal first block size */
	txg_node_t zl_dirty_link; /* protected by dp_dirty_zilogs list */
	uint64_t zl_dirty_max_txg; /* highest txg used to dirty zilog */

	kmutex_t zl_lwb_io_lock; /* protect following members */
	uint64_t zl_lwb_inflight[TXG_SIZE]; /* io issued, but not done */
	kcondvar_t zl_lwb_io_cv; /* signal when the flush is done */
	uint64_t zl_lwb_max_issued_txg; /* max txg when lwb io issued */

	/*
	* Max block size for this ZIL. Note that this can not be changed
	* while the ZIL is in use because consumers (ZPL/zvol) need to take
	* this into account when deciding between WR_COPIED and WR_NEED_COPY
	* (see zil_max_copied_data()).
	*/
	uint64_t zl_max_block_size;

	/* Pointer for per dataset zil sums */
	zil_sums_t *zl_sums;
	};

	typedef struct zil_bp_node {
	dva_t zn_dva;
	avl_node_t zn_node;
	} zil_bp_node_t;

	#ifdef __cplusplus
	}
	#endif

	#endif /* _SYS_ZIL_IMPL_H */
	diff --git a/sys/contrib/openzfs/lib/libspl/include/assert.h b/sys/contrib/openzfs/lib/libspl/include/assert.h
	index af4957dfbaa6..57f5719c1ac1 100644
	--- a/sys/contrib/openzfs/lib/libspl/include/assert.h
	+++ b/sys/contrib/openzfs/lib/libspl/include/assert.h
	@@ -1,165 +1,176 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License, Version 1.0 only
	* (the "License"). You may not use this file except in compliance
	* with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright 2007 Sun Microsystems, Inc. All rights reserved.
	* Use is subject to license terms.
	*/

	#include_next <assert.h>

	#ifndef _LIBSPL_ASSERT_H
	#define _LIBSPL_ASSERT_H

	#include <stdio.h>
	#include <stdlib.h>
	#include <stdarg.h>
	#include <sys/types.h>

	/* Workaround for non-Clang compilers */
	#ifndef __has_feature
	#define __has_feature(x) 0
	#endif

	/* We need to workaround libspl_set_assert_ok() that we have for zdb */
	#if __has_feature(attribute_analyzer_noreturn) \|\| defined(__COVERITY__)
	#define NORETURN __attribute__((__noreturn__))
	#else
	#define NORETURN
	#endif

	/* Set to non-zero to avoid abort()ing on an assertion failure */
	extern void libspl_set_assert_ok(boolean_t val);

	/* printf version of libspl_assert */
	extern void libspl_assertf(const char file, const char func, int line,
	const char *format, ...) NORETURN __attribute__((format(printf, 4, 5)));

	static inline int
	libspl_assert(const char buf, const char file, const char *func, int line)
	{
	libspl_assertf(file, func, line, "%s", buf);
	return (0);
	}

	#ifdef verify
	#undef verify
	#endif

	#define PANIC(fmt, a...) \
	libspl_assertf(__FILE__, __FUNCTION__, __LINE__, fmt, ## a)

	#define VERIFY(cond) \
	(void) ((!(cond)) && \
	libspl_assert(#cond, __FILE__, __FUNCTION__, __LINE__))
	#define verify(cond) \
	(void) ((!(cond)) && \
	libspl_assert(#cond, __FILE__, __FUNCTION__, __LINE__))

	#define VERIFY3B(LEFT, OP, RIGHT) \
	do { \
	const boolean_t __left = (boolean_t)(LEFT); \
	const boolean_t __right = (boolean_t)(RIGHT); \
	if (!(__left OP __right)) \
	libspl_assertf(__FILE__, __FUNCTION__, __LINE__, \
	"%s %s %s (0x%llx %s 0x%llx)", #LEFT, #OP, #RIGHT, \
	(u_longlong_t)__left, #OP, (u_longlong_t)__right); \
	} while (0)

	#define VERIFY3S(LEFT, OP, RIGHT) \
	do { \
	const int64_t __left = (int64_t)(LEFT); \
	const int64_t __right = (int64_t)(RIGHT); \
	if (!(__left OP __right)) \
	libspl_assertf(__FILE__, __FUNCTION__, __LINE__, \
	"%s %s %s (0x%llx %s 0x%llx)", #LEFT, #OP, #RIGHT, \
	(u_longlong_t)__left, #OP, (u_longlong_t)__right); \
	} while (0)

	#define VERIFY3U(LEFT, OP, RIGHT) \
	do { \
	const uint64_t __left = (uint64_t)(LEFT); \
	const uint64_t __right = (uint64_t)(RIGHT); \
	if (!(__left OP __right)) \
	libspl_assertf(__FILE__, __FUNCTION__, __LINE__, \
	"%s %s %s (0x%llx %s 0x%llx)", #LEFT, #OP, #RIGHT, \
	(u_longlong_t)__left, #OP, (u_longlong_t)__right); \
	} while (0)

	#define VERIFY3P(LEFT, OP, RIGHT) \
	do { \
	const uintptr_t __left = (uintptr_t)(LEFT); \
	const uintptr_t __right = (uintptr_t)(RIGHT); \
	if (!(__left OP __right)) \
	libspl_assertf(__FILE__, __FUNCTION__, __LINE__, \
	- "%s %s %s (0x%llx %s 0x%llx)", #LEFT, #OP, #RIGHT, \
	- (u_longlong_t)__left, #OP, (u_longlong_t)__right); \
	+ "%s %s %s (%p %s %p)", #LEFT, #OP, #RIGHT, \
	+ (void )__left, #OP, (void )__right); \
	} while (0)

	#define VERIFY0(LEFT) \
	do { \
	const uint64_t __left = (uint64_t)(LEFT); \
	if (!(__left == 0)) \
	libspl_assertf(__FILE__, __FUNCTION__, __LINE__, \
	"%s == 0 (0x%llx == 0)", #LEFT, \
	(u_longlong_t)__left); \
	} while (0)

	+#define VERIFY0P(LEFT) \
	+do { \
	+ const uintptr_t __left = (uintptr_t)(LEFT); \
	+ if (!(__left == 0)) \
	+ libspl_assertf(__FILE__, __FUNCTION__, __LINE__, \
	+ "%s == 0 (%p == 0)", #LEFT, \
	+ (void *)__left); \
	+} while (0)
	+
	#ifdef assert
	#undef assert
	#endif

	#ifdef NDEBUG
	#define ASSERT3B(x, y, z) \
	((void) sizeof ((uintptr_t)(x)), (void) sizeof ((uintptr_t)(z)))
	#define ASSERT3S(x, y, z) \
	((void) sizeof ((uintptr_t)(x)), (void) sizeof ((uintptr_t)(z)))
	#define ASSERT3U(x, y, z) \
	((void) sizeof ((uintptr_t)(x)), (void) sizeof ((uintptr_t)(z)))
	#define ASSERT3P(x, y, z) \
	((void) sizeof ((uintptr_t)(x)), (void) sizeof ((uintptr_t)(z)))
	#define ASSERT0(x) ((void) sizeof ((uintptr_t)(x)))
	+#define ASSERT0P(x) ((void) sizeof ((uintptr_t)(x)))
	#define ASSERT(x) ((void) sizeof ((uintptr_t)(x)))
	#define assert(x) ((void) sizeof ((uintptr_t)(x)))
	#define IMPLY(A, B) \
	((void) sizeof ((uintptr_t)(A)), (void) sizeof ((uintptr_t)(B)))
	#define EQUIV(A, B) \
	((void) sizeof ((uintptr_t)(A)), (void) sizeof ((uintptr_t)(B)))
	#else
	#define ASSERT3B VERIFY3B
	#define ASSERT3S VERIFY3S
	#define ASSERT3U VERIFY3U
	#define ASSERT3P VERIFY3P
	#define ASSERT0 VERIFY0
	+#define ASSERT0P VERIFY0P
	#define ASSERT VERIFY
	#define assert VERIFY
	#define IMPLY(A, B) \
	((void)(((!(A)) \|\| (B)) \|\| \
	libspl_assert("(" #A ") implies (" #B ")", \
	__FILE__, __FUNCTION__, __LINE__)))
	#define EQUIV(A, B) \
	((void)((!!(A) == !!(B)) \|\| \
	libspl_assert("(" #A ") is equivalent to (" #B ")", \
	__FILE__, __FUNCTION__, __LINE__)))

	#endif /* NDEBUG */

	#endif /* _LIBSPL_ASSERT_H */
	diff --git a/sys/contrib/openzfs/lib/libuutil/uu_list.c b/sys/contrib/openzfs/lib/libuutil/uu_list.c
	index 0ca6f05205e9..aa8b129cc22a 100644
	--- a/sys/contrib/openzfs/lib/libuutil/uu_list.c
	+++ b/sys/contrib/openzfs/lib/libuutil/uu_list.c
	@@ -1,716 +1,722 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright 2008 Sun Microsystems, Inc. All rights reserved.
	* Use is subject to license terms.
	*/



	#include "libuutil_common.h"

	#include <stdlib.h>
	#include <string.h>
	#include <unistd.h>
	#include <sys/time.h>

	#define ELEM_TO_NODE(lp, e) \
	((uu_list_node_impl_t *)((uintptr_t)(e) + (lp)->ul_offset))

	#define NODE_TO_ELEM(lp, n) \
	((void *)((uintptr_t)(n) - (lp)->ul_offset))

	/*
	* uu_list_index_ts define a location for insertion. They are simply a
	* pointer to the object after the insertion point. We store a mark
	* in the low-bits of the index, to help prevent mistakes.
	*
	* When debugging, the index mark changes on every insert and delete, to
	* catch stale references.
	*/
	#define INDEX_MAX (sizeof (uintptr_t) - 1)
	#define INDEX_NEXT(m) (((m) == INDEX_MAX)? 1 : ((m) + 1) & INDEX_MAX)

	#define INDEX_TO_NODE(i) ((uu_list_node_impl_t *)((i) & ~INDEX_MAX))
	#define NODE_TO_INDEX(p, n) (((uintptr_t)(n) & ~INDEX_MAX) \| (p)->ul_index)
	#define INDEX_VALID(p, i) (((i) & INDEX_MAX) == (p)->ul_index)
	#define INDEX_CHECK(i) (((i) & INDEX_MAX) != 0)

	#define POOL_TO_MARKER(pp) ((void *)((uintptr_t)(pp) \| 1))

	static uu_list_pool_t uu_null_lpool = { &uu_null_lpool, &uu_null_lpool };
	static pthread_mutex_t uu_lpool_list_lock = PTHREAD_MUTEX_INITIALIZER;

	uu_list_pool_t *
	uu_list_pool_create(const char *name, size_t objsize,
	size_t nodeoffset, uu_compare_fn_t *compare_func, uint32_t flags)
	{
	uu_list_pool_t pp, next, *prev;

	if (name == NULL \|\|
	uu_check_name(name, UU_NAME_DOMAIN) == -1 \|\|
	nodeoffset + sizeof (uu_list_node_t) > objsize) {
	uu_set_error(UU_ERROR_INVALID_ARGUMENT);
	return (NULL);
	}

	if (flags & ~UU_LIST_POOL_DEBUG) {
	uu_set_error(UU_ERROR_UNKNOWN_FLAG);
	return (NULL);
	}

	pp = uu_zalloc(sizeof (uu_list_pool_t));
	if (pp == NULL) {
	uu_set_error(UU_ERROR_NO_MEMORY);
	return (NULL);
	}

	(void) strlcpy(pp->ulp_name, name, sizeof (pp->ulp_name));
	pp->ulp_nodeoffset = nodeoffset;
	pp->ulp_objsize = objsize;
	pp->ulp_cmp = compare_func;
	if (flags & UU_LIST_POOL_DEBUG)
	pp->ulp_debug = 1;
	pp->ulp_last_index = 0;

	(void) pthread_mutex_init(&pp->ulp_lock, NULL);

	pp->ulp_null_list.ul_next = &pp->ulp_null_list;
	pp->ulp_null_list.ul_prev = &pp->ulp_null_list;

	(void) pthread_mutex_lock(&uu_lpool_list_lock);
	pp->ulp_next = next = &uu_null_lpool;
	pp->ulp_prev = prev = next->ulp_prev;
	next->ulp_prev = pp;
	prev->ulp_next = pp;
	(void) pthread_mutex_unlock(&uu_lpool_list_lock);

	return (pp);
	}

	void
	uu_list_pool_destroy(uu_list_pool_t *pp)
	{
	if (pp->ulp_debug) {
	if (pp->ulp_null_list.ul_next != &pp->ulp_null_list \|\|
	pp->ulp_null_list.ul_prev != &pp->ulp_null_list) {
	uu_panic("uu_list_pool_destroy: Pool \"%.*s\" (%p) has "
	"outstanding lists, or is corrupt.\n",
	(int)sizeof (pp->ulp_name), pp->ulp_name,
	(void *)pp);
	}
	}
	(void) pthread_mutex_lock(&uu_lpool_list_lock);
	pp->ulp_next->ulp_prev = pp->ulp_prev;
	pp->ulp_prev->ulp_next = pp->ulp_next;
	(void) pthread_mutex_unlock(&uu_lpool_list_lock);
	pp->ulp_prev = NULL;
	pp->ulp_next = NULL;
	uu_free(pp);
	}

	void
	uu_list_node_init(void base, uu_list_node_t np_arg, uu_list_pool_t *pp)
	{
	uu_list_node_impl_t np = (uu_list_node_impl_t )np_arg;

	if (pp->ulp_debug) {
	uintptr_t offset = (uintptr_t)np - (uintptr_t)base;
	if (offset + sizeof (*np) > pp->ulp_objsize) {
	uu_panic("uu_list_node_init(%p, %p, %p (\"%s\")): "
	"offset %ld doesn't fit in object (size %ld)\n",
	base, (void )np, (void )pp, pp->ulp_name,
	(long)offset, (long)pp->ulp_objsize);
	}
	if (offset != pp->ulp_nodeoffset) {
	uu_panic("uu_list_node_init(%p, %p, %p (\"%s\")): "
	"offset %ld doesn't match pool's offset (%ld)\n",
	base, (void )np, (void )pp, pp->ulp_name,
	(long)offset, (long)pp->ulp_objsize);
	}
	}
	np->uln_next = POOL_TO_MARKER(pp);
	np->uln_prev = NULL;
	}

	void
	uu_list_node_fini(void base, uu_list_node_t np_arg, uu_list_pool_t *pp)
	{
	uu_list_node_impl_t np = (uu_list_node_impl_t )np_arg;

	if (pp->ulp_debug) {
	if (np->uln_next == NULL &&
	np->uln_prev == NULL) {
	uu_panic("uu_list_node_fini(%p, %p, %p (\"%s\")): "
	"node already finied\n",
	base, (void )np_arg, (void )pp, pp->ulp_name);
	}
	if (np->uln_next != POOL_TO_MARKER(pp) \|\|
	np->uln_prev != NULL) {
	uu_panic("uu_list_node_fini(%p, %p, %p (\"%s\")): "
	"node corrupt or on list\n",
	base, (void )np_arg, (void )pp, pp->ulp_name);
	}
	}
	np->uln_next = NULL;
	np->uln_prev = NULL;
	}

	uu_list_t *
	uu_list_create(uu_list_pool_t pp, void parent, uint32_t flags)
	{
	uu_list_t lp, next, *prev;

	if (flags & ~(UU_LIST_DEBUG \| UU_LIST_SORTED)) {
	uu_set_error(UU_ERROR_UNKNOWN_FLAG);
	return (NULL);
	}

	if ((flags & UU_LIST_SORTED) && pp->ulp_cmp == NULL) {
	if (pp->ulp_debug)
	uu_panic("uu_list_create(%p, ...): requested "
	"UU_LIST_SORTED, but pool has no comparison func\n",
	(void *)pp);
	uu_set_error(UU_ERROR_NOT_SUPPORTED);
	return (NULL);
	}

	lp = uu_zalloc(sizeof (*lp));
	if (lp == NULL) {
	uu_set_error(UU_ERROR_NO_MEMORY);
	return (NULL);
	}

	lp->ul_pool = pp;
	lp->ul_parent = parent;
	lp->ul_offset = pp->ulp_nodeoffset;
	lp->ul_debug = pp->ulp_debug \|\| (flags & UU_LIST_DEBUG);
	lp->ul_sorted = (flags & UU_LIST_SORTED);
	lp->ul_numnodes = 0;
	lp->ul_index = (pp->ulp_last_index = INDEX_NEXT(pp->ulp_last_index));

	lp->ul_null_node.uln_next = &lp->ul_null_node;
	lp->ul_null_node.uln_prev = &lp->ul_null_node;

	lp->ul_null_walk.ulw_next = &lp->ul_null_walk;
	lp->ul_null_walk.ulw_prev = &lp->ul_null_walk;

	(void) pthread_mutex_lock(&pp->ulp_lock);
	next = &pp->ulp_null_list;
	prev = next->ul_prev;
	lp->ul_next = next;
	lp->ul_prev = prev;
	next->ul_prev = lp;
	prev->ul_next = lp;
	(void) pthread_mutex_unlock(&pp->ulp_lock);

	return (lp);
	}

	void
	uu_list_destroy(uu_list_t *lp)
	{
	uu_list_pool_t *pp = lp->ul_pool;

	if (lp->ul_debug) {
	if (lp->ul_null_node.uln_next != &lp->ul_null_node \|\|
	lp->ul_null_node.uln_prev != &lp->ul_null_node) {
	uu_panic("uu_list_destroy(%p): list not empty\n",
	(void *)lp);
	}
	if (lp->ul_numnodes != 0) {
	uu_panic("uu_list_destroy(%p): numnodes is nonzero, "
	"but list is empty\n", (void *)lp);
	}
	if (lp->ul_null_walk.ulw_next != &lp->ul_null_walk \|\|
	lp->ul_null_walk.ulw_prev != &lp->ul_null_walk) {
	uu_panic("uu_list_destroy(%p): outstanding walkers\n",
	(void *)lp);
	}
	}

	(void) pthread_mutex_lock(&pp->ulp_lock);
	lp->ul_next->ul_prev = lp->ul_prev;
	lp->ul_prev->ul_next = lp->ul_next;
	(void) pthread_mutex_unlock(&pp->ulp_lock);
	lp->ul_prev = NULL;
	lp->ul_next = NULL;
	lp->ul_pool = NULL;
	uu_free(lp);
	}

	static void
	list_insert(uu_list_t lp, uu_list_node_impl_t np, uu_list_node_impl_t *prev,
	uu_list_node_impl_t *next)
	{
	if (lp->ul_debug) {
	if (next->uln_prev != prev \|\| prev->uln_next != next)
	uu_panic("insert(%p): internal error: %p and %p not "
	"neighbors\n", (void )lp, (void )next,
	(void *)prev);

	if (np->uln_next != POOL_TO_MARKER(lp->ul_pool) \|\|
	np->uln_prev != NULL) {
	uu_panic("insert(%p): elem %p node %p corrupt, "
	"not initialized, or already in a list.\n",
	(void )lp, NODE_TO_ELEM(lp, np), (void )np);
	}
	/*
	* invalidate outstanding uu_list_index_ts.
	*/
	lp->ul_index = INDEX_NEXT(lp->ul_index);
	}
	np->uln_next = next;
	np->uln_prev = prev;
	next->uln_prev = np;
	prev->uln_next = np;

	lp->ul_numnodes++;
	}

	void
	uu_list_insert(uu_list_t lp, void elem, uu_list_index_t idx)
	{
	uu_list_node_impl_t *np;

	np = INDEX_TO_NODE(idx);
	if (np == NULL)
	np = &lp->ul_null_node;

	if (lp->ul_debug) {
	if (!INDEX_VALID(lp, idx))
	uu_panic("uu_list_insert(%p, %p, %p): %s\n",
	(void )lp, elem, (void )idx,
	INDEX_CHECK(idx)? "outdated index" :
	"invalid index");
	if (np->uln_prev == NULL)
	uu_panic("uu_list_insert(%p, %p, %p): out-of-date "
	"index\n", (void )lp, elem, (void )idx);
	}

	list_insert(lp, ELEM_TO_NODE(lp, elem), np->uln_prev, np);
	}

	void *
	uu_list_find(uu_list_t lp, void elem, void private, uu_list_index_t out)
	{
	int sorted = lp->ul_sorted;
	uu_compare_fn_t *func = lp->ul_pool->ulp_cmp;
	uu_list_node_impl_t *np;

	if (func == NULL) {
	if (out != NULL)
	*out = 0;
	uu_set_error(UU_ERROR_NOT_SUPPORTED);
	return (NULL);
	}
	for (np = lp->ul_null_node.uln_next; np != &lp->ul_null_node;
	np = np->uln_next) {
	void *ep = NODE_TO_ELEM(lp, np);
	int cmp = func(ep, elem, private);
	if (cmp == 0) {
	if (out != NULL)
	*out = NODE_TO_INDEX(lp, np);
	return (ep);
	}
	if (sorted && cmp > 0) {
	if (out != NULL)
	*out = NODE_TO_INDEX(lp, np);
	return (NULL);
	}
	}
	if (out != NULL)
	*out = NODE_TO_INDEX(lp, 0);
	return (NULL);
	}

	void *
	uu_list_nearest_next(uu_list_t *lp, uu_list_index_t idx)
	{
	uu_list_node_impl_t *np = INDEX_TO_NODE(idx);

	if (np == NULL)
	np = &lp->ul_null_node;

	if (lp->ul_debug) {
	if (!INDEX_VALID(lp, idx))
	uu_panic("uu_list_nearest_next(%p, %p): %s\n",
	(void )lp, (void )idx,
	INDEX_CHECK(idx)? "outdated index" :
	"invalid index");
	if (np->uln_prev == NULL)
	uu_panic("uu_list_nearest_next(%p, %p): out-of-date "
	"index\n", (void )lp, (void )idx);
	}

	if (np == &lp->ul_null_node)
	return (NULL);
	else
	return (NODE_TO_ELEM(lp, np));
	}

	void *
	uu_list_nearest_prev(uu_list_t *lp, uu_list_index_t idx)
	{
	uu_list_node_impl_t *np = INDEX_TO_NODE(idx);

	if (np == NULL)
	np = &lp->ul_null_node;

	if (lp->ul_debug) {
	if (!INDEX_VALID(lp, idx))
	uu_panic("uu_list_nearest_prev(%p, %p): %s\n",
	(void )lp, (void )idx, INDEX_CHECK(idx)?
	"outdated index" : "invalid index");
	if (np->uln_prev == NULL)
	uu_panic("uu_list_nearest_prev(%p, %p): out-of-date "
	"index\n", (void )lp, (void )idx);
	}

	if ((np = np->uln_prev) == &lp->ul_null_node)
	return (NULL);
	else
	return (NODE_TO_ELEM(lp, np));
	}

	static void
	list_walk_init(uu_list_walk_t wp, uu_list_t lp, uint32_t flags)
	{
	uu_list_walk_t next, prev;

	int robust = (flags & UU_WALK_ROBUST);
	int direction = (flags & UU_WALK_REVERSE)? -1 : 1;

	(void) memset(wp, 0, sizeof (*wp));
	wp->ulw_list = lp;
	wp->ulw_robust = robust;
	wp->ulw_dir = direction;
	if (direction > 0)
	wp->ulw_next_result = lp->ul_null_node.uln_next;
	else
	wp->ulw_next_result = lp->ul_null_node.uln_prev;

	if (lp->ul_debug \|\| robust) {
	/*
	* Add this walker to the list's list of walkers so
	* uu_list_remove() can advance us if somebody tries to
	* remove ulw_next_result.
	*/
	wp->ulw_next = next = &lp->ul_null_walk;
	wp->ulw_prev = prev = next->ulw_prev;
	next->ulw_prev = wp;
	prev->ulw_next = wp;
	}
	}

	static uu_list_node_impl_t *
	list_walk_advance(uu_list_walk_t wp, uu_list_t lp)
	{
	uu_list_node_impl_t *np = wp->ulw_next_result;
	uu_list_node_impl_t *next;

	if (np == &lp->ul_null_node)
	return (NULL);

	next = (wp->ulw_dir > 0)? np->uln_next : np->uln_prev;

	wp->ulw_next_result = next;
	return (np);
	}

	static void
	list_walk_fini(uu_list_walk_t *wp)
	{
	/* GLXXX debugging? */
	if (wp->ulw_next != NULL) {
	wp->ulw_next->ulw_prev = wp->ulw_prev;
	wp->ulw_prev->ulw_next = wp->ulw_next;
	wp->ulw_next = NULL;
	wp->ulw_prev = NULL;
	}
	wp->ulw_list = NULL;
	wp->ulw_next_result = NULL;
	}

	uu_list_walk_t *
	uu_list_walk_start(uu_list_t *lp, uint32_t flags)
	{
	uu_list_walk_t *wp;

	if (flags & ~(UU_WALK_ROBUST \| UU_WALK_REVERSE)) {
	uu_set_error(UU_ERROR_UNKNOWN_FLAG);
	return (NULL);
	}

	wp = uu_zalloc(sizeof (*wp));
	if (wp == NULL) {
	uu_set_error(UU_ERROR_NO_MEMORY);
	return (NULL);
	}

	list_walk_init(wp, lp, flags);
	return (wp);
	}

	void *
	uu_list_walk_next(uu_list_walk_t *wp)
	{
	uu_list_t *lp = wp->ulw_list;
	uu_list_node_impl_t *np = list_walk_advance(wp, lp);

	if (np == NULL)
	return (NULL);

	return (NODE_TO_ELEM(lp, np));
	}

	void
	uu_list_walk_end(uu_list_walk_t *wp)
	{
	list_walk_fini(wp);
	uu_free(wp);
	}

	int
	uu_list_walk(uu_list_t lp, uu_walk_fn_t func, void *private, uint32_t flags)
	{
	uu_list_node_impl_t *np;

	int status = UU_WALK_NEXT;

	int robust = (flags & UU_WALK_ROBUST);
	int reverse = (flags & UU_WALK_REVERSE);

	if (flags & ~(UU_WALK_ROBUST \| UU_WALK_REVERSE)) {
	uu_set_error(UU_ERROR_UNKNOWN_FLAG);
	return (-1);
	}

	if (lp->ul_debug \|\| robust) {
	- uu_list_walk_t my_walk;
	+ uu_list_walk_t *my_walk;
	void *e;

	- list_walk_init(&my_walk, lp, flags);
	+ my_walk = uu_zalloc(sizeof (*my_walk));
	+ if (my_walk == NULL)
	+ return (-1);
	+
	+ list_walk_init(my_walk, lp, flags);
	while (status == UU_WALK_NEXT &&
	- (e = uu_list_walk_next(&my_walk)) != NULL)
	+ (e = uu_list_walk_next(my_walk)) != NULL)
	status = (*func)(e, private);
	- list_walk_fini(&my_walk);
	+ list_walk_fini(my_walk);
	+
	+ uu_free(my_walk);
	} else {
	if (!reverse) {
	for (np = lp->ul_null_node.uln_next;
	status == UU_WALK_NEXT && np != &lp->ul_null_node;
	np = np->uln_next) {
	status = (*func)(NODE_TO_ELEM(lp, np), private);
	}
	} else {
	for (np = lp->ul_null_node.uln_prev;
	status == UU_WALK_NEXT && np != &lp->ul_null_node;
	np = np->uln_prev) {
	status = (*func)(NODE_TO_ELEM(lp, np), private);
	}
	}
	}
	if (status >= 0)
	return (0);
	uu_set_error(UU_ERROR_CALLBACK_FAILED);
	return (-1);
	}

	void
	uu_list_remove(uu_list_t lp, void elem)
	{
	uu_list_node_impl_t *np = ELEM_TO_NODE(lp, elem);
	uu_list_walk_t *wp;

	if (lp->ul_debug) {
	if (np->uln_prev == NULL)
	uu_panic("uu_list_remove(%p, %p): elem not on list\n",
	(void *)lp, elem);
	/*
	* invalidate outstanding uu_list_index_ts.
	*/
	lp->ul_index = INDEX_NEXT(lp->ul_index);
	}

	/*
	* robust walkers must be advanced. In debug mode, non-robust
	* walkers are also on the list. If there are any, it's an error.
	*/
	for (wp = lp->ul_null_walk.ulw_next; wp != &lp->ul_null_walk;
	wp = wp->ulw_next) {
	if (wp->ulw_robust) {
	if (np == wp->ulw_next_result)
	(void) list_walk_advance(wp, lp);
	} else if (wp->ulw_next_result != NULL) {
	uu_panic("uu_list_remove(%p, %p): active non-robust "
	"walker\n", (void *)lp, elem);
	}
	}

	np->uln_next->uln_prev = np->uln_prev;
	np->uln_prev->uln_next = np->uln_next;

	lp->ul_numnodes--;

	np->uln_next = POOL_TO_MARKER(lp->ul_pool);
	np->uln_prev = NULL;
	}

	void *
	uu_list_teardown(uu_list_t lp, void *cookie)
	{
	void *ep;

	/*
	* XXX: disable list modification until list is empty
	*/
	if (lp->ul_debug && *cookie != NULL)
	uu_panic("uu_list_teardown(%p, %p): unexpected cookie\n",
	(void )lp, (void )cookie);

	ep = uu_list_first(lp);
	if (ep)
	uu_list_remove(lp, ep);
	return (ep);
	}

	int
	uu_list_insert_before(uu_list_t lp, void target, void *elem)
	{
	uu_list_node_impl_t *np = ELEM_TO_NODE(lp, target);

	if (target == NULL)
	np = &lp->ul_null_node;

	if (lp->ul_debug) {
	if (np->uln_prev == NULL)
	uu_panic("uu_list_insert_before(%p, %p, %p): %p is "
	"not currently on a list\n",
	(void *)lp, target, elem, target);
	}
	if (lp->ul_sorted) {
	if (lp->ul_debug)
	uu_panic("uu_list_insert_before(%p, ...): list is "
	"UU_LIST_SORTED\n", (void *)lp);
	uu_set_error(UU_ERROR_NOT_SUPPORTED);
	return (-1);
	}

	list_insert(lp, ELEM_TO_NODE(lp, elem), np->uln_prev, np);
	return (0);
	}

	int
	uu_list_insert_after(uu_list_t lp, void target, void *elem)
	{
	uu_list_node_impl_t *np = ELEM_TO_NODE(lp, target);

	if (target == NULL)
	np = &lp->ul_null_node;

	if (lp->ul_debug) {
	if (np->uln_prev == NULL)
	uu_panic("uu_list_insert_after(%p, %p, %p): %p is "
	"not currently on a list\n",
	(void *)lp, target, elem, target);
	}
	if (lp->ul_sorted) {
	if (lp->ul_debug)
	uu_panic("uu_list_insert_after(%p, ...): list is "
	"UU_LIST_SORTED\n", (void *)lp);
	uu_set_error(UU_ERROR_NOT_SUPPORTED);
	return (-1);
	}

	list_insert(lp, ELEM_TO_NODE(lp, elem), np, np->uln_next);
	return (0);
	}

	size_t
	uu_list_numnodes(uu_list_t *lp)
	{
	return (lp->ul_numnodes);
	}

	void *
	uu_list_first(uu_list_t *lp)
	{
	uu_list_node_impl_t *n = lp->ul_null_node.uln_next;
	if (n == &lp->ul_null_node)
	return (NULL);
	return (NODE_TO_ELEM(lp, n));
	}

	void *
	uu_list_last(uu_list_t *lp)
	{
	uu_list_node_impl_t *n = lp->ul_null_node.uln_prev;
	if (n == &lp->ul_null_node)
	return (NULL);
	return (NODE_TO_ELEM(lp, n));
	}

	void *
	uu_list_next(uu_list_t lp, void elem)
	{
	uu_list_node_impl_t *n = ELEM_TO_NODE(lp, elem);

	n = n->uln_next;
	if (n == &lp->ul_null_node)
	return (NULL);
	return (NODE_TO_ELEM(lp, n));
	}

	void *
	uu_list_prev(uu_list_t lp, void elem)
	{
	uu_list_node_impl_t *n = ELEM_TO_NODE(lp, elem);

	n = n->uln_prev;
	if (n == &lp->ul_null_node)
	return (NULL);
	return (NODE_TO_ELEM(lp, n));
	}

	/*
	* called from uu_lockup() and uu_release(), as part of our fork1()-safety.
	*/
	void
	uu_list_lockup(void)
	{
	uu_list_pool_t *pp;

	(void) pthread_mutex_lock(&uu_lpool_list_lock);
	for (pp = uu_null_lpool.ulp_next; pp != &uu_null_lpool;
	pp = pp->ulp_next)
	(void) pthread_mutex_lock(&pp->ulp_lock);
	}

	void
	uu_list_release(void)
	{
	uu_list_pool_t *pp;

	for (pp = uu_null_lpool.ulp_next; pp != &uu_null_lpool;
	pp = pp->ulp_next)
	(void) pthread_mutex_unlock(&pp->ulp_lock);
	(void) pthread_mutex_unlock(&uu_lpool_list_lock);
	}
	diff --git a/sys/contrib/openzfs/lib/libzfs/libzfs.abi b/sys/contrib/openzfs/lib/libzfs/libzfs.abi
	index 9bb8f6a47de1..8bedfe72294c 100644
	--- a/sys/contrib/openzfs/lib/libzfs/libzfs.abi
	+++ b/sys/contrib/openzfs/lib/libzfs/libzfs.abi
	@@ -1,9497 +1,9549 @@
	<abi-corpus version='2.0' architecture='elf-amd-x86_64' soname='libzfs.so.4'>
	<elf-needed>
	<dependency name='libzfs_core.so.3'/>
	<dependency name='libnvpair.so.3'/>
	<dependency name='libuuid.so.1'/>
	<dependency name='libblkid.so.1'/>
	<dependency name='libudev.so.1'/>
	<dependency name='libuutil.so.3'/>
	<dependency name='libm.so.6'/>
	<dependency name='libcrypto.so.3'/>
	<dependency name='libz.so.1'/>
	<dependency name='libc.so.6'/>
	<dependency name='ld-linux-x86-64.so.2'/>
	</elf-needed>
	<elf-function-symbols>
	<elf-symbol name='_sol_getmntent' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_16' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_16_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_32' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_32_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_64' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_64_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_8' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_8_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_char' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_char_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_int' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_int_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_long' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_long_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_ptr' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_ptr_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_short' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_add_short_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_and_16' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_and_16_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_and_32' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_and_32_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_and_64' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_and_64_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_and_8' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_and_8_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_and_uchar' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_and_uchar_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_and_uint' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_and_uint_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_and_ulong' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_and_ulong_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_and_ushort' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_and_ushort_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_cas_16' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_cas_32' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_cas_64' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_cas_8' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_cas_ptr' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_cas_uchar' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_cas_uint' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_cas_ulong' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_cas_ushort' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_clear_long_excl' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_dec_16' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_dec_16_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_dec_32' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_dec_32_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_dec_64' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_dec_64_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_dec_8' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_dec_8_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_dec_uchar' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_dec_uchar_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_dec_uint' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_dec_uint_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_dec_ulong' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_dec_ulong_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_dec_ushort' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_dec_ushort_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_inc_16' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_inc_16_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_inc_32' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_inc_32_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_inc_64' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_inc_64_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_inc_8' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_inc_8_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_inc_uchar' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_inc_uchar_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_inc_uint' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_inc_uint_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_inc_ulong' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_inc_ulong_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_inc_ushort' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_inc_ushort_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_or_16' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_or_16_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_or_32' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_or_32_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_or_64' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_or_64_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_or_8' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_or_8_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_or_uchar' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_or_uchar_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_or_uint' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_or_uint_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_or_ulong' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_or_ulong_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_or_ushort' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_or_ushort_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_set_long_excl' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_16' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_16_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_32' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_32_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_64' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_64_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_8' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_8_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_char' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_char_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_int' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_int_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_long' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_long_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_ptr' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_ptr_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_short' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_sub_short_nv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_swap_16' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_swap_32' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_swap_64' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_swap_8' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_swap_ptr' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_swap_uchar' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_swap_uint' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_swap_ulong' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='atomic_swap_ushort' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_add' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_create' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_destroy' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_destroy_nodes' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_find' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_first' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_insert' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_insert_here' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_is_empty' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_last' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_nearest' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_numnodes' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_remove' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_swap' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_update' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_update_gt' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_update_lt' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='avl_walk' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='bookmark_namecheck' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='cityhash4' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='color_end' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='color_start' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='dataset_namecheck' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='dataset_nestcheck' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='efi_alloc_and_init' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='efi_alloc_and_read' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='efi_err_check' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='efi_free' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='efi_rescan' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='efi_use_whole_disk' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='efi_write' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='entity_namecheck' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_2_byteswap' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_2_incremental_byteswap' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_2_incremental_native' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_2_native' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_4_byteswap' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_4_fini' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_4_impl_set' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_4_incremental_byteswap' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_4_incremental_native' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_4_init' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_4_native' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_4_native_varsize' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_init' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fsleep' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='get_dataset_depth' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='get_system_hostid' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='getexecname' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='getextmntent' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='getmntany' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='getprop_uint64' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='getzoneid' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='is_mounted' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='is_mpath_whole_disk' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libpc_error_description' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libspl_assertf' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libspl_set_assert_ok' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_add_handle' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_envvar_is_set' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_errno' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_error_action' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_error_description' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_error_init' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_fini' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_free_str_array' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_init' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_mnttab_add' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_mnttab_cache' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_mnttab_find' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_mnttab_fini' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_mnttab_init' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_mnttab_remove' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_print_on_error' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_run_process' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_run_process_get_stdout' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_run_process_get_stdout_nopath' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_create' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_destroy' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_head' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_insert_after' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_insert_before' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_insert_head' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_insert_tail' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_is_empty' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_link_active' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_link_init' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_link_replace' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_move_tail' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_next' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_prev' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_remove' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_remove_head' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_remove_tail' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='list_tail' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='membar_consumer' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='membar_enter' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='membar_exit' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='membar_producer' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='membar_sync' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='mkdirp' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='mountpoint_namecheck' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='permset_namecheck' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='pool_namecheck' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='print_timestamp' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='printf_color' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='sa_commit_shares' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='sa_disable_share' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='sa_enable_share' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='sa_errorstr' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='sa_is_shared' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='sa_truncate_shares' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='sa_validate_shareopts' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='snapshot_namecheck' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='spl_pagesize' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='strlcat' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='strlcpy' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='tpool_abandon' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='tpool_create' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='tpool_destroy' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='tpool_dispatch' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='tpool_member' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='tpool_resume' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='tpool_suspend' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='tpool_suspended' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='tpool_wait' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='update_vdev_config_dev_strs' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='update_vdev_config_dev_sysfs_path' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='use_color' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='vdev_expand_proplist' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='vdev_name_to_prop' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='vdev_prop_align_right' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='vdev_prop_column_name' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='vdev_prop_default_numeric' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='vdev_prop_default_string' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='vdev_prop_get_table' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='vdev_prop_get_type' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='vdev_prop_index_to_string' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='vdev_prop_init' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='vdev_prop_random_value' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='vdev_prop_readonly' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='vdev_prop_string_to_index' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='vdev_prop_to_name' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='vdev_prop_user' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='vdev_prop_values' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfeature_depends_on' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfeature_is_supported' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfeature_is_valid_guid' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfeature_lookup_guid' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfeature_lookup_name' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_adjust_mount_options' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_allocatable_devs' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_append_partition' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_basename' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_bookmark_exists' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_clone' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_close' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_commit_shares' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_component_namecheck' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_create' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_create_ancestors' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_crypto_attempt_load_keys' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_crypto_clone_check' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_crypto_create' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_crypto_get_encryption_root' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_crypto_load_key' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_crypto_rewrap' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_crypto_unload_key' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_dataset_exists' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_dataset_name_hidden' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_deleg_canonicalize_perm' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_deleg_verify_nvlist' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_deleg_whokey' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_destroy' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_destroy_snaps' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_destroy_snaps_nvl' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_destroy_snaps_nvl_os' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_dev_flush' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_dev_is_dm' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_dev_is_whole_disk' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_device_get_devid' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_device_get_physical' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_dirnamelen' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_expand_proplist' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_foreach_mountpoint' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_get_all_props' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_get_clones_nvl' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_get_enclosure_sysfs_path' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_get_fsacl' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_get_handle' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_get_holds' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_get_name' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_get_pool_handle' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_get_pool_name' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_get_recvd_props' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_get_type' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_get_underlying_path' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_get_underlying_type' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_get_user_props' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_handle_dup' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_hold' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_hold_nvl' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_ioctl' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_is_mounted' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_is_shared' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_isnumber' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_iter_bookmarks' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_iter_bookmarks_v2' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_iter_children' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_iter_children_v2' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_iter_dependents' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_iter_dependents_v2' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_iter_filesystems' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_iter_filesystems_v2' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_iter_mounted' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_iter_root' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_iter_snapshots' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_iter_snapshots_sorted' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_iter_snapshots_sorted_v2' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_iter_snapshots_v2' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_iter_snapspec' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_iter_snapspec_v2' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_mod_supported' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_mount' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_mount_at' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_mount_delegation_check' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_name_to_prop' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_name_valid' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_nicebytes' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_nicenum' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_nicenum_format' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_niceraw' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_nicestrtonum' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_nicetime' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_open' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_parent_name' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_parse_mount_options' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_path_to_zhandle' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_promote' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_align_right' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_column_name' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_default_numeric' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_default_string' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_delegatable' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_encryption_key_param' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_get' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_get_int' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_get_numeric' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_get_recvd' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_get_table' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_get_type' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_get_userquota' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_get_userquota_int' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_get_written' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_get_written_int' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_index_to_string' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_inherit' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_inheritable' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_init' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_is_string' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_random_value' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_readonly' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_set' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_set_list' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_set_list_flags' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_setonce' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_string_to_index' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_to_name' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_user' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_userquota' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_valid_for_type' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_valid_keylocation' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_values' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_visible' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prop_written' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_prune_proplist' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_receive' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_refresh_properties' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_release' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_rename' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_resolve_shortname' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_rollback' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_save_arguments' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_send' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_send_one' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_send_progress' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_send_resume' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_send_resume_token_to_nvlist' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_send_saved' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_set_fsacl' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_setproctitle' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_setproctitle_init' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_share' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_show_diffs' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_smb_acl_add' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_smb_acl_purge' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_smb_acl_remove' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_smb_acl_rename' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_snapshot' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_snapshot_nvl' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_spa_version' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_spa_version_map' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_special_devs' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_standard_error' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_strcmp_pathname' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_strip_partition' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_strip_path' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_truncate_shares' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_type_to_name' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_unmount' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_unmountall' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_unshare' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_unshareall' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_userns' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_userspace' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_valid_proplist' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_version_kernel' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_version_print' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_version_userland' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_wait_status' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_zpl_version_map' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_add' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_checkpoint' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_clear' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_clear_label' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_close' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_create' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_default_search_paths' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_destroy' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_disable_datasets' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_disable_datasets_os' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_disable_volume_os' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_discard_checkpoint' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_disk_wait' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_dump_ddt' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_enable_datasets' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_events_clear' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_events_next' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_events_seek' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_expand_proplist' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_explain_recover' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_export' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_export_force' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_feature_init' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_find_config' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_find_vdev' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_find_vdev_by_physpath' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_free_handles' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_all_vdev_props' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_bootenv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_config' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_errlog' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_features' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_handle' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_history' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_load_policy' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_name' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_prop' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_prop_int' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_state' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_state_str' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_status' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_userprop' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_vdev_prop' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_get_vdev_prop_value' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_getenv_int' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_history_unpack' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_import' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_import_props' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_import_status' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_in_use' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_initialize' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_initialize_wait' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_is_draid_spare' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_iter' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_label_disk' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_label_disk_wait' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_load_compat' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_log_history' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_name_to_prop' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_obj_to_path' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_obj_to_path_ds' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_open' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_open_canfail' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_pool_state_to_name' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prepare_and_label_disk' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prepare_disk' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_print_unsup_feat' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_align_right' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_column_name' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_default_numeric' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_default_string' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_feature' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_get_feature' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_get_table' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_get_type' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_index_to_string' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_init' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_random_value' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_readonly' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_setonce' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_string_to_index' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_to_name' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_unsupported' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_values' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_prop_vdev' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_props_refresh' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_read_label' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_refresh_stats' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_reguid' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_reopen_one' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_scan' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_search_import' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_set_bootenv' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_set_prop' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_set_vdev_prop' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_skip_pool' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_state_to_name' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_sync_one' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_trim' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_upgrade' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_attach' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_clear' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_degrade' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_detach' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_fault' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_indirect_size' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_name' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_offline' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_online' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_path_to_guid' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_remove' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_remove_cancel' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_remove_wanted' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_script_alloc_env' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_script_free_env' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_set_removed_state' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_vdev_split' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_wait' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zpool_wait_status' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_free_list' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_get_list' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_index_to_string' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_iter' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_iter_common' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_name_to_prop' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_print_one_property' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_random_value' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_register_hidden' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_register_impl' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_register_index' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_register_number' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_register_string' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_string_to_index' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_valid_char' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_valid_for_type' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_values' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zprop_width' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zvol_volsize_to_reservation' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	</elf-function-symbols>
	<elf-variable-symbols>
	<elf-symbol name='efi_debug' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_4_abd_ops' size='24' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_4_avx2_ops' size='128' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_4_avx512bw_ops' size='128' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_4_avx512f_ops' size='128' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_4_sse2_ops' size='128' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_4_ssse3_ops' size='128' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_4_superscalar4_ops' size='128' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='fletcher_4_superscalar_ops' size='128' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='libzfs_config_ops' size='16' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='sa_protocol_names' size='16' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='spa_feature_table' size='2184' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfeature_checks_disable' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_deleg_perm_tab' size='512' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_history_event_names' size='328' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_max_dataset_nesting' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	<elf-symbol name='zfs_userquota_prop_prefixes' size='96' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
	</elf-variable-symbols>
	<abi-instr address-size='64' path='lib/libefi/rdwr_efi.c' language='LANG_C99'>
	<typedef-decl name='uInt' type-id='f0981eeb' id='09110a74'/>
	<var-decl name='efi_debug' type-id='95e97e5e' mangled-name='efi_debug' visibility='default' elf-symbol-id='efi_debug'/>
	<function-decl name='uuid_generate' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='cf536864'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='uuid_is_null' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='354f7eb9'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='crc32' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5bbcce85'/>
	<parameter type-id='e8cb3e0e'/>
	<parameter type-id='09110a74'/>
	<return type-id='5bbcce85'/>
	</function-decl>
	<function-decl name='efi_err_check' mangled-name='efi_err_check' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='efi_err_check'>
	<parameter type-id='0d8119a8' name='vtoc'/>
	<return type-id='48b5725f'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libshare/libshare.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='b99c00c9' size-in-bits='128' id='2d6895a3'>
	<subrange length='2' type-id='7359adad' id='52efc4ef'/>
	</array-type-def>
	<var-decl name='sa_protocol_names' type-id='2d6895a3' mangled-name='sa_protocol_names' visibility='default' elf-symbol-id='sa_protocol_names'/>
	<type-decl name='unsigned long int' size-in-bits='64' id='7359adad'/>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libshare/nfs.c' language='LANG_C99'>
	<function-decl name='rename' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='memchr' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='flock' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='fchmod' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='e1c52942'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='mkdir' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='e1c52942'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libshare/os/linux/nfs.c' language='LANG_C99'>
	<class-decl name='sa_share_impl' size-in-bits='192' is-struct='yes' visibility='default' id='72b09bf8'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='sa_zfsname' type-id='80f4b756' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='sa_mountpoint' type-id='80f4b756' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='sa_shareopts' type-id='80f4b756' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='sa_share_impl_t' type-id='946a2c6b' id='a48b47d0'/>
	<class-decl name='sa_fstype_t' size-in-bits='384' is-struct='yes' naming-typedef-id='639af739' visibility='default' id='944afa86'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='enable_share' type-id='2f78a9c1' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='disable_share' type-id='2f78a9c1' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='is_shared' type-id='81020bc2' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='validate_shareopts' type-id='f194a8fb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='commit_shares' type-id='797ee7da' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='truncate_shares' type-id='5d51038b' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='sa_fstype_t' type-id='944afa86' id='639af739'/>
	<qualified-type-def type-id='639af739' const='yes' id='d19dbca9'/>
	<qualified-type-def type-id='72b09bf8' const='yes' id='484950e3'/>
	<pointer-type-def type-id='484950e3' size-in-bits='64' id='946a2c6b'/>
	<pointer-type-def type-id='276427e1' size-in-bits='64' id='1db260e5'/>
	<qualified-type-def type-id='1db260e5' const='yes' id='797ee7da'/>
	<pointer-type-def type-id='5113b296' size-in-bits='64' id='70487b28'/>
	<qualified-type-def type-id='70487b28' const='yes' id='f194a8fb'/>
	<pointer-type-def type-id='c13578bc' size-in-bits='64' id='fa1f29ce'/>
	<qualified-type-def type-id='fa1f29ce' const='yes' id='2f78a9c1'/>
	<pointer-type-def type-id='723e6cf2' size-in-bits='64' id='1d99e49c'/>
	<pointer-type-def type-id='86373eb1' size-in-bits='64' id='f337456d'/>
	<qualified-type-def type-id='f337456d' const='yes' id='81020bc2'/>
	<qualified-type-def type-id='953b12f8' const='yes' id='5d51038b'/>
	<var-decl name='libshare_nfs_type' type-id='d19dbca9' visibility='default'/>
	<function-decl name='nfs_escape_mountpoint' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='9b23c9ad'/>
	<parameter type-id='37e3bd22'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nfs_is_shared_impl' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='a48b47d0'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='nfs_toggle_share' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='a48b47d0'/>
	<parameter type-id='1d99e49c'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nfs_reset_shares' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-type size-in-bits='64' id='276427e1'>
	<return type-id='95e97e5e'/>
	</function-type>
	<function-type size-in-bits='64' id='5113b296'>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-type>
	<function-type size-in-bits='64' id='c13578bc'>
	<parameter type-id='a48b47d0'/>
	<return type-id='95e97e5e'/>
	</function-type>
	<function-type size-in-bits='64' id='723e6cf2'>
	<parameter type-id='a48b47d0'/>
	<parameter type-id='822cd80b'/>
	<return type-id='95e97e5e'/>
	</function-type>
	<function-type size-in-bits='64' id='86373eb1'>
	<parameter type-id='a48b47d0'/>
	<return type-id='c19b74c3'/>
	</function-type>
	<function-type size-in-bits='64' id='ee076206'>
	<return type-id='48b5725f'/>
	</function-type>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libshare/os/linux/smb.c' language='LANG_C99'>
	<var-decl name='libshare_smb_type' type-id='d19dbca9' visibility='default'/>
	<function-decl name='__fgets_chk' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='266fe297'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='e75a27e9'/>
	<return type-id='26a90f95'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libspl/assert.c' language='LANG_C99'>
	<function-decl name='libspl_set_assert_ok' mangled-name='libspl_set_assert_ok' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libspl_set_assert_ok'>
	<parameter type-id='c19b74c3' name='val'/>
	<return type-id='48b5725f'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libspl/atomic.c' language='LANG_C99'>
	<typedef-decl name='int8_t' type-id='2171a512' id='ee31ee44'/>
	<typedef-decl name='__int8_t' type-id='28577a57' id='2171a512'/>
	<qualified-type-def type-id='149c6638' volatile='yes' id='5120c5f7'/>
	<pointer-type-def type-id='5120c5f7' size-in-bits='64' id='93977ae7'/>
	<qualified-type-def type-id='b96825af' volatile='yes' id='84ff7d66'/>
	<pointer-type-def type-id='84ff7d66' size-in-bits='64' id='aa323ea4'/>
	<qualified-type-def type-id='ee1f298e' volatile='yes' id='6f7e09cb'/>
	<pointer-type-def type-id='6f7e09cb' size-in-bits='64' id='64698d33'/>
	<function-decl name='atomic_inc_8' mangled-name='atomic_inc_8' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_inc_8'>
	<parameter type-id='aa323ea4' name='target'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_inc_16' mangled-name='atomic_inc_16' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_inc_16'>
	<parameter type-id='93977ae7' name='target'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_inc_32' mangled-name='atomic_inc_32' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_inc_32'>
	<parameter type-id='3a147f31' name='target'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_inc_ulong' mangled-name='atomic_inc_ulong' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_inc_ulong'>
	<parameter type-id='64698d33' name='target'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_dec_8' mangled-name='atomic_dec_8' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_dec_8'>
	<parameter type-id='aa323ea4' name='target'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_dec_16' mangled-name='atomic_dec_16' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_dec_16'>
	<parameter type-id='93977ae7' name='target'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_dec_32' mangled-name='atomic_dec_32' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_dec_32'>
	<parameter type-id='3a147f31' name='target'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_dec_ulong' mangled-name='atomic_dec_ulong' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_dec_ulong'>
	<parameter type-id='64698d33' name='target'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_add_ptr' mangled-name='atomic_add_ptr' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_add_ptr'>
	<parameter type-id='fe09dd29' name='target'/>
	<parameter type-id='79a0948f' name='bits'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_add_8' mangled-name='atomic_add_8' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_add_8'>
	<parameter type-id='aa323ea4' name='target'/>
	<parameter type-id='ee31ee44' name='bits'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_add_16' mangled-name='atomic_add_16' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_add_16'>
	<parameter type-id='93977ae7' name='target'/>
	<parameter type-id='23bd8cb5' name='bits'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_add_32' mangled-name='atomic_add_32' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_add_32'>
	<parameter type-id='3a147f31' name='target'/>
	<parameter type-id='3ff5601b' name='bits'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_sub_ptr' mangled-name='atomic_sub_ptr' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_sub_ptr'>
	<parameter type-id='fe09dd29' name='target'/>
	<parameter type-id='79a0948f' name='bits'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_sub_8' mangled-name='atomic_sub_8' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_sub_8'>
	<parameter type-id='aa323ea4' name='target'/>
	<parameter type-id='ee31ee44' name='bits'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_sub_16' mangled-name='atomic_sub_16' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_sub_16'>
	<parameter type-id='93977ae7' name='target'/>
	<parameter type-id='23bd8cb5' name='bits'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_sub_32' mangled-name='atomic_sub_32' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_sub_32'>
	<parameter type-id='3a147f31' name='target'/>
	<parameter type-id='3ff5601b' name='bits'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_or_8' mangled-name='atomic_or_8' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_or_8'>
	<parameter type-id='aa323ea4' name='target'/>
	<parameter type-id='b96825af' name='bits'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_or_16' mangled-name='atomic_or_16' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_or_16'>
	<parameter type-id='93977ae7' name='target'/>
	<parameter type-id='149c6638' name='bits'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_or_32' mangled-name='atomic_or_32' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_or_32'>
	<parameter type-id='3a147f31' name='target'/>
	<parameter type-id='8f92235e' name='bits'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_or_ulong' mangled-name='atomic_or_ulong' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_or_ulong'>
	<parameter type-id='64698d33' name='target'/>
	<parameter type-id='ee1f298e' name='bits'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_and_8' mangled-name='atomic_and_8' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_and_8'>
	<parameter type-id='aa323ea4' name='target'/>
	<parameter type-id='b96825af' name='bits'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_and_16' mangled-name='atomic_and_16' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_and_16'>
	<parameter type-id='93977ae7' name='target'/>
	<parameter type-id='149c6638' name='bits'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_and_32' mangled-name='atomic_and_32' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_and_32'>
	<parameter type-id='3a147f31' name='target'/>
	<parameter type-id='8f92235e' name='bits'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_and_ulong' mangled-name='atomic_and_ulong' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_and_ulong'>
	<parameter type-id='64698d33' name='target'/>
	<parameter type-id='ee1f298e' name='bits'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='atomic_inc_8_nv' mangled-name='atomic_inc_8_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_inc_8_nv'>
	<parameter type-id='aa323ea4' name='target'/>
	<return type-id='b96825af'/>
	</function-decl>
	<function-decl name='atomic_inc_16_nv' mangled-name='atomic_inc_16_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_inc_16_nv'>
	<parameter type-id='93977ae7' name='target'/>
	<return type-id='149c6638'/>
	</function-decl>
	<function-decl name='atomic_inc_32_nv' mangled-name='atomic_inc_32_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_inc_32_nv'>
	<parameter type-id='3a147f31' name='target'/>
	<return type-id='8f92235e'/>
	</function-decl>
	<function-decl name='atomic_inc_ulong_nv' mangled-name='atomic_inc_ulong_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_inc_ulong_nv'>
	<parameter type-id='64698d33' name='target'/>
	<return type-id='ee1f298e'/>
	</function-decl>
	<function-decl name='atomic_dec_8_nv' mangled-name='atomic_dec_8_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_dec_8_nv'>
	<parameter type-id='aa323ea4' name='target'/>
	<return type-id='b96825af'/>
	</function-decl>
	<function-decl name='atomic_dec_16_nv' mangled-name='atomic_dec_16_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_dec_16_nv'>
	<parameter type-id='93977ae7' name='target'/>
	<return type-id='149c6638'/>
	</function-decl>
	<function-decl name='atomic_dec_32_nv' mangled-name='atomic_dec_32_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_dec_32_nv'>
	<parameter type-id='3a147f31' name='target'/>
	<return type-id='8f92235e'/>
	</function-decl>
	<function-decl name='atomic_dec_ulong_nv' mangled-name='atomic_dec_ulong_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_dec_ulong_nv'>
	<parameter type-id='64698d33' name='target'/>
	<return type-id='ee1f298e'/>
	</function-decl>
	<function-decl name='atomic_add_ptr_nv' mangled-name='atomic_add_ptr_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_add_ptr_nv'>
	<parameter type-id='fe09dd29' name='target'/>
	<parameter type-id='79a0948f' name='bits'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='atomic_add_8_nv' mangled-name='atomic_add_8_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_add_8_nv'>
	<parameter type-id='aa323ea4' name='target'/>
	<parameter type-id='ee31ee44' name='bits'/>
	<return type-id='b96825af'/>
	</function-decl>
	<function-decl name='atomic_add_16_nv' mangled-name='atomic_add_16_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_add_16_nv'>
	<parameter type-id='93977ae7' name='target'/>
	<parameter type-id='23bd8cb5' name='bits'/>
	<return type-id='149c6638'/>
	</function-decl>
	<function-decl name='atomic_add_32_nv' mangled-name='atomic_add_32_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_add_32_nv'>
	<parameter type-id='3a147f31' name='target'/>
	<parameter type-id='3ff5601b' name='bits'/>
	<return type-id='8f92235e'/>
	</function-decl>
	<function-decl name='atomic_add_long_nv' mangled-name='atomic_add_long_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_add_long_nv'>
	<parameter type-id='64698d33' name='target'/>
	<parameter type-id='bd54fe1a' name='bits'/>
	<return type-id='ee1f298e'/>
	</function-decl>
	<function-decl name='atomic_sub_ptr_nv' mangled-name='atomic_sub_ptr_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_sub_ptr_nv'>
	<parameter type-id='fe09dd29' name='target'/>
	<parameter type-id='79a0948f' name='bits'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='atomic_sub_8_nv' mangled-name='atomic_sub_8_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_sub_8_nv'>
	<parameter type-id='aa323ea4' name='target'/>
	<parameter type-id='ee31ee44' name='bits'/>
	<return type-id='b96825af'/>
	</function-decl>
	<function-decl name='atomic_sub_16_nv' mangled-name='atomic_sub_16_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_sub_16_nv'>
	<parameter type-id='93977ae7' name='target'/>
	<parameter type-id='23bd8cb5' name='bits'/>
	<return type-id='149c6638'/>
	</function-decl>
	<function-decl name='atomic_sub_32_nv' mangled-name='atomic_sub_32_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_sub_32_nv'>
	<parameter type-id='3a147f31' name='target'/>
	<parameter type-id='3ff5601b' name='bits'/>
	<return type-id='8f92235e'/>
	</function-decl>
	<function-decl name='atomic_sub_long_nv' mangled-name='atomic_sub_long_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_sub_long_nv'>
	<parameter type-id='64698d33' name='target'/>
	<parameter type-id='bd54fe1a' name='bits'/>
	<return type-id='ee1f298e'/>
	</function-decl>
	<function-decl name='atomic_or_8_nv' mangled-name='atomic_or_8_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_or_8_nv'>
	<parameter type-id='aa323ea4' name='target'/>
	<parameter type-id='b96825af' name='bits'/>
	<return type-id='b96825af'/>
	</function-decl>
	<function-decl name='atomic_or_16_nv' mangled-name='atomic_or_16_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_or_16_nv'>
	<parameter type-id='93977ae7' name='target'/>
	<parameter type-id='149c6638' name='bits'/>
	<return type-id='149c6638'/>
	</function-decl>
	<function-decl name='atomic_or_32_nv' mangled-name='atomic_or_32_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_or_32_nv'>
	<parameter type-id='3a147f31' name='target'/>
	<parameter type-id='8f92235e' name='bits'/>
	<return type-id='8f92235e'/>
	</function-decl>
	<function-decl name='atomic_or_ulong_nv' mangled-name='atomic_or_ulong_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_or_ulong_nv'>
	<parameter type-id='64698d33' name='target'/>
	<parameter type-id='ee1f298e' name='bits'/>
	<return type-id='ee1f298e'/>
	</function-decl>
	<function-decl name='atomic_and_8_nv' mangled-name='atomic_and_8_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_and_8_nv'>
	<parameter type-id='aa323ea4' name='target'/>
	<parameter type-id='b96825af' name='bits'/>
	<return type-id='b96825af'/>
	</function-decl>
	<function-decl name='atomic_and_16_nv' mangled-name='atomic_and_16_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_and_16_nv'>
	<parameter type-id='93977ae7' name='target'/>
	<parameter type-id='149c6638' name='bits'/>
	<return type-id='149c6638'/>
	</function-decl>
	<function-decl name='atomic_and_32_nv' mangled-name='atomic_and_32_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_and_32_nv'>
	<parameter type-id='3a147f31' name='target'/>
	<parameter type-id='8f92235e' name='bits'/>
	<return type-id='8f92235e'/>
	</function-decl>
	<function-decl name='atomic_and_ulong_nv' mangled-name='atomic_and_ulong_nv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_and_ulong_nv'>
	<parameter type-id='64698d33' name='target'/>
	<parameter type-id='ee1f298e' name='bits'/>
	<return type-id='ee1f298e'/>
	</function-decl>
	<function-decl name='atomic_cas_ptr' mangled-name='atomic_cas_ptr' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_cas_ptr'>
	<parameter type-id='fe09dd29' name='target'/>
	<parameter type-id='eaa32e2f' name='exp'/>
	<parameter type-id='eaa32e2f' name='des'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='atomic_cas_8' mangled-name='atomic_cas_8' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_cas_8'>
	<parameter type-id='aa323ea4' name='target'/>
	<parameter type-id='b96825af' name='exp'/>
	<parameter type-id='b96825af' name='des'/>
	<return type-id='b96825af'/>
	</function-decl>
	<function-decl name='atomic_cas_16' mangled-name='atomic_cas_16' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_cas_16'>
	<parameter type-id='93977ae7' name='target'/>
	<parameter type-id='149c6638' name='exp'/>
	<parameter type-id='149c6638' name='des'/>
	<return type-id='149c6638'/>
	</function-decl>
	<function-decl name='atomic_cas_32' mangled-name='atomic_cas_32' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_cas_32'>
	<parameter type-id='3a147f31' name='target'/>
	<parameter type-id='8f92235e' name='exp'/>
	<parameter type-id='8f92235e' name='des'/>
	<return type-id='8f92235e'/>
	</function-decl>
	<function-decl name='atomic_cas_ulong' mangled-name='atomic_cas_ulong' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_cas_ulong'>
	<parameter type-id='64698d33' name='target'/>
	<parameter type-id='ee1f298e' name='exp'/>
	<parameter type-id='ee1f298e' name='des'/>
	<return type-id='ee1f298e'/>
	</function-decl>
	<function-decl name='atomic_swap_8' mangled-name='atomic_swap_8' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_swap_8'>
	<parameter type-id='aa323ea4' name='target'/>
	<parameter type-id='b96825af' name='bits'/>
	<return type-id='b96825af'/>
	</function-decl>
	<function-decl name='atomic_swap_16' mangled-name='atomic_swap_16' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_swap_16'>
	<parameter type-id='93977ae7' name='target'/>
	<parameter type-id='149c6638' name='bits'/>
	<return type-id='149c6638'/>
	</function-decl>
	<function-decl name='atomic_swap_ulong' mangled-name='atomic_swap_ulong' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_swap_ulong'>
	<parameter type-id='64698d33' name='target'/>
	<parameter type-id='ee1f298e' name='bits'/>
	<return type-id='ee1f298e'/>
	</function-decl>
	<function-decl name='atomic_swap_ptr' mangled-name='atomic_swap_ptr' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_swap_ptr'>
	<parameter type-id='fe09dd29' name='target'/>
	<parameter type-id='eaa32e2f' name='bits'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='atomic_set_long_excl' mangled-name='atomic_set_long_excl' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_set_long_excl'>
	<parameter type-id='64698d33' name='target'/>
	<parameter type-id='3502e3ff' name='value'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='atomic_clear_long_excl' mangled-name='atomic_clear_long_excl' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_clear_long_excl'>
	<parameter type-id='64698d33' name='target'/>
	<parameter type-id='3502e3ff' name='value'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='membar_enter' mangled-name='membar_enter' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='membar_enter'>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='membar_consumer' mangled-name='membar_consumer' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='membar_consumer'>
	<return type-id='48b5725f'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libspl/getexecname.c' language='LANG_C99'>
	<function-decl name='getexecname' mangled-name='getexecname' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='getexecname'>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='getexecname_impl' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='26a90f95'/>
	<return type-id='79a0948f'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libspl/list.c' language='LANG_C99'>
	<typedef-decl name='list_node_t' type-id='b0b5e45e' id='b21843b2'/>
	<typedef-decl name='list_t' type-id='e824dae9' id='0899125f'/>
	<class-decl name='list_node' size-in-bits='128' is-struct='yes' visibility='default' id='b0b5e45e'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='next' type-id='b03eadb4' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='prev' type-id='b03eadb4' visibility='default'/>
	</data-member>
	</class-decl>
	- <class-decl name='list' size-in-bits='256' is-struct='yes' visibility='default' id='e824dae9'>
	+ <class-decl name='list' size-in-bits='192' is-struct='yes' visibility='default' id='e824dae9'>
	<data-member access='public' layout-offset-in-bits='0'>
	- <var-decl name='list_size' type-id='b59d7dce' visibility='default'/>
	- </data-member>
	- <data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='list_offset' type-id='b59d7dce' visibility='default'/>
	</data-member>
	- <data-member access='public' layout-offset-in-bits='128'>
	+ <data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='list_head' type-id='b0b5e45e' visibility='default'/>
	</data-member>
	</class-decl>
	<pointer-type-def type-id='b0b5e45e' size-in-bits='64' id='b03eadb4'/>
	<pointer-type-def type-id='b21843b2' size-in-bits='64' id='ccc38265'/>
	<pointer-type-def type-id='0899125f' size-in-bits='64' id='352ec160'/>
	<function-decl name='list_create' mangled-name='list_create' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_create'>
	<parameter type-id='352ec160' name='list'/>
	<parameter type-id='b59d7dce' name='size'/>
	<parameter type-id='b59d7dce' name='offset'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='list_destroy' mangled-name='list_destroy' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_destroy'>
	<parameter type-id='352ec160' name='list'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='list_insert_after' mangled-name='list_insert_after' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_insert_after'>
	<parameter type-id='352ec160' name='list'/>
	<parameter type-id='eaa32e2f' name='object'/>
	<parameter type-id='eaa32e2f' name='nobject'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='list_insert_before' mangled-name='list_insert_before' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_insert_before'>
	<parameter type-id='352ec160' name='list'/>
	<parameter type-id='eaa32e2f' name='object'/>
	<parameter type-id='eaa32e2f' name='nobject'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='list_insert_head' mangled-name='list_insert_head' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_insert_head'>
	<parameter type-id='352ec160' name='list'/>
	<parameter type-id='eaa32e2f' name='object'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='list_insert_tail' mangled-name='list_insert_tail' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_insert_tail'>
	<parameter type-id='352ec160' name='list'/>
	<parameter type-id='eaa32e2f' name='object'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='list_remove' mangled-name='list_remove' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_remove'>
	<parameter type-id='352ec160' name='list'/>
	<parameter type-id='eaa32e2f' name='object'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='list_remove_head' mangled-name='list_remove_head' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_remove_head'>
	<parameter type-id='352ec160' name='list'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='list_remove_tail' mangled-name='list_remove_tail' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_remove_tail'>
	<parameter type-id='352ec160' name='list'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='list_head' mangled-name='list_head' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_head'>
	<parameter type-id='352ec160' name='list'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='list_tail' mangled-name='list_tail' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_tail'>
	<parameter type-id='352ec160' name='list'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='list_next' mangled-name='list_next' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_next'>
	<parameter type-id='352ec160' name='list'/>
	<parameter type-id='eaa32e2f' name='object'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='list_prev' mangled-name='list_prev' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_prev'>
	<parameter type-id='352ec160' name='list'/>
	<parameter type-id='eaa32e2f' name='object'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='list_move_tail' mangled-name='list_move_tail' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_move_tail'>
	<parameter type-id='352ec160' name='dst'/>
	<parameter type-id='352ec160' name='src'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='list_link_replace' mangled-name='list_link_replace' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_link_replace'>
	<parameter type-id='ccc38265' name='lold'/>
	<parameter type-id='ccc38265' name='lnew'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='list_link_init' mangled-name='list_link_init' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_link_init'>
	<parameter type-id='ccc38265' name='ln'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='list_link_active' mangled-name='list_link_active' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_link_active'>
	<parameter type-id='ccc38265' name='ln'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='list_is_empty' mangled-name='list_is_empty' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='list_is_empty'>
	<parameter type-id='352ec160' name='list'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libspl/mkdirp.c' language='LANG_C99'>
	<typedef-decl name='wchar_t' type-id='95e97e5e' id='928221d2'/>
	<qualified-type-def type-id='928221d2' const='yes' id='effb3702'/>
	<pointer-type-def type-id='effb3702' size-in-bits='64' id='f077d3f8'/>
	<qualified-type-def type-id='f077d3f8' restrict='yes' id='598aab80'/>
	<pointer-type-def type-id='928221d2' size-in-bits='64' id='323d93c1'/>
	<qualified-type-def type-id='323d93c1' restrict='yes' id='f1358bc3'/>
	<function-decl name='__mbstowcs_chk' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='f1358bc3'/>
	<parameter type-id='9d26089a'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='b59d7dce'/>
	</function-decl>
	<function-decl name='__wcstombs_chk' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='266fe297'/>
	<parameter type-id='598aab80'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='b59d7dce'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libspl/os/linux/getmntany.c' language='LANG_C99'>
	<pointer-type-def type-id='56fe4a37' size-in-bits='64' id='b6b61d2f'/>
	<qualified-type-def type-id='b6b61d2f' restrict='yes' id='3cad23cd'/>
	<function-decl name='getmntent_r' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='e75a27e9'/>
	<parameter type-id='3cad23cd'/>
	<parameter type-id='266fe297'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='b6b61d2f'/>
	</function-decl>
	<function-decl name='feof' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='822cd80b'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libspl/timestamp.c' language='LANG_C99'>
	<typedef-decl name='nl_item' type-id='95e97e5e' id='03b79a94'/>
	<function-decl name='nl_langinfo' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='03b79a94'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='print_timestamp' mangled-name='print_timestamp' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='print_timestamp'>
	<parameter type-id='3502e3ff' name='timestamp_fmt'/>
	<return type-id='48b5725f'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libtpool/thread_pool.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='8901473c' size-in-bits='576' id='f5da478b'>
	<subrange length='1' type-id='7359adad' id='52f813b4'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='49ef3ffd' size-in-bits='1024' id='a14403f5'>
	<subrange length='16' type-id='7359adad' id='848d0938'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='a84c031d' size-in-bits='384' id='36d7f119'>
	<subrange length='48' type-id='7359adad' id='8f6d2a81'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='bd54fe1a' size-in-bits='512' id='5d4efd44'>
	<subrange length='8' type-id='7359adad' id='56e0c0b1'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='f0981eeb' size-in-bits='64' id='0d532ec1'>
	<subrange length='2' type-id='7359adad' id='52efc4ef'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='eaa32e2f' size-in-bits='256' id='209ef23f'>
	<subrange length='4' type-id='7359adad' id='16fe7105'/>
	</array-type-def>
	<class-decl name='__cancel_jmp_buf_tag' size-in-bits='576' is-struct='yes' visibility='default' id='8901473c'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='__cancel_jmp_buf' type-id='379a1ab7' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='512'>
	<var-decl name='__mask_was_saved' type-id='95e97e5e' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='__pthread_unwind_buf_t' size-in-bits='832' is-struct='yes' naming-typedef-id='4423cf7f' visibility='default' id='a0abc656'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='__cancel_jmp_buf' type-id='f5da478b' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='576'>
	<var-decl name='__pad' type-id='209ef23f' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='__pthread_unwind_buf_t' type-id='a0abc656' id='4423cf7f'/>
	<union-decl name='__atomic_wide_counter' size-in-bits='64' naming-typedef-id='f3b40860' visibility='default' id='613ce450'>
	<data-member access='public'>
	<var-decl name='__value64' type-id='3a47d82b' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='__value32' type-id='e7f43f72' visibility='default'/>
	</data-member>
	</union-decl>
	<class-decl name='__anonymous_struct__' size-in-bits='64' is-struct='yes' is-anonymous='yes' visibility='default' id='e7f43f72'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='__low' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='__high' type-id='f0981eeb' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='__atomic_wide_counter' type-id='613ce450' id='f3b40860'/>
	<typedef-decl name='__cpu_mask' type-id='7359adad' id='49ef3ffd'/>
	<class-decl name='cpu_set_t' size-in-bits='1024' is-struct='yes' naming-typedef-id='8037c762' visibility='default' id='1f20d231'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='__bits' type-id='a14403f5' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='cpu_set_t' type-id='1f20d231' id='8037c762'/>
	<union-decl name='pthread_condattr_t' size-in-bits='32' naming-typedef-id='836265dd' visibility='default' id='33dd3aad'>
	<data-member access='public'>
	<var-decl name='__size' type-id='8e0573fd' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='__align' type-id='95e97e5e' visibility='default'/>
	</data-member>
	</union-decl>
	<typedef-decl name='pthread_condattr_t' type-id='33dd3aad' id='836265dd'/>
	<union-decl name='pthread_cond_t' size-in-bits='384' naming-typedef-id='62fab762' visibility='default' id='cbb12c12'>
	<data-member access='public'>
	<var-decl name='__data' type-id='c987b47c' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='__size' type-id='36d7f119' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='__align' type-id='1eb56b1e' visibility='default'/>
	</data-member>
	</union-decl>
	<typedef-decl name='pthread_cond_t' type-id='cbb12c12' id='62fab762'/>
	<typedef-decl name='__jmp_buf' type-id='5d4efd44' id='379a1ab7'/>
	<class-decl name='__pthread_cond_s' size-in-bits='384' is-struct='yes' visibility='default' id='c987b47c'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='__wseq' type-id='f3b40860' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='__g1_start' type-id='f3b40860' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='__g_refs' type-id='0d532ec1' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='__g_size' type-id='0d532ec1' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='__g1_orig_size' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='288'>
	<var-decl name='__wrefs' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='__g_signals' type-id='0d532ec1' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='sched_param' size-in-bits='32' is-struct='yes' visibility='default' id='0897719a'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='sched_priority' type-id='95e97e5e' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='tpool_job_t' type-id='3b8579e5' id='66a0afc9'/>
	<class-decl name='tpool_job' size-in-bits='192' is-struct='yes' visibility='default' id='3b8579e5'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='tpj_next' type-id='f32b30e4' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='tpj_func' type-id='b7f9d8e6' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='tpj_arg' type-id='eaa32e2f' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='tpool_active_t' type-id='c8d086f4' id='6fcda10e'/>
	<class-decl name='tpool_active' size-in-bits='128' is-struct='yes' visibility='default' id='c8d086f4'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='tpa_next' type-id='ad33e5e7' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='tpa_tid' type-id='4051f5e7' visibility='default'/>
	</data-member>
	</class-decl>
	<pointer-type-def type-id='8901473c' size-in-bits='64' id='eb91b7ea'/>
	<pointer-type-def type-id='4423cf7f' size-in-bits='64' id='ba7c727c'/>
	<pointer-type-def type-id='b9c97942' size-in-bits='64' id='bbf06c47'/>
	<qualified-type-def type-id='bbf06c47' restrict='yes' id='65e6ec45'/>
	<qualified-type-def type-id='b9c97942' const='yes' id='191f6b72'/>
	<pointer-type-def type-id='191f6b72' size-in-bits='64' id='e475fb88'/>
	<qualified-type-def type-id='e475fb88' restrict='yes' id='5a8729d0'/>
	<qualified-type-def type-id='8037c762' const='yes' id='f50ea9b2'/>
	<pointer-type-def type-id='f50ea9b2' size-in-bits='64' id='5e14fa48'/>
	<qualified-type-def type-id='836265dd' const='yes' id='7d24c58d'/>
	<pointer-type-def type-id='7d24c58d' size-in-bits='64' id='a7e325e5'/>
	<qualified-type-def type-id='a7e325e5' restrict='yes' id='4c428e67'/>
	<qualified-type-def type-id='0897719a' const='yes' id='c4a7b189'/>
	<pointer-type-def type-id='c4a7b189' size-in-bits='64' id='36fca399'/>
	<qualified-type-def type-id='36fca399' restrict='yes' id='37e4897b'/>
	<qualified-type-def type-id='e05e8614' restrict='yes' id='0be2e71c'/>
	<pointer-type-def type-id='8037c762' size-in-bits='64' id='d74a6869'/>
	<qualified-type-def type-id='7292109c' restrict='yes' id='6942f6a4'/>
	<qualified-type-def type-id='7347a39e' restrict='yes' id='578ba182'/>
	<pointer-type-def type-id='62fab762' size-in-bits='64' id='db285b03'/>
	<qualified-type-def type-id='db285b03' restrict='yes' id='2a468b41'/>
	<qualified-type-def type-id='18c91f9e' restrict='yes' id='6e745582'/>
	<pointer-type-def type-id='0897719a' size-in-bits='64' id='23cbcb08'/>
	<qualified-type-def type-id='23cbcb08' restrict='yes' id='b09b2050'/>
	<pointer-type-def type-id='6fcda10e' size-in-bits='64' id='ad33e5e7'/>
	<pointer-type-def type-id='66a0afc9' size-in-bits='64' id='f32b30e4'/>
	<qualified-type-def type-id='63e171df' restrict='yes' id='9e7a3a7d'/>
	<function-decl name='pthread_self' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='4051f5e7'/>
	</function-decl>
	<function-decl name='pthread_attr_init' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='7347a39e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_destroy' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='7347a39e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_getdetachstate' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='540db505'/>
	<parameter type-id='7292109c'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_setdetachstate' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='7347a39e'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_getguardsize' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='540db505'/>
	<parameter type-id='78c01427'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_setguardsize' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='7347a39e'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_getschedparam' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='e1815e87'/>
	<parameter type-id='b09b2050'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_setschedparam' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='578ba182'/>
	<parameter type-id='37e4897b'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_getschedpolicy' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='e1815e87'/>
	<parameter type-id='6942f6a4'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_setschedpolicy' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='7347a39e'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_getinheritsched' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='e1815e87'/>
	<parameter type-id='6942f6a4'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_setinheritsched' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='7347a39e'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_getscope' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='e1815e87'/>
	<parameter type-id='6942f6a4'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_setscope' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='7347a39e'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_getstack' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='e1815e87'/>
	<parameter type-id='9e7a3a7d'/>
	<parameter type-id='d19b2c25'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_setstack' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='7347a39e'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_setaffinity_np' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='7347a39e'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='5e14fa48'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_attr_getaffinity_np' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='540db505'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='d74a6869'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_setcancelstate' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='7292109c'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_setcanceltype' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='7292109c'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='__pthread_register_cancel' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='ba7c727c'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='__pthread_unregister_cancel' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='ba7c727c'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='__pthread_unwind_next' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='ba7c727c'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='pthread_cond_init' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='2a468b41'/>
	<parameter type-id='4c428e67'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_cond_signal' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='db285b03'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_cond_broadcast' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='db285b03'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_cond_wait' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='2a468b41'/>
	<parameter type-id='6e745582'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_cond_timedwait' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='2a468b41'/>
	<parameter type-id='6e745582'/>
	<parameter type-id='0be2e71c'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='__sysconf' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<return type-id='bd54fe1a'/>
	</function-decl>
	<function-decl name='pthread_sigmask' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='5a8729d0'/>
	<parameter type-id='65e6ec45'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='tpool_abandon' mangled-name='tpool_abandon' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='tpool_abandon'>
	<parameter type-id='9cf59a50' name='tpool'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='tpool_suspend' mangled-name='tpool_suspend' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='tpool_suspend'>
	<parameter type-id='9cf59a50' name='tpool'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='tpool_suspended' mangled-name='tpool_suspended' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='tpool_suspended'>
	<parameter type-id='9cf59a50' name='tpool'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='tpool_resume' mangled-name='tpool_resume' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='tpool_resume'>
	<parameter type-id='9cf59a50' name='tpool'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='tpool_member' mangled-name='tpool_member' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='tpool_member'>
	<parameter type-id='9cf59a50' name='tpool'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<pointer-type-def type-id='b1bbf10d' size-in-bits='64' id='9cf59a50'/>
	<typedef-decl name='tpool_t' type-id='88d1b7f9' id='b1bbf10d'/>
	<class-decl name='tpool' size-in-bits='2496' is-struct='yes' visibility='default' id='88d1b7f9'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='tp_forw' type-id='9cf59a50' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='tp_back' type-id='9cf59a50' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='tp_mutex' type-id='7a6844eb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='tp_busycv' type-id='62fab762' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='832'>
	<var-decl name='tp_workcv' type-id='62fab762' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1216'>
	<var-decl name='tp_waitcv' type-id='62fab762' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1600'>
	<var-decl name='tp_active' type-id='ad33e5e7' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1664'>
	<var-decl name='tp_head' type-id='f32b30e4' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1728'>
	<var-decl name='tp_tail' type-id='f32b30e4' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1792'>
	<var-decl name='tp_attr' type-id='7d8569fd' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2240'>
	<var-decl name='tp_flags' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2272'>
	<var-decl name='tp_linger' type-id='3502e3ff' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2304'>
	<var-decl name='tp_njobs' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2336'>
	<var-decl name='tp_minimum' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2368'>
	<var-decl name='tp_maximum' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2400'>
	<var-decl name='tp_current' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2432'>
	<var-decl name='tp_idle' type-id='95e97e5e' visibility='default'/>
	</data-member>
	</class-decl>
	<function-type size-in-bits='64' id='c5c76c9c'>
	<parameter type-id='eaa32e2f'/>
	<return type-id='48b5725f'/>
	</function-type>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzfs/libzfs_changelist.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='bf311473' size-in-bits='128' id='f0f65199'>
	<subrange length='2' type-id='7359adad' id='52efc4ef'/>
	</array-type-def>
	<type-decl name='char' size-in-bits='8' id='a84c031d'/>
	<array-type-def dimensions='1' type-id='a84c031d' size-in-bits='8192' id='b54ce520'>
	<subrange length='1024' type-id='7359adad' id='c60446f8'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='a84c031d' size-in-bits='2048' id='d1617432'>
	<subrange length='256' type-id='7359adad' id='36e5b9fa'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='a84c031d' size-in-bits='320' id='36c46961'>
	<subrange length='40' type-id='7359adad' id='8f80b239'/>
	</array-type-def>
	<class-decl name='re_dfa_t' is-struct='yes' visibility='default' is-declaration-only='yes' id='b48d2441'/>
	<class-decl name='uu_avl' is-struct='yes' visibility='default' is-declaration-only='yes' id='4af029d1'/>
	<class-decl name='uu_avl_pool' is-struct='yes' visibility='default' is-declaration-only='yes' id='12a530a8'/>
	<class-decl name='uu_avl_walk' is-struct='yes' visibility='default' is-declaration-only='yes' id='e70a39e3'/>
	<type-decl name='int' size-in-bits='32' id='95e97e5e'/>
	<type-decl name='long int' size-in-bits='64' id='bd54fe1a'/>
	<type-decl name='long long int' size-in-bits='64' id='1eb56b1e'/>
	<type-decl name='short int' size-in-bits='16' id='a2185560'/>
	<array-type-def dimensions='1' type-id='e475ab95' size-in-bits='192' id='0ce65a8b'>
	<subrange length='3' type-id='7359adad' id='56f209d2'/>
	</array-type-def>
	<type-decl name='unnamed-enum-underlying-type-32' is-anonymous='yes' size-in-bits='32' alignment-in-bits='32' id='9cac1fee'/>
	<type-decl name='unsigned char' size-in-bits='8' id='002ac4a6'/>
	<type-decl name='unsigned int' size-in-bits='32' id='f0981eeb'/>
	<type-decl name='unsigned long int' size-in-bits='64' id='7359adad'/>
	<type-decl name='void' id='48b5725f'/>
	<typedef-decl name='uu_compare_fn_t' type-id='add6e811' id='40f93560'/>
	<typedef-decl name='uu_avl_pool_t' type-id='12a530a8' id='7f84e390'/>
	<typedef-decl name='uu_avl_t' type-id='4af029d1' id='bb7f0973'/>
	<class-decl name='uu_avl_node' size-in-bits='192' is-struct='yes' visibility='default' id='f65f4326'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='uan_opaque' type-id='0ce65a8b' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='uu_avl_node_t' type-id='f65f4326' id='73a65116'/>
	<typedef-decl name='uu_avl_walk_t' type-id='e70a39e3' id='edd8457b'/>
	<typedef-decl name='uu_avl_index_t' type-id='e475ab95' id='5d7f5fc8'/>
	<typedef-decl name='zfs_handle_t' type-id='f6ee4445' id='775509eb'/>
	<typedef-decl name='zpool_handle_t' type-id='67002a8a' id='b1efc708'/>
	<typedef-decl name='libzfs_handle_t' type-id='c8a9d9d8' id='95942d0c'/>
	<typedef-decl name='zfs_iter_f' type-id='5571cde4' id='d8e49ab9'/>
	<typedef-decl name='avl_tree_t' type-id='b351119f' id='f20fbd51'/>
	<class-decl name='avl_node' size-in-bits='192' is-struct='yes' visibility='default' id='428b67b3'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='avl_child' type-id='f0f65199' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='avl_pcb' type-id='e475ab95' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='avl_tree' size-in-bits='320' is-struct='yes' visibility='default' id='b351119f'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='avl_root' type-id='bf311473' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='avl_compar' type-id='585e1de9' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='avl_offset' type-id='b59d7dce' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='avl_numnodes' type-id='ee1f298e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='avl_pad' type-id='b59d7dce' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='dmu_objset_stats' size-in-bits='2304' is-struct='yes' visibility='default' id='098f0221'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='dds_num_clones' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='dds_creation_txg' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='dds_guid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='dds_type' type-id='230f1e16' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='224'>
	<var-decl name='dds_is_snapshot' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='232'>
	<var-decl name='dds_inconsistent' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='240'>
	<var-decl name='dds_redacted' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='248'>
	<var-decl name='dds_origin' type-id='d1617432' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='dmu_objset_stats_t' type-id='098f0221' id='b2c14f17'/>
	<enum-decl name='zfs_type_t' naming-typedef-id='2e45de5d' id='5d8f7321'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='ZFS_TYPE_INVALID' value='0'/>
	<enumerator name='ZFS_TYPE_FILESYSTEM' value='1'/>
	<enumerator name='ZFS_TYPE_SNAPSHOT' value='2'/>
	<enumerator name='ZFS_TYPE_VOLUME' value='4'/>
	<enumerator name='ZFS_TYPE_POOL' value='8'/>
	<enumerator name='ZFS_TYPE_BOOKMARK' value='16'/>
	<enumerator name='ZFS_TYPE_VDEV' value='32'/>
	</enum-decl>
	<typedef-decl name='zfs_type_t' type-id='5d8f7321' id='2e45de5d'/>
	<enum-decl name='dmu_objset_type' id='6b1b19f9'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='DMU_OST_NONE' value='0'/>
	<enumerator name='DMU_OST_META' value='1'/>
	<enumerator name='DMU_OST_ZFS' value='2'/>
	<enumerator name='DMU_OST_ZVOL' value='3'/>
	<enumerator name='DMU_OST_OTHER' value='4'/>
	<enumerator name='DMU_OST_ANY' value='5'/>
	<enumerator name='DMU_OST_NUMTYPES' value='6'/>
	</enum-decl>
	<typedef-decl name='dmu_objset_type_t' type-id='6b1b19f9' id='230f1e16'/>
	<enum-decl name='zfs_prop_t' naming-typedef-id='58603c44' id='4b000d60'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='ZPROP_CONT' value='-2'/>
	<enumerator name='ZPROP_INVAL' value='-1'/>
	<enumerator name='ZPROP_USERPROP' value='-1'/>
	<enumerator name='ZFS_PROP_TYPE' value='0'/>
	<enumerator name='ZFS_PROP_CREATION' value='1'/>
	<enumerator name='ZFS_PROP_USED' value='2'/>
	<enumerator name='ZFS_PROP_AVAILABLE' value='3'/>
	<enumerator name='ZFS_PROP_REFERENCED' value='4'/>
	<enumerator name='ZFS_PROP_COMPRESSRATIO' value='5'/>
	<enumerator name='ZFS_PROP_MOUNTED' value='6'/>
	<enumerator name='ZFS_PROP_ORIGIN' value='7'/>
	<enumerator name='ZFS_PROP_QUOTA' value='8'/>
	<enumerator name='ZFS_PROP_RESERVATION' value='9'/>
	<enumerator name='ZFS_PROP_VOLSIZE' value='10'/>
	<enumerator name='ZFS_PROP_VOLBLOCKSIZE' value='11'/>
	<enumerator name='ZFS_PROP_RECORDSIZE' value='12'/>
	<enumerator name='ZFS_PROP_MOUNTPOINT' value='13'/>
	<enumerator name='ZFS_PROP_SHARENFS' value='14'/>
	<enumerator name='ZFS_PROP_CHECKSUM' value='15'/>
	<enumerator name='ZFS_PROP_COMPRESSION' value='16'/>
	<enumerator name='ZFS_PROP_ATIME' value='17'/>
	<enumerator name='ZFS_PROP_DEVICES' value='18'/>
	<enumerator name='ZFS_PROP_EXEC' value='19'/>
	<enumerator name='ZFS_PROP_SETUID' value='20'/>
	<enumerator name='ZFS_PROP_READONLY' value='21'/>
	<enumerator name='ZFS_PROP_ZONED' value='22'/>
	<enumerator name='ZFS_PROP_SNAPDIR' value='23'/>
	<enumerator name='ZFS_PROP_ACLMODE' value='24'/>
	<enumerator name='ZFS_PROP_ACLINHERIT' value='25'/>
	<enumerator name='ZFS_PROP_CREATETXG' value='26'/>
	<enumerator name='ZFS_PROP_NAME' value='27'/>
	<enumerator name='ZFS_PROP_CANMOUNT' value='28'/>
	<enumerator name='ZFS_PROP_ISCSIOPTIONS' value='29'/>
	<enumerator name='ZFS_PROP_XATTR' value='30'/>
	<enumerator name='ZFS_PROP_NUMCLONES' value='31'/>
	<enumerator name='ZFS_PROP_COPIES' value='32'/>
	<enumerator name='ZFS_PROP_VERSION' value='33'/>
	<enumerator name='ZFS_PROP_UTF8ONLY' value='34'/>
	<enumerator name='ZFS_PROP_NORMALIZE' value='35'/>
	<enumerator name='ZFS_PROP_CASE' value='36'/>
	<enumerator name='ZFS_PROP_VSCAN' value='37'/>
	<enumerator name='ZFS_PROP_NBMAND' value='38'/>
	<enumerator name='ZFS_PROP_SHARESMB' value='39'/>
	<enumerator name='ZFS_PROP_REFQUOTA' value='40'/>
	<enumerator name='ZFS_PROP_REFRESERVATION' value='41'/>
	<enumerator name='ZFS_PROP_GUID' value='42'/>
	<enumerator name='ZFS_PROP_PRIMARYCACHE' value='43'/>
	<enumerator name='ZFS_PROP_SECONDARYCACHE' value='44'/>
	<enumerator name='ZFS_PROP_USEDSNAP' value='45'/>
	<enumerator name='ZFS_PROP_USEDDS' value='46'/>
	<enumerator name='ZFS_PROP_USEDCHILD' value='47'/>
	<enumerator name='ZFS_PROP_USEDREFRESERV' value='48'/>
	<enumerator name='ZFS_PROP_USERACCOUNTING' value='49'/>
	<enumerator name='ZFS_PROP_STMF_SHAREINFO' value='50'/>
	<enumerator name='ZFS_PROP_DEFER_DESTROY' value='51'/>
	<enumerator name='ZFS_PROP_USERREFS' value='52'/>
	<enumerator name='ZFS_PROP_LOGBIAS' value='53'/>
	<enumerator name='ZFS_PROP_UNIQUE' value='54'/>
	<enumerator name='ZFS_PROP_OBJSETID' value='55'/>
	<enumerator name='ZFS_PROP_DEDUP' value='56'/>
	<enumerator name='ZFS_PROP_MLSLABEL' value='57'/>
	<enumerator name='ZFS_PROP_SYNC' value='58'/>
	<enumerator name='ZFS_PROP_DNODESIZE' value='59'/>
	<enumerator name='ZFS_PROP_REFRATIO' value='60'/>
	<enumerator name='ZFS_PROP_WRITTEN' value='61'/>
	<enumerator name='ZFS_PROP_CLONES' value='62'/>
	<enumerator name='ZFS_PROP_LOGICALUSED' value='63'/>
	<enumerator name='ZFS_PROP_LOGICALREFERENCED' value='64'/>
	<enumerator name='ZFS_PROP_INCONSISTENT' value='65'/>
	<enumerator name='ZFS_PROP_VOLMODE' value='66'/>
	<enumerator name='ZFS_PROP_FILESYSTEM_LIMIT' value='67'/>
	<enumerator name='ZFS_PROP_SNAPSHOT_LIMIT' value='68'/>
	<enumerator name='ZFS_PROP_FILESYSTEM_COUNT' value='69'/>
	<enumerator name='ZFS_PROP_SNAPSHOT_COUNT' value='70'/>
	<enumerator name='ZFS_PROP_SNAPDEV' value='71'/>
	<enumerator name='ZFS_PROP_ACLTYPE' value='72'/>
	<enumerator name='ZFS_PROP_SELINUX_CONTEXT' value='73'/>
	<enumerator name='ZFS_PROP_SELINUX_FSCONTEXT' value='74'/>
	<enumerator name='ZFS_PROP_SELINUX_DEFCONTEXT' value='75'/>
	<enumerator name='ZFS_PROP_SELINUX_ROOTCONTEXT' value='76'/>
	<enumerator name='ZFS_PROP_RELATIME' value='77'/>
	<enumerator name='ZFS_PROP_REDUNDANT_METADATA' value='78'/>
	<enumerator name='ZFS_PROP_OVERLAY' value='79'/>
	<enumerator name='ZFS_PROP_PREV_SNAP' value='80'/>
	<enumerator name='ZFS_PROP_RECEIVE_RESUME_TOKEN' value='81'/>
	<enumerator name='ZFS_PROP_ENCRYPTION' value='82'/>
	<enumerator name='ZFS_PROP_KEYLOCATION' value='83'/>
	<enumerator name='ZFS_PROP_KEYFORMAT' value='84'/>
	<enumerator name='ZFS_PROP_PBKDF2_SALT' value='85'/>
	<enumerator name='ZFS_PROP_PBKDF2_ITERS' value='86'/>
	<enumerator name='ZFS_PROP_ENCRYPTION_ROOT' value='87'/>
	<enumerator name='ZFS_PROP_KEY_GUID' value='88'/>
	<enumerator name='ZFS_PROP_KEYSTATUS' value='89'/>
	<enumerator name='ZFS_PROP_REMAPTXG' value='90'/>
	<enumerator name='ZFS_PROP_SPECIAL_SMALL_BLOCKS' value='91'/>
	<enumerator name='ZFS_PROP_IVSET_GUID' value='92'/>
	<enumerator name='ZFS_PROP_REDACTED' value='93'/>
	<enumerator name='ZFS_PROP_REDACT_SNAPS' value='94'/>
	<enumerator name='ZFS_PROP_SNAPSHOTS_CHANGED' value='95'/>
	- <enumerator name='ZFS_NUM_PROPS' value='96'/>
	+ <enumerator name='ZFS_PROP_PREFETCH' value='96'/>
	+ <enumerator name='ZFS_NUM_PROPS' value='97'/>
	</enum-decl>
	<typedef-decl name='zfs_prop_t' type-id='4b000d60' id='58603c44'/>
	<enum-decl name='zprop_source_t' naming-typedef-id='a2256d42' id='5903f80e'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='ZPROP_SRC_NONE' value='1'/>
	<enumerator name='ZPROP_SRC_DEFAULT' value='2'/>
	<enumerator name='ZPROP_SRC_TEMPORARY' value='4'/>
	<enumerator name='ZPROP_SRC_LOCAL' value='8'/>
	<enumerator name='ZPROP_SRC_INHERITED' value='16'/>
	<enumerator name='ZPROP_SRC_RECEIVED' value='32'/>
	</enum-decl>
	<typedef-decl name='zprop_source_t' type-id='5903f80e' id='a2256d42'/>
	<class-decl name='nvlist' size-in-bits='192' is-struct='yes' visibility='default' id='ac266fd9'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='nvl_version' type-id='3ff5601b' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='nvl_nvflag' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='nvl_priv' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='nvl_flag' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='160'>
	<var-decl name='nvl_pad' type-id='3ff5601b' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='nvlist_t' type-id='ac266fd9' id='8e8d4be3'/>
	<enum-decl name='sa_protocol' id='9155d4b5'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='SA_PROTOCOL_NFS' value='0'/>
	<enumerator name='SA_PROTOCOL_SMB' value='1'/>
	<enumerator name='SA_PROTOCOL_COUNT' value='2'/>
	</enum-decl>
	<enum-decl name='boolean_t' naming-typedef-id='c19b74c3' id='f58c8277'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='B_FALSE' value='0'/>
	<enumerator name='B_TRUE' value='1'/>
	</enum-decl>
	<typedef-decl name='boolean_t' type-id='f58c8277' id='c19b74c3'/>
	<typedef-decl name='uint_t' type-id='f0981eeb' id='3502e3ff'/>
	<typedef-decl name='ulong_t' type-id='7359adad' id='ee1f298e'/>
	<typedef-decl name='longlong_t' type-id='1eb56b1e' id='9b3ff54f'/>
	<typedef-decl name='diskaddr_t' type-id='9b3ff54f' id='804dc465'/>
	<typedef-decl name='zoneid_t' type-id='3502e3ff' id='4da03624'/>
	<typedef-decl name='__re_long_size_t' type-id='7359adad' id='ba516949'/>
	<typedef-decl name='reg_syntax_t' type-id='7359adad' id='1b72c3b3'/>
	<class-decl name='re_pattern_buffer' size-in-bits='512' is-struct='yes' visibility='default' id='19fc9a8c'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='buffer' type-id='33976309' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='allocated' type-id='ba516949' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='used' type-id='ba516949' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='syntax' type-id='1b72c3b3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='fastmap' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='translate' type-id='cf536864' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='re_nsub' type-id='b59d7dce' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='can_be_null' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='449'>
	<var-decl name='regs_allocated' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='451'>
	<var-decl name='fastmap_accurate' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='452'>
	<var-decl name='no_sub' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='453'>
	<var-decl name='not_bol' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='454'>
	<var-decl name='not_eol' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='455'>
	<var-decl name='newline_anchor' type-id='f0981eeb' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='regex_t' type-id='19fc9a8c' id='aca3bac8'/>
	<typedef-decl name='uintptr_t' type-id='7359adad' id='e475ab95'/>
	<union-decl name='pthread_mutex_t' size-in-bits='320' naming-typedef-id='7a6844eb' visibility='default' id='70681f9b'>
	<data-member access='public'>
	<var-decl name='__data' type-id='4c734837' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='__size' type-id='36c46961' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='__align' type-id='bd54fe1a' visibility='default'/>
	</data-member>
	</union-decl>
	<typedef-decl name='pthread_mutex_t' type-id='70681f9b' id='7a6844eb'/>
	<typedef-decl name='int32_t' type-id='33f57a65' id='3ff5601b'/>
	<typedef-decl name='uint8_t' type-id='c51d6389' id='b96825af'/>
	<typedef-decl name='uint32_t' type-id='62f1140c' id='8f92235e'/>
	<typedef-decl name='uint64_t' type-id='8910171f' id='9c313c2d'/>
	<class-decl name='__pthread_mutex_s' size-in-bits='320' is-struct='yes' visibility='default' id='4c734837'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='__lock' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='__count' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='__owner' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='96'>
	<var-decl name='__nusers' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='__kind' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='160'>
	<var-decl name='__spins' type-id='a2185560' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='176'>
	<var-decl name='__elision' type-id='a2185560' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='__list' type-id='518fb49c' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='__pthread_internal_list' size-in-bits='128' is-struct='yes' visibility='default' id='0e01899c'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='__prev' type-id='4d98cd5a' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='__next' type-id='4d98cd5a' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='__pthread_list_t' type-id='0e01899c' id='518fb49c'/>
	<typedef-decl name='__uint8_t' type-id='002ac4a6' id='c51d6389'/>
	<typedef-decl name='__int32_t' type-id='95e97e5e' id='33f57a65'/>
	<typedef-decl name='__uint32_t' type-id='f0981eeb' id='62f1140c'/>
	<typedef-decl name='__uint64_t' type-id='7359adad' id='8910171f'/>
	<typedef-decl name='size_t' type-id='7359adad' id='b59d7dce'/>
	<class-decl name='libzfs_handle' size-in-bits='18240' is-struct='yes' visibility='default' id='c8a9d9d8'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='libzfs_error' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='libzfs_fd' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='libzfs_pool_handles' type-id='4c81de99' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='libzfs_ns_avlpool' type-id='de82c773' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='libzfs_ns_avl' type-id='a5c21a38' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='libzfs_ns_gen' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='libzfs_desc_active' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='352'>
	<var-decl name='libzfs_action' type-id='b54ce520' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='8544'>
	<var-decl name='libzfs_desc' type-id='b54ce520' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='16736'>
	<var-decl name='libzfs_printerr' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='16768'>
	<var-decl name='libzfs_mnttab_enable' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='16832'>
	<var-decl name='libzfs_mnttab_cache_lock' type-id='7a6844eb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='17152'>
	<var-decl name='libzfs_mnttab_cache' type-id='f20fbd51' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='17472'>
	<var-decl name='libzfs_pool_iter' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='17504'>
	<var-decl name='libzfs_prop_debug' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='17536'>
	<var-decl name='libzfs_urire' type-id='aca3bac8' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='18048'>
	<var-decl name='libzfs_max_nvlist' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='18112'>
	<var-decl name='libfetch' type-id='eaa32e2f' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='18176'>
	<var-decl name='libfetch_load_error' type-id='26a90f95' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='zfs_handle' size-in-bits='4928' is-struct='yes' visibility='default' id='f6ee4445'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='zfs_hdl' type-id='b0382bb3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='zpool_hdl' type-id='4c81de99' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='zfs_name' type-id='d1617432' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2176'>
	<var-decl name='zfs_type' type-id='2e45de5d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2208'>
	<var-decl name='zfs_head_type' type-id='2e45de5d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2240'>
	<var-decl name='zfs_dmustats' type-id='b2c14f17' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='4544'>
	<var-decl name='zfs_props' type-id='5ce45b60' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='4608'>
	<var-decl name='zfs_user_props' type-id='5ce45b60' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='4672'>
	<var-decl name='zfs_recvd_props' type-id='5ce45b60' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='4736'>
	<var-decl name='zfs_mntcheck' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='4800'>
	<var-decl name='zfs_mntopts' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='4864'>
	<var-decl name='zfs_props_table' type-id='ae3e8ca6' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='zpool_handle' size-in-bits='2560' is-struct='yes' visibility='default' id='67002a8a'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='zpool_hdl' type-id='b0382bb3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='zpool_next' type-id='4c81de99' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='zpool_name' type-id='d1617432' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2176'>
	<var-decl name='zpool_state' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2240'>
	<var-decl name='zpool_config_size' type-id='b59d7dce' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2304'>
	<var-decl name='zpool_config' type-id='5ce45b60' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2368'>
	<var-decl name='zpool_old_config' type-id='5ce45b60' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2432'>
	<var-decl name='zpool_props' type-id='5ce45b60' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2496'>
	<var-decl name='zpool_start_block' type-id='804dc465' visibility='default'/>
	</data-member>
	</class-decl>
	<pointer-type-def type-id='0e01899c' size-in-bits='64' id='4d98cd5a'/>
	<pointer-type-def type-id='428b67b3' size-in-bits='64' id='bf311473'/>
	<pointer-type-def type-id='a84c031d' size-in-bits='64' id='26a90f95'/>
	<pointer-type-def type-id='26a90f95' size-in-bits='64' id='9b23c9ad'/>
	<qualified-type-def type-id='a84c031d' const='yes' id='9b45d938'/>
	<pointer-type-def type-id='9b45d938' size-in-bits='64' id='80f4b756'/>
	<qualified-type-def type-id='9155d4b5' const='yes' id='9f2c1699'/>
	<pointer-type-def type-id='9f2c1699' size-in-bits='64' id='4567bbc9'/>
	<qualified-type-def type-id='775509eb' const='yes' id='5eadf2db'/>
	<pointer-type-def type-id='5eadf2db' size-in-bits='64' id='fcd57163'/>
	<pointer-type-def type-id='96ee24a5' size-in-bits='64' id='585e1de9'/>
	<pointer-type-def type-id='cb9628fa' size-in-bits='64' id='5571cde4'/>
	<pointer-type-def type-id='95942d0c' size-in-bits='64' id='b0382bb3'/>
	<pointer-type-def type-id='8e8d4be3' size-in-bits='64' id='5ce45b60'/>
	<pointer-type-def type-id='b48d2441' size-in-bits='64' id='33976309'/>
	<pointer-type-def type-id='b96825af' size-in-bits='64' id='ae3e8ca6'/>
	<pointer-type-def type-id='002ac4a6' size-in-bits='64' id='cf536864'/>
	<pointer-type-def type-id='5d7f5fc8' size-in-bits='64' id='813a2225'/>
	<pointer-type-def type-id='73a65116' size-in-bits='64' id='2dc35b9d'/>
	<pointer-type-def type-id='7f84e390' size-in-bits='64' id='de82c773'/>
	<pointer-type-def type-id='bb7f0973' size-in-bits='64' id='a5c21a38'/>
	<pointer-type-def type-id='edd8457b' size-in-bits='64' id='5842d146'/>
	<pointer-type-def type-id='40f93560' size-in-bits='64' id='d502b39f'/>
	<pointer-type-def type-id='48b5725f' size-in-bits='64' id='eaa32e2f'/>
	<pointer-type-def type-id='775509eb' size-in-bits='64' id='9200a744'/>
	<pointer-type-def type-id='b1efc708' size-in-bits='64' id='4c81de99'/>
	<pointer-type-def type-id='a2256d42' size-in-bits='64' id='debc6aa3'/>
	<class-decl name='re_dfa_t' is-struct='yes' visibility='default' is-declaration-only='yes' id='b48d2441'/>
	<class-decl name='uu_avl' is-struct='yes' visibility='default' is-declaration-only='yes' id='4af029d1'/>
	<class-decl name='uu_avl_pool' is-struct='yes' visibility='default' is-declaration-only='yes' id='12a530a8'/>
	<class-decl name='uu_avl_walk' is-struct='yes' visibility='default' is-declaration-only='yes' id='e70a39e3'/>
	<function-decl name='uu_avl_pool_create' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='d502b39f'/>
	<parameter type-id='8f92235e'/>
	<return type-id='de82c773'/>
	</function-decl>
	<function-decl name='uu_avl_pool_destroy' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='de82c773'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='uu_avl_node_init' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='2dc35b9d'/>
	<parameter type-id='de82c773'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='uu_avl_create' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='de82c773'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='8f92235e'/>
	<return type-id='a5c21a38'/>
	</function-decl>
	<function-decl name='uu_avl_destroy' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='a5c21a38'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='uu_avl_last' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='a5c21a38'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='uu_avl_walk_start' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='a5c21a38'/>
	<parameter type-id='8f92235e'/>
	<return type-id='5842d146'/>
	</function-decl>
	<function-decl name='uu_avl_walk_next' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5842d146'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='uu_avl_walk_end' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5842d146'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='uu_avl_find' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='a5c21a38'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='813a2225'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='uu_avl_insert' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='a5c21a38'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='5d7f5fc8'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='uu_avl_remove' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='a5c21a38'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_get_handle' mangled-name='zfs_get_handle' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_get_handle'>
	<parameter type-id='9200a744'/>
	<return type-id='b0382bb3'/>
	</function-decl>
	<function-decl name='zfs_open' mangled-name='zfs_open' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_open'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='9200a744'/>
	</function-decl>
	<function-decl name='zfs_close' mangled-name='zfs_close' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_close'>
	<parameter type-id='9200a744'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_get_name' mangled-name='zfs_get_name' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_get_name'>
	<parameter type-id='fcd57163'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zfs_prop_get' mangled-name='zfs_prop_get' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_get'>
	<parameter type-id='9200a744'/>
	<parameter type-id='58603c44'/>
	<parameter type-id='26a90f95'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='debc6aa3'/>
	<parameter type-id='26a90f95'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='c19b74c3'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prop_get_int' mangled-name='zfs_prop_get_int' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_get_int'>
	<parameter type-id='9200a744'/>
	<parameter type-id='58603c44'/>
	<return type-id='9c313c2d'/>
	</function-decl>
	<function-decl name='zfs_iter_children_v2' mangled-name='zfs_iter_children_v2' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_iter_children_v2'>
	<parameter type-id='9200a744'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='d8e49ab9'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_iter_dependents_v2' mangled-name='zfs_iter_dependents_v2' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_iter_dependents_v2'>
	<parameter type-id='9200a744'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='d8e49ab9'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_iter_mounted' mangled-name='zfs_iter_mounted' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_iter_mounted'>
	<parameter type-id='9200a744'/>
	<parameter type-id='d8e49ab9'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_refresh_properties' mangled-name='zfs_refresh_properties' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_refresh_properties'>
	<parameter type-id='9200a744'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_is_mounted' mangled-name='zfs_is_mounted' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_is_mounted'>
	<parameter type-id='9200a744'/>
	<parameter type-id='9b23c9ad'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_mount' mangled-name='zfs_mount' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_mount'>
	<parameter type-id='9200a744'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_unmount' mangled-name='zfs_unmount' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_unmount'>
	<parameter type-id='9200a744'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_is_shared' mangled-name='zfs_is_shared' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_is_shared'>
	<parameter type-id='9200a744'/>
	<parameter type-id='9b23c9ad'/>
	<parameter type-id='4567bbc9'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_share' mangled-name='zfs_share' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_share'>
	<parameter type-id='9200a744'/>
	<parameter type-id='4567bbc9'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_unshare' mangled-name='zfs_unshare' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_unshare'>
	<parameter type-id='9200a744'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='4567bbc9'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_commit_shares' mangled-name='zfs_commit_shares' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_commit_shares'>
	<parameter type-id='4567bbc9'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='getzoneid' mangled-name='getzoneid' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='getzoneid'>
	<return type-id='4da03624'/>
	</function-decl>
	<function-decl name='sa_commit_shares' mangled-name='sa_commit_shares' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='sa_commit_shares'>
	<parameter type-id='9155d4b5'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='strlcat' mangled-name='strlcat' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='strlcat'>
	<parameter type-id='26a90f95'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='b59d7dce'/>
	</function-decl>
	<function-decl name='strlcpy' mangled-name='strlcpy' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='strlcpy'>
	<parameter type-id='26a90f95'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='b59d7dce'/>
	</function-decl>
	<function-decl name='free' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='eaa32e2f'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='strcmp' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='strncmp' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='strlen' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<return type-id='b59d7dce'/>
	</function-decl>
	<function-decl name='zfs_error' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_alloc' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='remove_mountpoint' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='9200a744'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-type size-in-bits='64' id='96ee24a5'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-type>
	<function-type size-in-bits='64' id='add6e811'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-type>
	<function-type size-in-bits='64' id='cb9628fa'>
	<parameter type-id='9200a744'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-type>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzfs/libzfs_config.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='a84c031d' size-in-bits='32768' id='d16c6df4'>
	<subrange length='4096' type-id='7359adad' id='bc1b5ddc'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='a84c031d' size-in-bits='65536' id='163f6aa5'>
	<subrange length='8192' type-id='7359adad' id='c88f397d'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='a84c031d' size-in-bits='infinite' id='e84913bd'>
	<subrange length='infinite' type-id='7359adad' id='031f2035'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='9c313c2d' size-in-bits='128' id='c1c22e6c'>
	<subrange length='2' type-id='7359adad' id='52efc4ef'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='b96825af' size-in-bits='24' id='d3490169'>
	<subrange length='3' type-id='7359adad' id='56f209d2'/>
	</array-type-def>
	<type-decl name='variadic parameter type' id='2c1145c5'/>
	<typedef-decl name='zpool_iter_f' type-id='3aebb66f' id='fa476e62'/>
	<enum-decl name='data_type_t' naming-typedef-id='8d0687d2' id='aeeae136'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='DATA_TYPE_DONTCARE' value='-1'/>
	<enumerator name='DATA_TYPE_UNKNOWN' value='0'/>
	<enumerator name='DATA_TYPE_BOOLEAN' value='1'/>
	<enumerator name='DATA_TYPE_BYTE' value='2'/>
	<enumerator name='DATA_TYPE_INT16' value='3'/>
	<enumerator name='DATA_TYPE_UINT16' value='4'/>
	<enumerator name='DATA_TYPE_INT32' value='5'/>
	<enumerator name='DATA_TYPE_UINT32' value='6'/>
	<enumerator name='DATA_TYPE_INT64' value='7'/>
	<enumerator name='DATA_TYPE_UINT64' value='8'/>
	<enumerator name='DATA_TYPE_STRING' value='9'/>
	<enumerator name='DATA_TYPE_BYTE_ARRAY' value='10'/>
	<enumerator name='DATA_TYPE_INT16_ARRAY' value='11'/>
	<enumerator name='DATA_TYPE_UINT16_ARRAY' value='12'/>
	<enumerator name='DATA_TYPE_INT32_ARRAY' value='13'/>
	<enumerator name='DATA_TYPE_UINT32_ARRAY' value='14'/>
	<enumerator name='DATA_TYPE_INT64_ARRAY' value='15'/>
	<enumerator name='DATA_TYPE_UINT64_ARRAY' value='16'/>
	<enumerator name='DATA_TYPE_STRING_ARRAY' value='17'/>
	<enumerator name='DATA_TYPE_HRTIME' value='18'/>
	<enumerator name='DATA_TYPE_NVLIST' value='19'/>
	<enumerator name='DATA_TYPE_NVLIST_ARRAY' value='20'/>
	<enumerator name='DATA_TYPE_BOOLEAN_VALUE' value='21'/>
	<enumerator name='DATA_TYPE_INT8' value='22'/>
	<enumerator name='DATA_TYPE_UINT8' value='23'/>
	<enumerator name='DATA_TYPE_BOOLEAN_ARRAY' value='24'/>
	<enumerator name='DATA_TYPE_INT8_ARRAY' value='25'/>
	<enumerator name='DATA_TYPE_UINT8_ARRAY' value='26'/>
	<enumerator name='DATA_TYPE_DOUBLE' value='27'/>
	</enum-decl>
	<typedef-decl name='data_type_t' type-id='aeeae136' id='8d0687d2'/>
	<class-decl name='nvpair' size-in-bits='128' is-struct='yes' visibility='default' id='1c34e459'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='nvp_size' type-id='3ff5601b' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='nvp_name_sz' type-id='23bd8cb5' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='48'>
	<var-decl name='nvp_reserve' type-id='23bd8cb5' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='nvp_value_elem' type-id='3ff5601b' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='96'>
	<var-decl name='nvp_type' type-id='8d0687d2' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='nvp_name' type-id='e84913bd' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='nvpair_t' type-id='1c34e459' id='57928edf'/>
	<class-decl name='drr_begin' size-in-bits='2432' is-struct='yes' visibility='default' id='09fcdc01'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='drr_magic' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='drr_versioninfo' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='drr_creation_time' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='drr_type' type-id='230f1e16' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='224'>
	<var-decl name='drr_flags' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='drr_toguid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='drr_fromguid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='drr_toname' type-id='d1617432' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='zinject_record' size-in-bits='2816' is-struct='yes' visibility='default' id='3216f820'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='zi_objset' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='zi_object' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='zi_start' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='zi_end' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='zi_guid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='zi_level' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='352'>
	<var-decl name='zi_error' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='zi_type' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='zi_freq' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='480'>
	<var-decl name='zi_failfast' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='512'>
	<var-decl name='zi_func' type-id='d1617432' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2560'>
	<var-decl name='zi_iotype' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2592'>
	<var-decl name='zi_duration' type-id='3ff5601b' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2624'>
	<var-decl name='zi_timer' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2688'>
	<var-decl name='zi_nlanes' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2752'>
	<var-decl name='zi_cmd' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2784'>
	<var-decl name='zi_dvas' type-id='8f92235e' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zinject_record_t' type-id='3216f820' id='a4301ca6'/>
	<class-decl name='zfs_share' size-in-bits='256' is-struct='yes' visibility='default' id='feb6f2da'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='z_exportdata' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='z_sharedata' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='z_sharetype' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='z_sharemax' type-id='9c313c2d' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zfs_share_t' type-id='feb6f2da' id='ee5cec36'/>
	<class-decl name='zfs_cmd' size-in-bits='109952' is-struct='yes' visibility='default' id='3522cd69'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='zc_name' type-id='d16c6df4' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32768'>
	<var-decl name='zc_nvlist_src' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32832'>
	<var-decl name='zc_nvlist_src_size' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32896'>
	<var-decl name='zc_nvlist_dst' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32960'>
	<var-decl name='zc_nvlist_dst_size' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='33024'>
	<var-decl name='zc_nvlist_dst_filled' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='33056'>
	<var-decl name='zc_pad2' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='33088'>
	<var-decl name='zc_history' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='33152'>
	<var-decl name='zc_value' type-id='163f6aa5' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='98688'>
	<var-decl name='zc_string' type-id='d1617432' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='100736'>
	<var-decl name='zc_guid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='100800'>
	<var-decl name='zc_nvlist_conf' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='100864'>
	<var-decl name='zc_nvlist_conf_size' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='100928'>
	<var-decl name='zc_cookie' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='100992'>
	<var-decl name='zc_objset_type' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='101056'>
	<var-decl name='zc_perm_action' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='101120'>
	<var-decl name='zc_history_len' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='101184'>
	<var-decl name='zc_history_offset' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='101248'>
	<var-decl name='zc_obj' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='101312'>
	<var-decl name='zc_iflags' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='101376'>
	<var-decl name='zc_share' type-id='ee5cec36' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='101632'>
	<var-decl name='zc_objset_stats' type-id='b2c14f17' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='103936'>
	<var-decl name='zc_begin_record' type-id='09fcdc01' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='106368'>
	<var-decl name='zc_inject_record' type-id='a4301ca6' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='109184'>
	<var-decl name='zc_defer_destroy' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='109216'>
	<var-decl name='zc_flags' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='109248'>
	<var-decl name='zc_action_handle' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='109312'>
	<var-decl name='zc_cleanup_fd' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='109344'>
	<var-decl name='zc_simple' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='109352'>
	<var-decl name='zc_pad' type-id='d3490169' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='109376'>
	<var-decl name='zc_sendobj' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='109440'>
	<var-decl name='zc_fromobj' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='109504'>
	<var-decl name='zc_createtxg' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='109568'>
	<var-decl name='zc_stat' type-id='0371a9c7' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='109888'>
	<var-decl name='zc_zoneid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zfs_cmd_t' type-id='3522cd69' id='a5559cdd'/>
	<class-decl name='zfs_stat' size-in-bits='320' is-struct='yes' visibility='default' id='6417f0b9'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='zs_gen' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='zs_mode' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='zs_links' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='zs_ctime' type-id='c1c22e6c' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zfs_stat_t' type-id='6417f0b9' id='0371a9c7'/>
	<typedef-decl name='int16_t' type-id='03896e23' id='23bd8cb5'/>
	<typedef-decl name='__int16_t' type-id='a2185560' id='03896e23'/>
	<pointer-type-def type-id='c19b74c3' size-in-bits='64' id='37e3bd22'/>
	<qualified-type-def type-id='8e8d4be3' const='yes' id='693c3853'/>
	<pointer-type-def type-id='693c3853' size-in-bits='64' id='22cce67b'/>
	<qualified-type-def type-id='57928edf' const='yes' id='642ee20f'/>
	<pointer-type-def type-id='642ee20f' size-in-bits='64' id='dace003f'/>
	<pointer-type-def type-id='2bce87e3' size-in-bits='64' id='3aebb66f'/>
	<pointer-type-def type-id='95e97e5e' size-in-bits='64' id='7292109c'/>
	<pointer-type-def type-id='5ce45b60' size-in-bits='64' id='857bb57e'/>
	<pointer-type-def type-id='57928edf' size-in-bits='64' id='3fa542f0'/>
	<pointer-type-def type-id='eaa32e2f' size-in-bits='64' id='63e171df'/>
	<pointer-type-def type-id='3522cd69' size-in-bits='64' id='b65f7fd1'/>
	<pointer-type-def type-id='a5559cdd' size-in-bits='64' id='e4ec4540'/>
	<pointer-type-def type-id='4c81de99' size-in-bits='64' id='237193c9'/>
	<function-decl name='uu_avl_first' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='a5c21a38'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='uu_avl_next' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='a5c21a38'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='uu_avl_teardown' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='a5c21a38'/>
	<parameter type-id='63e171df'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='zfs_standard_error' mangled-name='zfs_standard_error' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_standard_error'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_ioctl' mangled-name='zfs_ioctl' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_ioctl'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='b65f7fd1'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_free' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='nvlist_dup' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='22cce67b'/>
	<parameter type-id='857bb57e'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_lookup_nvlist' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_exists' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='22cce67b'/>
	<parameter type-id='80f4b756'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='nvlist_next_nvpair' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='dace003f'/>
	<return type-id='3fa542f0'/>
	</function-decl>
	<function-decl name='nvpair_name' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='dace003f'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='fnvpair_value_nvlist' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='3fa542f0'/>
	<return type-id='5ce45b60'/>
	</function-decl>
	<function-decl name='libspl_assertf' mangled-name='libspl_assertf' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libspl_assertf'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter is-variadic='yes'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='__errno_location' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='7292109c'/>
	</function-decl>
	<function-decl name='dcgettext' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='getenv' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='strchr' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='zpool_get_config' mangled-name='zpool_get_config' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_config'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='857bb57e' name='oldconfig'/>
	<return type-id='5ce45b60'/>
	</function-decl>
	<function-decl name='zpool_get_features' mangled-name='zpool_get_features' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_features'>
	<parameter type-id='4c81de99' name='zhp'/>
	<return type-id='5ce45b60'/>
	</function-decl>
	<function-decl name='zpool_refresh_stats' mangled-name='zpool_refresh_stats' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_refresh_stats'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='37e3bd22' name='missing'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_skip_pool' mangled-name='zpool_skip_pool' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_skip_pool'>
	<parameter type-id='80f4b756' name='poolname'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zpool_iter' mangled-name='zpool_iter' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_iter'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='fa476e62' name='func'/>
	<parameter type-id='eaa32e2f' name='data'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_iter_root' mangled-name='zfs_iter_root' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_iter_root'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='d8e49ab9' name='func'/>
	<parameter type-id='eaa32e2f' name='data'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_strdup' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='80f4b756'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='no_memory' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zcmd_alloc_dst_nvlist' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='e4ec4540'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zcmd_expand_dst_nvlist' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='e4ec4540'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zcmd_read_dst_nvlist' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='e4ec4540'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zcmd_free_nvlists' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='e4ec4540'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='make_dataset_handle' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='80f4b756'/>
	<return type-id='9200a744'/>
	</function-decl>
	<function-decl name='zpool_open_silent' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='237193c9'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-type size-in-bits='64' id='2bce87e3'>
	<parameter type-id='4c81de99'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-type>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzfs/libzfs_crypto.c' language='LANG_C99'>
	+ <array-type-def dimensions='1' type-id='38b51b3c' size-in-bits='832' id='02b72c00'>
	+ <subrange length='13' type-id='7359adad' id='487fded1'/>
	+ </array-type-def>
	<array-type-def dimensions='1' type-id='fb7c6451' size-in-bits='256' id='64177143'>
	<subrange length='32' type-id='7359adad' id='ae5bde82'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='a84c031d' size-in-bits='8' id='89feb1ec'>
	<subrange length='1' type-id='7359adad' id='52f813b4'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='a84c031d' size-in-bits='160' id='664ac0b7'>
	<subrange length='20' type-id='7359adad' id='fdca39cf'/>
	</array-type-def>
	<class-decl name='_IO_codecvt' is-struct='yes' visibility='default' is-declaration-only='yes' id='a4036571'/>
	<class-decl name='_IO_marker' is-struct='yes' visibility='default' is-declaration-only='yes' id='010ae0b9'/>
	<class-decl name='_IO_wide_data' is-struct='yes' visibility='default' is-declaration-only='yes' id='79bd3751'/>
	+ <class-decl name='__locale_data' is-struct='yes' visibility='default' is-declaration-only='yes' id='23de8b96'/>
	+ <array-type-def dimensions='1' type-id='80f4b756' size-in-bits='832' id='39e6f84a'>
	+ <subrange length='13' type-id='7359adad' id='487fded1'/>
	+ </array-type-def>
	<array-type-def dimensions='1' type-id='95e97e5e' size-in-bits='896' id='47394ee0'>
	<subrange length='28' type-id='7359adad' id='3db583d7'/>
	</array-type-def>
	<type-decl name='signed char' size-in-bits='8' id='28577a57'/>
	<array-type-def dimensions='1' type-id='7359adad' size-in-bits='1024' id='d2baa450'>
	<subrange length='16' type-id='7359adad' id='848d0938'/>
	</array-type-def>
	<type-decl name='unsigned short int' size-in-bits='16' id='8efea9e5'/>
	<enum-decl name='zpool_prop_t' naming-typedef-id='5d0c23fb' id='af1ba157'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='ZPOOL_PROP_INVAL' value='-1'/>
	<enumerator name='ZPOOL_PROP_NAME' value='0'/>
	<enumerator name='ZPOOL_PROP_SIZE' value='1'/>
	<enumerator name='ZPOOL_PROP_CAPACITY' value='2'/>
	<enumerator name='ZPOOL_PROP_ALTROOT' value='3'/>
	<enumerator name='ZPOOL_PROP_HEALTH' value='4'/>
	<enumerator name='ZPOOL_PROP_GUID' value='5'/>
	<enumerator name='ZPOOL_PROP_VERSION' value='6'/>
	<enumerator name='ZPOOL_PROP_BOOTFS' value='7'/>
	<enumerator name='ZPOOL_PROP_DELEGATION' value='8'/>
	<enumerator name='ZPOOL_PROP_AUTOREPLACE' value='9'/>
	<enumerator name='ZPOOL_PROP_CACHEFILE' value='10'/>
	<enumerator name='ZPOOL_PROP_FAILUREMODE' value='11'/>
	<enumerator name='ZPOOL_PROP_LISTSNAPS' value='12'/>
	<enumerator name='ZPOOL_PROP_AUTOEXPAND' value='13'/>
	<enumerator name='ZPOOL_PROP_DEDUPDITTO' value='14'/>
	<enumerator name='ZPOOL_PROP_DEDUPRATIO' value='15'/>
	<enumerator name='ZPOOL_PROP_FREE' value='16'/>
	<enumerator name='ZPOOL_PROP_ALLOCATED' value='17'/>
	<enumerator name='ZPOOL_PROP_READONLY' value='18'/>
	<enumerator name='ZPOOL_PROP_ASHIFT' value='19'/>
	<enumerator name='ZPOOL_PROP_COMMENT' value='20'/>
	<enumerator name='ZPOOL_PROP_EXPANDSZ' value='21'/>
	<enumerator name='ZPOOL_PROP_FREEING' value='22'/>
	<enumerator name='ZPOOL_PROP_FRAGMENTATION' value='23'/>
	<enumerator name='ZPOOL_PROP_LEAKED' value='24'/>
	<enumerator name='ZPOOL_PROP_MAXBLOCKSIZE' value='25'/>
	<enumerator name='ZPOOL_PROP_TNAME' value='26'/>
	<enumerator name='ZPOOL_PROP_MAXDNODESIZE' value='27'/>
	<enumerator name='ZPOOL_PROP_MULTIHOST' value='28'/>
	<enumerator name='ZPOOL_PROP_CHECKPOINT' value='29'/>
	<enumerator name='ZPOOL_PROP_LOAD_GUID' value='30'/>
	<enumerator name='ZPOOL_PROP_AUTOTRIM' value='31'/>
	<enumerator name='ZPOOL_PROP_COMPATIBILITY' value='32'/>
	<enumerator name='ZPOOL_PROP_BCLONEUSED' value='33'/>
	<enumerator name='ZPOOL_PROP_BCLONESAVED' value='34'/>
	<enumerator name='ZPOOL_PROP_BCLONERATIO' value='35'/>
	<enumerator name='ZPOOL_NUM_PROPS' value='36'/>
	</enum-decl>
	<typedef-decl name='zpool_prop_t' type-id='af1ba157' id='5d0c23fb'/>
	<typedef-decl name='regoff_t' type-id='95e97e5e' id='54a2a2a8'/>
	<class-decl name='regmatch_t' size-in-bits='64' is-struct='yes' naming-typedef-id='1b941664' visibility='default' id='4f932615'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='rm_so' type-id='54a2a2a8' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='rm_eo' type-id='54a2a2a8' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='regmatch_t' type-id='4f932615' id='1b941664'/>
	<typedef-decl name='__sighandler_t' type-id='03347643' id='8cdd9566'/>
	<typedef-decl name='ssize_t' type-id='41060289' id='79a0948f'/>
	<class-decl name='sigaction' size-in-bits='1216' is-struct='yes' visibility='default' id='fe391c48'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='__sigaction_handler' type-id='ac5ab595' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='sa_mask' type-id='b9c97942' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1088'>
	<var-decl name='sa_flags' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1152'>
	<var-decl name='sa_restorer' type-id='953b12f8' visibility='default'/>
	</data-member>
	</class-decl>
	<union-decl name='__anonymous_union__' size-in-bits='64' is-anonymous='yes' visibility='default' id='ac5ab595'>
	<data-member access='public'>
	<var-decl name='sa_handler' type-id='8cdd9566' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='sa_sigaction' type-id='6e756877' visibility='default'/>
	</data-member>
	</union-decl>
	<class-decl name='termios' size-in-bits='480' is-struct='yes' visibility='default' id='ad55d2bc'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='c_iflag' type-id='241ce6f8' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='c_oflag' type-id='241ce6f8' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='c_cflag' type-id='241ce6f8' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='96'>
	<var-decl name='c_lflag' type-id='241ce6f8' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='c_line' type-id='fb7c6451' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='136'>
	<var-decl name='c_cc' type-id='64177143' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='416'>
	<var-decl name='c_ispeed' type-id='6a8e8a14' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='c_ospeed' type-id='6a8e8a14' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='cc_t' type-id='002ac4a6' id='fb7c6451'/>
	<typedef-decl name='speed_t' type-id='f0981eeb' id='6a8e8a14'/>
	<typedef-decl name='tcflag_t' type-id='f0981eeb' id='241ce6f8'/>
	<typedef-decl name='__uid_t' type-id='f0981eeb' id='cc5fcceb'/>
	<typedef-decl name='__off_t' type-id='bd54fe1a' id='79989e9c'/>
	<typedef-decl name='__off64_t' type-id='bd54fe1a' id='724e4de6'/>
	<typedef-decl name='__pid_t' type-id='95e97e5e' id='3629bad8'/>
	<typedef-decl name='__clock_t' type-id='bd54fe1a' id='4d66c6d7'/>
	<typedef-decl name='__ssize_t' type-id='bd54fe1a' id='41060289'/>
	<typedef-decl name='FILE' type-id='ec1ed955' id='aa12d1ba'/>
	+ <class-decl name='__locale_struct' size-in-bits='1856' is-struct='yes' visibility='default' id='90cc1ce3'>
	+ <data-member access='public' layout-offset-in-bits='0'>
	+ <var-decl name='__locales' type-id='02b72c00' visibility='default'/>
	+ </data-member>
	+ <data-member access='public' layout-offset-in-bits='832'>
	+ <var-decl name='__ctype_b' type-id='31347b7a' visibility='default'/>
	+ </data-member>
	+ <data-member access='public' layout-offset-in-bits='896'>
	+ <var-decl name='__ctype_tolower' type-id='6d60f45d' visibility='default'/>
	+ </data-member>
	+ <data-member access='public' layout-offset-in-bits='960'>
	+ <var-decl name='__ctype_toupper' type-id='6d60f45d' visibility='default'/>
	+ </data-member>
	+ <data-member access='public' layout-offset-in-bits='1024'>
	+ <var-decl name='__names' type-id='39e6f84a' visibility='default'/>
	+ </data-member>
	+ </class-decl>
	+ <typedef-decl name='__locale_t' type-id='f01e1813' id='b7ac9b5f'/>
	<class-decl name='__sigset_t' size-in-bits='1024' is-struct='yes' naming-typedef-id='b9c97942' visibility='default' id='2616147f'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='__val' type-id='d2baa450' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='__sigset_t' type-id='2616147f' id='b9c97942'/>
	<union-decl name='sigval' size-in-bits='64' visibility='default' id='a094b870'>
	<data-member access='public'>
	<var-decl name='sival_int' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='sival_ptr' type-id='eaa32e2f' visibility='default'/>
	</data-member>
	</union-decl>
	<typedef-decl name='__sigval_t' type-id='a094b870' id='eabacd01'/>
	+ <typedef-decl name='locale_t' type-id='b7ac9b5f' id='973a4f8d'/>
	<class-decl name='siginfo_t' size-in-bits='1024' is-struct='yes' naming-typedef-id='cb681f62' visibility='default' id='d8149419'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='si_signo' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='si_errno' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='si_code' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='96'>
	<var-decl name='__pad0' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='_sifields' type-id='ac5ab596' visibility='default'/>
	</data-member>
	</class-decl>
	<union-decl name='__anonymous_union__1' size-in-bits='896' is-anonymous='yes' visibility='default' id='ac5ab596'>
	<data-member access='public'>
	<var-decl name='_pad' type-id='47394ee0' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='_kill' type-id='e7f43f73' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='_timer' type-id='e7f43f74' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='_rt' type-id='e7f43f75' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='_sigchld' type-id='e7f43f76' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='_sigfault' type-id='e7f43f77' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='_sigpoll' type-id='e7f43f78' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='_sigsys' type-id='e7f43f79' visibility='default'/>
	</data-member>
	</union-decl>
	<class-decl name='__anonymous_struct__1' size-in-bits='64' is-struct='yes' is-anonymous='yes' visibility='default' id='e7f43f73'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='si_pid' type-id='3629bad8' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='si_uid' type-id='cc5fcceb' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='__anonymous_struct__2' size-in-bits='128' is-struct='yes' is-anonymous='yes' visibility='default' id='e7f43f74'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='si_tid' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='si_overrun' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='si_sigval' type-id='eabacd01' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='__anonymous_struct__3' size-in-bits='128' is-struct='yes' is-anonymous='yes' visibility='default' id='e7f43f75'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='si_pid' type-id='3629bad8' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='si_uid' type-id='cc5fcceb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='si_sigval' type-id='eabacd01' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='__anonymous_struct__4' size-in-bits='256' is-struct='yes' is-anonymous='yes' visibility='default' id='e7f43f76'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='si_pid' type-id='3629bad8' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='si_uid' type-id='cc5fcceb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='si_status' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='si_utime' type-id='4d66c6d7' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='si_stime' type-id='4d66c6d7' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='__anonymous_struct__5' size-in-bits='256' is-struct='yes' is-anonymous='yes' visibility='default' id='e7f43f77'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='si_addr' type-id='eaa32e2f' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='si_addr_lsb' type-id='a2185560' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='_bounds' type-id='ac5ab597' visibility='default'/>
	</data-member>
	</class-decl>
	<union-decl name='__anonymous_union__2' size-in-bits='128' is-anonymous='yes' visibility='default' id='ac5ab597'>
	<data-member access='public'>
	<var-decl name='_addr_bnd' type-id='e7f43f7a' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='_pkey' type-id='62f1140c' visibility='default'/>
	</data-member>
	</union-decl>
	<class-decl name='__anonymous_struct__6' size-in-bits='128' is-struct='yes' is-anonymous='yes' visibility='default' id='e7f43f7a'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='_lower' type-id='eaa32e2f' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='_upper' type-id='eaa32e2f' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='__anonymous_struct__7' size-in-bits='128' is-struct='yes' is-anonymous='yes' visibility='default' id='e7f43f78'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='si_band' type-id='bd54fe1a' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='si_fd' type-id='95e97e5e' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='__anonymous_struct__8' size-in-bits='128' is-struct='yes' is-anonymous='yes' visibility='default' id='e7f43f79'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='_call_addr' type-id='eaa32e2f' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='_syscall' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='96'>
	<var-decl name='_arch' type-id='f0981eeb' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='siginfo_t' type-id='d8149419' id='cb681f62'/>
	<typedef-decl name='sigset_t' type-id='b9c97942' id='daf33c64'/>
	<typedef-decl name='_IO_lock_t' type-id='48b5725f' id='bb4788fa'/>
	<class-decl name='_IO_FILE' size-in-bits='1728' is-struct='yes' visibility='default' id='ec1ed955'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='_flags' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='_IO_read_ptr' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='_IO_read_end' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='_IO_read_base' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='_IO_write_base' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='_IO_write_ptr' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='_IO_write_end' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='_IO_buf_base' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='512'>
	<var-decl name='_IO_buf_end' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='576'>
	<var-decl name='_IO_save_base' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='640'>
	<var-decl name='_IO_backup_base' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='704'>
	<var-decl name='_IO_save_end' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='768'>
	<var-decl name='_markers' type-id='e4c6fa61' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='832'>
	<var-decl name='_chain' type-id='dca988a5' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='896'>
	<var-decl name='_fileno' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='928'>
	<var-decl name='_flags2' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='960'>
	<var-decl name='_old_offset' type-id='79989e9c' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1024'>
	<var-decl name='_cur_column' type-id='8efea9e5' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1040'>
	<var-decl name='_vtable_offset' type-id='28577a57' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1048'>
	<var-decl name='_shortbuf' type-id='89feb1ec' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1088'>
	<var-decl name='_lock' type-id='cecf4ea7' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1152'>
	<var-decl name='_offset' type-id='724e4de6' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1216'>
	<var-decl name='_codecvt' type-id='570f8c59' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1280'>
	<var-decl name='_wide_data' type-id='c65a1f29' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1344'>
	<var-decl name='_freeres_list' type-id='dca988a5' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1408'>
	<var-decl name='_freeres_buf' type-id='eaa32e2f' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1472'>
	<var-decl name='__pad5' type-id='b59d7dce' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1536'>
	<var-decl name='_mode' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1568'>
	<var-decl name='_unused2' type-id='664ac0b7' visibility='default'/>
	</data-member>
	</class-decl>
	<pointer-type-def type-id='aa12d1ba' size-in-bits='64' id='822cd80b'/>
	<qualified-type-def type-id='822cd80b' restrict='yes' id='e75a27e9'/>
	<pointer-type-def type-id='ec1ed955' size-in-bits='64' id='dca988a5'/>
	<pointer-type-def type-id='a4036571' size-in-bits='64' id='570f8c59'/>
	<pointer-type-def type-id='bb4788fa' size-in-bits='64' id='cecf4ea7'/>
	<pointer-type-def type-id='010ae0b9' size-in-bits='64' id='e4c6fa61'/>
	<pointer-type-def type-id='79bd3751' size-in-bits='64' id='c65a1f29'/>
	+ <pointer-type-def type-id='23de8b96' size-in-bits='64' id='38b51b3c'/>
	+ <pointer-type-def type-id='90cc1ce3' size-in-bits='64' id='f01e1813'/>
	<qualified-type-def type-id='9b23c9ad' restrict='yes' id='8c85230f'/>
	<qualified-type-def type-id='80f4b756' restrict='yes' id='9d26089a'/>
	<pointer-type-def type-id='80f4b756' size-in-bits='64' id='7d3cd834'/>
	+ <qualified-type-def type-id='95e97e5e' const='yes' id='2448a865'/>
	+ <pointer-type-def type-id='2448a865' size-in-bits='64' id='6d60f45d'/>
	<qualified-type-def type-id='aca3bac8' const='yes' id='2498fd78'/>
	<pointer-type-def type-id='2498fd78' size-in-bits='64' id='eed6c816'/>
	<qualified-type-def type-id='eed6c816' restrict='yes' id='a431a9da'/>
	<qualified-type-def type-id='fe391c48' const='yes' id='14a93b33'/>
	<pointer-type-def type-id='14a93b33' size-in-bits='64' id='9f68085b'/>
	<qualified-type-def type-id='9f68085b' restrict='yes' id='e2a5e6f9'/>
	<qualified-type-def type-id='ad55d2bc' const='yes' id='a46bf13f'/>
	<pointer-type-def type-id='a46bf13f' size-in-bits='64' id='eaec840f'/>
	<qualified-type-def type-id='002ac4a6' const='yes' id='ea86de29'/>
	<pointer-type-def type-id='ea86de29' size-in-bits='64' id='354f7eb9'/>
	<qualified-type-def type-id='8efea9e5' const='yes' id='3beb2af4'/>
	<pointer-type-def type-id='3beb2af4' size-in-bits='64' id='31347b7a'/>
	<pointer-type-def type-id='31347b7a' size-in-bits='64' id='c59e1ef0'/>
	<pointer-type-def type-id='1b941664' size-in-bits='64' id='7e2979d5'/>
	<qualified-type-def type-id='7e2979d5' restrict='yes' id='fc212857'/>
	<pointer-type-def type-id='fe391c48' size-in-bits='64' id='568dd84e'/>
	<qualified-type-def type-id='568dd84e' restrict='yes' id='3d8ee6f2'/>
	<pointer-type-def type-id='cb681f62' size-in-bits='64' id='185869c1'/>
	<pointer-type-def type-id='daf33c64' size-in-bits='64' id='9e80f729'/>
	<pointer-type-def type-id='b59d7dce' size-in-bits='64' id='78c01427'/>
	<qualified-type-def type-id='78c01427' restrict='yes' id='d19b2c25'/>
	<pointer-type-def type-id='ad55d2bc' size-in-bits='64' id='665a4eda'/>
	<pointer-type-def type-id='9c313c2d' size-in-bits='64' id='5d6479ae'/>
	<pointer-type-def type-id='ae3e8ca6' size-in-bits='64' id='d8774064'/>
	<pointer-type-def type-id='3502e3ff' size-in-bits='64' id='4dd26a40'/>
	<pointer-type-def type-id='ee076206' size-in-bits='64' id='953b12f8'/>
	<pointer-type-def type-id='f712e2b7' size-in-bits='64' id='03347643'/>
	<pointer-type-def type-id='ef70d893' size-in-bits='64' id='6e756877'/>
	<qualified-type-def type-id='eaa32e2f' restrict='yes' id='1b7446cd'/>
	<class-decl name='_IO_codecvt' is-struct='yes' visibility='default' is-declaration-only='yes' id='a4036571'/>
	<class-decl name='_IO_marker' is-struct='yes' visibility='default' is-declaration-only='yes' id='010ae0b9'/>
	<class-decl name='_IO_wide_data' is-struct='yes' visibility='default' is-declaration-only='yes' id='79bd3751'/>
	+ <class-decl name='__locale_data' is-struct='yes' visibility='default' is-declaration-only='yes' id='23de8b96'/>
	<function-decl name='zpool_get_prop_int' mangled-name='zpool_get_prop_int' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_prop_int'>
	<parameter type-id='4c81de99'/>
	<parameter type-id='5d0c23fb'/>
	<parameter type-id='debc6aa3'/>
	<return type-id='9c313c2d'/>
	</function-decl>
	<function-decl name='zfs_handle_dup' mangled-name='zfs_handle_dup' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_handle_dup'>
	<parameter type-id='9200a744'/>
	<return type-id='9200a744'/>
	</function-decl>
	<function-decl name='zfs_valid_proplist' mangled-name='zfs_valid_proplist' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_valid_proplist'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='2e45de5d'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='9200a744'/>
	<parameter type-id='4c81de99'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='80f4b756'/>
	<return type-id='5ce45b60'/>
	</function-decl>
	<function-decl name='zfs_prop_to_name' mangled-name='zfs_prop_to_name' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_to_name'>
	<parameter type-id='58603c44'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zfs_iter_filesystems_v2' mangled-name='zfs_iter_filesystems_v2' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_iter_filesystems_v2'>
	<parameter type-id='9200a744'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='d8e49ab9'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_parent_name' mangled-name='zfs_parent_name' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_parent_name'>
	<parameter type-id='9200a744'/>
	<parameter type-id='26a90f95'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_load_key' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='ae3e8ca6'/>
	<parameter type-id='3502e3ff'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_unload_key' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_change_key' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='ae3e8ca6'/>
	<parameter type-id='3502e3ff'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_name_to_prop' mangled-name='zfs_name_to_prop' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_name_to_prop'>
	<parameter type-id='80f4b756'/>
	<return type-id='58603c44'/>
	</function-decl>
	<function-decl name='nvlist_add_uint64' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='9c313c2d'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_add_string' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_lookup_uint64' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='22cce67b'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='5d6479ae'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_lookup_string' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='22cce67b'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='7d3cd834'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='fnvlist_alloc' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='5ce45b60'/>
	</function-decl>
	<function-decl name='__ctype_b_loc' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='c59e1ef0'/>
	</function-decl>
	<function-decl name='dlopen' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='dlsym' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='1b7446cd'/>
	<parameter type-id='9d26089a'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='dlerror' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='26a90f95'/>
	</function-decl>
	+ <function-decl name='uselocale' visibility='default' binding='global' size-in-bits='64'>
	+ <parameter type-id='973a4f8d'/>
	+ <return type-id='973a4f8d'/>
	+ </function-decl>
	<function-decl name='PKCS5_PBKDF2_HMAC_SHA1' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='354f7eb9'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='cf536864'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='regexec' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='a431a9da'/>
	<parameter type-id='9d26089a'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='fc212857'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='kill' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='3629bad8'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='sigemptyset' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='9e80f729'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='sigaction' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='e2a5e6f9'/>
	<parameter type-id='3d8ee6f2'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='fclose' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='822cd80b'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='fflush' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='822cd80b'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='fdopen' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<return type-id='822cd80b'/>
	</function-decl>
	<function-decl name='fputc' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='822cd80b'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='__getdelim' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='8c85230f'/>
	<parameter type-id='d19b2c25'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='e75a27e9'/>
	<return type-id='41060289'/>
	</function-decl>
	<function-decl name='rewind' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='822cd80b'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='ferror' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='822cd80b'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='fileno' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='822cd80b'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='malloc' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b59d7dce'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='calloc' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='strdup' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<return type-id='26a90f95'/>
	</function-decl>
	- <function-decl name='strerror' visibility='default' binding='global' size-in-bits='64'>
	+ <function-decl name='strerror_l' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	+ <parameter type-id='973a4f8d'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='tcgetattr' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='665a4eda'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='tcsetattr' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='eaec840f'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='close' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='getpid' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='3629bad8'/>
	</function-decl>
	<function-decl name='isatty' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='unlink' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='__open_too_many_args' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='__open_missing_mode' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='__printf_chk' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter is-variadic='yes'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='__asprintf_chk' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='8c85230f'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='9d26089a'/>
	<parameter is-variadic='yes'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='__fread_chk' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='1b7446cd'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='e75a27e9'/>
	<return type-id='b59d7dce'/>
	</function-decl>
	<function-decl name='__read_chk' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='79a0948f'/>
	</function-decl>
	<function-decl name='zfs_crypto_get_encryption_root' mangled-name='zfs_crypto_get_encryption_root' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_crypto_get_encryption_root'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='37e3bd22' name='is_encroot'/>
	<parameter type-id='26a90f95' name='buf'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_crypto_create' mangled-name='zfs_crypto_create' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_crypto_create'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='26a90f95' name='parent_name'/>
	<parameter type-id='5ce45b60' name='props'/>
	<parameter type-id='5ce45b60' name='pool_props'/>
	<parameter type-id='c19b74c3' name='stdin_available'/>
	<parameter type-id='d8774064' name='wkeydata_out'/>
	<parameter type-id='4dd26a40' name='wkeylen_out'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_crypto_clone_check' mangled-name='zfs_crypto_clone_check' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_crypto_clone_check'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='9200a744' name='origin_zhp'/>
	<parameter type-id='26a90f95' name='parent_name'/>
	<parameter type-id='5ce45b60' name='props'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_crypto_attempt_load_keys' mangled-name='zfs_crypto_attempt_load_keys' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_crypto_attempt_load_keys'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='80f4b756' name='fsname'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_crypto_load_key' mangled-name='zfs_crypto_load_key' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_crypto_load_key'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='c19b74c3' name='noop'/>
	<parameter type-id='80f4b756' name='alt_keylocation'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_crypto_unload_key' mangled-name='zfs_crypto_unload_key' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_crypto_unload_key'>
	<parameter type-id='9200a744' name='zhp'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_crypto_rewrap' mangled-name='zfs_crypto_rewrap' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_crypto_rewrap'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='5ce45b60' name='raw_props'/>
	<parameter type-id='c19b74c3' name='inheritkey'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_error_aux' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='80f4b756'/>
	<parameter is-variadic='yes'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-type size-in-bits='64' id='f712e2b7'>
	<parameter type-id='95e97e5e'/>
	<return type-id='48b5725f'/>
	</function-type>
	<function-type size-in-bits='64' id='ef70d893'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='185869c1'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='48b5725f'/>
	</function-type>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzfs/libzfs_dataset.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='a84c031d' size-in-bits='32' id='8e0573fd'>
	<subrange length='4' type-id='7359adad' id='16fe7105'/>
	</array-type-def>
	<class-decl name='prop_changelist' is-struct='yes' visibility='default' is-declaration-only='yes' id='d86edc51'/>
	<class-decl name='zprop_list' size-in-bits='448' is-struct='yes' visibility='default' id='bd9b4291'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='pl_prop' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='pl_user_prop' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='pl_next' type-id='9f1a1109' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='pl_all' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='pl_width' type-id='b59d7dce' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='pl_recvd_width' type-id='b59d7dce' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='pl_fixed' type-id='c19b74c3' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zprop_list_t' type-id='bd9b4291' id='bdb8ac4f'/>
	<class-decl name='renameflags' size-in-bits='32' is-struct='yes' visibility='default' id='7aee5792'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='recursive' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1'>
	<var-decl name='nounmount' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2'>
	<var-decl name='forceunmount' type-id='f0981eeb' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='renameflags_t' type-id='7aee5792' id='067170c2'/>
	<typedef-decl name='zfs_userspace_cb_t' type-id='ca64ff60' id='16c5f410'/>
	<enum-decl name='lzc_dataset_type' id='bc9887f1'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='LZC_DATSET_TYPE_ZFS' value='2'/>
	<enumerator name='LZC_DATSET_TYPE_ZVOL' value='3'/>
	</enum-decl>
	<typedef-decl name='avl_index_t' type-id='e475ab95' id='fba6cb51'/>
	<enum-decl name='zfs_userquota_prop_t' naming-typedef-id='279fde6a' id='5258d2f6'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='ZFS_PROP_USERUSED' value='0'/>
	<enumerator name='ZFS_PROP_USERQUOTA' value='1'/>
	<enumerator name='ZFS_PROP_GROUPUSED' value='2'/>
	<enumerator name='ZFS_PROP_GROUPQUOTA' value='3'/>
	<enumerator name='ZFS_PROP_USEROBJUSED' value='4'/>
	<enumerator name='ZFS_PROP_USEROBJQUOTA' value='5'/>
	<enumerator name='ZFS_PROP_GROUPOBJUSED' value='6'/>
	<enumerator name='ZFS_PROP_GROUPOBJQUOTA' value='7'/>
	<enumerator name='ZFS_PROP_PROJECTUSED' value='8'/>
	<enumerator name='ZFS_PROP_PROJECTQUOTA' value='9'/>
	<enumerator name='ZFS_PROP_PROJECTOBJUSED' value='10'/>
	<enumerator name='ZFS_PROP_PROJECTOBJQUOTA' value='11'/>
	<enumerator name='ZFS_NUM_USERQUOTA_PROPS' value='12'/>
	</enum-decl>
	<typedef-decl name='zfs_userquota_prop_t' type-id='5258d2f6' id='279fde6a'/>
	<enum-decl name='zfs_wait_activity_t' naming-typedef-id='3024501a' id='527d5dc6'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='ZFS_WAIT_DELETEQ' value='0'/>
	<enumerator name='ZFS_WAIT_NUM_ACTIVITIES' value='1'/>
	</enum-decl>
	<typedef-decl name='zfs_wait_activity_t' type-id='527d5dc6' id='3024501a'/>
	<enum-decl name='namecheck_err_t' naming-typedef-id='8e0af06e' id='f43bbcda'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='NAME_ERR_LEADING_SLASH' value='0'/>
	<enumerator name='NAME_ERR_EMPTY_COMPONENT' value='1'/>
	<enumerator name='NAME_ERR_TRAILING_SLASH' value='2'/>
	<enumerator name='NAME_ERR_INVALCHAR' value='3'/>
	<enumerator name='NAME_ERR_MULTIPLE_DELIMITERS' value='4'/>
	<enumerator name='NAME_ERR_NOLETTER' value='5'/>
	<enumerator name='NAME_ERR_RESERVED' value='6'/>
	<enumerator name='NAME_ERR_DISKLIKE' value='7'/>
	<enumerator name='NAME_ERR_TOOLONG' value='8'/>
	<enumerator name='NAME_ERR_SELF_REF' value='9'/>
	<enumerator name='NAME_ERR_PARENT_REF' value='10'/>
	<enumerator name='NAME_ERR_NO_AT' value='11'/>
	<enumerator name='NAME_ERR_NO_POUND' value='12'/>
	</enum-decl>
	<typedef-decl name='namecheck_err_t' type-id='f43bbcda' id='8e0af06e'/>
	<enum-decl name='zprop_type_t' naming-typedef-id='31429eff' id='87676253'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='PROP_TYPE_NUMBER' value='0'/>
	<enumerator name='PROP_TYPE_STRING' value='1'/>
	<enumerator name='PROP_TYPE_INDEX' value='2'/>
	</enum-decl>
	<typedef-decl name='zprop_type_t' type-id='87676253' id='31429eff'/>
	<class-decl name='mnttab' size-in-bits='256' is-struct='yes' visibility='default' id='1b055409'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='mnt_special' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='mnt_mountp' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='mnt_fstype' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='mnt_mntopts' type-id='26a90f95' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='group' size-in-bits='256' is-struct='yes' visibility='default' id='01a1b934'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='gr_name' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='gr_passwd' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='gr_gid' type-id='d94ec6d9' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='gr_mem' type-id='9b23c9ad' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='mntent' size-in-bits='320' is-struct='yes' visibility='default' id='56fe4a37'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='mnt_fsname' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='mnt_dir' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='mnt_type' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='mnt_opts' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='mnt_freq' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='288'>
	<var-decl name='mnt_passno' type-id='95e97e5e' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='passwd' size-in-bits='384' is-struct='yes' visibility='default' id='a63d15a3'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='pw_name' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='pw_passwd' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='pw_uid' type-id='cc5fcceb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='160'>
	<var-decl name='pw_gid' type-id='d94ec6d9' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='pw_gecos' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='pw_dir' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='pw_shell' type-id='26a90f95' visibility='default'/>
	</data-member>
	</class-decl>
	<union-decl name='pthread_mutexattr_t' size-in-bits='32' naming-typedef-id='8afd6070' visibility='default' id='7300eb00'>
	<data-member access='public'>
	<var-decl name='__size' type-id='8e0573fd' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='__align' type-id='95e97e5e' visibility='default'/>
	</data-member>
	</union-decl>
	<typedef-decl name='pthread_mutexattr_t' type-id='7300eb00' id='8afd6070'/>
	<typedef-decl name='int64_t' type-id='0c9942d2' id='9da381c4'/>
	<typedef-decl name='__int64_t' type-id='bd54fe1a' id='0c9942d2'/>
	<typedef-decl name='__gid_t' type-id='f0981eeb' id='d94ec6d9'/>
	<typedef-decl name='__time_t' type-id='bd54fe1a' id='65eda9c0'/>
	<class-decl name='tm' size-in-bits='448' is-struct='yes' visibility='default' id='dddf6ca2'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='tm_sec' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='tm_min' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='tm_hour' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='96'>
	<var-decl name='tm_mday' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='tm_mon' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='160'>
	<var-decl name='tm_year' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='tm_wday' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='224'>
	<var-decl name='tm_yday' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='tm_isdst' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='tm_gmtoff' type-id='bd54fe1a' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='tm_zone' type-id='80f4b756' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='time_t' type-id='65eda9c0' id='c9d12d66'/>
	<typedef-decl name='uid_t' type-id='cc5fcceb' id='354978ed'/>
	<typedef-decl name='prop_changelist_t' type-id='d86edc51' id='eae6431d'/>
	<pointer-type-def type-id='fba6cb51' size-in-bits='64' id='32adbf30'/>
	<pointer-type-def type-id='f20fbd51' size-in-bits='64' id='a3681dea'/>
	<qualified-type-def type-id='26a90f95' restrict='yes' id='266fe297'/>
	<qualified-type-def type-id='56fe4a37' const='yes' id='a75125ce'/>
	<pointer-type-def type-id='a75125ce' size-in-bits='64' id='48bea5ec'/>
	<qualified-type-def type-id='8afd6070' const='yes' id='1d853360'/>
	<pointer-type-def type-id='1d853360' size-in-bits='64' id='c2afbd7e'/>
	<qualified-type-def type-id='c9d12d66' const='yes' id='588b3216'/>
	<pointer-type-def type-id='588b3216' size-in-bits='64' id='9f201474'/>
	<qualified-type-def type-id='9f201474' restrict='yes' id='d6e2847c'/>
	<qualified-type-def type-id='dddf6ca2' const='yes' id='e824a34f'/>
	<pointer-type-def type-id='e824a34f' size-in-bits='64' id='d6ad37ff'/>
	<qualified-type-def type-id='d6ad37ff' restrict='yes' id='f8c6051d'/>
	<qualified-type-def type-id='9c313c2d' const='yes' id='c3b7ba7d'/>
	<pointer-type-def type-id='c3b7ba7d' size-in-bits='64' id='713a56f5'/>
	<pointer-type-def type-id='01a1b934' size-in-bits='64' id='566b3f52'/>
	+ <qualified-type-def type-id='566b3f52' restrict='yes' id='c878edd6'/>
	+ <pointer-type-def type-id='566b3f52' size-in-bits='64' id='82d4e9e8'/>
	+ <qualified-type-def type-id='82d4e9e8' restrict='yes' id='aa19c230'/>
	<pointer-type-def type-id='7e291ce6' size-in-bits='64' id='ca64ff60'/>
	<pointer-type-def type-id='9da381c4' size-in-bits='64' id='cb785ebf'/>
	<pointer-type-def type-id='1b055409' size-in-bits='64' id='9d424d31'/>
	<pointer-type-def type-id='8e0af06e' size-in-bits='64' id='053457bd'/>
	<pointer-type-def type-id='857bb57e' size-in-bits='64' id='75be733c'/>
	<pointer-type-def type-id='a63d15a3' size-in-bits='64' id='a195f4a3'/>
	+ <qualified-type-def type-id='a195f4a3' restrict='yes' id='33518961'/>
	+ <pointer-type-def type-id='a195f4a3' size-in-bits='64' id='e80ff3ab'/>
	+ <qualified-type-def type-id='e80ff3ab' restrict='yes' id='8f2c7109'/>
	<pointer-type-def type-id='eae6431d' size-in-bits='64' id='0d41d328'/>
	<pointer-type-def type-id='7a6844eb' size-in-bits='64' id='18c91f9e'/>
	<pointer-type-def type-id='dddf6ca2' size-in-bits='64' id='d915a820'/>
	<qualified-type-def type-id='d915a820' restrict='yes' id='f099ad08'/>
	<pointer-type-def type-id='5d6479ae' size-in-bits='64' id='892b4acc'/>
	<pointer-type-def type-id='bd9b4291' size-in-bits='64' id='9f1a1109'/>
	<pointer-type-def type-id='bdb8ac4f' size-in-bits='64' id='3a9b2288'/>
	<pointer-type-def type-id='3a9b2288' size-in-bits='64' id='e4378506'/>
	<class-decl name='prop_changelist' is-struct='yes' visibility='default' is-declaration-only='yes' id='d86edc51'/>
	<function-decl name='zpool_open' mangled-name='zpool_open' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_open'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='80f4b756'/>
	<return type-id='4c81de99'/>
	</function-decl>
	<function-decl name='zpool_open_canfail' mangled-name='zpool_open_canfail' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_open_canfail'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='80f4b756'/>
	<return type-id='4c81de99'/>
	</function-decl>
	<function-decl name='zpool_close' mangled-name='zpool_close' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_close'>
	<parameter type-id='4c81de99'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zpool_get_name' mangled-name='zpool_get_name' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_name'>
	<parameter type-id='4c81de99'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zpool_get_prop' mangled-name='zpool_get_prop' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_prop'>
	<parameter type-id='4c81de99'/>
	<parameter type-id='5d0c23fb'/>
	<parameter type-id='26a90f95'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='debc6aa3'/>
	<parameter type-id='c19b74c3'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prop_default_string' mangled-name='zfs_prop_default_string' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_default_string'>
	<parameter type-id='58603c44'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zfs_prop_default_numeric' mangled-name='zfs_prop_default_numeric' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_default_numeric'>
	<parameter type-id='58603c44'/>
	<return type-id='9c313c2d'/>
	</function-decl>
	<function-decl name='zpool_prop_get_feature' mangled-name='zpool_prop_get_feature' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_get_feature'>
	<parameter type-id='4c81de99'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='26a90f95'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_iter_snapshots_v2' mangled-name='zfs_iter_snapshots_v2' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_iter_snapshots_v2'>
	<parameter type-id='9200a744'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='d8e49ab9'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='9c313c2d'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_iter_bookmarks_v2' mangled-name='zfs_iter_bookmarks_v2' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_iter_bookmarks_v2'>
	<parameter type-id='9200a744'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='d8e49ab9'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_destroy_snaps_nvl_os' mangled-name='zfs_destroy_snaps_nvl_os' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_destroy_snaps_nvl_os'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='5ce45b60'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_nicestrtonum' mangled-name='zfs_nicestrtonum' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_nicestrtonum'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='5d6479ae'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_snapshot' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_create' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='bc9887f1'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='ae3e8ca6'/>
	<parameter type-id='3502e3ff'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_clone' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='5ce45b60'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_promote' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='26a90f95'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_destroy_snaps' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_get_bookmarks' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_destroy_bookmarks' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_hold' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_release' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_get_holds' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_exists' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='lzc_rollback_to' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_destroy' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_channel_program_nosync' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_wait_fs' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='3024501a'/>
	<parameter type-id='37e3bd22'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_nicebytes' mangled-name='zfs_nicebytes' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_nicebytes'>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='26a90f95'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_nicenum' mangled-name='zfs_nicenum' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_nicenum'>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='26a90f95'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='avl_create' mangled-name='avl_create' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_create'>
	<parameter type-id='a3681dea'/>
	<parameter type-id='585e1de9'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='avl_find' mangled-name='avl_find' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_find'>
	<parameter type-id='a3681dea'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='32adbf30'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='avl_add' mangled-name='avl_add' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_add'>
	<parameter type-id='a3681dea'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='avl_remove' mangled-name='avl_remove' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_remove'>
	<parameter type-id='a3681dea'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='avl_numnodes' mangled-name='avl_numnodes' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_numnodes'>
	<parameter type-id='a3681dea'/>
	<return type-id='ee1f298e'/>
	</function-decl>
	<function-decl name='avl_destroy_nodes' mangled-name='avl_destroy_nodes' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_destroy_nodes'>
	<parameter type-id='a3681dea'/>
	<parameter type-id='63e171df'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='avl_destroy' mangled-name='avl_destroy' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_destroy'>
	<parameter type-id='a3681dea'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_prop_readonly' mangled-name='zfs_prop_readonly' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_readonly'>
	<parameter type-id='58603c44'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_prop_inheritable' mangled-name='zfs_prop_inheritable' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_inheritable'>
	<parameter type-id='58603c44'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_prop_setonce' mangled-name='zfs_prop_setonce' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_setonce'>
	<parameter type-id='58603c44'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_prop_encryption_key_param' mangled-name='zfs_prop_encryption_key_param' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_encryption_key_param'>
	<parameter type-id='58603c44'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_prop_valid_keylocation' mangled-name='zfs_prop_valid_keylocation' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_valid_keylocation'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='c19b74c3'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_prop_user' mangled-name='zfs_prop_user' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_user'>
	<parameter type-id='80f4b756'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_prop_userquota' mangled-name='zfs_prop_userquota' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_userquota'>
	<parameter type-id='80f4b756'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_prop_written' mangled-name='zfs_prop_written' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_written'>
	<parameter type-id='80f4b756'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_prop_index_to_string' mangled-name='zfs_prop_index_to_string' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_index_to_string'>
	<parameter type-id='58603c44'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='7d3cd834'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prop_valid_for_type' mangled-name='zfs_prop_valid_for_type' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_valid_for_type'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='2e45de5d'/>
	<parameter type-id='c19b74c3'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='nvlist_alloc' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='857bb57e'/>
	<parameter type-id='3502e3ff'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_size' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='78c01427'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_pack' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='9b23c9ad'/>
	<parameter type-id='78c01427'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_unpack' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='26a90f95'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='857bb57e'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_add_nvlist' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='22cce67b'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_add_uint64_array' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='713a56f5'/>
	<parameter type-id='3502e3ff'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_remove' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='8d0687d2'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_remove_all' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_lookup_int64' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='22cce67b'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='cb785ebf'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_lookup_uint64_array' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='892b4acc'/>
	<parameter type-id='4dd26a40'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_lookup_nvlist_array' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='75be733c'/>
	<parameter type-id='4dd26a40'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_empty' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='22cce67b'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='nvpair_type' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='dace003f'/>
	<return type-id='8d0687d2'/>
	</function-decl>
	<function-decl name='nvpair_value_uint64' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='dace003f'/>
	<parameter type-id='5d6479ae'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvpair_value_string' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='dace003f'/>
	<parameter type-id='7d3cd834'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='fnvlist_free' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fnvlist_add_boolean' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fnvlist_add_uint64' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='9c313c2d'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fnvlist_add_string' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fnvlist_add_nvlist' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='5ce45b60'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fnvlist_lookup_uint64' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='22cce67b'/>
	<parameter type-id='80f4b756'/>
	<return type-id='9c313c2d'/>
	</function-decl>
	<function-decl name='fnvlist_lookup_string' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='22cce67b'/>
	<parameter type-id='80f4b756'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='fnvlist_lookup_nvlist' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<return type-id='5ce45b60'/>
	</function-decl>
	<function-decl name='fnvpair_value_int32' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='dace003f'/>
	<return type-id='3ff5601b'/>
	</function-decl>
	<function-decl name='fnvpair_value_uint64' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='dace003f'/>
	<return type-id='9c313c2d'/>
	</function-decl>
	<function-decl name='entity_namecheck' mangled-name='entity_namecheck' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='entity_namecheck'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='053457bd'/>
	<parameter type-id='26a90f95'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='dataset_nestcheck' mangled-name='dataset_nestcheck' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='dataset_nestcheck'>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='mountpoint_namecheck' mangled-name='mountpoint_namecheck' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='mountpoint_namecheck'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='053457bd'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prop_get_type' mangled-name='zfs_prop_get_type' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_get_type'>
	<parameter type-id='58603c44'/>
	<return type-id='31429eff'/>
	</function-decl>
	<function-decl name='sa_validate_shareopts' mangled-name='sa_validate_shareopts' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='sa_validate_shareopts'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='9155d4b5'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='getmntany' mangled-name='getmntany' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='getmntany'>
	<parameter type-id='822cd80b'/>
	<parameter type-id='9d424d31'/>
	<parameter type-id='9d424d31'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='_sol_getmntent' mangled-name='_sol_getmntent' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_sol_getmntent'>
	<parameter type-id='822cd80b'/>
	<parameter type-id='9d424d31'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	- <function-decl name='getgrnam' visibility='default' binding='global' size-in-bits='64'>
	- <parameter type-id='80f4b756'/>
	- <return type-id='566b3f52'/>
	+ <function-decl name='getgrnam_r' visibility='default' binding='global' size-in-bits='64'>
	+ <parameter type-id='9d26089a'/>
	+ <parameter type-id='c878edd6'/>
	+ <parameter type-id='266fe297'/>
	+ <parameter type-id='b59d7dce'/>
	+ <parameter type-id='aa19c230'/>
	+ <return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='hasmntopt' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='48bea5ec'/>
	<parameter type-id='80f4b756'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='pthread_mutex_init' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='18c91f9e'/>
	<parameter type-id='c2afbd7e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_mutex_destroy' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='18c91f9e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_mutex_lock' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='18c91f9e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_mutex_unlock' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='18c91f9e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	- <function-decl name='getpwnam' visibility='default' binding='global' size-in-bits='64'>
	- <parameter type-id='80f4b756'/>
	- <return type-id='a195f4a3'/>
	+ <function-decl name='getpwnam_r' visibility='default' binding='global' size-in-bits='64'>
	+ <parameter type-id='9d26089a'/>
	+ <parameter type-id='33518961'/>
	+ <parameter type-id='266fe297'/>
	+ <parameter type-id='b59d7dce'/>
	+ <parameter type-id='8f2c7109'/>
	+ <return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='strtol' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='9d26089a'/>
	<parameter type-id='8c85230f'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='bd54fe1a'/>
	</function-decl>
	<function-decl name='strtoul' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='9d26089a'/>
	<parameter type-id='8c85230f'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='7359adad'/>
	</function-decl>
	<function-decl name='abort' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='strrchr' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='strcspn' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<return type-id='b59d7dce'/>
	</function-decl>
	<function-decl name='strstr' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='strsep' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='8c85230f'/>
	<parameter type-id='9d26089a'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='strftime' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='266fe297'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='9d26089a'/>
	<parameter type-id='f8c6051d'/>
	<return type-id='b59d7dce'/>
	</function-decl>
	<function-decl name='localtime_r' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='d6e2847c'/>
	<parameter type-id='f099ad08'/>
	<return type-id='d915a820'/>
	</function-decl>
	<function-decl name='__fprintf_chk' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='e75a27e9'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='9d26089a'/>
	<parameter is-variadic='yes'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='ioctl' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='7359adad'/>
	<parameter is-variadic='yes'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_type_to_name' mangled-name='zfs_type_to_name' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_type_to_name'>
	<parameter type-id='2e45de5d' name='type'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zfs_name_valid' mangled-name='zfs_name_valid' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_name_valid'>
	<parameter type-id='80f4b756' name='name'/>
	<parameter type-id='2e45de5d' name='type'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_free_handles' mangled-name='zpool_free_handles' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_free_handles'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_bookmark_exists' mangled-name='zfs_bookmark_exists' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_bookmark_exists'>
	<parameter type-id='80f4b756' name='path'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='libzfs_mnttab_init' mangled-name='libzfs_mnttab_init' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_mnttab_init'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='libzfs_mnttab_fini' mangled-name='libzfs_mnttab_fini' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_mnttab_fini'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='libzfs_mnttab_cache' mangled-name='libzfs_mnttab_cache' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_mnttab_cache'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='c19b74c3' name='enable'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='libzfs_mnttab_find' mangled-name='libzfs_mnttab_find' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_mnttab_find'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='80f4b756' name='fsname'/>
	<parameter type-id='9d424d31' name='entry'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='libzfs_mnttab_add' mangled-name='libzfs_mnttab_add' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_mnttab_add'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='80f4b756' name='special'/>
	<parameter type-id='80f4b756' name='mountp'/>
	<parameter type-id='80f4b756' name='mntopts'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='libzfs_mnttab_remove' mangled-name='libzfs_mnttab_remove' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_mnttab_remove'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='80f4b756' name='fsname'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_spa_version' mangled-name='zfs_spa_version' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_spa_version'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='7292109c' name='spa_version'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prop_set' mangled-name='zfs_prop_set' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_set'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='80f4b756' name='propname'/>
	<parameter type-id='80f4b756' name='propval'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prop_set_list' mangled-name='zfs_prop_set_list' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_set_list'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='5ce45b60' name='props'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prop_set_list_flags' mangled-name='zfs_prop_set_list_flags' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_set_list_flags'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='5ce45b60' name='props'/>
	<parameter type-id='95e97e5e' name='flags'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prop_inherit' mangled-name='zfs_prop_inherit' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_inherit'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='80f4b756' name='propname'/>
	<parameter type-id='c19b74c3' name='received'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='getprop_uint64' mangled-name='getprop_uint64' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='getprop_uint64'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='58603c44' name='prop'/>
	<parameter type-id='7d3cd834' name='source'/>
	<return type-id='9c313c2d'/>
	</function-decl>
	<function-decl name='zfs_prop_get_recvd' mangled-name='zfs_prop_get_recvd' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_get_recvd'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='80f4b756' name='propname'/>
	<parameter type-id='26a90f95' name='propbuf'/>
	<parameter type-id='b59d7dce' name='proplen'/>
	<parameter type-id='c19b74c3' name='literal'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_get_clones_nvl' mangled-name='zfs_get_clones_nvl' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_get_clones_nvl'>
	<parameter type-id='9200a744' name='zhp'/>
	<return type-id='5ce45b60'/>
	</function-decl>
	<function-decl name='zfs_prop_get_numeric' mangled-name='zfs_prop_get_numeric' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_get_numeric'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='58603c44' name='prop'/>
	<parameter type-id='5d6479ae' name='value'/>
	<parameter type-id='debc6aa3' name='src'/>
	<parameter type-id='26a90f95' name='statbuf'/>
	<parameter type-id='b59d7dce' name='statlen'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prop_get_userquota_int' mangled-name='zfs_prop_get_userquota_int' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_get_userquota_int'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='80f4b756' name='propname'/>
	<parameter type-id='5d6479ae' name='propvalue'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prop_get_userquota' mangled-name='zfs_prop_get_userquota' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_get_userquota'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='80f4b756' name='propname'/>
	<parameter type-id='26a90f95' name='propbuf'/>
	<parameter type-id='95e97e5e' name='proplen'/>
	<parameter type-id='c19b74c3' name='literal'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prop_get_written_int' mangled-name='zfs_prop_get_written_int' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_get_written_int'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='80f4b756' name='propname'/>
	<parameter type-id='5d6479ae' name='propvalue'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prop_get_written' mangled-name='zfs_prop_get_written' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_get_written'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='80f4b756' name='propname'/>
	<parameter type-id='26a90f95' name='propbuf'/>
	<parameter type-id='95e97e5e' name='proplen'/>
	<parameter type-id='c19b74c3' name='literal'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_get_pool_name' mangled-name='zfs_get_pool_name' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_get_pool_name'>
	<parameter type-id='fcd57163' name='zhp'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zfs_get_type' mangled-name='zfs_get_type' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_get_type'>
	<parameter type-id='fcd57163' name='zhp'/>
	<return type-id='2e45de5d'/>
	</function-decl>
	<function-decl name='zfs_get_underlying_type' mangled-name='zfs_get_underlying_type' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_get_underlying_type'>
	<parameter type-id='fcd57163' name='zhp'/>
	<return type-id='2e45de5d'/>
	</function-decl>
	<function-decl name='zfs_dataset_exists' mangled-name='zfs_dataset_exists' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_dataset_exists'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='2e45de5d' name='types'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_create_ancestors' mangled-name='zfs_create_ancestors' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_create_ancestors'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='80f4b756' name='path'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_create' mangled-name='zfs_create' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_create'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='2e45de5d' name='type'/>
	<parameter type-id='5ce45b60' name='props'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_destroy' mangled-name='zfs_destroy' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_destroy'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='c19b74c3' name='defer'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_destroy_snaps' mangled-name='zfs_destroy_snaps' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_destroy_snaps'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='26a90f95' name='snapname'/>
	<parameter type-id='c19b74c3' name='defer'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_destroy_snaps_nvl' mangled-name='zfs_destroy_snaps_nvl' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_destroy_snaps_nvl'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='5ce45b60' name='snaps'/>
	<parameter type-id='c19b74c3' name='defer'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_clone' mangled-name='zfs_clone' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_clone'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='80f4b756' name='target'/>
	<parameter type-id='5ce45b60' name='props'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_promote' mangled-name='zfs_promote' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_promote'>
	<parameter type-id='9200a744' name='zhp'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_snapshot_nvl' mangled-name='zfs_snapshot_nvl' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_snapshot_nvl'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='5ce45b60' name='snaps'/>
	<parameter type-id='5ce45b60' name='props'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_snapshot' mangled-name='zfs_snapshot' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_snapshot'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='c19b74c3' name='recursive'/>
	<parameter type-id='5ce45b60' name='props'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_rollback' mangled-name='zfs_rollback' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_rollback'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='9200a744' name='snap'/>
	<parameter type-id='c19b74c3' name='force'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_rename' mangled-name='zfs_rename' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_rename'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='80f4b756' name='target'/>
	<parameter type-id='067170c2' name='flags'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_get_all_props' mangled-name='zfs_get_all_props' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_get_all_props'>
	<parameter type-id='9200a744' name='zhp'/>
	<return type-id='5ce45b60'/>
	</function-decl>
	<function-decl name='zfs_get_recvd_props' mangled-name='zfs_get_recvd_props' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_get_recvd_props'>
	<parameter type-id='9200a744' name='zhp'/>
	<return type-id='5ce45b60'/>
	</function-decl>
	<function-decl name='zfs_get_user_props' mangled-name='zfs_get_user_props' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_get_user_props'>
	<parameter type-id='9200a744' name='zhp'/>
	<return type-id='5ce45b60'/>
	</function-decl>
	<function-decl name='zfs_expand_proplist' mangled-name='zfs_expand_proplist' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_expand_proplist'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='e4378506' name='plp'/>
	<parameter type-id='c19b74c3' name='received'/>
	<parameter type-id='c19b74c3' name='literal'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prune_proplist' mangled-name='zfs_prune_proplist' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prune_proplist'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='ae3e8ca6' name='props'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_smb_acl_add' mangled-name='zfs_smb_acl_add' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_smb_acl_add'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='26a90f95' name='dataset'/>
	<parameter type-id='26a90f95' name='path'/>
	<parameter type-id='26a90f95' name='resource'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_smb_acl_remove' mangled-name='zfs_smb_acl_remove' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_smb_acl_remove'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='26a90f95' name='dataset'/>
	<parameter type-id='26a90f95' name='path'/>
	<parameter type-id='26a90f95' name='resource'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_smb_acl_purge' mangled-name='zfs_smb_acl_purge' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_smb_acl_purge'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='26a90f95' name='dataset'/>
	<parameter type-id='26a90f95' name='path'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_smb_acl_rename' mangled-name='zfs_smb_acl_rename' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_smb_acl_rename'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='26a90f95' name='dataset'/>
	<parameter type-id='26a90f95' name='path'/>
	<parameter type-id='26a90f95' name='oldname'/>
	<parameter type-id='26a90f95' name='newname'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_userspace' mangled-name='zfs_userspace' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_userspace'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='279fde6a' name='type'/>
	<parameter type-id='16c5f410' name='func'/>
	<parameter type-id='eaa32e2f' name='arg'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_hold' mangled-name='zfs_hold' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_hold'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='80f4b756' name='snapname'/>
	<parameter type-id='80f4b756' name='tag'/>
	<parameter type-id='c19b74c3' name='recursive'/>
	<parameter type-id='95e97e5e' name='cleanup_fd'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_hold_nvl' mangled-name='zfs_hold_nvl' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_hold_nvl'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='95e97e5e' name='cleanup_fd'/>
	<parameter type-id='5ce45b60' name='holds'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_release' mangled-name='zfs_release' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_release'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='80f4b756' name='snapname'/>
	<parameter type-id='80f4b756' name='tag'/>
	<parameter type-id='c19b74c3' name='recursive'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_get_fsacl' mangled-name='zfs_get_fsacl' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_get_fsacl'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='857bb57e' name='nvl'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_set_fsacl' mangled-name='zfs_set_fsacl' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_set_fsacl'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='c19b74c3' name='un'/>
	<parameter type-id='5ce45b60' name='nvl'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_get_holds' mangled-name='zfs_get_holds' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_get_holds'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='857bb57e' name='nvl'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zvol_volsize_to_reservation' mangled-name='zvol_volsize_to_reservation' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zvol_volsize_to_reservation'>
	<parameter type-id='4c81de99' name='zph'/>
	<parameter type-id='9c313c2d' name='volsize'/>
	<parameter type-id='5ce45b60' name='props'/>
	<return type-id='9c313c2d'/>
	</function-decl>
	<function-decl name='zfs_wait_status' mangled-name='zfs_wait_status' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_wait_status'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='3024501a' name='activity'/>
	<parameter type-id='37e3bd22' name='missing'/>
	<parameter type-id='37e3bd22' name='waited'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_error_fmt' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter is-variadic='yes'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_standard_error_fmt' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter is-variadic='yes'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_setprop_error' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='58603c44'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='26a90f95'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zprop_parse_value' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='3fa542f0'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='2e45de5d'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='7d3cd834'/>
	<parameter type-id='5d6479ae'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zprop_expand_list' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='e4378506'/>
	<parameter type-id='2e45de5d'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zcmd_write_src_nvlist' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='e4ec4540'/>
	<parameter type-id='5ce45b60'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='changelist_prefix' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='0d41d328'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='changelist_postfix' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='0d41d328'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='changelist_rename' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='0d41d328'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='changelist_remove' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='0d41d328'/>
	<parameter type-id='80f4b756'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='changelist_free' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='0d41d328'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='changelist_gather' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='9200a744'/>
	<parameter type-id='58603c44'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='0d41d328'/>
	</function-decl>
	<function-decl name='changelist_haszonedchild' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='0d41d328'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_name_valid' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='80f4b756'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-type size-in-bits='64' id='7e291ce6'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='354978ed'/>
	<parameter type-id='9c313c2d'/>
	<return type-id='95e97e5e'/>
	</function-type>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzfs/libzfs_diff.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='a84c031d' size-in-bits='448' id='6093ff7c'>
	<subrange length='56' type-id='7359adad' id='f8137894'/>
	</array-type-def>
	<typedef-decl name='pthread_t' type-id='7359adad' id='4051f5e7'/>
	<union-decl name='pthread_attr_t' size-in-bits='448' visibility='default' id='b63afacd'>
	<data-member access='public'>
	<var-decl name='__size' type-id='6093ff7c' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='__align' type-id='bd54fe1a' visibility='default'/>
	</data-member>
	</union-decl>
	<typedef-decl name='pthread_attr_t' type-id='b63afacd' id='7d8569fd'/>
	<class-decl name='differ_info' size-in-bits='9088' is-struct='yes' visibility='default' id='d41965ee'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='zhp' type-id='9200a744' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='fromsnap' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='frommnt' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='tosnap' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='tomnt' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='ds' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='dsmnt' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='tmpsnap' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='512'>
	<var-decl name='errbuf' type-id='b54ce520' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='8704'>
	<var-decl name='isclone' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='8736'>
	<var-decl name='scripted' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='8768'>
	<var-decl name='classify' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='8800'>
	<var-decl name='timestamped' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='8832'>
	<var-decl name='no_mangle' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='8896'>
	<var-decl name='shares' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='8960'>
	<var-decl name='zerr' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='8992'>
	<var-decl name='cleanupfd' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='9024'>
	<var-decl name='outputfd' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='9056'>
	<var-decl name='datafd' type-id='95e97e5e' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='differ_info_t' type-id='d41965ee' id='e8525f0e'/>
	<qualified-type-def type-id='7d8569fd' const='yes' id='e06dee2d'/>
	<pointer-type-def type-id='e06dee2d' size-in-bits='64' id='540db505'/>
	<qualified-type-def type-id='540db505' restrict='yes' id='e1815e87'/>
	<pointer-type-def type-id='e8525f0e' size-in-bits='64' id='ee78f675'/>
	<pointer-type-def type-id='4051f5e7' size-in-bits='64' id='e01b5462'/>
	<qualified-type-def type-id='e01b5462' restrict='yes' id='cc338b26'/>
	<pointer-type-def type-id='cd5d79f4' size-in-bits='64' id='5ad9edb6'/>
	<function-decl name='is_mounted' mangled-name='is_mounted' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='is_mounted'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='9b23c9ad'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='color_start' mangled-name='color_start' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='color_start'>
	<parameter type-id='80f4b756'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='color_end' mangled-name='color_end' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='color_end'>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='pthread_create' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='cc338b26'/>
	<parameter type-id='e1815e87'/>
	<parameter type-id='5ad9edb6'/>
	<parameter type-id='1b7446cd'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_join' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='4051f5e7'/>
	<parameter type-id='63e171df'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pthread_cancel' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='4051f5e7'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='fputs' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='9d26089a'/>
	<parameter type-id='e75a27e9'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pipe2' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='7292109c'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_show_diffs' mangled-name='zfs_show_diffs' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_show_diffs'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='95e97e5e' name='outfd'/>
	<parameter type-id='80f4b756' name='fromsnap'/>
	<parameter type-id='80f4b756' name='tosnap'/>
	<parameter type-id='95e97e5e' name='flags'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_asprintf' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='80f4b756'/>
	<parameter is-variadic='yes'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='zfs_validate_name' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='c19b74c3'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='find_shares_object' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='ee78f675'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-type size-in-bits='64' id='cd5d79f4'>
	<parameter type-id='eaa32e2f'/>
	<return type-id='eaa32e2f'/>
	</function-type>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzfs/libzfs_import.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='03085adc' size-in-bits='192' id='083f8d58'>
	<subrange length='3' type-id='7359adad' id='56f209d2'/>
	</array-type-def>
	<typedef-decl name='refresh_config_func_t' type-id='29f040d2' id='b7c58eaa'/>
	<typedef-decl name='pool_active_func_t' type-id='baa42fef' id='de5d1d8f'/>
	<class-decl name='pool_config_ops' size-in-bits='128' is-struct='yes' visibility='default' id='8b092c69'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='pco_refresh_config' type-id='e7c00489' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='pco_pool_active' type-id='9eadf5e0' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='pool_config_ops_t' type-id='1a21babe' id='b1e62775'/>
	<enum-decl name='pool_state' id='4871ac24'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='POOL_STATE_ACTIVE' value='0'/>
	<enumerator name='POOL_STATE_EXPORTED' value='1'/>
	<enumerator name='POOL_STATE_DESTROYED' value='2'/>
	<enumerator name='POOL_STATE_SPARE' value='3'/>
	<enumerator name='POOL_STATE_L2CACHE' value='4'/>
	<enumerator name='POOL_STATE_UNINITIALIZED' value='5'/>
	<enumerator name='POOL_STATE_UNAVAIL' value='6'/>
	<enumerator name='POOL_STATE_POTENTIALLY_ACTIVE' value='7'/>
	</enum-decl>
	<typedef-decl name='pool_state_t' type-id='4871ac24' id='084a08a3'/>
	<class-decl name='stat64' size-in-bits='1152' is-struct='yes' visibility='default' id='0bbec9cd'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='st_dev' type-id='35ed8932' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='st_ino' type-id='71288a47' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='st_nlink' type-id='80f0b9df' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='st_mode' type-id='e1c52942' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='224'>
	<var-decl name='st_uid' type-id='cc5fcceb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='st_gid' type-id='d94ec6d9' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='288'>
	<var-decl name='__pad0' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='st_rdev' type-id='35ed8932' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='st_size' type-id='79989e9c' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='st_blksize' type-id='d3f10a7f' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='512'>
	<var-decl name='st_blocks' type-id='4e711bf1' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='576'>
	<var-decl name='st_atim' type-id='a9c79a1f' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='704'>
	<var-decl name='st_mtim' type-id='a9c79a1f' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='832'>
	<var-decl name='st_ctim' type-id='a9c79a1f' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='960'>
	<var-decl name='__glibc_reserved' type-id='083f8d58' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='__dev_t' type-id='7359adad' id='35ed8932'/>
	<typedef-decl name='__ino64_t' type-id='7359adad' id='71288a47'/>
	<typedef-decl name='__mode_t' type-id='f0981eeb' id='e1c52942'/>
	<typedef-decl name='__nlink_t' type-id='7359adad' id='80f0b9df'/>
	<typedef-decl name='__blksize_t' type-id='bd54fe1a' id='d3f10a7f'/>
	<typedef-decl name='__blkcnt64_t' type-id='bd54fe1a' id='4e711bf1'/>
	<typedef-decl name='__syscall_slong_t' type-id='bd54fe1a' id='03085adc'/>
	<class-decl name='timespec' size-in-bits='128' is-struct='yes' visibility='default' id='a9c79a1f'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='tv_sec' type-id='65eda9c0' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='tv_nsec' type-id='03085adc' visibility='default'/>
	</data-member>
	</class-decl>
	<qualified-type-def type-id='8b092c69' const='yes' id='1a21babe'/>
	<pointer-type-def type-id='de5d1d8f' size-in-bits='64' id='9eadf5e0'/>
	<pointer-type-def type-id='084a08a3' size-in-bits='64' id='b9ea57b8'/>
	<pointer-type-def type-id='b7c58eaa' size-in-bits='64' id='e7c00489'/>
	<pointer-type-def type-id='0bbec9cd' size-in-bits='64' id='62f7a03d'/>
	<var-decl name='libzfs_config_ops' type-id='b1e62775' mangled-name='libzfs_config_ops' visibility='default' elf-symbol-id='libzfs_config_ops'/>
	<function-decl name='zpool_read_label' mangled-name='zpool_read_label' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_read_label'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='857bb57e'/>
	<parameter type-id='7292109c'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='pwrite64' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='724e4de6'/>
	<return type-id='79a0948f'/>
	</function-decl>
	<function-decl name='__pread64_chk' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='724e4de6'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='79a0948f'/>
	</function-decl>
	<function-decl name='fstat64' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='62f7a03d'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zcmd_write_conf_nvlist' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='e4ec4540'/>
	<parameter type-id='5ce45b60'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zpool_clear_label' mangled-name='zpool_clear_label' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_clear_label'>
	<parameter type-id='95e97e5e' name='fd'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_in_use' mangled-name='zpool_in_use' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_in_use'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='95e97e5e' name='fd'/>
	<parameter type-id='b9ea57b8' name='state'/>
	<parameter type-id='9b23c9ad' name='namestr'/>
	<parameter type-id='37e3bd22' name='inuse'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-type size-in-bits='64' id='baa42fef'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='37e3bd22'/>
	<return type-id='95e97e5e'/>
	</function-type>
	<function-type size-in-bits='64' id='29f040d2'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='5ce45b60'/>
	<return type-id='5ce45b60'/>
	</function-type>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzfs/libzfs_iter.c' language='LANG_C99'>
	<pointer-type-def type-id='b351119f' size-in-bits='64' id='716943c7'/>
	<function-decl name='avl_first' mangled-name='avl_first' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_first'>
	<parameter type-id='a3681dea'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='avl_walk' mangled-name='avl_walk' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_walk'>
	<parameter type-id='716943c7'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='make_dataset_handle_zc' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='e4ec4540'/>
	<return type-id='9200a744'/>
	</function-decl>
	<function-decl name='make_dataset_simple_handle_zc' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='9200a744'/>
	<parameter type-id='e4ec4540'/>
	<return type-id='9200a744'/>
	</function-decl>
	<function-decl name='make_bookmark_handle' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='9200a744'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='5ce45b60'/>
	<return type-id='9200a744'/>
	</function-decl>
	<function-decl name='zfs_iter_filesystems' mangled-name='zfs_iter_filesystems' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_iter_filesystems'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='d8e49ab9' name='func'/>
	<parameter type-id='eaa32e2f' name='data'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_iter_snapshots' mangled-name='zfs_iter_snapshots' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_iter_snapshots'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='c19b74c3' name='simple'/>
	<parameter type-id='d8e49ab9' name='func'/>
	<parameter type-id='eaa32e2f' name='data'/>
	<parameter type-id='9c313c2d' name='min_txg'/>
	<parameter type-id='9c313c2d' name='max_txg'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_iter_bookmarks' mangled-name='zfs_iter_bookmarks' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_iter_bookmarks'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='d8e49ab9' name='func'/>
	<parameter type-id='eaa32e2f' name='data'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_iter_snapshots_sorted' mangled-name='zfs_iter_snapshots_sorted' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_iter_snapshots_sorted'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='d8e49ab9' name='callback'/>
	<parameter type-id='eaa32e2f' name='data'/>
	<parameter type-id='9c313c2d' name='min_txg'/>
	<parameter type-id='9c313c2d' name='max_txg'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_iter_snapshots_sorted_v2' mangled-name='zfs_iter_snapshots_sorted_v2' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_iter_snapshots_sorted_v2'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='95e97e5e' name='flags'/>
	<parameter type-id='d8e49ab9' name='callback'/>
	<parameter type-id='eaa32e2f' name='data'/>
	<parameter type-id='9c313c2d' name='min_txg'/>
	<parameter type-id='9c313c2d' name='max_txg'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_iter_snapspec' mangled-name='zfs_iter_snapspec' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_iter_snapspec'>
	<parameter type-id='9200a744' name='fs_zhp'/>
	<parameter type-id='80f4b756' name='spec_orig'/>
	<parameter type-id='d8e49ab9' name='func'/>
	<parameter type-id='eaa32e2f' name='arg'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_iter_snapspec_v2' mangled-name='zfs_iter_snapspec_v2' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_iter_snapspec_v2'>
	<parameter type-id='9200a744' name='fs_zhp'/>
	<parameter type-id='95e97e5e' name='flags'/>
	<parameter type-id='80f4b756' name='spec_orig'/>
	<parameter type-id='d8e49ab9' name='func'/>
	<parameter type-id='eaa32e2f' name='arg'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_iter_children' mangled-name='zfs_iter_children' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_iter_children'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='d8e49ab9' name='func'/>
	<parameter type-id='eaa32e2f' name='data'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_iter_dependents' mangled-name='zfs_iter_dependents' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_iter_dependents'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='c19b74c3' name='allowrecursion'/>
	<parameter type-id='d8e49ab9' name='func'/>
	<parameter type-id='eaa32e2f' name='data'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzfs/libzfs_mount.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='6028cbfe' size-in-bits='256' id='b39b9aa7'>
	<subrange length='4' type-id='7359adad' id='16fe7105'/>
	</array-type-def>
	<class-decl name='__dirstream' is-struct='yes' visibility='default' is-declaration-only='yes' id='20cd73f2'/>
	<class-decl name='tpool' size-in-bits='2496' is-struct='yes' visibility='default' id='88d1b7f9'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='tp_forw' type-id='9cf59a50' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='tp_back' type-id='9cf59a50' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='tp_mutex' type-id='7a6844eb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='tp_busycv' type-id='62fab762' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='832'>
	<var-decl name='tp_workcv' type-id='62fab762' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1216'>
	<var-decl name='tp_waitcv' type-id='62fab762' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1600'>
	<var-decl name='tp_active' type-id='ad33e5e7' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1664'>
	<var-decl name='tp_head' type-id='f32b30e4' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1728'>
	<var-decl name='tp_tail' type-id='f32b30e4' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1792'>
	<var-decl name='tp_attr' type-id='7d8569fd' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2240'>
	<var-decl name='tp_flags' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2272'>
	<var-decl name='tp_linger' type-id='3502e3ff' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2304'>
	<var-decl name='tp_njobs' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2336'>
	<var-decl name='tp_minimum' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2368'>
	<var-decl name='tp_maximum' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2400'>
	<var-decl name='tp_current' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2432'>
	<var-decl name='tp_idle' type-id='95e97e5e' visibility='default'/>
	</data-member>
	</class-decl>
	<array-type-def dimensions='1' type-id='95e97e5e' size-in-bits='64' id='e4266c7e'>
	<subrange length='2' type-id='7359adad' id='52efc4ef'/>
	</array-type-def>
	<class-decl name='get_all_cb' size-in-bits='192' is-struct='yes' visibility='default' id='803dac95'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='cb_handles' type-id='4507922a' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='cb_alloc' type-id='b59d7dce' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='cb_used' type-id='b59d7dce' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='get_all_cb_t' type-id='803dac95' id='9b293607'/>
	<typedef-decl name='tpool_t' type-id='88d1b7f9' id='b1bbf10d'/>
	<typedef-decl name='DIR' type-id='20cd73f2' id='54a5d683'/>
	<typedef-decl name='mode_t' type-id='e1c52942' id='d50d396c'/>
	<typedef-decl name='__compar_fn_t' type-id='585e1de9' id='aba7edd8'/>
	<class-decl name='dirent64' size-in-bits='2240' is-struct='yes' visibility='default' id='5725d813'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='d_ino' type-id='71288a47' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='d_off' type-id='724e4de6' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='d_reclen' type-id='8efea9e5' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='144'>
	<var-decl name='d_type' type-id='002ac4a6' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='152'>
	<var-decl name='d_name' type-id='d1617432' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='statfs64' size-in-bits='960' is-struct='yes' visibility='default' id='a2a6be1a'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='f_type' type-id='6028cbfe' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='f_bsize' type-id='6028cbfe' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='f_blocks' type-id='95fe1a02' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='f_bfree' type-id='95fe1a02' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='f_bavail' type-id='95fe1a02' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='f_files' type-id='0c3a4dde' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='f_ffree' type-id='0c3a4dde' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='f_fsid' type-id='0f35d263' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='512'>
	<var-decl name='f_namelen' type-id='6028cbfe' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='576'>
	<var-decl name='f_frsize' type-id='6028cbfe' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='640'>
	<var-decl name='f_flags' type-id='6028cbfe' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='704'>
	<var-decl name='f_spare' type-id='b39b9aa7' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='stat' size-in-bits='1152' is-struct='yes' visibility='default' id='aafc373f'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='st_dev' type-id='35ed8932' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='st_ino' type-id='e43e523d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='st_nlink' type-id='80f0b9df' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='st_mode' type-id='e1c52942' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='224'>
	<var-decl name='st_uid' type-id='cc5fcceb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='st_gid' type-id='d94ec6d9' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='288'>
	<var-decl name='__pad0' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='st_rdev' type-id='35ed8932' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='st_size' type-id='79989e9c' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='st_blksize' type-id='d3f10a7f' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='512'>
	<var-decl name='st_blocks' type-id='dbc43803' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='576'>
	<var-decl name='st_atim' type-id='a9c79a1f' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='704'>
	<var-decl name='st_mtim' type-id='a9c79a1f' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='832'>
	<var-decl name='st_ctim' type-id='a9c79a1f' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='960'>
	<var-decl name='__glibc_reserved' type-id='083f8d58' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='__ino_t' type-id='7359adad' id='e43e523d'/>
	<class-decl name='__fsid_t' size-in-bits='64' is-struct='yes' naming-typedef-id='0f35d263' visibility='default' id='ea35c84a'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='__val' type-id='e4266c7e' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='__fsid_t' type-id='ea35c84a' id='0f35d263'/>
	<typedef-decl name='__blkcnt_t' type-id='bd54fe1a' id='dbc43803'/>
	<typedef-decl name='__fsblkcnt64_t' type-id='7359adad' id='95fe1a02'/>
	<typedef-decl name='__fsfilcnt64_t' type-id='7359adad' id='0c3a4dde'/>
	<typedef-decl name='__fsword_t' type-id='bd54fe1a' id='6028cbfe'/>
	<pointer-type-def type-id='54a5d683' size-in-bits='64' id='f09217ba'/>
	<pointer-type-def type-id='5725d813' size-in-bits='64' id='07b96073'/>
	<pointer-type-def type-id='9b293607' size-in-bits='64' id='77bf1784'/>
	<pointer-type-def type-id='7d8569fd' size-in-bits='64' id='7347a39e'/>
	<pointer-type-def type-id='aafc373f' size-in-bits='64' id='4330df87'/>
	<qualified-type-def type-id='4330df87' restrict='yes' id='73665405'/>
	<pointer-type-def type-id='a2a6be1a' size-in-bits='64' id='7fd094c8'/>
	<pointer-type-def type-id='b1bbf10d' size-in-bits='64' id='9cf59a50'/>
	<pointer-type-def type-id='c5c76c9c' size-in-bits='64' id='b7f9d8e6'/>
	<pointer-type-def type-id='9200a744' size-in-bits='64' id='4507922a'/>
	<class-decl name='__dirstream' is-struct='yes' visibility='default' is-declaration-only='yes' id='20cd73f2'/>
	<function-decl name='zpool_disable_datasets_os' mangled-name='zpool_disable_datasets_os' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_disable_datasets_os'>
	<parameter type-id='4c81de99'/>
	<parameter type-id='c19b74c3'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zpool_disable_volume_os' mangled-name='zpool_disable_volume_os' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_disable_volume_os'>
	<parameter type-id='80f4b756'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='tpool_create' mangled-name='tpool_create' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='tpool_create'>
	<parameter type-id='3502e3ff'/>
	<parameter type-id='3502e3ff'/>
	<parameter type-id='3502e3ff'/>
	<parameter type-id='7347a39e'/>
	<return type-id='9cf59a50'/>
	</function-decl>
	<function-decl name='tpool_dispatch' mangled-name='tpool_dispatch' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='tpool_dispatch'>
	<parameter type-id='9cf59a50'/>
	<parameter type-id='b7f9d8e6'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='tpool_destroy' mangled-name='tpool_destroy' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='tpool_destroy'>
	<parameter type-id='9cf59a50'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='tpool_wait' mangled-name='tpool_wait' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='tpool_wait'>
	<parameter type-id='9cf59a50'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='mkdirp' mangled-name='mkdirp' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='mkdirp'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='d50d396c'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='sa_errorstr' mangled-name='sa_errorstr' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='sa_errorstr'>
	<parameter type-id='95e97e5e'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='sa_enable_share' mangled-name='sa_enable_share' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='sa_enable_share'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='9155d4b5'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='sa_disable_share' mangled-name='sa_disable_share' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='sa_disable_share'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='9155d4b5'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='sa_is_shared' mangled-name='sa_is_shared' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='sa_is_shared'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='9155d4b5'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='sa_truncate_shares' mangled-name='sa_truncate_shares' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='sa_truncate_shares'>
	<parameter type-id='9155d4b5'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fdopendir' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<return type-id='f09217ba'/>
	</function-decl>
	<function-decl name='closedir' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='f09217ba'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='readdir64' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='f09217ba'/>
	<return type-id='07b96073'/>
	</function-decl>
	<function-decl name='qsort' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='aba7edd8'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='rmdir' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='__openat_too_many_args' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='__openat_missing_mode' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='statfs64' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='7fd094c8'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_realloc' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='changelist_unshare' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='0d41d328'/>
	<parameter type-id='4567bbc9'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='do_mount' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='9200a744'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='do_unmount' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='9200a744'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_mount_at' mangled-name='zfs_mount_at' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_mount_at'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='80f4b756' name='options'/>
	<parameter type-id='95e97e5e' name='flags'/>
	<parameter type-id='80f4b756' name='mountpoint'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_unmountall' mangled-name='zfs_unmountall' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_unmountall'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='95e97e5e' name='flags'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_truncate_shares' mangled-name='zfs_truncate_shares' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_truncate_shares'>
	<parameter type-id='4567bbc9' name='proto'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_unshareall' mangled-name='zfs_unshareall' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_unshareall'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='4567bbc9' name='proto'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='libzfs_add_handle' mangled-name='libzfs_add_handle' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_add_handle'>
	<parameter type-id='77bf1784' name='cbp'/>
	<parameter type-id='9200a744' name='zhp'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_foreach_mountpoint' mangled-name='zfs_foreach_mountpoint' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_foreach_mountpoint'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='4507922a' name='handles'/>
	<parameter type-id='b59d7dce' name='num_handles'/>
	<parameter type-id='d8e49ab9' name='func'/>
	<parameter type-id='eaa32e2f' name='data'/>
	<parameter type-id='c19b74c3' name='parallel'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zpool_enable_datasets' mangled-name='zpool_enable_datasets' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_enable_datasets'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='mntopts'/>
	<parameter type-id='95e97e5e' name='flags'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_disable_datasets' mangled-name='zpool_disable_datasets' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_disable_datasets'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='c19b74c3' name='force'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzfs/libzfs_pool.c' language='LANG_C99'>
	<type-decl name='long long unsigned int' size-in-bits='64' id='3a47d82b'/>
	<class-decl name='splitflags' size-in-bits='64' is-struct='yes' visibility='default' id='dc01bf52'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='dryrun' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1'>
	<var-decl name='import' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='name_flags' type-id='95e97e5e' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='splitflags_t' type-id='dc01bf52' id='325c1e34'/>
	<class-decl name='trimflags' size-in-bits='192' is-struct='yes' visibility='default' id='8ef58008'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='fullpool' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='secure' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='wait' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='rate' type-id='9c313c2d' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='trimflags_t' type-id='8ef58008' id='a093cbb8'/>
	<enum-decl name='zpool_status_t' naming-typedef-id='d3dd6294' id='5e770b40'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='ZPOOL_STATUS_CORRUPT_CACHE' value='0'/>
	<enumerator name='ZPOOL_STATUS_MISSING_DEV_R' value='1'/>
	<enumerator name='ZPOOL_STATUS_MISSING_DEV_NR' value='2'/>
	<enumerator name='ZPOOL_STATUS_CORRUPT_LABEL_R' value='3'/>
	<enumerator name='ZPOOL_STATUS_CORRUPT_LABEL_NR' value='4'/>
	<enumerator name='ZPOOL_STATUS_BAD_GUID_SUM' value='5'/>
	<enumerator name='ZPOOL_STATUS_CORRUPT_POOL' value='6'/>
	<enumerator name='ZPOOL_STATUS_CORRUPT_DATA' value='7'/>
	<enumerator name='ZPOOL_STATUS_FAILING_DEV' value='8'/>
	<enumerator name='ZPOOL_STATUS_VERSION_NEWER' value='9'/>
	<enumerator name='ZPOOL_STATUS_HOSTID_MISMATCH' value='10'/>
	<enumerator name='ZPOOL_STATUS_HOSTID_ACTIVE' value='11'/>
	<enumerator name='ZPOOL_STATUS_HOSTID_REQUIRED' value='12'/>
	<enumerator name='ZPOOL_STATUS_IO_FAILURE_WAIT' value='13'/>
	<enumerator name='ZPOOL_STATUS_IO_FAILURE_CONTINUE' value='14'/>
	<enumerator name='ZPOOL_STATUS_IO_FAILURE_MMP' value='15'/>
	<enumerator name='ZPOOL_STATUS_BAD_LOG' value='16'/>
	<enumerator name='ZPOOL_STATUS_ERRATA' value='17'/>
	<enumerator name='ZPOOL_STATUS_UNSUP_FEAT_READ' value='18'/>
	<enumerator name='ZPOOL_STATUS_UNSUP_FEAT_WRITE' value='19'/>
	<enumerator name='ZPOOL_STATUS_FAULTED_DEV_R' value='20'/>
	<enumerator name='ZPOOL_STATUS_FAULTED_DEV_NR' value='21'/>
	<enumerator name='ZPOOL_STATUS_VERSION_OLDER' value='22'/>
	<enumerator name='ZPOOL_STATUS_FEAT_DISABLED' value='23'/>
	<enumerator name='ZPOOL_STATUS_RESILVERING' value='24'/>
	<enumerator name='ZPOOL_STATUS_OFFLINE_DEV' value='25'/>
	<enumerator name='ZPOOL_STATUS_REMOVED_DEV' value='26'/>
	<enumerator name='ZPOOL_STATUS_REBUILDING' value='27'/>
	<enumerator name='ZPOOL_STATUS_REBUILD_SCRUB' value='28'/>
	<enumerator name='ZPOOL_STATUS_NON_NATIVE_ASHIFT' value='29'/>
	<enumerator name='ZPOOL_STATUS_COMPATIBILITY_ERR' value='30'/>
	<enumerator name='ZPOOL_STATUS_INCOMPATIBLE_FEAT' value='31'/>
	<enumerator name='ZPOOL_STATUS_OK' value='32'/>
	</enum-decl>
	<typedef-decl name='zpool_status_t' type-id='5e770b40' id='d3dd6294'/>
	<enum-decl name='zpool_compat_status_t' naming-typedef-id='901b78d1' id='20676925'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='ZPOOL_COMPATIBILITY_OK' value='0'/>
	<enumerator name='ZPOOL_COMPATIBILITY_WARNTOKEN' value='1'/>
	<enumerator name='ZPOOL_COMPATIBILITY_BADTOKEN' value='2'/>
	<enumerator name='ZPOOL_COMPATIBILITY_BADFILE' value='3'/>
	<enumerator name='ZPOOL_COMPATIBILITY_NOFILES' value='4'/>
	</enum-decl>
	<typedef-decl name='zpool_compat_status_t' type-id='20676925' id='901b78d1'/>
	<enum-decl name='vdev_prop_t' naming-typedef-id='5aa5c90c' id='1573bec8'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='VDEV_PROP_INVAL' value='-1'/>
	<enumerator name='VDEV_PROP_USERPROP' value='-1'/>
	<enumerator name='VDEV_PROP_NAME' value='0'/>
	<enumerator name='VDEV_PROP_CAPACITY' value='1'/>
	<enumerator name='VDEV_PROP_STATE' value='2'/>
	<enumerator name='VDEV_PROP_GUID' value='3'/>
	<enumerator name='VDEV_PROP_ASIZE' value='4'/>
	<enumerator name='VDEV_PROP_PSIZE' value='5'/>
	<enumerator name='VDEV_PROP_ASHIFT' value='6'/>
	<enumerator name='VDEV_PROP_SIZE' value='7'/>
	<enumerator name='VDEV_PROP_FREE' value='8'/>
	<enumerator name='VDEV_PROP_ALLOCATED' value='9'/>
	<enumerator name='VDEV_PROP_COMMENT' value='10'/>
	<enumerator name='VDEV_PROP_EXPANDSZ' value='11'/>
	<enumerator name='VDEV_PROP_FRAGMENTATION' value='12'/>
	<enumerator name='VDEV_PROP_BOOTSIZE' value='13'/>
	<enumerator name='VDEV_PROP_PARITY' value='14'/>
	<enumerator name='VDEV_PROP_PATH' value='15'/>
	<enumerator name='VDEV_PROP_DEVID' value='16'/>
	<enumerator name='VDEV_PROP_PHYS_PATH' value='17'/>
	<enumerator name='VDEV_PROP_ENC_PATH' value='18'/>
	<enumerator name='VDEV_PROP_FRU' value='19'/>
	<enumerator name='VDEV_PROP_PARENT' value='20'/>
	<enumerator name='VDEV_PROP_CHILDREN' value='21'/>
	<enumerator name='VDEV_PROP_NUMCHILDREN' value='22'/>
	<enumerator name='VDEV_PROP_READ_ERRORS' value='23'/>
	<enumerator name='VDEV_PROP_WRITE_ERRORS' value='24'/>
	<enumerator name='VDEV_PROP_CHECKSUM_ERRORS' value='25'/>
	<enumerator name='VDEV_PROP_INITIALIZE_ERRORS' value='26'/>
	<enumerator name='VDEV_PROP_OPS_NULL' value='27'/>
	<enumerator name='VDEV_PROP_OPS_READ' value='28'/>
	<enumerator name='VDEV_PROP_OPS_WRITE' value='29'/>
	<enumerator name='VDEV_PROP_OPS_FREE' value='30'/>
	<enumerator name='VDEV_PROP_OPS_CLAIM' value='31'/>
	<enumerator name='VDEV_PROP_OPS_TRIM' value='32'/>
	<enumerator name='VDEV_PROP_BYTES_NULL' value='33'/>
	<enumerator name='VDEV_PROP_BYTES_READ' value='34'/>
	<enumerator name='VDEV_PROP_BYTES_WRITE' value='35'/>
	<enumerator name='VDEV_PROP_BYTES_FREE' value='36'/>
	<enumerator name='VDEV_PROP_BYTES_CLAIM' value='37'/>
	<enumerator name='VDEV_PROP_BYTES_TRIM' value='38'/>
	<enumerator name='VDEV_PROP_REMOVING' value='39'/>
	<enumerator name='VDEV_PROP_ALLOCATING' value='40'/>
	<enumerator name='VDEV_PROP_FAILFAST' value='41'/>
	<enumerator name='VDEV_PROP_CHECKSUM_N' value='42'/>
	<enumerator name='VDEV_PROP_CHECKSUM_T' value='43'/>
	<enumerator name='VDEV_PROP_IO_N' value='44'/>
	<enumerator name='VDEV_PROP_IO_T' value='45'/>
	- <enumerator name='VDEV_NUM_PROPS' value='46'/>
	+ <enumerator name='VDEV_PROP_RAIDZ_EXPANDING' value='46'/>
	+ <enumerator name='VDEV_PROP_SLOW_IO_N' value='47'/>
	+ <enumerator name='VDEV_PROP_SLOW_IO_T' value='48'/>
	+ <enumerator name='VDEV_NUM_PROPS' value='49'/>
	</enum-decl>
	<typedef-decl name='vdev_prop_t' type-id='1573bec8' id='5aa5c90c'/>
	<class-decl name='zpool_load_policy' size-in-bits='256' is-struct='yes' visibility='default' id='2f65b36f'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='zlp_rewind' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='zlp_maxmeta' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='zlp_maxdata' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='zlp_txg' type-id='9c313c2d' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zpool_load_policy_t' type-id='2f65b36f' id='d11b7617'/>
	<enum-decl name='vdev_state' id='21566197'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='VDEV_STATE_UNKNOWN' value='0'/>
	<enumerator name='VDEV_STATE_CLOSED' value='1'/>
	<enumerator name='VDEV_STATE_OFFLINE' value='2'/>
	<enumerator name='VDEV_STATE_REMOVED' value='3'/>
	<enumerator name='VDEV_STATE_CANT_OPEN' value='4'/>
	<enumerator name='VDEV_STATE_FAULTED' value='5'/>
	<enumerator name='VDEV_STATE_DEGRADED' value='6'/>
	<enumerator name='VDEV_STATE_HEALTHY' value='7'/>
	</enum-decl>
	<typedef-decl name='vdev_state_t' type-id='21566197' id='35acf840'/>
	<enum-decl name='vdev_aux' id='7f5bcca4'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='VDEV_AUX_NONE' value='0'/>
	<enumerator name='VDEV_AUX_OPEN_FAILED' value='1'/>
	<enumerator name='VDEV_AUX_CORRUPT_DATA' value='2'/>
	<enumerator name='VDEV_AUX_NO_REPLICAS' value='3'/>
	<enumerator name='VDEV_AUX_BAD_GUID_SUM' value='4'/>
	<enumerator name='VDEV_AUX_TOO_SMALL' value='5'/>
	<enumerator name='VDEV_AUX_BAD_LABEL' value='6'/>
	<enumerator name='VDEV_AUX_VERSION_NEWER' value='7'/>
	<enumerator name='VDEV_AUX_VERSION_OLDER' value='8'/>
	<enumerator name='VDEV_AUX_UNSUP_FEAT' value='9'/>
	<enumerator name='VDEV_AUX_SPARED' value='10'/>
	<enumerator name='VDEV_AUX_ERR_EXCEEDED' value='11'/>
	<enumerator name='VDEV_AUX_IO_FAILURE' value='12'/>
	<enumerator name='VDEV_AUX_BAD_LOG' value='13'/>
	<enumerator name='VDEV_AUX_EXTERNAL' value='14'/>
	<enumerator name='VDEV_AUX_SPLIT_POOL' value='15'/>
	<enumerator name='VDEV_AUX_BAD_ASHIFT' value='16'/>
	<enumerator name='VDEV_AUX_EXTERNAL_PERSIST' value='17'/>
	<enumerator name='VDEV_AUX_ACTIVE' value='18'/>
	<enumerator name='VDEV_AUX_CHILDREN_OFFLINE' value='19'/>
	<enumerator name='VDEV_AUX_ASHIFT_TOO_BIG' value='20'/>
	</enum-decl>
	<typedef-decl name='vdev_aux_t' type-id='7f5bcca4' id='9d774e0b'/>
	<enum-decl name='pool_scan_func' id='1b092565'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='POOL_SCAN_NONE' value='0'/>
	<enumerator name='POOL_SCAN_SCRUB' value='1'/>
	<enumerator name='POOL_SCAN_RESILVER' value='2'/>
	<enumerator name='POOL_SCAN_ERRORSCRUB' value='3'/>
	<enumerator name='POOL_SCAN_FUNCS' value='4'/>
	</enum-decl>
	<typedef-decl name='pool_scan_func_t' type-id='1b092565' id='7313fbe2'/>
	<enum-decl name='pool_scrub_cmd' id='a1474cbd'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='POOL_SCRUB_NORMAL' value='0'/>
	<enumerator name='POOL_SCRUB_PAUSE' value='1'/>
	<enumerator name='POOL_SCRUB_FLAGS_END' value='2'/>
	</enum-decl>
	<typedef-decl name='pool_scrub_cmd_t' type-id='a1474cbd' id='b51cf3c2'/>
	<enum-decl name='zpool_errata' id='d9abbf54'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='ZPOOL_ERRATA_NONE' value='0'/>
	<enumerator name='ZPOOL_ERRATA_ZOL_2094_SCRUB' value='1'/>
	<enumerator name='ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY' value='2'/>
	<enumerator name='ZPOOL_ERRATA_ZOL_6845_ENCRYPTION' value='3'/>
	<enumerator name='ZPOOL_ERRATA_ZOL_8308_ENCRYPTION' value='4'/>
	</enum-decl>
	<typedef-decl name='zpool_errata_t' type-id='d9abbf54' id='688c495b'/>
	<enum-decl name='pool_initialize_func' id='5c246ad4'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='POOL_INITIALIZE_START' value='0'/>
	<enumerator name='POOL_INITIALIZE_CANCEL' value='1'/>
	<enumerator name='POOL_INITIALIZE_SUSPEND' value='2'/>
	<enumerator name='POOL_INITIALIZE_UNINIT' value='3'/>
	<enumerator name='POOL_INITIALIZE_FUNCS' value='4'/>
	</enum-decl>
	<typedef-decl name='pool_initialize_func_t' type-id='5c246ad4' id='7063e1ab'/>
	<enum-decl name='pool_trim_func' id='54ed608a'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='POOL_TRIM_START' value='0'/>
	<enumerator name='POOL_TRIM_CANCEL' value='1'/>
	<enumerator name='POOL_TRIM_SUSPEND' value='2'/>
	<enumerator name='POOL_TRIM_FUNCS' value='3'/>
	</enum-decl>
	<typedef-decl name='pool_trim_func_t' type-id='54ed608a' id='b1146b8d'/>
	<enum-decl name='zpool_wait_activity_t' naming-typedef-id='73446457' id='849338e3'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='ZPOOL_WAIT_CKPT_DISCARD' value='0'/>
	<enumerator name='ZPOOL_WAIT_FREE' value='1'/>
	<enumerator name='ZPOOL_WAIT_INITIALIZE' value='2'/>
	<enumerator name='ZPOOL_WAIT_REPLACE' value='3'/>
	<enumerator name='ZPOOL_WAIT_REMOVE' value='4'/>
	<enumerator name='ZPOOL_WAIT_RESILVER' value='5'/>
	<enumerator name='ZPOOL_WAIT_SCRUB' value='6'/>
	<enumerator name='ZPOOL_WAIT_TRIM' value='7'/>
	<enumerator name='ZPOOL_WAIT_NUM_ACTIVITIES' value='8'/>
	</enum-decl>
	<typedef-decl name='zpool_wait_activity_t' type-id='849338e3' id='73446457'/>
	<enum-decl name='spa_feature' id='33ecb627'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='SPA_FEATURE_NONE' value='-1'/>
	<enumerator name='SPA_FEATURE_ASYNC_DESTROY' value='0'/>
	<enumerator name='SPA_FEATURE_EMPTY_BPOBJ' value='1'/>
	<enumerator name='SPA_FEATURE_LZ4_COMPRESS' value='2'/>
	<enumerator name='SPA_FEATURE_MULTI_VDEV_CRASH_DUMP' value='3'/>
	<enumerator name='SPA_FEATURE_SPACEMAP_HISTOGRAM' value='4'/>
	<enumerator name='SPA_FEATURE_ENABLED_TXG' value='5'/>
	<enumerator name='SPA_FEATURE_HOLE_BIRTH' value='6'/>
	<enumerator name='SPA_FEATURE_EXTENSIBLE_DATASET' value='7'/>
	<enumerator name='SPA_FEATURE_EMBEDDED_DATA' value='8'/>
	<enumerator name='SPA_FEATURE_BOOKMARKS' value='9'/>
	<enumerator name='SPA_FEATURE_FS_SS_LIMIT' value='10'/>
	<enumerator name='SPA_FEATURE_LARGE_BLOCKS' value='11'/>
	<enumerator name='SPA_FEATURE_LARGE_DNODE' value='12'/>
	<enumerator name='SPA_FEATURE_SHA512' value='13'/>
	<enumerator name='SPA_FEATURE_SKEIN' value='14'/>
	<enumerator name='SPA_FEATURE_EDONR' value='15'/>
	<enumerator name='SPA_FEATURE_USEROBJ_ACCOUNTING' value='16'/>
	<enumerator name='SPA_FEATURE_ENCRYPTION' value='17'/>
	<enumerator name='SPA_FEATURE_PROJECT_QUOTA' value='18'/>
	<enumerator name='SPA_FEATURE_DEVICE_REMOVAL' value='19'/>
	<enumerator name='SPA_FEATURE_OBSOLETE_COUNTS' value='20'/>
	<enumerator name='SPA_FEATURE_POOL_CHECKPOINT' value='21'/>
	<enumerator name='SPA_FEATURE_SPACEMAP_V2' value='22'/>
	<enumerator name='SPA_FEATURE_ALLOCATION_CLASSES' value='23'/>
	<enumerator name='SPA_FEATURE_RESILVER_DEFER' value='24'/>
	<enumerator name='SPA_FEATURE_BOOKMARK_V2' value='25'/>
	<enumerator name='SPA_FEATURE_REDACTION_BOOKMARKS' value='26'/>
	<enumerator name='SPA_FEATURE_REDACTED_DATASETS' value='27'/>
	<enumerator name='SPA_FEATURE_BOOKMARK_WRITTEN' value='28'/>
	<enumerator name='SPA_FEATURE_LOG_SPACEMAP' value='29'/>
	<enumerator name='SPA_FEATURE_LIVELIST' value='30'/>
	<enumerator name='SPA_FEATURE_DEVICE_REBUILD' value='31'/>
	<enumerator name='SPA_FEATURE_ZSTD_COMPRESS' value='32'/>
	<enumerator name='SPA_FEATURE_DRAID' value='33'/>
	<enumerator name='SPA_FEATURE_ZILSAXATTR' value='34'/>
	<enumerator name='SPA_FEATURE_HEAD_ERRLOG' value='35'/>
	<enumerator name='SPA_FEATURE_BLAKE3' value='36'/>
	<enumerator name='SPA_FEATURE_BLOCK_CLONING' value='37'/>
	<enumerator name='SPA_FEATURE_AVZ_V2' value='38'/>
	<enumerator name='SPA_FEATURES' value='39'/>
	</enum-decl>
	<typedef-decl name='spa_feature_t' type-id='33ecb627' id='d6618c78'/>
	<qualified-type-def type-id='22cce67b' const='yes' id='d2816df0'/>
	<pointer-type-def type-id='d2816df0' size-in-bits='64' id='3bbfee2e'/>
	<qualified-type-def type-id='b96825af' const='yes' id='2b61797f'/>
	<pointer-type-def type-id='2b61797f' size-in-bits='64' id='9f7200cf'/>
	<pointer-type-def type-id='d6618c78' size-in-bits='64' id='a8425263'/>
	<qualified-type-def type-id='62f7a03d' restrict='yes' id='f1cadedf'/>
	<pointer-type-def type-id='a093cbb8' size-in-bits='64' id='b13f38c3'/>
	<pointer-type-def type-id='35acf840' size-in-bits='64' id='17f3480d'/>
	<pointer-type-def type-id='688c495b' size-in-bits='64' id='cec6f2e4'/>
	<pointer-type-def type-id='d11b7617' size-in-bits='64' id='23432aaa'/>
	<function-decl name='zpool_get_handle' mangled-name='zpool_get_handle' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_handle'>
	<parameter type-id='4c81de99'/>
	<return type-id='b0382bb3'/>
	</function-decl>
	<function-decl name='zpool_prop_to_name' mangled-name='zpool_prop_to_name' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_to_name'>
	<parameter type-id='5d0c23fb'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='vdev_prop_to_name' mangled-name='vdev_prop_to_name' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='vdev_prop_to_name'>
	<parameter type-id='5aa5c90c'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='vdev_prop_user' mangled-name='vdev_prop_user' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='vdev_prop_user'>
	<parameter type-id='80f4b756'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zpool_get_status' mangled-name='zpool_get_status' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_status'>
	<parameter type-id='4c81de99'/>
	<parameter type-id='7d3cd834'/>
	<parameter type-id='cec6f2e4'/>
	<return type-id='d3dd6294'/>
	</function-decl>
	<function-decl name='zpool_prop_default_string' mangled-name='zpool_prop_default_string' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_default_string'>
	<parameter type-id='5d0c23fb'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zpool_prop_default_numeric' mangled-name='zpool_prop_default_numeric' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_default_numeric'>
	<parameter type-id='5d0c23fb'/>
	<return type-id='9c313c2d'/>
	</function-decl>
	<function-decl name='libzfs_envvar_is_set' mangled-name='libzfs_envvar_is_set' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_envvar_is_set'>
	<parameter type-id='80f4b756'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='lzc_initialize' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='7063e1ab'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_trim' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='b1146b8d'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_sync' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_reopen' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='c19b74c3'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_pool_checkpoint' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_pool_checkpoint_discard' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_wait' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='73446457'/>
	<parameter type-id='37e3bd22'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_wait_tag' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='73446457'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='37e3bd22'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_set_bootenv' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='22cce67b'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_get_bootenv' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_get_vdev_prop' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_set_vdev_prop' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_resolve_shortname' mangled-name='zfs_resolve_shortname' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_resolve_shortname'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='26a90f95'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_strip_partition' mangled-name='zfs_strip_partition' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_strip_partition'>
	<parameter type-id='80f4b756'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='zfs_strip_path' mangled-name='zfs_strip_path' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_strip_path'>
	<parameter type-id='80f4b756'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zfs_strcmp_pathname' mangled-name='zfs_strcmp_pathname' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_strcmp_pathname'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_history_unpack' mangled-name='zpool_history_unpack' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_history_unpack'>
	<parameter type-id='26a90f95'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='5d6479ae'/>
	<parameter type-id='75be733c'/>
	<parameter type-id='4dd26a40'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_basename' mangled-name='zfs_basename' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_basename'>
	<parameter type-id='80f4b756'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zpool_name_to_prop' mangled-name='zpool_name_to_prop' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_name_to_prop'>
	<parameter type-id='80f4b756'/>
	<return type-id='5d0c23fb'/>
	</function-decl>
	<function-decl name='zpool_prop_readonly' mangled-name='zpool_prop_readonly' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_readonly'>
	<parameter type-id='5d0c23fb'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zpool_prop_setonce' mangled-name='zpool_prop_setonce' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_setonce'>
	<parameter type-id='5d0c23fb'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zpool_prop_feature' mangled-name='zpool_prop_feature' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_feature'>
	<parameter type-id='80f4b756'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zpool_prop_index_to_string' mangled-name='zpool_prop_index_to_string' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_index_to_string'>
	<parameter type-id='5d0c23fb'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='7d3cd834'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='vdev_name_to_prop' mangled-name='vdev_name_to_prop' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='vdev_name_to_prop'>
	<parameter type-id='80f4b756'/>
	<return type-id='5aa5c90c'/>
	</function-decl>
	<function-decl name='vdev_prop_default_string' mangled-name='vdev_prop_default_string' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='vdev_prop_default_string'>
	<parameter type-id='5aa5c90c'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='vdev_prop_default_numeric' mangled-name='vdev_prop_default_numeric' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='vdev_prop_default_numeric'>
	<parameter type-id='5aa5c90c'/>
	<return type-id='9c313c2d'/>
	</function-decl>
	<function-decl name='vdev_prop_readonly' mangled-name='vdev_prop_readonly' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='vdev_prop_readonly'>
	<parameter type-id='5aa5c90c'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='vdev_prop_index_to_string' mangled-name='vdev_prop_index_to_string' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='vdev_prop_index_to_string'>
	<parameter type-id='5aa5c90c'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='7d3cd834'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_prop_vdev' mangled-name='zpool_prop_vdev' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_vdev'>
	<parameter type-id='80f4b756'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='nvlist_add_nvpair' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='3fa542f0'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_add_uint8_array' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='9f7200cf'/>
	<parameter type-id='3502e3ff'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_add_nvlist_array' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='3bbfee2e'/>
	<parameter type-id='3502e3ff'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvpair_value_nvlist' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='3fa542f0'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='fnvlist_add_boolean_value' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='c19b74c3'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fnvlist_add_int64' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='9da381c4'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fnvlist_add_nvlist_array' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='3bbfee2e'/>
	<parameter type-id='3502e3ff'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fnvlist_lookup_uint64_array' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='4dd26a40'/>
	<return type-id='5d6479ae'/>
	</function-decl>
	<function-decl name='fnvpair_value_int64' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='dace003f'/>
	<return type-id='9da381c4'/>
	</function-decl>
	<function-decl name='fnvpair_value_string' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='dace003f'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zfeature_is_supported' mangled-name='zfeature_is_supported' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfeature_is_supported'>
	<parameter type-id='80f4b756'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfeature_lookup_guid' mangled-name='zfeature_lookup_guid' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfeature_lookup_guid'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='a8425263'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfeature_lookup_name' mangled-name='zfeature_lookup_name' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfeature_lookup_name'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='a8425263'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_get_load_policy' mangled-name='zpool_get_load_policy' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_load_policy'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='23432aaa'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='pool_namecheck' mangled-name='pool_namecheck' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='pool_namecheck'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='053457bd'/>
	<parameter type-id='26a90f95'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_prop_get_type' mangled-name='zpool_prop_get_type' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_get_type'>
	<parameter type-id='5d0c23fb'/>
	<return type-id='31429eff'/>
	</function-decl>
	<function-decl name='vdev_prop_get_type' mangled-name='vdev_prop_get_type' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='vdev_prop_get_type'>
	<parameter type-id='5aa5c90c'/>
	<return type-id='31429eff'/>
	</function-decl>
	<function-decl name='get_system_hostid' mangled-name='get_system_hostid' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='get_system_hostid'>
	<return type-id='7359adad'/>
	</function-decl>
	<function-decl name='strtoull' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='9d26089a'/>
	<parameter type-id='8c85230f'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='3a47d82b'/>
	</function-decl>
	<function-decl name='memcmp' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='strtok_r' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='266fe297'/>
	<parameter type-id='9d26089a'/>
	<parameter type-id='8c85230f'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='__realpath_chk' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='9d26089a'/>
	<parameter type-id='266fe297'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='munmap' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='stat64' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='9d26089a'/>
	<parameter type-id='f1cadedf'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_standard_error' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_standard_error_fmt' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter is-variadic='yes'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_relabel_disk' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_props_refresh' mangled-name='zpool_props_refresh' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_props_refresh'>
	<parameter type-id='4c81de99' name='zhp'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_state_to_name' mangled-name='zpool_state_to_name' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_state_to_name'>
	<parameter type-id='35acf840' name='state'/>
	<parameter type-id='9d774e0b' name='aux'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zpool_pool_state_to_name' mangled-name='zpool_pool_state_to_name' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_pool_state_to_name'>
	<parameter type-id='084a08a3' name='state'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zpool_get_state_str' mangled-name='zpool_get_state_str' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_state_str'>
	<parameter type-id='4c81de99' name='zhp'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zpool_get_userprop' mangled-name='zpool_get_userprop' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_userprop'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='propname'/>
	<parameter type-id='26a90f95' name='buf'/>
	<parameter type-id='b59d7dce' name='len'/>
	<parameter type-id='debc6aa3' name='srctype'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_set_prop' mangled-name='zpool_set_prop' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_set_prop'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='propname'/>
	<parameter type-id='80f4b756' name='propval'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_expand_proplist' mangled-name='zpool_expand_proplist' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_expand_proplist'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='e4378506' name='plp'/>
	<parameter type-id='2e45de5d' name='type'/>
	<parameter type-id='c19b74c3' name='literal'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='vdev_expand_proplist' mangled-name='vdev_expand_proplist' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='vdev_expand_proplist'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='vdevname'/>
	<parameter type-id='e4378506' name='plp'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_get_state' mangled-name='zpool_get_state' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_state'>
	<parameter type-id='4c81de99' name='zhp'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_is_draid_spare' mangled-name='zpool_is_draid_spare' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_is_draid_spare'>
	<parameter type-id='80f4b756' name='name'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zpool_create' mangled-name='zpool_create' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_create'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='80f4b756' name='pool'/>
	<parameter type-id='5ce45b60' name='nvroot'/>
	<parameter type-id='5ce45b60' name='props'/>
	<parameter type-id='5ce45b60' name='fsprops'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_destroy' mangled-name='zpool_destroy' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_destroy'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='log_str'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_checkpoint' mangled-name='zpool_checkpoint' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_checkpoint'>
	<parameter type-id='4c81de99' name='zhp'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_discard_checkpoint' mangled-name='zpool_discard_checkpoint' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_discard_checkpoint'>
	<parameter type-id='4c81de99' name='zhp'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_add' mangled-name='zpool_add' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_add'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='5ce45b60' name='nvroot'/>
	+ <parameter type-id='c19b74c3' name='ashift_check'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_export' mangled-name='zpool_export' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_export'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='c19b74c3' name='force'/>
	<parameter type-id='80f4b756' name='log_str'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_export_force' mangled-name='zpool_export_force' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_export_force'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='log_str'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_explain_recover' mangled-name='zpool_explain_recover' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_explain_recover'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='80f4b756' name='name'/>
	<parameter type-id='95e97e5e' name='reason'/>
	<parameter type-id='5ce45b60' name='config'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zpool_import' mangled-name='zpool_import' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_import'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='5ce45b60' name='config'/>
	<parameter type-id='80f4b756' name='newname'/>
	<parameter type-id='26a90f95' name='altroot'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_print_unsup_feat' mangled-name='zpool_print_unsup_feat' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_print_unsup_feat'>
	<parameter type-id='5ce45b60' name='config'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zpool_import_props' mangled-name='zpool_import_props' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_import_props'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='5ce45b60' name='config'/>
	<parameter type-id='80f4b756' name='newname'/>
	<parameter type-id='5ce45b60' name='props'/>
	<parameter type-id='95e97e5e' name='flags'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_initialize' mangled-name='zpool_initialize' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_initialize'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='7063e1ab' name='cmd_type'/>
	<parameter type-id='5ce45b60' name='vds'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_initialize_wait' mangled-name='zpool_initialize_wait' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_initialize_wait'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='7063e1ab' name='cmd_type'/>
	<parameter type-id='5ce45b60' name='vds'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_trim' mangled-name='zpool_trim' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_trim'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='b1146b8d' name='cmd_type'/>
	<parameter type-id='5ce45b60' name='vds'/>
	<parameter type-id='b13f38c3' name='trim_flags'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_scan' mangled-name='zpool_scan' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_scan'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='7313fbe2' name='func'/>
	<parameter type-id='b51cf3c2' name='cmd'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_find_vdev_by_physpath' mangled-name='zpool_find_vdev_by_physpath' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_find_vdev_by_physpath'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='ppath'/>
	<parameter type-id='37e3bd22' name='avail_spare'/>
	<parameter type-id='37e3bd22' name='l2cache'/>
	<parameter type-id='37e3bd22' name='log'/>
	<return type-id='5ce45b60'/>
	</function-decl>
	<function-decl name='zpool_find_vdev' mangled-name='zpool_find_vdev' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_find_vdev'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='37e3bd22' name='avail_spare'/>
	<parameter type-id='37e3bd22' name='l2cache'/>
	<parameter type-id='37e3bd22' name='log'/>
	<return type-id='5ce45b60'/>
	</function-decl>
	<function-decl name='zpool_vdev_path_to_guid' mangled-name='zpool_vdev_path_to_guid' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_path_to_guid'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='path'/>
	<return type-id='9c313c2d'/>
	</function-decl>
	<function-decl name='zpool_vdev_online' mangled-name='zpool_vdev_online' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_online'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='95e97e5e' name='flags'/>
	<parameter type-id='17f3480d' name='newstate'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_vdev_offline' mangled-name='zpool_vdev_offline' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_offline'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='c19b74c3' name='istmp'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_vdev_remove_wanted' mangled-name='zpool_vdev_remove_wanted' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_remove_wanted'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='path'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_vdev_fault' mangled-name='zpool_vdev_fault' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_fault'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='9c313c2d' name='guid'/>
	<parameter type-id='9d774e0b' name='aux'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_vdev_degrade' mangled-name='zpool_vdev_degrade' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_degrade'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='9c313c2d' name='guid'/>
	<parameter type-id='9d774e0b' name='aux'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_vdev_set_removed_state' mangled-name='zpool_vdev_set_removed_state' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_set_removed_state'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='9c313c2d' name='guid'/>
	<parameter type-id='9d774e0b' name='aux'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_vdev_attach' mangled-name='zpool_vdev_attach' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_attach'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='old_disk'/>
	<parameter type-id='80f4b756' name='new_disk'/>
	<parameter type-id='5ce45b60' name='nvroot'/>
	<parameter type-id='95e97e5e' name='replacing'/>
	<parameter type-id='c19b74c3' name='rebuild'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_vdev_detach' mangled-name='zpool_vdev_detach' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_detach'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='path'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_vdev_split' mangled-name='zpool_vdev_split' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_split'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='26a90f95' name='newname'/>
	<parameter type-id='857bb57e' name='newroot'/>
	<parameter type-id='5ce45b60' name='props'/>
	<parameter type-id='325c1e34' name='flags'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_vdev_remove' mangled-name='zpool_vdev_remove' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_remove'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='path'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_vdev_remove_cancel' mangled-name='zpool_vdev_remove_cancel' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_remove_cancel'>
	<parameter type-id='4c81de99' name='zhp'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_vdev_indirect_size' mangled-name='zpool_vdev_indirect_size' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_indirect_size'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='5d6479ae' name='sizep'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_clear' mangled-name='zpool_clear' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_clear'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='5ce45b60' name='rewindnvl'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_vdev_clear' mangled-name='zpool_vdev_clear' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_clear'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='9c313c2d' name='guid'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_reguid' mangled-name='zpool_reguid' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_reguid'>
	<parameter type-id='4c81de99' name='zhp'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_reopen_one' mangled-name='zpool_reopen_one' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_reopen_one'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='eaa32e2f' name='data'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_sync_one' mangled-name='zpool_sync_one' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_sync_one'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='eaa32e2f' name='data'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_vdev_name' mangled-name='zpool_vdev_name' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_name'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='5ce45b60' name='nv'/>
	<parameter type-id='95e97e5e' name='name_flags'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='zpool_get_errlog' mangled-name='zpool_get_errlog' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_errlog'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='857bb57e' name='nverrlistp'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_upgrade' mangled-name='zpool_upgrade' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_upgrade'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='9c313c2d' name='new_version'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_save_arguments' mangled-name='zfs_save_arguments' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_save_arguments'>
	<parameter type-id='95e97e5e' name='argc'/>
	<parameter type-id='9b23c9ad' name='argv'/>
	<parameter type-id='26a90f95' name='string'/>
	<parameter type-id='95e97e5e' name='len'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zpool_log_history' mangled-name='zpool_log_history' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_log_history'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='80f4b756' name='message'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_get_history' mangled-name='zpool_get_history' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_history'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='857bb57e' name='nvhisp'/>
	<parameter type-id='5d6479ae' name='off'/>
	<parameter type-id='37e3bd22' name='eof'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_events_next' mangled-name='zpool_events_next' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_events_next'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='857bb57e' name='nvp'/>
	<parameter type-id='7292109c' name='dropped'/>
	<parameter type-id='f0981eeb' name='flags'/>
	<parameter type-id='95e97e5e' name='zevent_fd'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_events_clear' mangled-name='zpool_events_clear' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_events_clear'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='7292109c' name='count'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_events_seek' mangled-name='zpool_events_seek' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_events_seek'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='9c313c2d' name='eid'/>
	<parameter type-id='95e97e5e' name='zevent_fd'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_obj_to_path' mangled-name='zpool_obj_to_path' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_obj_to_path'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='9c313c2d' name='dsobj'/>
	<parameter type-id='9c313c2d' name='obj'/>
	<parameter type-id='26a90f95' name='pathname'/>
	<parameter type-id='b59d7dce' name='len'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zpool_obj_to_path_ds' mangled-name='zpool_obj_to_path_ds' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_obj_to_path_ds'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='9c313c2d' name='dsobj'/>
	<parameter type-id='9c313c2d' name='obj'/>
	<parameter type-id='26a90f95' name='pathname'/>
	<parameter type-id='b59d7dce' name='len'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zpool_wait' mangled-name='zpool_wait' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_wait'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='73446457' name='activity'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_wait_status' mangled-name='zpool_wait_status' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_wait_status'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='73446457' name='activity'/>
	<parameter type-id='37e3bd22' name='missing'/>
	<parameter type-id='37e3bd22' name='waited'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_set_bootenv' mangled-name='zpool_set_bootenv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_set_bootenv'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='22cce67b' name='envmap'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_get_bootenv' mangled-name='zpool_get_bootenv' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_bootenv'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='857bb57e' name='nvlp'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_load_compat' mangled-name='zpool_load_compat' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_load_compat'>
	<parameter type-id='80f4b756' name='compat'/>
	<parameter type-id='37e3bd22' name='features'/>
	<parameter type-id='26a90f95' name='report'/>
	<parameter type-id='b59d7dce' name='rlen'/>
	<return type-id='901b78d1'/>
	</function-decl>
	<function-decl name='zpool_get_vdev_prop_value' mangled-name='zpool_get_vdev_prop_value' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_vdev_prop_value'>
	<parameter type-id='5ce45b60' name='nvprop'/>
	<parameter type-id='5aa5c90c' name='prop'/>
	<parameter type-id='26a90f95' name='prop_name'/>
	<parameter type-id='26a90f95' name='buf'/>
	<parameter type-id='b59d7dce' name='len'/>
	<parameter type-id='debc6aa3' name='srctype'/>
	<parameter type-id='c19b74c3' name='literal'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_get_vdev_prop' mangled-name='zpool_get_vdev_prop' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_vdev_prop'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='vdevname'/>
	<parameter type-id='5aa5c90c' name='prop'/>
	<parameter type-id='26a90f95' name='prop_name'/>
	<parameter type-id='26a90f95' name='buf'/>
	<parameter type-id='b59d7dce' name='len'/>
	<parameter type-id='debc6aa3' name='srctype'/>
	<parameter type-id='c19b74c3' name='literal'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_get_all_vdev_props' mangled-name='zpool_get_all_vdev_props' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_all_vdev_props'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='vdevname'/>
	<parameter type-id='857bb57e' name='outnvl'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_set_vdev_prop' mangled-name='zpool_set_vdev_prop' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_set_vdev_prop'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='vdevname'/>
	<parameter type-id='80f4b756' name='propname'/>
	<parameter type-id='80f4b756' name='propval'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzfs/libzfs_sendrecv.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='9c313c2d' size-in-bits='2176' id='8c2bcad1'>
	<subrange length='34' type-id='7359adad' id='6a6a7e00'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='9c313c2d' size-in-bits='256' id='85c64d26'>
	<subrange length='4' type-id='7359adad' id='16fe7105'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='b96825af' size-in-bits='96' id='fa8ef949'>
	<subrange length='12' type-id='7359adad' id='84827bdc'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='b96825af' size-in-bits='128' id='fa9986a5'>
	<subrange length='16' type-id='7359adad' id='848d0938'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='b96825af' size-in-bits='40' id='0f4ddd0b'>
	<subrange length='5' type-id='7359adad' id='53010e10'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='b96825af' size-in-bits='48' id='0f562bd0'>
	<subrange length='6' type-id='7359adad' id='52fa524b'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='b96825af' size-in-bits='64' id='13339fda'>
	<subrange length='8' type-id='7359adad' id='56e0c0b1'/>
	</array-type-def>
	<class-decl name='sendflags' size-in-bits='576' is-struct='yes' visibility='default' id='f6aa15be'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='verbosity' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='replicate' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='skipmissing' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='96'>
	<var-decl name='doall' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='fromorigin' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='160'>
	<var-decl name='pad' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='props' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='224'>
	<var-decl name='dryrun' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='parsable' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='288'>
	<var-decl name='progress' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='progressastitle' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='352'>
	<var-decl name='largeblock' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='embed_data' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='416'>
	<var-decl name='compress' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='raw' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='480'>
	<var-decl name='backup' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='512'>
	<var-decl name='holds' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='544'>
	<var-decl name='saved' type-id='c19b74c3' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='sendflags_t' type-id='f6aa15be' id='945467e6'/>
	<typedef-decl name='snapfilter_cb_t' type-id='d2a5e211' id='3d3ffb69'/>
	<class-decl name='recvflags' size-in-bits='448' is-struct='yes' visibility='default' id='34a384dc'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='verbose' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='isprefix' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='istail' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='96'>
	<var-decl name='dryrun' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='force' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='160'>
	<var-decl name='canmountoff' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='resumable' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='224'>
	<var-decl name='byteswap' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='nomount' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='288'>
	<var-decl name='holds' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='skipholds' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='352'>
	<var-decl name='domount' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='forceunmount' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='416'>
	<var-decl name='heal' type-id='c19b74c3' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='recvflags_t' type-id='34a384dc' id='9e59d1d4'/>
	<enum-decl name='lzc_send_flags' id='bfbd3c8e'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='LZC_SEND_FLAG_EMBED_DATA' value='1'/>
	<enumerator name='LZC_SEND_FLAG_LARGE_BLOCK' value='2'/>
	<enumerator name='LZC_SEND_FLAG_COMPRESS' value='4'/>
	<enumerator name='LZC_SEND_FLAG_RAW' value='8'/>
	<enumerator name='LZC_SEND_FLAG_SAVED' value='16'/>
	</enum-decl>
	- <class-decl name='ddt_key' size-in-bits='320' is-struct='yes' visibility='default' id='e0a4a1cb'>
	+ <class-decl name='ddt_key_t' size-in-bits='320' is-struct='yes' naming-typedef-id='67f6d2cf' visibility='default' id='5fae1718'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='ddk_cksum' type-id='39730d0b' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='ddk_prop' type-id='9c313c2d' visibility='default'/>
	</data-member>
	</class-decl>
	- <typedef-decl name='ddt_key_t' type-id='e0a4a1cb' id='67f6d2cf'/>
	+ <typedef-decl name='ddt_key_t' type-id='5fae1718' id='67f6d2cf'/>
	<enum-decl name='dmu_object_type' id='04b3b0b9'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='DMU_OT_NONE' value='0'/>
	<enumerator name='DMU_OT_OBJECT_DIRECTORY' value='1'/>
	<enumerator name='DMU_OT_OBJECT_ARRAY' value='2'/>
	<enumerator name='DMU_OT_PACKED_NVLIST' value='3'/>
	<enumerator name='DMU_OT_PACKED_NVLIST_SIZE' value='4'/>
	<enumerator name='DMU_OT_BPOBJ' value='5'/>
	<enumerator name='DMU_OT_BPOBJ_HDR' value='6'/>
	<enumerator name='DMU_OT_SPACE_MAP_HEADER' value='7'/>
	<enumerator name='DMU_OT_SPACE_MAP' value='8'/>
	<enumerator name='DMU_OT_INTENT_LOG' value='9'/>
	<enumerator name='DMU_OT_DNODE' value='10'/>
	<enumerator name='DMU_OT_OBJSET' value='11'/>
	<enumerator name='DMU_OT_DSL_DIR' value='12'/>
	<enumerator name='DMU_OT_DSL_DIR_CHILD_MAP' value='13'/>
	<enumerator name='DMU_OT_DSL_DS_SNAP_MAP' value='14'/>
	<enumerator name='DMU_OT_DSL_PROPS' value='15'/>
	<enumerator name='DMU_OT_DSL_DATASET' value='16'/>
	<enumerator name='DMU_OT_ZNODE' value='17'/>
	<enumerator name='DMU_OT_OLDACL' value='18'/>
	<enumerator name='DMU_OT_PLAIN_FILE_CONTENTS' value='19'/>
	<enumerator name='DMU_OT_DIRECTORY_CONTENTS' value='20'/>
	<enumerator name='DMU_OT_MASTER_NODE' value='21'/>
	<enumerator name='DMU_OT_UNLINKED_SET' value='22'/>
	<enumerator name='DMU_OT_ZVOL' value='23'/>
	<enumerator name='DMU_OT_ZVOL_PROP' value='24'/>
	<enumerator name='DMU_OT_PLAIN_OTHER' value='25'/>
	<enumerator name='DMU_OT_UINT64_OTHER' value='26'/>
	<enumerator name='DMU_OT_ZAP_OTHER' value='27'/>
	<enumerator name='DMU_OT_ERROR_LOG' value='28'/>
	<enumerator name='DMU_OT_SPA_HISTORY' value='29'/>
	<enumerator name='DMU_OT_SPA_HISTORY_OFFSETS' value='30'/>
	<enumerator name='DMU_OT_POOL_PROPS' value='31'/>
	<enumerator name='DMU_OT_DSL_PERMS' value='32'/>
	<enumerator name='DMU_OT_ACL' value='33'/>
	<enumerator name='DMU_OT_SYSACL' value='34'/>
	<enumerator name='DMU_OT_FUID' value='35'/>
	<enumerator name='DMU_OT_FUID_SIZE' value='36'/>
	<enumerator name='DMU_OT_NEXT_CLONES' value='37'/>
	<enumerator name='DMU_OT_SCAN_QUEUE' value='38'/>
	<enumerator name='DMU_OT_USERGROUP_USED' value='39'/>
	<enumerator name='DMU_OT_USERGROUP_QUOTA' value='40'/>
	<enumerator name='DMU_OT_USERREFS' value='41'/>
	<enumerator name='DMU_OT_DDT_ZAP' value='42'/>
	<enumerator name='DMU_OT_DDT_STATS' value='43'/>
	<enumerator name='DMU_OT_SA' value='44'/>
	<enumerator name='DMU_OT_SA_MASTER_NODE' value='45'/>
	<enumerator name='DMU_OT_SA_ATTR_REGISTRATION' value='46'/>
	<enumerator name='DMU_OT_SA_ATTR_LAYOUTS' value='47'/>
	<enumerator name='DMU_OT_SCAN_XLATE' value='48'/>
	<enumerator name='DMU_OT_DEDUP' value='49'/>
	<enumerator name='DMU_OT_DEADLIST' value='50'/>
	<enumerator name='DMU_OT_DEADLIST_HDR' value='51'/>
	<enumerator name='DMU_OT_DSL_CLONES' value='52'/>
	<enumerator name='DMU_OT_BPOBJ_SUBOBJ' value='53'/>
	<enumerator name='DMU_OT_NUMTYPES' value='54'/>
	<enumerator name='DMU_OTN_UINT8_DATA' value='128'/>
	<enumerator name='DMU_OTN_UINT8_METADATA' value='192'/>
	<enumerator name='DMU_OTN_UINT16_DATA' value='129'/>
	<enumerator name='DMU_OTN_UINT16_METADATA' value='193'/>
	<enumerator name='DMU_OTN_UINT32_DATA' value='130'/>
	<enumerator name='DMU_OTN_UINT32_METADATA' value='194'/>
	<enumerator name='DMU_OTN_UINT64_DATA' value='131'/>
	<enumerator name='DMU_OTN_UINT64_METADATA' value='195'/>
	<enumerator name='DMU_OTN_ZAP_DATA' value='132'/>
	<enumerator name='DMU_OTN_ZAP_METADATA' value='196'/>
	<enumerator name='DMU_OTN_UINT8_ENC_DATA' value='160'/>
	<enumerator name='DMU_OTN_UINT8_ENC_METADATA' value='224'/>
	<enumerator name='DMU_OTN_UINT16_ENC_DATA' value='161'/>
	<enumerator name='DMU_OTN_UINT16_ENC_METADATA' value='225'/>
	<enumerator name='DMU_OTN_UINT32_ENC_DATA' value='162'/>
	<enumerator name='DMU_OTN_UINT32_ENC_METADATA' value='226'/>
	<enumerator name='DMU_OTN_UINT64_ENC_DATA' value='163'/>
	<enumerator name='DMU_OTN_UINT64_ENC_METADATA' value='227'/>
	<enumerator name='DMU_OTN_ZAP_ENC_DATA' value='164'/>
	<enumerator name='DMU_OTN_ZAP_ENC_METADATA' value='228'/>
	</enum-decl>
	<typedef-decl name='dmu_object_type_t' type-id='04b3b0b9' id='5c9d8906'/>
	<class-decl name='zio_cksum' size-in-bits='256' is-struct='yes' visibility='default' id='1d53e28b'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='zc_word' type-id='85c64d26' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zio_cksum_t' type-id='1d53e28b' id='39730d0b'/>
	<class-decl name='dmu_replay_record' size-in-bits='2496' is-struct='yes' visibility='default' id='781a52d7'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='drr_type' type-id='08f5ca17' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='drr_payloadlen' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='drr_u' type-id='ac5ab598' visibility='default'/>
	</data-member>
	</class-decl>
	<enum-decl name='__anonymous_enum__' is-anonymous='yes' id='08f5ca17'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='DRR_BEGIN' value='0'/>
	<enumerator name='DRR_OBJECT' value='1'/>
	<enumerator name='DRR_FREEOBJECTS' value='2'/>
	<enumerator name='DRR_WRITE' value='3'/>
	<enumerator name='DRR_FREE' value='4'/>
	<enumerator name='DRR_END' value='5'/>
	<enumerator name='DRR_WRITE_BYREF' value='6'/>
	<enumerator name='DRR_SPILL' value='7'/>
	<enumerator name='DRR_WRITE_EMBEDDED' value='8'/>
	<enumerator name='DRR_OBJECT_RANGE' value='9'/>
	<enumerator name='DRR_REDACT' value='10'/>
	<enumerator name='DRR_NUMTYPES' value='11'/>
	</enum-decl>
	<union-decl name='__anonymous_union__' size-in-bits='2432' is-anonymous='yes' visibility='default' id='ac5ab598'>
	<data-member access='public'>
	<var-decl name='drr_begin' type-id='09fcdc01' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='drr_end' type-id='6ee25631' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='drr_object' type-id='f9ad530b' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='drr_freeobjects' type-id='a27d958e' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='drr_write' type-id='4cc69e4b' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='drr_free' type-id='c836cfd2' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='drr_write_byref' type-id='e511cdce' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='drr_spill' type-id='1e69a80a' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='drr_write_embedded' type-id='98b1345e' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='drr_object_range' type-id='aba1f9e1' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='drr_redact' type-id='50389039' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='drr_checksum' type-id='a5fe3647' visibility='default'/>
	</data-member>
	</union-decl>
	<class-decl name='drr_end' size-in-bits='320' is-struct='yes' visibility='default' id='6ee25631'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='drr_checksum' type-id='39730d0b' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='drr_toguid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='drr_object' size-in-bits='448' is-struct='yes' visibility='default' id='f9ad530b'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='drr_object' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='drr_type' type-id='5c9d8906' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='96'>
	<var-decl name='drr_bonustype' type-id='5c9d8906' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='drr_blksz' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='160'>
	<var-decl name='drr_bonuslen' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='drr_checksumtype' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='200'>
	<var-decl name='drr_compress' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='208'>
	<var-decl name='drr_dn_slots' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='216'>
	<var-decl name='drr_flags' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='224'>
	<var-decl name='drr_raw_bonuslen' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='drr_toguid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='drr_indblkshift' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='328'>
	<var-decl name='drr_nlevels' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='336'>
	<var-decl name='drr_nblkptr' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='344'>
	<var-decl name='drr_pad' type-id='0f4ddd0b' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='drr_maxblkid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='drr_freeobjects' size-in-bits='192' is-struct='yes' visibility='default' id='a27d958e'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='drr_firstobj' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='drr_numobjs' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='drr_toguid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='drr_write' size-in-bits='1088' is-struct='yes' visibility='default' id='4cc69e4b'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='drr_object' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='drr_type' type-id='5c9d8906' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='96'>
	<var-decl name='drr_pad' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='drr_offset' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='drr_logical_size' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='drr_toguid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='drr_checksumtype' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='328'>
	<var-decl name='drr_flags' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='336'>
	<var-decl name='drr_compressiontype' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='344'>
	<var-decl name='drr_pad2' type-id='0f4ddd0b' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='drr_key' type-id='67f6d2cf' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='704'>
	<var-decl name='drr_compressed_size' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='768'>
	<var-decl name='drr_salt' type-id='13339fda' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='832'>
	<var-decl name='drr_iv' type-id='fa8ef949' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='928'>
	<var-decl name='drr_mac' type-id='fa9986a5' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='drr_free' size-in-bits='256' is-struct='yes' visibility='default' id='c836cfd2'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='drr_object' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='drr_offset' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='drr_length' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='drr_toguid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='drr_write_byref' size-in-bits='832' is-struct='yes' visibility='default' id='e511cdce'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='drr_object' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='drr_offset' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='drr_length' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='drr_toguid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='drr_refguid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='drr_refobject' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='drr_refoffset' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='drr_checksumtype' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='456'>
	<var-decl name='drr_flags' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='464'>
	<var-decl name='drr_pad2' type-id='0f562bd0' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='512'>
	<var-decl name='drr_key' type-id='67f6d2cf' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='drr_spill' size-in-bits='640' is-struct='yes' visibility='default' id='1e69a80a'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='drr_object' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='drr_length' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='drr_toguid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='drr_flags' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='200'>
	<var-decl name='drr_compressiontype' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='208'>
	<var-decl name='drr_pad' type-id='0f562bd0' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='drr_compressed_size' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='drr_salt' type-id='13339fda' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='drr_iv' type-id='fa8ef949' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='480'>
	<var-decl name='drr_mac' type-id='fa9986a5' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='608'>
	<var-decl name='drr_type' type-id='5c9d8906' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='drr_write_embedded' size-in-bits='384' is-struct='yes' visibility='default' id='98b1345e'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='drr_object' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='drr_offset' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='drr_length' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='drr_toguid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='drr_compression' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='264'>
	<var-decl name='drr_etype' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='272'>
	<var-decl name='drr_pad' type-id='0f562bd0' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='drr_lsize' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='352'>
	<var-decl name='drr_psize' type-id='8f92235e' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='drr_object_range' size-in-bits='512' is-struct='yes' visibility='default' id='aba1f9e1'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='drr_firstobj' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='drr_numslots' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='drr_toguid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='drr_salt' type-id='13339fda' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='drr_iv' type-id='fa8ef949' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='352'>
	<var-decl name='drr_mac' type-id='fa9986a5' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='480'>
	<var-decl name='drr_flags' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='488'>
	<var-decl name='drr_pad' type-id='d3490169' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='drr_redact' size-in-bits='256' is-struct='yes' visibility='default' id='50389039'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='drr_object' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='drr_offset' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='drr_length' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='drr_toguid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='drr_checksum' size-in-bits='2432' is-struct='yes' visibility='default' id='a5fe3647'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='drr_pad' type-id='8c2bcad1' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='2176'>
	<var-decl name='drr_checksum' type-id='39730d0b' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='__clockid_t' type-id='95e97e5e' id='08f9a87a'/>
	<typedef-decl name='clockid_t' type-id='08f9a87a' id='a1c3b834'/>
	<typedef-decl name='Byte' type-id='002ac4a6' id='efb9ba06'/>
	<typedef-decl name='uLong' type-id='7359adad' id='5bbcce85'/>
	<typedef-decl name='Bytef' type-id='efb9ba06' id='c1606520'/>
	<typedef-decl name='uLongf' type-id='5bbcce85' id='4d39af59'/>
	<pointer-type-def type-id='c1606520' size-in-bits='64' id='4c667223'/>
	<qualified-type-def type-id='c1606520' const='yes' id='a6124a50'/>
	<pointer-type-def type-id='a6124a50' size-in-bits='64' id='e8cb3e0e'/>
	<qualified-type-def type-id='781a52d7' const='yes' id='413ab2b8'/>
	<pointer-type-def type-id='413ab2b8' size-in-bits='64' id='41671bd6'/>
	<pointer-type-def type-id='c70fa2e8' size-in-bits='64' id='2e711a2a'/>
	<pointer-type-def type-id='3ff5601b' size-in-bits='64' id='4aafb922'/>
	<pointer-type-def type-id='9e59d1d4' size-in-bits='64' id='4ea84b4f'/>
	<pointer-type-def type-id='945467e6' size-in-bits='64' id='8def7735'/>
	<pointer-type-def type-id='3d3ffb69' size-in-bits='64' id='72a26210'/>
	<pointer-type-def type-id='c9d12d66' size-in-bits='64' id='b2eb2c3f'/>
	<pointer-type-def type-id='a9c79a1f' size-in-bits='64' id='3d83ba87'/>
	<pointer-type-def type-id='4d39af59' size-in-bits='64' id='60db3356'/>
	<pointer-type-def type-id='39730d0b' size-in-bits='64' id='c24fc2ee'/>
	<function-decl name='nvlist_print' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='822cd80b'/>
	<parameter type-id='5ce45b60'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_get_pool_handle' mangled-name='zfs_get_pool_handle' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_get_pool_handle'>
	<parameter type-id='fcd57163'/>
	<return type-id='4c81de99'/>
	</function-decl>
	<function-decl name='lzc_send_wrapper' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='2e711a2a'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_send_redacted' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='bfbd3c8e'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_send_resume_redacted' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='bfbd3c8e'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_receive_with_cmdprops' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='ae3e8ca6'/>
	<parameter type-id='3502e3ff'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='41671bd6'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='5d6479ae'/>
	<parameter type-id='5d6479ae'/>
	<parameter type-id='5d6479ae'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_receive_with_heal' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='ae3e8ca6'/>
	<parameter type-id='3502e3ff'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='41671bd6'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='5d6479ae'/>
	<parameter type-id='5d6479ae'/>
	<parameter type-id='5d6479ae'/>
	<parameter type-id='857bb57e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_send_space' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='bfbd3c8e'/>
	<parameter type-id='5d6479ae'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_send_space_resume_redacted' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='bfbd3c8e'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='5d6479ae'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='lzc_rename' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_setproctitle' mangled-name='zfs_setproctitle' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_setproctitle'>
	<parameter type-id='80f4b756'/>
	<parameter is-variadic='yes'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='avl_insert' mangled-name='avl_insert' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_insert'>
	<parameter type-id='a3681dea'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='fba6cb51'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='nvlist_add_boolean' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvlist_lookup_boolean' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='22cce67b'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='nvpair_value_int32' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='dace003f'/>
	<parameter type-id='4aafb922'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='fnvlist_size' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<return type-id='b59d7dce'/>
	</function-decl>
	<function-decl name='fnvlist_merge' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='5ce45b60'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fnvlist_remove' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='80f4b756'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fnvlist_lookup_boolean_value' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='22cce67b'/>
	<parameter type-id='80f4b756'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='fletcher_4_native_varsize' mangled-name='fletcher_4_native_varsize' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='fletcher_4_native_varsize'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='c24fc2ee'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fletcher_4_incremental_native' mangled-name='fletcher_4_incremental_native' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='fletcher_4_incremental_native'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='fletcher_4_incremental_byteswap' mangled-name='fletcher_4_incremental_byteswap' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='fletcher_4_incremental_byteswap'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='perror' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='strndup' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='time' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b2eb2c3f'/>
	<return type-id='c9d12d66'/>
	</function-decl>
	<function-decl name='clock_gettime' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='a1c3b834'/>
	<parameter type-id='3d83ba87'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='write' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='79a0948f'/>
	</function-decl>
	<function-decl name='sleep' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='f0981eeb'/>
	<return type-id='f0981eeb'/>
	</function-decl>
	<function-decl name='uncompress' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='4c667223'/>
	<parameter type-id='60db3356'/>
	<parameter type-id='e8cb3e0e'/>
	<parameter type-id='5bbcce85'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='create_parents' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='26a90f95'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_send_progress' mangled-name='zfs_send_progress' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_send_progress'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='95e97e5e' name='fd'/>
	<parameter type-id='5d6479ae' name='bytes_written'/>
	<parameter type-id='5d6479ae' name='blocks_visited'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_send_resume_token_to_nvlist' mangled-name='zfs_send_resume_token_to_nvlist' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_send_resume_token_to_nvlist'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='80f4b756' name='token'/>
	<return type-id='5ce45b60'/>
	</function-decl>
	<function-decl name='zfs_send_resume' mangled-name='zfs_send_resume' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_send_resume'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='8def7735' name='flags'/>
	<parameter type-id='95e97e5e' name='outfd'/>
	<parameter type-id='80f4b756' name='resume_token'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_send_saved' mangled-name='zfs_send_saved' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_send_saved'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='8def7735' name='flags'/>
	<parameter type-id='95e97e5e' name='outfd'/>
	<parameter type-id='80f4b756' name='resume_token'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_send' mangled-name='zfs_send' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_send'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='80f4b756' name='fromsnap'/>
	<parameter type-id='80f4b756' name='tosnap'/>
	<parameter type-id='8def7735' name='flags'/>
	<parameter type-id='95e97e5e' name='outfd'/>
	<parameter type-id='72a26210' name='filter_func'/>
	<parameter type-id='eaa32e2f' name='cb_arg'/>
	<parameter type-id='857bb57e' name='debugnvp'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_send_one' mangled-name='zfs_send_one' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_send_one'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='80f4b756' name='from'/>
	<parameter type-id='95e97e5e' name='fd'/>
	<parameter type-id='8def7735' name='flags'/>
	<parameter type-id='80f4b756' name='redactbook'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_receive' mangled-name='zfs_receive' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_receive'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='80f4b756' name='tosnap'/>
	<parameter type-id='5ce45b60' name='props'/>
	<parameter type-id='4ea84b4f' name='flags'/>
	<parameter type-id='95e97e5e' name='infd'/>
	<parameter type-id='a3681dea' name='stream_avl'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-type size-in-bits='64' id='c70fa2e8'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-type>
	<function-type size-in-bits='64' id='d2a5e211'>
	<parameter type-id='9200a744'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='c19b74c3'/>
	</function-type>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzfs/libzfs_status.c' language='LANG_C99'>
	<function-decl name='zpool_import_status' mangled-name='zpool_import_status' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_import_status'>
	<parameter type-id='5ce45b60' name='config'/>
	<parameter type-id='7d3cd834' name='msgid'/>
	<parameter type-id='cec6f2e4' name='errata'/>
	<return type-id='d3dd6294'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzfs/libzfs_util.c' language='LANG_C99'>
	<class-decl name='__va_list_tag' size-in-bits='192' is-struct='yes' visibility='default' id='d5027220'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='gp_offset' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='fp_offset' type-id='f0981eeb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='overflow_arg_area' type-id='eaa32e2f' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='reg_save_area' type-id='eaa32e2f' visibility='default'/>
	</data-member>
	</class-decl>
	<type-decl name='double' size-in-bits='64' id='a0eb0f08'/>
	<array-type-def dimensions='1' type-id='95e97e5e' size-in-bits='192' id='e41bdf22'>
	<subrange length='6' type-id='7359adad' id='52fa524b'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='19cefcee' size-in-bits='160' alignment-in-bits='32' id='3fcf57d2'>
	<subrange length='5' type-id='7359adad' id='53010e10'/>
	</array-type-def>
	<enum-decl name='zfs_get_column_t' naming-typedef-id='19cefcee' id='223bdcaa'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='GET_COL_NONE' value='0'/>
	<enumerator name='GET_COL_NAME' value='1'/>
	<enumerator name='GET_COL_PROPERTY' value='2'/>
	<enumerator name='GET_COL_VALUE' value='3'/>
	<enumerator name='GET_COL_RECVD' value='4'/>
	<enumerator name='GET_COL_SOURCE' value='5'/>
	</enum-decl>
	<typedef-decl name='zfs_get_column_t' type-id='223bdcaa' id='19cefcee'/>
	<class-decl name='vdev_cbdata' size-in-bits='192' is-struct='yes' visibility='default' id='b8006be8'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='cb_name_flags' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='cb_names' type-id='9b23c9ad' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='cb_names_count' type-id='f0981eeb' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='vdev_cbdata_t' type-id='b8006be8' id='a9679c94'/>
	<class-decl name='zprop_get_cbdata' size-in-bits='832' is-struct='yes' visibility='default' id='f3d3c319'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='cb_sources' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='cb_columns' type-id='3fcf57d2' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='cb_colwidths' type-id='e41bdf22' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='cb_scripted' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='416'>
	<var-decl name='cb_literal' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='cb_first' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='512'>
	<var-decl name='cb_proplist' type-id='3a9b2288' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='576'>
	<var-decl name='cb_type' type-id='2e45de5d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='640'>
	<var-decl name='cb_vdevs' type-id='a9679c94' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zprop_get_cbdata_t' type-id='f3d3c319' id='f3d87113'/>
	<typedef-decl name='zprop_func' type-id='2e711a2a' id='1ec3747a'/>
	<enum-decl name='zprop_attr_t' naming-typedef-id='999701cc' id='77d05200'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='PROP_DEFAULT' value='0'/>
	<enumerator name='PROP_READONLY' value='1'/>
	<enumerator name='PROP_INHERIT' value='2'/>
	<enumerator name='PROP_ONETIME' value='3'/>
	<enumerator name='PROP_ONETIME_DEFAULT' value='4'/>
	</enum-decl>
	<typedef-decl name='zprop_attr_t' type-id='77d05200' id='999701cc'/>
	<class-decl name='zfs_index' size-in-bits='128' is-struct='yes' visibility='default' id='87957af9'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='pi_name' type-id='80f4b756' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='pi_value' type-id='9c313c2d' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zprop_index_t' type-id='87957af9' id='64636ce3'/>
	<class-decl name='zprop_desc_t' size-in-bits='640' is-struct='yes' naming-typedef-id='ffa52b96' visibility='default' id='bbff5e4b'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='pd_name' type-id='80f4b756' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='pd_propnum' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='96'>
	<var-decl name='pd_proptype' type-id='31429eff' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='pd_strdefault' type-id='80f4b756' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='pd_numdefault' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='pd_attr' type-id='999701cc' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='288'>
	<var-decl name='pd_types' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='pd_values' type-id='80f4b756' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='pd_colname' type-id='80f4b756' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='pd_rightalign' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='449'>
	<var-decl name='pd_visible' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='450'>
	<var-decl name='pd_zfs_mod_supported' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='451'>
	<var-decl name='pd_always_flex' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='512'>
	<var-decl name='pd_table' type-id='c8bc397b' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='576'>
	<var-decl name='pd_table_size' type-id='b59d7dce' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zprop_desc_t' type-id='bbff5e4b' id='ffa52b96'/>
	<class-decl name='extmnttab' size-in-bits='320' is-struct='yes' visibility='default' id='0c544dc0'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='mnt_special' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='mnt_mountp' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='mnt_fstype' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='mnt_mntopts' type-id='26a90f95' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='mnt_major' type-id='3502e3ff' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='288'>
	<var-decl name='mnt_minor' type-id='3502e3ff' visibility='default'/>
	</data-member>
	</class-decl>
	<pointer-type-def type-id='d5027220' size-in-bits='64' id='b7f2d5e6'/>
	<qualified-type-def type-id='26a90f95' const='yes' id='57de658a'/>
	<pointer-type-def type-id='57de658a' size-in-bits='64' id='f319fae0'/>
	<pointer-type-def type-id='9b23c9ad' size-in-bits='64' id='c0563f85'/>
	<qualified-type-def type-id='33f57a65' const='yes' id='21fd6035'/>
	<pointer-type-def type-id='21fd6035' size-in-bits='64' id='a0de50cd'/>
	<pointer-type-def type-id='a0de50cd' size-in-bits='64' id='24f95ba5'/>
	<qualified-type-def type-id='64636ce3' const='yes' id='072f7953'/>
	<pointer-type-def type-id='072f7953' size-in-bits='64' id='c8bc397b'/>
	<pointer-type-def type-id='0c544dc0' size-in-bits='64' id='394fc496'/>
	<pointer-type-def type-id='aca3bac8' size-in-bits='64' id='d33f11cb'/>
	<qualified-type-def type-id='d33f11cb' restrict='yes' id='5c53ba29'/>
	<pointer-type-def type-id='ffa52b96' size-in-bits='64' id='76c8174b'/>
	<pointer-type-def type-id='f3d87113' size-in-bits='64' id='0d2a0670'/>
	<function-decl name='zpool_label_disk' mangled-name='zpool_label_disk' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_label_disk'>
	<parameter type-id='b0382bb3'/>
	<parameter type-id='4c81de99'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_version_kernel' mangled-name='zfs_version_kernel' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_version_kernel'>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='libzfs_core_init' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='libzfs_core_fini' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_get_underlying_path' mangled-name='zfs_get_underlying_path' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_get_underlying_path'>
	<parameter type-id='80f4b756'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='zpool_prop_unsupported' mangled-name='zpool_prop_unsupported' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_unsupported'>
	<parameter type-id='80f4b756'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zpool_feature_init' mangled-name='zpool_feature_init' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_feature_init'>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fletcher_4_init' mangled-name='fletcher_4_init' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='fletcher_4_init'>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fletcher_4_fini' mangled-name='fletcher_4_fini' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='fletcher_4_fini'>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_prop_init' mangled-name='zfs_prop_init' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_init'>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_prop_get_table' mangled-name='zfs_prop_get_table' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_get_table'>
	<return type-id='76c8174b'/>
	</function-decl>
	<function-decl name='zpool_prop_init' mangled-name='zpool_prop_init' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_init'>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zpool_prop_get_table' mangled-name='zpool_prop_get_table' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_get_table'>
	<return type-id='76c8174b'/>
	</function-decl>
	<function-decl name='vdev_prop_init' mangled-name='vdev_prop_init' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='vdev_prop_init'>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zprop_iter_common' mangled-name='zprop_iter_common' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_iter_common'>
	<parameter type-id='1ec3747a'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='2e45de5d'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zprop_name_to_prop' mangled-name='zprop_name_to_prop' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_name_to_prop'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='2e45de5d'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zprop_string_to_index' mangled-name='zprop_string_to_index' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_string_to_index'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='5d6479ae'/>
	<parameter type-id='2e45de5d'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zprop_values' mangled-name='zprop_values' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_values'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='2e45de5d'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zprop_width' mangled-name='zprop_width' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_width'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='37e3bd22'/>
	<parameter type-id='2e45de5d'/>
	<return type-id='b59d7dce'/>
	</function-decl>
	<function-decl name='zprop_valid_for_type' mangled-name='zprop_valid_for_type' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_valid_for_type'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='2e45de5d'/>
	<parameter type-id='c19b74c3'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='getextmntent' mangled-name='getextmntent' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='getextmntent'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='394fc496'/>
	<parameter type-id='62f7a03d'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='__ctype_toupper_loc' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='24f95ba5'/>
	</function-decl>
	<function-decl name='dlclose' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='regcomp' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5c53ba29'/>
	<parameter type-id='9d26089a'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='regfree' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='d33f11cb'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='puts' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='strtod' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='9d26089a'/>
	<parameter type-id='8c85230f'/>
	<return type-id='a0eb0f08'/>
	</function-decl>
	<function-decl name='realloc' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='exit' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='strnlen' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='b59d7dce'/>
	</function-decl>
	<function-decl name='strncasecmp' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='access' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='dup2' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='execve' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='f319fae0'/>
	<parameter type-id='f319fae0'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='execv' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='f319fae0'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='execvp' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='f319fae0'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='execvpe' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='f319fae0'/>
	<parameter type-id='f319fae0'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='_exit' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fork' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='3629bad8'/>
	</function-decl>
	<function-decl name='pow' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='a0eb0f08'/>
	<parameter type-id='a0eb0f08'/>
	<return type-id='a0eb0f08'/>
	</function-decl>
	<function-decl name='__vfprintf_chk' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='e75a27e9'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='9d26089a'/>
	<parameter type-id='b7f2d5e6'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='__vasprintf_chk' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='8c85230f'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='9d26089a'/>
	<parameter type-id='b7f2d5e6'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='waitpid' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='3629bad8'/>
	<parameter type-id='7292109c'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='3629bad8'/>
	</function-decl>
	<function-decl name='namespace_clear' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b0382bb3'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='libzfs_load_module' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='libzfs_errno' mangled-name='libzfs_errno' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_errno'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='libzfs_error_action' mangled-name='libzfs_error_action' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_error_action'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='libzfs_error_description' mangled-name='libzfs_error_description' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_error_description'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='libzfs_print_on_error' mangled-name='libzfs_print_on_error' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_print_on_error'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='c19b74c3' name='printerr'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='libzfs_run_process' mangled-name='libzfs_run_process' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_run_process'>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='9b23c9ad' name='argv'/>
	<parameter type-id='95e97e5e' name='flags'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='libzfs_run_process_get_stdout' mangled-name='libzfs_run_process_get_stdout' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_run_process_get_stdout'>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='9b23c9ad' name='argv'/>
	<parameter type-id='9b23c9ad' name='env'/>
	<parameter type-id='c0563f85' name='lines'/>
	<parameter type-id='7292109c' name='lines_cnt'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='libzfs_run_process_get_stdout_nopath' mangled-name='libzfs_run_process_get_stdout_nopath' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_run_process_get_stdout_nopath'>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='9b23c9ad' name='argv'/>
	<parameter type-id='9b23c9ad' name='env'/>
	<parameter type-id='c0563f85' name='lines'/>
	<parameter type-id='7292109c' name='lines_cnt'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='libzfs_free_str_array' mangled-name='libzfs_free_str_array' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_free_str_array'>
	<parameter type-id='9b23c9ad' name='strs'/>
	<parameter type-id='95e97e5e' name='count'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='libzfs_init' mangled-name='libzfs_init' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_init'>
	<return type-id='b0382bb3'/>
	</function-decl>
	<function-decl name='libzfs_fini' mangled-name='libzfs_fini' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_fini'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_path_to_zhandle' mangled-name='zfs_path_to_zhandle' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_path_to_zhandle'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='2e45de5d' name='argtype'/>
	<return type-id='9200a744'/>
	</function-decl>
	<function-decl name='zprop_print_one_property' mangled-name='zprop_print_one_property' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_print_one_property'>
	<parameter type-id='80f4b756' name='name'/>
	<parameter type-id='0d2a0670' name='cbp'/>
	<parameter type-id='80f4b756' name='propname'/>
	<parameter type-id='80f4b756' name='value'/>
	<parameter type-id='a2256d42' name='sourcetype'/>
	<parameter type-id='80f4b756' name='source'/>
	<parameter type-id='80f4b756' name='recvd_value'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zprop_get_list' mangled-name='zprop_get_list' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_get_list'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='26a90f95' name='props'/>
	<parameter type-id='e4378506' name='listp'/>
	<parameter type-id='2e45de5d' name='type'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zprop_free_list' mangled-name='zprop_free_list' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_free_list'>
	<parameter type-id='3a9b2288' name='pl'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zprop_iter' mangled-name='zprop_iter' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_iter'>
	<parameter type-id='1ec3747a' name='func'/>
	<parameter type-id='eaa32e2f' name='cb'/>
	<parameter type-id='c19b74c3' name='show_all'/>
	<parameter type-id='c19b74c3' name='ordered'/>
	<parameter type-id='2e45de5d' name='type'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_version_userland' mangled-name='zfs_version_userland' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_version_userland'>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zfs_version_print' mangled-name='zfs_version_print' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_version_print'>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='use_color' mangled-name='use_color' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='use_color'>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='printf_color' mangled-name='printf_color' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='printf_color'>
	<parameter type-id='80f4b756' name='color'/>
	<parameter type-id='80f4b756' name='format'/>
	<parameter is-variadic='yes'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_vdev_script_alloc_env' mangled-name='zpool_vdev_script_alloc_env' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_script_alloc_env'>
	<parameter type-id='80f4b756' name='pool_name'/>
	<parameter type-id='80f4b756' name='vdev_path'/>
	<parameter type-id='80f4b756' name='vdev_upath'/>
	<parameter type-id='80f4b756' name='vdev_enc_sysfs_path'/>
	<parameter type-id='80f4b756' name='opt_key'/>
	<parameter type-id='80f4b756' name='opt_val'/>
	<return type-id='9b23c9ad'/>
	</function-decl>
	<function-decl name='zpool_vdev_script_free_env' mangled-name='zpool_vdev_script_free_env' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_script_free_env'>
	<parameter type-id='9b23c9ad' name='env'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zpool_prepare_disk' mangled-name='zpool_prepare_disk' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prepare_disk'>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='5ce45b60' name='vdev_nv'/>
	<parameter type-id='80f4b756' name='prepare_str'/>
	<parameter type-id='c0563f85' name='lines'/>
	<parameter type-id='7292109c' name='lines_cnt'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_prepare_and_label_disk' mangled-name='zpool_prepare_and_label_disk' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prepare_and_label_disk'>
	<parameter type-id='b0382bb3' name='hdl'/>
	<parameter type-id='4c81de99' name='zhp'/>
	<parameter type-id='80f4b756' name='name'/>
	<parameter type-id='5ce45b60' name='vdev_nv'/>
	<parameter type-id='80f4b756' name='prepare_str'/>
	<parameter type-id='c0563f85' name='lines'/>
	<parameter type-id='7292109c' name='lines_cnt'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzfs/os/linux/libzfs_mount_os.c' language='LANG_C99'>
	<pointer-type-def type-id='7359adad' size-in-bits='64' id='1d2c2b85'/>
	<function-decl name='geteuid' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='cc5fcceb'/>
	</function-decl>
	<function-decl name='mount' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='7359adad'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='umount2' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_parse_mount_options' mangled-name='zfs_parse_mount_options' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_parse_mount_options'>
	<parameter type-id='80f4b756' name='mntopts'/>
	<parameter type-id='1d2c2b85' name='mntflags'/>
	<parameter type-id='1d2c2b85' name='zfsflags'/>
	<parameter type-id='95e97e5e' name='sloppy'/>
	<parameter type-id='26a90f95' name='badopt'/>
	<parameter type-id='26a90f95' name='mtabopt'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_adjust_mount_options' mangled-name='zfs_adjust_mount_options' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_adjust_mount_options'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='80f4b756' name='mntpoint'/>
	<parameter type-id='26a90f95' name='mntopts'/>
	<parameter type-id='26a90f95' name='mtabopt'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_mount_delegation_check' mangled-name='zfs_mount_delegation_check' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_mount_delegation_check'>
	<return type-id='95e97e5e'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzfs/os/linux/libzfs_pool_os.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='a84c031d' size-in-bits='288' id='16e6f2c6'>
	<subrange length='36' type-id='7359adad' id='ae666bde'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='a65ae39c' size-in-bits='960' id='fa198beb'>
	<subrange length='1' type-id='7359adad' id='52f813b4'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='3502e3ff' size-in-bits='384' id='dba89ba3'>
	<subrange length='12' type-id='7359adad' id='84827bdc'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='3502e3ff' size-in-bits='256' id='01d84ed4'>
	<subrange length='8' type-id='7359adad' id='56e0c0b1'/>
	</array-type-def>
	<class-decl name='dk_part' size-in-bits='960' is-struct='yes' visibility='default' id='a65ae39c'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='p_start' type-id='804dc465' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='p_size' type-id='804dc465' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='p_guid' type-id='214f32ea' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='p_tag' type-id='d908a348' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='272'>
	<var-decl name='p_flag' type-id='d908a348' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='288'>
	<var-decl name='p_name' type-id='16e6f2c6' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='576'>
	<var-decl name='p_uguid' type-id='214f32ea' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='704'>
	<var-decl name='p_resv' type-id='01d84ed4' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='dk_gpt' size-in-bits='1920' is-struct='yes' visibility='default' id='dd4a2e5a'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='efi_version' type-id='3502e3ff' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='efi_nparts' type-id='3502e3ff' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='efi_part_size' type-id='3502e3ff' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='96'>
	<var-decl name='efi_lbasize' type-id='3502e3ff' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='efi_last_lba' type-id='804dc465' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='efi_first_u_lba' type-id='804dc465' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='efi_last_u_lba' type-id='804dc465' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='efi_disk_uguid' type-id='214f32ea' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='efi_flags' type-id='3502e3ff' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='480'>
	<var-decl name='efi_reserved1' type-id='3502e3ff' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='512'>
	<var-decl name='efi_altern_lba' type-id='804dc465' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='576'>
	<var-decl name='efi_reserved' type-id='dba89ba3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='960'>
	<var-decl name='efi_parts' type-id='fa198beb' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='uuid' size-in-bits='128' is-struct='yes' visibility='default' id='214f32ea'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='time_low' type-id='8f92235e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='time_mid' type-id='149c6638' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='48'>
	<var-decl name='time_hi_and_version' type-id='149c6638' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='clock_seq_hi_and_reserved' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='72'>
	<var-decl name='clock_seq_low' type-id='b96825af' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='80'>
	<var-decl name='node_addr' type-id='0f562bd0' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='ushort_t' type-id='8efea9e5' id='d908a348'/>
	<typedef-decl name='uint16_t' type-id='253c2d2a' id='149c6638'/>
	<typedef-decl name='__uint16_t' type-id='8efea9e5' id='253c2d2a'/>
	<pointer-type-def type-id='dd4a2e5a' size-in-bits='64' id='0d8119a8'/>
	<pointer-type-def type-id='0d8119a8' size-in-bits='64' id='c43b27a6'/>
	<function-decl name='zpool_label_disk_wait' mangled-name='zpool_label_disk_wait' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_label_disk_wait'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_append_partition' mangled-name='zfs_append_partition' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_append_partition'>
	<parameter type-id='26a90f95'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='efi_alloc_and_init' mangled-name='efi_alloc_and_init' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='efi_alloc_and_init'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='8f92235e'/>
	<parameter type-id='c43b27a6'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='efi_alloc_and_read' mangled-name='efi_alloc_and_read' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='efi_alloc_and_read'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='c43b27a6'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='efi_write' mangled-name='efi_write' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='efi_write'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='0d8119a8'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='efi_rescan' mangled-name='efi_rescan' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='efi_rescan'>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='efi_free' mangled-name='efi_free' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='efi_free'>
	<parameter type-id='0d8119a8'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='efi_use_whole_disk' mangled-name='efi_use_whole_disk' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='efi_use_whole_disk'>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='rand' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='fsync' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzfs/os/linux/libzfs_util_os.c' language='LANG_C99'>
	<class-decl name='itimerspec' size-in-bits='256' is-struct='yes' visibility='default' id='acbdbcc6'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='it_interval' type-id='a9c79a1f' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='it_value' type-id='a9c79a1f' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='nfds_t' type-id='7359adad' id='555eef66'/>
	<class-decl name='pollfd' size-in-bits='64' is-struct='yes' visibility='default' id='b440e872'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='fd' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='events' type-id='a2185560' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='48'>
	<var-decl name='revents' type-id='a2185560' visibility='default'/>
	</data-member>
	</class-decl>
	<qualified-type-def type-id='acbdbcc6' const='yes' id='4ba62af7'/>
	<pointer-type-def type-id='4ba62af7' size-in-bits='64' id='f39579e7'/>
	<pointer-type-def type-id='acbdbcc6' size-in-bits='64' id='116842ac'/>
	<pointer-type-def type-id='b440e872' size-in-bits='64' id='3ac36db0'/>
	<function-decl name='__poll_chk' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='3ac36db0'/>
	<parameter type-id='555eef66'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='7359adad'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='inotify_init1' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='inotify_add_watch' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='8f92235e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='timerfd_create' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='08f9a87a'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='timerfd_settime' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='f39579e7'/>
	<parameter type-id='116842ac'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='libzfs_error_init' mangled-name='libzfs_error_init' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_error_init'>
	<parameter type-id='95e97e5e' name='error'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zfs_userns' mangled-name='zfs_userns' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_userns'>
	<parameter type-id='9200a744' name='zhp'/>
	<parameter type-id='80f4b756' name='nspath'/>
	<parameter type-id='95e97e5e' name='attach'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzutil/os/linux/zutil_device_path_os.c' language='LANG_C99'>
	<class-decl name='udev' is-struct='yes' visibility='default' is-declaration-only='yes' id='e4a7fb7f'/>
	<class-decl name='udev_device' is-struct='yes' visibility='default' is-declaration-only='yes' id='640b33ca'/>
	<pointer-type-def type-id='e4a7fb7f' size-in-bits='64' id='025eefe7'/>
	<pointer-type-def type-id='640b33ca' size-in-bits='64' id='b32bae08'/>
	<class-decl name='udev' is-struct='yes' visibility='default' is-declaration-only='yes' id='e4a7fb7f'/>
	<class-decl name='udev_device' is-struct='yes' visibility='default' is-declaration-only='yes' id='640b33ca'/>
	<function-decl name='udev_new' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='025eefe7'/>
	</function-decl>
	<function-decl name='udev_device_unref' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b32bae08'/>
	<return type-id='b32bae08'/>
	</function-decl>
	<function-decl name='udev_device_new_from_subsystem_sysname' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='025eefe7'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<return type-id='b32bae08'/>
	</function-decl>
	<function-decl name='udev_device_get_property_value' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b32bae08'/>
	<parameter type-id='80f4b756'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='__readlink_chk' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='9d26089a'/>
	<parameter type-id='266fe297'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='79a0948f'/>
	</function-decl>
	<function-decl name='zfs_get_enclosure_sysfs_path' mangled-name='zfs_get_enclosure_sysfs_path' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_get_enclosure_sysfs_path'>
	<parameter type-id='80f4b756' name='dev_name'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='zfs_dev_is_dm' mangled-name='zfs_dev_is_dm' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_dev_is_dm'>
	<parameter type-id='80f4b756' name='dev_name'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_dev_is_whole_disk' mangled-name='zfs_dev_is_whole_disk' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_dev_is_whole_disk'>
	<parameter type-id='80f4b756' name='dev_name'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='is_mpath_whole_disk' mangled-name='is_mpath_whole_disk' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='is_mpath_whole_disk'>
	<parameter type-id='80f4b756' name='path'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzutil/os/linux/zutil_import_os.c' language='LANG_C99'>
	<class-decl name='blkid_struct_cache' is-struct='yes' visibility='default' is-declaration-only='yes' id='09286066'/>
	<class-decl name='blkid_struct_dev' is-struct='yes' visibility='default' is-declaration-only='yes' id='86223623'/>
	<class-decl name='blkid_struct_dev_iterate' is-struct='yes' visibility='default' is-declaration-only='yes' id='d88420d6'/>
	<class-decl name='udev_list_entry' is-struct='yes' visibility='default' is-declaration-only='yes' id='e7dbdca3'/>
	<typedef-decl name='pool_vdev_iter_f' type-id='6c16a6c8' id='dff793e0'/>
	<typedef-decl name='blkid_dev' type-id='8433f053' id='f47b023a'/>
	<typedef-decl name='blkid_cache' type-id='940e3afc' id='0882dfdf'/>
	<typedef-decl name='blkid_dev_iterate' type-id='b8fa2efc' id='f4760fa7'/>
	<typedef-decl name='__useconds_t' type-id='f0981eeb' id='4e80d4b1'/>
	<pointer-type-def type-id='0882dfdf' size-in-bits='64' id='2e3e7caa'/>
	<pointer-type-def type-id='f47b023a' size-in-bits='64' id='d87f9b75'/>
	<pointer-type-def type-id='09286066' size-in-bits='64' id='940e3afc'/>
	<pointer-type-def type-id='86223623' size-in-bits='64' id='8433f053'/>
	<pointer-type-def type-id='d88420d6' size-in-bits='64' id='b8fa2efc'/>
	<pointer-type-def type-id='2ec2411e' size-in-bits='64' id='6c16a6c8'/>
	<pointer-type-def type-id='e7dbdca3' size-in-bits='64' id='deabd0d3'/>
	<class-decl name='blkid_struct_cache' is-struct='yes' visibility='default' is-declaration-only='yes' id='09286066'/>
	<class-decl name='blkid_struct_dev' is-struct='yes' visibility='default' is-declaration-only='yes' id='86223623'/>
	<class-decl name='blkid_struct_dev_iterate' is-struct='yes' visibility='default' is-declaration-only='yes' id='d88420d6'/>
	<class-decl name='udev_list_entry' is-struct='yes' visibility='default' is-declaration-only='yes' id='e7dbdca3'/>
	<function-decl name='for_each_vdev_in_nvlist' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='dff793e0'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='label_paths' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5507783b'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='7d3cd834'/>
	<parameter type-id='7d3cd834'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zutil_alloc' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5507783b'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='zutil_strdup' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5507783b'/>
	<parameter type-id='80f4b756'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='slice_cache_compare' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='blkid_put_cache' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='0882dfdf'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='blkid_get_cache' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='2e3e7caa'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='blkid_dev_devname' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='f47b023a'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='blkid_dev_iterate_begin' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='0882dfdf'/>
	<return type-id='f4760fa7'/>
	</function-decl>
	<function-decl name='blkid_dev_set_search' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='f4760fa7'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='blkid_dev_next' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='f4760fa7'/>
	<parameter type-id='d87f9b75'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='blkid_dev_iterate_end' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='f4760fa7'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='blkid_probe_all_new' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='0882dfdf'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='udev_unref' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='025eefe7'/>
	<return type-id='025eefe7'/>
	</function-decl>
	<function-decl name='udev_list_entry_get_next' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='deabd0d3'/>
	<return type-id='deabd0d3'/>
	</function-decl>
	<function-decl name='udev_list_entry_get_name' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='deabd0d3'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='udev_device_get_parent_with_subsystem_devtype' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b32bae08'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<return type-id='b32bae08'/>
	</function-decl>
	<function-decl name='udev_device_get_devlinks_list_entry' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='b32bae08'/>
	<return type-id='deabd0d3'/>
	</function-decl>
	<function-decl name='sched_yield' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='usleep' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='4e80d4b1'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_dev_flush' mangled-name='zfs_dev_flush' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_dev_flush'>
	<parameter type-id='95e97e5e' name='fd'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_device_get_devid' mangled-name='zfs_device_get_devid' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_device_get_devid'>
	<parameter type-id='b32bae08' name='dev'/>
	<parameter type-id='26a90f95' name='bufptr'/>
	<parameter type-id='b59d7dce' name='buflen'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_device_get_physical' mangled-name='zfs_device_get_physical' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_device_get_physical'>
	<parameter type-id='b32bae08' name='dev'/>
	<parameter type-id='26a90f95' name='bufptr'/>
	<parameter type-id='b59d7dce' name='buflen'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_disk_wait' mangled-name='zpool_disk_wait' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_disk_wait'>
	<parameter type-id='80f4b756' name='path'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='update_vdev_config_dev_sysfs_path' mangled-name='update_vdev_config_dev_sysfs_path' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='update_vdev_config_dev_sysfs_path'>
	<parameter type-id='5ce45b60' name='nv'/>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='80f4b756' name='key'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-type size-in-bits='64' id='2ec2411e'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='5ce45b60'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-type>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzutil/os/linux/zutil_setproctitle.c' language='LANG_C99'>
	<function-decl name='warnx' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter is-variadic='yes'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='setenv' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='95e97e5e'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='clearenv' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_setproctitle_init' mangled-name='zfs_setproctitle_init' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_setproctitle_init'>
	<parameter type-id='95e97e5e' name='argc'/>
	<parameter type-id='9b23c9ad' name='argv'/>
	<parameter type-id='9b23c9ad' name='envp'/>
	<return type-id='48b5725f'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzutil/zutil_device_path.c' language='LANG_C99'>
	<pointer-type-def type-id='b99c00c9' size-in-bits='64' id='13956559'/>
	<function-decl name='zpool_default_search_paths' mangled-name='zpool_default_search_paths' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_default_search_paths'>
	<parameter type-id='78c01427'/>
	<return type-id='13956559'/>
	</function-decl>
	<function-decl name='strspn' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<return type-id='b59d7dce'/>
	</function-decl>
	<function-decl name='zfs_dirnamelen' mangled-name='zfs_dirnamelen' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_dirnamelen'>
	<parameter type-id='80f4b756' name='path'/>
	<return type-id='79a0948f'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzutil/zutil_import.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='a84c031d' size-in-bits='256' id='16dc656a'>
	<subrange length='32' type-id='7359adad' id='ae5bde82'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='95e97e5e' size-in-bits='384' id='73b82f0f'>
	<subrange length='12' type-id='7359adad' id='84827bdc'/>
	</array-type-def>
	<class-decl name='importargs' size-in-bits='448' is-struct='yes' visibility='default' id='7ac83801'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='path' type-id='9b23c9ad' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='paths' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='poolname' type-id='80f4b756' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='guid' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='cachefile' type-id='80f4b756' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='can_be_active' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='352'>
	<var-decl name='scan' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='policy' type-id='5ce45b60' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='importargs_t' type-id='7ac83801' id='7a842a6b'/>
	<class-decl name='libpc_handle' size-in-bits='8448' is-struct='yes' visibility='default' id='7c8737f0'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='lpc_error' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='lpc_printerr' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='lpc_open_access_error' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='96'>
	<var-decl name='lpc_desc_active' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='lpc_desc' type-id='b54ce520' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='8320'>
	<var-decl name='lpc_ops' type-id='f095e320' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='8384'>
	<var-decl name='lpc_lib_handle' type-id='eaa32e2f' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='libpc_handle_t' type-id='7c8737f0' id='8a70a786'/>
	<class-decl name='aiocb' size-in-bits='1344' is-struct='yes' visibility='default' id='e4957c49'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='aio_fildes' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='32'>
	<var-decl name='aio_lio_opcode' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='aio_reqprio' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='aio_buf' type-id='fe09dd29' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='aio_nbytes' type-id='b59d7dce' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='aio_sigevent' type-id='519bc206' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='768'>
	<var-decl name='__next_prio' type-id='924bbc81' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='832'>
	<var-decl name='__abs_prio' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='864'>
	<var-decl name='__policy' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='896'>
	<var-decl name='__error_code' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='960'>
	<var-decl name='__return_value' type-id='41060289' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1024'>
	<var-decl name='aio_offset' type-id='724e4de6' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='1088'>
	<var-decl name='__glibc_reserved' type-id='16dc656a' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='sigevent' size-in-bits='512' is-struct='yes' visibility='default' id='519bc206'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='sigev_value' type-id='eabacd01' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='sigev_signo' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='96'>
	<var-decl name='sigev_notify' type-id='95e97e5e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='_sigev_un' type-id='ac5ab599' visibility='default'/>
	</data-member>
	</class-decl>
	<union-decl name='__anonymous_union__' size-in-bits='384' is-anonymous='yes' visibility='default' id='ac5ab599'>
	<data-member access='public'>
	<var-decl name='_pad' type-id='73b82f0f' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='_tid' type-id='3629bad8' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='_sigev_thread' type-id='e7f43f7b' visibility='default'/>
	</data-member>
	</union-decl>
	<class-decl name='__anonymous_struct__' size-in-bits='128' is-struct='yes' is-anonymous='yes' visibility='default' id='e7f43f7b'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='_function' type-id='5f147c28' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='_attribute' type-id='7347a39e' visibility='default'/>
	</data-member>
	</class-decl>
	<pointer-type-def type-id='e4957c49' size-in-bits='64' id='924bbc81'/>
	<qualified-type-def type-id='924bbc81' const='yes' id='5499dcde'/>
	<pointer-type-def type-id='5499dcde' size-in-bits='64' id='2236d41c'/>
	<qualified-type-def type-id='2236d41c' restrict='yes' id='31488924'/>
	<pointer-type-def type-id='a3681dea' size-in-bits='64' id='fce6d540'/>
	<qualified-type-def type-id='e4957c49' const='yes' id='fced9da2'/>
	<pointer-type-def type-id='fced9da2' size-in-bits='64' id='b20efd18'/>
	<pointer-type-def type-id='7a842a6b' size-in-bits='64' id='07ee4a58'/>
	<pointer-type-def type-id='8a70a786' size-in-bits='64' id='5507783b'/>
	<pointer-type-def type-id='b1e62775' size-in-bits='64' id='f095e320'/>
	<pointer-type-def type-id='519bc206' size-in-bits='64' id='ef2f159c'/>
	<qualified-type-def type-id='ef2f159c' restrict='yes' id='de0eb5a4'/>
	<pointer-type-def type-id='f1abb096' size-in-bits='64' id='5f147c28'/>
	<qualified-type-def type-id='48b5725f' volatile='yes' id='b0b3cbf9'/>
	<pointer-type-def type-id='b0b3cbf9' size-in-bits='64' id='fe09dd29'/>
	<function-decl name='update_vdev_config_dev_strs' mangled-name='update_vdev_config_dev_strs' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='update_vdev_config_dev_strs'>
	<parameter type-id='5ce45b60'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='update_vdevs_config_dev_sysfs_path' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5ce45b60'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fnvlist_dup' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='22cce67b'/>
	<return type-id='5ce45b60'/>
	</function-decl>
	<function-decl name='spl_pagesize' mangled-name='spl_pagesize' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='spl_pagesize'>
	<return type-id='b59d7dce'/>
	</function-decl>
	<function-decl name='posix_memalign' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='63e171df'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='b59d7dce'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='sysconf' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='95e97e5e'/>
	<return type-id='bd54fe1a'/>
	</function-decl>
	<function-decl name='libpc_error_description' mangled-name='libpc_error_description' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libpc_error_description'>
	<parameter type-id='5507783b' name='hdl'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zpool_search_import' mangled-name='zpool_search_import' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_search_import'>
	<parameter type-id='5507783b' name='hdl'/>
	<parameter type-id='07ee4a58' name='import'/>
	<return type-id='5ce45b60'/>
	</function-decl>
	<function-decl name='zpool_find_config' mangled-name='zpool_find_config' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_find_config'>
	<parameter type-id='5507783b' name='hdl'/>
	<parameter type-id='80f4b756' name='target'/>
	<parameter type-id='857bb57e' name='configp'/>
	<parameter type-id='07ee4a58' name='args'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_find_import_blkid' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='5507783b'/>
	<parameter type-id='18c91f9e'/>
	<parameter type-id='fce6d540'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_open_func' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='eaa32e2f'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-type size-in-bits='64' id='f1abb096'>
	<parameter type-id='eabacd01'/>
	<return type-id='48b5725f'/>
	</function-type>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzutil/zutil_nicenum.c' language='LANG_C99'>
	<type-decl name='long double' size-in-bits='128' id='e095c704'/>
	<enum-decl name='zfs_nicenum_format' id='29cf1969'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='ZFS_NICENUM_1024' value='0'/>
	<enumerator name='ZFS_NICENUM_BYTES' value='1'/>
	<enumerator name='ZFS_NICENUM_TIME' value='2'/>
	<enumerator name='ZFS_NICENUM_RAW' value='3'/>
	<enumerator name='ZFS_NICENUM_RAWTIME' value='4'/>
	</enum-decl>
	<function-decl name='powl' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='e095c704'/>
	<parameter type-id='e095c704'/>
	<return type-id='e095c704'/>
	</function-decl>
	<function-decl name='floor' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='a0eb0f08'/>
	<return type-id='a0eb0f08'/>
	</function-decl>
	<function-decl name='zfs_isnumber' mangled-name='zfs_isnumber' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_isnumber'>
	<parameter type-id='80f4b756' name='str'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_nicenum_format' mangled-name='zfs_nicenum_format' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_nicenum_format'>
	<parameter type-id='9c313c2d' name='num'/>
	<parameter type-id='26a90f95' name='buf'/>
	<parameter type-id='b59d7dce' name='buflen'/>
	<parameter type-id='29cf1969' name='format'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_nicetime' mangled-name='zfs_nicetime' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_nicetime'>
	<parameter type-id='9c313c2d' name='num'/>
	<parameter type-id='26a90f95' name='buf'/>
	<parameter type-id='b59d7dce' name='buflen'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfs_niceraw' mangled-name='zfs_niceraw' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_niceraw'>
	<parameter type-id='9c313c2d' name='num'/>
	<parameter type-id='26a90f95' name='buf'/>
	<parameter type-id='b59d7dce' name='buflen'/>
	<return type-id='48b5725f'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='lib/libzutil/zutil_pool.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='853fd5dc' size-in-bits='32768' id='b505fc2f'>
	<subrange length='64' type-id='7359adad' id='b10be967'/>
	</array-type-def>
	<type-decl name='float' size-in-bits='32' id='a6c45d85'/>
	<class-decl name='ddt_stat' size-in-bits='512' is-struct='yes' visibility='default' id='65242dfe'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='dds_blocks' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='dds_lsize' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='dds_psize' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='dds_dsize' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='dds_ref_blocks' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='dds_ref_lsize' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='dds_ref_psize' type-id='9c313c2d' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='dds_ref_dsize' type-id='9c313c2d' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='ddt_stat_t' type-id='65242dfe' id='853fd5dc'/>
	<class-decl name='ddt_histogram' size-in-bits='32768' is-struct='yes' visibility='default' id='bc2b3086'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='ddh_stat' type-id='b505fc2f' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='ddt_histogram_t' type-id='bc2b3086' id='2d7fe832'/>
	<qualified-type-def type-id='2d7fe832' const='yes' id='ec92d602'/>
	<pointer-type-def type-id='ec92d602' size-in-bits='64' id='932720f8'/>
	<qualified-type-def type-id='853fd5dc' const='yes' id='764c298c'/>
	<pointer-type-def type-id='764c298c' size-in-bits='64' id='dfe59052'/>
	<qualified-type-def type-id='a9c79a1f' const='yes' id='cd087e36'/>
	<pointer-type-def type-id='cd087e36' size-in-bits='64' id='e05e8614'/>
	<function-decl name='nanosleep' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='e05e8614'/>
	<parameter type-id='3d83ba87'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_dump_ddt' mangled-name='zpool_dump_ddt' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_dump_ddt'>
	<parameter type-id='dfe59052' name='dds_total'/>
	<parameter type-id='932720f8' name='ddh'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fsleep' mangled-name='fsleep' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='fsleep'>
	<parameter type-id='a6c45d85' name='sec'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zpool_getenv_int' mangled-name='zpool_getenv_int' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_getenv_int'>
	<parameter type-id='80f4b756' name='env'/>
	<parameter type-id='95e97e5e' name='default_val'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='module/avl/avl.c' language='LANG_C99'>
	<function-decl name='avl_last' mangled-name='avl_last' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_last'>
	<parameter type-id='a3681dea' name='tree'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='avl_nearest' mangled-name='avl_nearest' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_nearest'>
	<parameter type-id='a3681dea' name='tree'/>
	<parameter type-id='fba6cb51' name='where'/>
	<parameter type-id='95e97e5e' name='direction'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='avl_insert_here' mangled-name='avl_insert_here' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_insert_here'>
	<parameter type-id='a3681dea' name='tree'/>
	<parameter type-id='eaa32e2f' name='new_data'/>
	<parameter type-id='eaa32e2f' name='here'/>
	<parameter type-id='95e97e5e' name='direction'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='avl_update_lt' mangled-name='avl_update_lt' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_update_lt'>
	<parameter type-id='a3681dea' name='t'/>
	<parameter type-id='eaa32e2f' name='obj'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='avl_update_gt' mangled-name='avl_update_gt' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_update_gt'>
	<parameter type-id='a3681dea' name='t'/>
	<parameter type-id='eaa32e2f' name='obj'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='avl_update' mangled-name='avl_update' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_update'>
	<parameter type-id='a3681dea' name='t'/>
	<parameter type-id='eaa32e2f' name='obj'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='avl_swap' mangled-name='avl_swap' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_swap'>
	<parameter type-id='a3681dea' name='tree1'/>
	<parameter type-id='a3681dea' name='tree2'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='avl_is_empty' mangled-name='avl_is_empty' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_is_empty'>
	<parameter type-id='a3681dea' name='tree'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='module/zcommon/cityhash.c' language='LANG_C99'>
	<function-decl name='cityhash4' mangled-name='cityhash4' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='cityhash4'>
	<parameter type-id='9c313c2d' name='w1'/>
	<parameter type-id='9c313c2d' name='w2'/>
	<parameter type-id='9c313c2d' name='w3'/>
	<parameter type-id='9c313c2d' name='w4'/>
	<return type-id='9c313c2d'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='module/zcommon/zfeature_common.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='83f29ca2' size-in-bits='17472' id='dd432c71'>
	<subrange length='39' type-id='7359adad' id='ae4a9561'/>
	</array-type-def>
	<enum-decl name='zfeature_flags' id='6db816a4'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='ZFEATURE_FLAG_READONLY_COMPAT' value='1'/>
	<enumerator name='ZFEATURE_FLAG_MOS' value='2'/>
	<enumerator name='ZFEATURE_FLAG_ACTIVATE_ON_ENABLE' value='4'/>
	<enumerator name='ZFEATURE_FLAG_PER_DATASET' value='8'/>
	</enum-decl>
	<typedef-decl name='zfeature_flags_t' type-id='6db816a4' id='fc329033'/>
	<enum-decl name='zfeature_type' id='c4fa2355'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='ZFEATURE_TYPE_BOOLEAN' value='0'/>
	<enumerator name='ZFEATURE_TYPE_UINT64_ARRAY' value='1'/>
	<enumerator name='ZFEATURE_NUM_TYPES' value='2'/>
	</enum-decl>
	<typedef-decl name='zfeature_type_t' type-id='c4fa2355' id='732d2bb2'/>
	<class-decl name='zfeature_info' size-in-bits='448' is-struct='yes' visibility='default' id='1178d146'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='fi_feature' type-id='d6618c78' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='fi_uname' type-id='80f4b756' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='fi_guid' type-id='80f4b756' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='fi_desc' type-id='80f4b756' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='fi_flags' type-id='fc329033' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='288'>
	<var-decl name='fi_zfs_mod_supported' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='fi_type' type-id='732d2bb2' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='fi_depends' type-id='1acff326' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zfeature_info_t' type-id='1178d146' id='83f29ca2'/>
	<typedef-decl name='__free_fn_t' type-id='b7f9d8e6' id='3ff5e51e'/>
	<class-decl name='dirent' size-in-bits='2240' is-struct='yes' visibility='default' id='611586a1'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='d_ino' type-id='71288a47' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='d_off' type-id='724e4de6' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='d_reclen' type-id='8efea9e5' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='144'>
	<var-decl name='d_type' type-id='002ac4a6' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='152'>
	<var-decl name='d_name' type-id='d1617432' visibility='default'/>
	</data-member>
	</class-decl>
	<class-decl name='zfs_mod_supported_features' size-in-bits='128' is-struct='yes' visibility='default' id='3eee3342'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='tree' type-id='eaa32e2f' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='all_features' type-id='c19b74c3' visibility='default'/>
	</data-member>
	</class-decl>
	<qualified-type-def type-id='d6618c78' const='yes' id='81a65028'/>
	<pointer-type-def type-id='81a65028' size-in-bits='64' id='1acff326'/>
	<qualified-type-def type-id='3eee3342' const='yes' id='0c1d5bbb'/>
	<pointer-type-def type-id='0c1d5bbb' size-in-bits='64' id='a3372543'/>
	<pointer-type-def type-id='611586a1' size-in-bits='64' id='2e243169'/>
	<qualified-type-def type-id='eaa32e2f' const='yes' id='83be723c'/>
	<pointer-type-def type-id='83be723c' size-in-bits='64' id='7acd98a2'/>
	<var-decl name='spa_feature_table' type-id='dd432c71' mangled-name='spa_feature_table' visibility='default' elf-symbol-id='spa_feature_table'/>
	<var-decl name='zfeature_checks_disable' type-id='c19b74c3' mangled-name='zfeature_checks_disable' visibility='default' elf-symbol-id='zfeature_checks_disable'/>
	<function-decl name='opendir' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<return type-id='f09217ba'/>
	</function-decl>
	<function-decl name='tsearch' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='63e171df'/>
	<parameter type-id='aba7edd8'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='tfind' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='7acd98a2'/>
	<parameter type-id='aba7edd8'/>
	<return type-id='eaa32e2f'/>
	</function-decl>
	<function-decl name='tdestroy' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='3ff5e51e'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zfeature_is_valid_guid' mangled-name='zfeature_is_valid_guid' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfeature_is_valid_guid'>
	<parameter type-id='80f4b756' name='name'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfeature_depends_on' mangled-name='zfeature_depends_on' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfeature_depends_on'>
	<parameter type-id='d6618c78' name='fid'/>
	<parameter type-id='d6618c78' name='check'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_mod_supported' mangled-name='zfs_mod_supported' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_mod_supported'>
	<parameter type-id='80f4b756' name='scope'/>
	<parameter type-id='80f4b756' name='name'/>
	<parameter type-id='a3372543' name='sfeatures'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='module/zcommon/zfs_comutil.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='b99c00c9' size-in-bits='2624' id='5ce15418'>
	<subrange length='41' type-id='7359adad' id='cb834f44'/>
	</array-type-def>
	<qualified-type-def type-id='80f4b756' const='yes' id='b99c00c9'/>
	<pointer-type-def type-id='8f92235e' size-in-bits='64' id='90421557'/>
	<function-decl name='nvpair_value_uint32' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='dace003f'/>
	<parameter type-id='90421557'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<var-decl name='zfs_history_event_names' type-id='5ce15418' mangled-name='zfs_history_event_names' visibility='default' elf-symbol-id='zfs_history_event_names'/>
	<function-decl name='strpbrk' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<return type-id='26a90f95'/>
	</function-decl>
	<function-decl name='zfs_allocatable_devs' mangled-name='zfs_allocatable_devs' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_allocatable_devs'>
	<parameter type-id='5ce45b60' name='nv'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_special_devs' mangled-name='zfs_special_devs' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_special_devs'>
	<parameter type-id='5ce45b60' name='nv'/>
	<parameter type-id='80f4b756' name='type'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_zpl_version_map' mangled-name='zfs_zpl_version_map' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_zpl_version_map'>
	<parameter type-id='95e97e5e' name='spa_version'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_spa_version_map' mangled-name='zfs_spa_version_map' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_spa_version_map'>
	<parameter type-id='95e97e5e' name='zpl_version'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_dataset_name_hidden' mangled-name='zfs_dataset_name_hidden' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_dataset_name_hidden'>
	<parameter type-id='80f4b756' name='name'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='module/zcommon/zfs_deleg.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='fa1870fd' size-in-bits='4096' id='59e94aca'>
	<subrange length='32' type-id='7359adad' id='ae5bde82'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='fa1870fd' size-in-bits='infinite' id='7c00e69d'>
	<subrange length='infinite' id='031f2035'/>
	</array-type-def>
	<enum-decl name='zfs_deleg_who_type_t' naming-typedef-id='36d4bd5a' id='b5fa5816'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='ZFS_DELEG_WHO_UNKNOWN' value='0'/>
	<enumerator name='ZFS_DELEG_USER' value='117'/>
	<enumerator name='ZFS_DELEG_USER_SETS' value='85'/>
	<enumerator name='ZFS_DELEG_GROUP' value='103'/>
	<enumerator name='ZFS_DELEG_GROUP_SETS' value='71'/>
	<enumerator name='ZFS_DELEG_EVERYONE' value='101'/>
	<enumerator name='ZFS_DELEG_EVERYONE_SETS' value='69'/>
	<enumerator name='ZFS_DELEG_CREATE' value='99'/>
	<enumerator name='ZFS_DELEG_CREATE_SETS' value='67'/>
	<enumerator name='ZFS_DELEG_NAMED_SET' value='115'/>
	<enumerator name='ZFS_DELEG_NAMED_SET_SETS' value='83'/>
	</enum-decl>
	<typedef-decl name='zfs_deleg_who_type_t' type-id='b5fa5816' id='36d4bd5a'/>
	<enum-decl name='zfs_deleg_note_t' naming-typedef-id='4613c173' id='729d4547'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='ZFS_DELEG_NOTE_CREATE' value='0'/>
	<enumerator name='ZFS_DELEG_NOTE_DESTROY' value='1'/>
	<enumerator name='ZFS_DELEG_NOTE_SNAPSHOT' value='2'/>
	<enumerator name='ZFS_DELEG_NOTE_ROLLBACK' value='3'/>
	<enumerator name='ZFS_DELEG_NOTE_CLONE' value='4'/>
	<enumerator name='ZFS_DELEG_NOTE_PROMOTE' value='5'/>
	<enumerator name='ZFS_DELEG_NOTE_RENAME' value='6'/>
	<enumerator name='ZFS_DELEG_NOTE_SEND' value='7'/>
	<enumerator name='ZFS_DELEG_NOTE_RECEIVE' value='8'/>
	<enumerator name='ZFS_DELEG_NOTE_ALLOW' value='9'/>
	<enumerator name='ZFS_DELEG_NOTE_USERPROP' value='10'/>
	<enumerator name='ZFS_DELEG_NOTE_MOUNT' value='11'/>
	<enumerator name='ZFS_DELEG_NOTE_SHARE' value='12'/>
	<enumerator name='ZFS_DELEG_NOTE_USERQUOTA' value='13'/>
	<enumerator name='ZFS_DELEG_NOTE_GROUPQUOTA' value='14'/>
	<enumerator name='ZFS_DELEG_NOTE_USERUSED' value='15'/>
	<enumerator name='ZFS_DELEG_NOTE_GROUPUSED' value='16'/>
	<enumerator name='ZFS_DELEG_NOTE_USEROBJQUOTA' value='17'/>
	<enumerator name='ZFS_DELEG_NOTE_GROUPOBJQUOTA' value='18'/>
	<enumerator name='ZFS_DELEG_NOTE_USEROBJUSED' value='19'/>
	<enumerator name='ZFS_DELEG_NOTE_GROUPOBJUSED' value='20'/>
	<enumerator name='ZFS_DELEG_NOTE_HOLD' value='21'/>
	<enumerator name='ZFS_DELEG_NOTE_RELEASE' value='22'/>
	<enumerator name='ZFS_DELEG_NOTE_DIFF' value='23'/>
	<enumerator name='ZFS_DELEG_NOTE_BOOKMARK' value='24'/>
	<enumerator name='ZFS_DELEG_NOTE_LOAD_KEY' value='25'/>
	<enumerator name='ZFS_DELEG_NOTE_CHANGE_KEY' value='26'/>
	<enumerator name='ZFS_DELEG_NOTE_PROJECTUSED' value='27'/>
	<enumerator name='ZFS_DELEG_NOTE_PROJECTQUOTA' value='28'/>
	<enumerator name='ZFS_DELEG_NOTE_PROJECTOBJUSED' value='29'/>
	<enumerator name='ZFS_DELEG_NOTE_PROJECTOBJQUOTA' value='30'/>
	<enumerator name='ZFS_DELEG_NOTE_NONE' value='31'/>
	</enum-decl>
	<typedef-decl name='zfs_deleg_note_t' type-id='729d4547' id='4613c173'/>
	<class-decl name='zfs_deleg_perm_tab' size-in-bits='128' is-struct='yes' visibility='default' id='5aa05c1f'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='z_perm' type-id='80f4b756' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='z_note' type-id='4613c173' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zfs_deleg_perm_tab_t' type-id='5aa05c1f' id='f3f851ad'/>
	<qualified-type-def type-id='f3f851ad' const='yes' id='fa1870fd'/>
	<var-decl name='zfs_deleg_perm_tab' type-id='7c00e69d' mangled-name='zfs_deleg_perm_tab' visibility='default' elf-symbol-id='zfs_deleg_perm_tab'/>
	<function-decl name='permset_namecheck' mangled-name='permset_namecheck' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='permset_namecheck'>
	<parameter type-id='80f4b756'/>
	<parameter type-id='053457bd'/>
	<parameter type-id='26a90f95'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prop_delegatable' mangled-name='zfs_prop_delegatable' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_delegatable'>
	<parameter type-id='58603c44'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_deleg_canonicalize_perm' mangled-name='zfs_deleg_canonicalize_perm' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_deleg_canonicalize_perm'>
	<parameter type-id='80f4b756' name='perm'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zfs_deleg_verify_nvlist' mangled-name='zfs_deleg_verify_nvlist' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_deleg_verify_nvlist'>
	<parameter type-id='5ce45b60' name='nvp'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_deleg_whokey' mangled-name='zfs_deleg_whokey' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_deleg_whokey'>
	<parameter type-id='26a90f95' name='attr'/>
	<parameter type-id='36d4bd5a' name='type'/>
	<parameter type-id='a84c031d' name='inheritchr'/>
	<parameter type-id='eaa32e2f' name='data'/>
	<return type-id='48b5725f'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='module/zcommon/zfs_fletcher.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='9c313c2d' size-in-bits='512' id='c5d13f42'>
	<subrange length='8' type-id='7359adad' id='56e0c0b1'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='90dbb6d6' size-in-bits='2048' id='16582e69'>
	<subrange length='4' type-id='7359adad' id='16fe7105'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='8240361c' size-in-bits='1024' id='481f90b1'>
	<subrange length='4' type-id='7359adad' id='16fe7105'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='7c1ab40c' size-in-bits='512' id='cbd91ec1'>
	<subrange length='4' type-id='7359adad' id='16fe7105'/>
	</array-type-def>
	<array-type-def dimensions='1' type-id='6d059eaa' size-in-bits='1024' id='729b6ebb'>
	<subrange length='4' type-id='7359adad' id='16fe7105'/>
	</array-type-def>
	<enum-decl name='zio_byteorder_t' naming-typedef-id='595a65ec' id='fc861be0'>
	<underlying-type type-id='9cac1fee'/>
	<enumerator name='ZIO_CHECKSUM_NATIVE' value='0'/>
	<enumerator name='ZIO_CHECKSUM_BYTESWAP' value='1'/>
	</enum-decl>
	<typedef-decl name='zio_byteorder_t' type-id='fc861be0' id='595a65ec'/>
	<class-decl name='zio_abd_checksum_data' size-in-bits='256' is-struct='yes' visibility='default' id='4bf4b004'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='acd_byteorder' type-id='595a65ec' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='acd_ctx' type-id='0f7df99e' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='acd_zcp' type-id='c24fc2ee' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='acd_private' type-id='eaa32e2f' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zio_abd_checksum_data_t' type-id='4bf4b004' id='74e39470'/>
	<typedef-decl name='zio_abd_checksum_init_t' type-id='a5444274' id='029a8ebe'/>
	<typedef-decl name='zio_abd_checksum_fini_t' type-id='a5444274' id='d6fd5c6c'/>
	<typedef-decl name='zio_abd_checksum_iter_t' type-id='f4a1892e' id='cefa0f4a'/>
	<class-decl name='zio_abd_checksum_func' size-in-bits='192' is-struct='yes' visibility='default' id='aa14691a'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='acf_init' type-id='0bcca125' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='acf_fini' type-id='bfe36153' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='acf_iter' type-id='1e276399' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zio_abd_checksum_func_t' type-id='3f8e8d11' id='c2eb138a'/>
	<class-decl name='zfs_fletcher_superscalar' size-in-bits='256' is-struct='yes' visibility='default' id='28efb250'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='v' type-id='85c64d26' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zfs_fletcher_superscalar_t' type-id='28efb250' id='6d059eaa'/>
	<class-decl name='zfs_fletcher_sse' size-in-bits='128' is-struct='yes' visibility='default' id='acd4019a'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='v' type-id='c1c22e6c' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zfs_fletcher_sse_t' type-id='acd4019a' id='7c1ab40c'/>
	<class-decl name='zfs_fletcher_avx' size-in-bits='256' is-struct='yes' visibility='default' id='8c208dfa'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='v' type-id='85c64d26' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zfs_fletcher_avx_t' type-id='8c208dfa' id='8240361c'/>
	<class-decl name='zfs_fletcher_avx512' size-in-bits='512' is-struct='yes' visibility='default' id='c6d0c382'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='v' type-id='c5d13f42' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='zfs_fletcher_avx512_t' type-id='c6d0c382' id='90dbb6d6'/>
	<union-decl name='fletcher_4_ctx' size-in-bits='2048' visibility='default' id='1f951ade'>
	<data-member access='public'>
	<var-decl name='scalar' type-id='39730d0b' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='superscalar' type-id='729b6ebb' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='sse' type-id='cbd91ec1' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='avx' type-id='481f90b1' visibility='default'/>
	</data-member>
	<data-member access='public'>
	<var-decl name='avx512' type-id='16582e69' visibility='default'/>
	</data-member>
	</union-decl>
	<typedef-decl name='fletcher_4_ctx_t' type-id='1f951ade' id='4b675395'/>
	<qualified-type-def type-id='aa14691a' const='yes' id='3f8e8d11'/>
	<pointer-type-def type-id='4b675395' size-in-bits='64' id='0f7df99e'/>
	<qualified-type-def type-id='8f92235e' volatile='yes' id='430e0681'/>
	<pointer-type-def type-id='430e0681' size-in-bits='64' id='3a147f31'/>
	<pointer-type-def type-id='74e39470' size-in-bits='64' id='eefe7427'/>
	<pointer-type-def type-id='d6fd5c6c' size-in-bits='64' id='bfe36153'/>
	<pointer-type-def type-id='029a8ebe' size-in-bits='64' id='0bcca125'/>
	<pointer-type-def type-id='cefa0f4a' size-in-bits='64' id='1e276399'/>
	<var-decl name='fletcher_4_abd_ops' type-id='c2eb138a' mangled-name='fletcher_4_abd_ops' visibility='default' elf-symbol-id='fletcher_4_abd_ops'/>
	<function-decl name='atomic_swap_32' mangled-name='atomic_swap_32' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='atomic_swap_32'>
	<parameter type-id='3a147f31'/>
	<parameter type-id='8f92235e'/>
	<return type-id='8f92235e'/>
	</function-decl>
	<function-decl name='membar_producer' mangled-name='membar_producer' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='membar_producer'>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fletcher_init' mangled-name='fletcher_init' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='fletcher_init'>
	<parameter type-id='c24fc2ee' name='zcp'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fletcher_2_incremental_native' mangled-name='fletcher_2_incremental_native' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='fletcher_2_incremental_native'>
	<parameter type-id='eaa32e2f' name='buf'/>
	<parameter type-id='b59d7dce' name='size'/>
	<parameter type-id='eaa32e2f' name='data'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='fletcher_2_native' mangled-name='fletcher_2_native' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='fletcher_2_native'>
	<parameter type-id='eaa32e2f' name='buf'/>
	<parameter type-id='9c313c2d' name='size'/>
	<parameter type-id='eaa32e2f' name='ctx_template'/>
	<parameter type-id='c24fc2ee' name='zcp'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fletcher_2_incremental_byteswap' mangled-name='fletcher_2_incremental_byteswap' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='fletcher_2_incremental_byteswap'>
	<parameter type-id='eaa32e2f' name='buf'/>
	<parameter type-id='b59d7dce' name='size'/>
	<parameter type-id='eaa32e2f' name='data'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='fletcher_2_byteswap' mangled-name='fletcher_2_byteswap' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='fletcher_2_byteswap'>
	<parameter type-id='eaa32e2f' name='buf'/>
	<parameter type-id='9c313c2d' name='size'/>
	<parameter type-id='eaa32e2f' name='ctx_template'/>
	<parameter type-id='c24fc2ee' name='zcp'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fletcher_4_impl_set' mangled-name='fletcher_4_impl_set' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='fletcher_4_impl_set'>
	<parameter type-id='80f4b756' name='val'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='fletcher_4_native' mangled-name='fletcher_4_native' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='fletcher_4_native'>
	<parameter type-id='eaa32e2f' name='buf'/>
	<parameter type-id='9c313c2d' name='size'/>
	<parameter type-id='eaa32e2f' name='ctx_template'/>
	<parameter type-id='c24fc2ee' name='zcp'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='fletcher_4_byteswap' mangled-name='fletcher_4_byteswap' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='fletcher_4_byteswap'>
	<parameter type-id='eaa32e2f' name='buf'/>
	<parameter type-id='9c313c2d' name='size'/>
	<parameter type-id='eaa32e2f' name='ctx_template'/>
	<parameter type-id='c24fc2ee' name='zcp'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-type size-in-bits='64' id='f4a1892e'>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='b59d7dce'/>
	<parameter type-id='eaa32e2f'/>
	<return type-id='95e97e5e'/>
	</function-type>
	<function-type size-in-bits='64' id='a5444274'>
	<parameter type-id='eefe7427'/>
	<return type-id='48b5725f'/>
	</function-type>
	</abi-instr>
	<abi-instr address-size='64' path='module/zcommon/zfs_fletcher_avx512.c' language='LANG_C99'>
	<typedef-decl name='fletcher_4_init_f' type-id='173aa527' id='b9ae1656'/>
	<typedef-decl name='fletcher_4_fini_f' type-id='0ad5b8a8' id='c4c1f4fc'/>
	<typedef-decl name='fletcher_4_compute_f' type-id='38147eff' id='ad1dc4cb'/>
	<class-decl name='fletcher_4_func' size-in-bits='1024' is-struct='yes' visibility='default' id='57f479a0'>
	<data-member access='public' layout-offset-in-bits='0'>
	<var-decl name='init_native' type-id='b9ae1656' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='64'>
	<var-decl name='fini_native' type-id='c4c1f4fc' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='128'>
	<var-decl name='compute_native' type-id='ad1dc4cb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='192'>
	<var-decl name='init_byteswap' type-id='b9ae1656' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='256'>
	<var-decl name='fini_byteswap' type-id='c4c1f4fc' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='320'>
	<var-decl name='compute_byteswap' type-id='ad1dc4cb' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='384'>
	<var-decl name='valid' type-id='297d38bc' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='448'>
	<var-decl name='uses_fpu' type-id='c19b74c3' visibility='default'/>
	</data-member>
	<data-member access='public' layout-offset-in-bits='512'>
	<var-decl name='name' type-id='80f4b756' visibility='default'/>
	</data-member>
	</class-decl>
	<typedef-decl name='fletcher_4_ops_t' type-id='57f479a0' id='eba91718'/>
	<qualified-type-def type-id='eba91718' const='yes' id='9eeabdc8'/>
	<pointer-type-def type-id='e9e61702' size-in-bits='64' id='297d38bc'/>
	<pointer-type-def type-id='fe40251b' size-in-bits='64' id='173aa527'/>
	<pointer-type-def type-id='17fb1f83' size-in-bits='64' id='38147eff'/>
	<pointer-type-def type-id='fb39e25e' size-in-bits='64' id='0ad5b8a8'/>
	<var-decl name='fletcher_4_avx512f_ops' type-id='9eeabdc8' mangled-name='fletcher_4_avx512f_ops' visibility='default' elf-symbol-id='fletcher_4_avx512f_ops'/>
	<var-decl name='fletcher_4_avx512bw_ops' type-id='9eeabdc8' mangled-name='fletcher_4_avx512bw_ops' visibility='default' elf-symbol-id='fletcher_4_avx512bw_ops'/>
	<function-type size-in-bits='64' id='e9e61702'>
	<return type-id='c19b74c3'/>
	</function-type>
	<function-type size-in-bits='64' id='fe40251b'>
	<parameter type-id='0f7df99e'/>
	<return type-id='48b5725f'/>
	</function-type>
	<function-type size-in-bits='64' id='17fb1f83'>
	<parameter type-id='0f7df99e'/>
	<parameter type-id='eaa32e2f'/>
	<parameter type-id='9c313c2d'/>
	<return type-id='48b5725f'/>
	</function-type>
	<function-type size-in-bits='64' id='fb39e25e'>
	<parameter type-id='0f7df99e'/>
	<parameter type-id='c24fc2ee'/>
	<return type-id='48b5725f'/>
	</function-type>
	</abi-instr>
	<abi-instr address-size='64' path='module/zcommon/zfs_fletcher_intel.c' language='LANG_C99'>
	<var-decl name='fletcher_4_avx2_ops' type-id='9eeabdc8' mangled-name='fletcher_4_avx2_ops' visibility='default' elf-symbol-id='fletcher_4_avx2_ops'/>
	</abi-instr>
	<abi-instr address-size='64' path='module/zcommon/zfs_fletcher_sse.c' language='LANG_C99'>
	<var-decl name='fletcher_4_sse2_ops' type-id='9eeabdc8' mangled-name='fletcher_4_sse2_ops' visibility='default' elf-symbol-id='fletcher_4_sse2_ops'/>
	<var-decl name='fletcher_4_ssse3_ops' type-id='9eeabdc8' mangled-name='fletcher_4_ssse3_ops' visibility='default' elf-symbol-id='fletcher_4_ssse3_ops'/>
	</abi-instr>
	<abi-instr address-size='64' path='module/zcommon/zfs_fletcher_superscalar.c' language='LANG_C99'>
	<var-decl name='fletcher_4_superscalar_ops' type-id='9eeabdc8' mangled-name='fletcher_4_superscalar_ops' visibility='default' elf-symbol-id='fletcher_4_superscalar_ops'/>
	</abi-instr>
	<abi-instr address-size='64' path='module/zcommon/zfs_fletcher_superscalar4.c' language='LANG_C99'>
	<var-decl name='fletcher_4_superscalar4_ops' type-id='9eeabdc8' mangled-name='fletcher_4_superscalar4_ops' visibility='default' elf-symbol-id='fletcher_4_superscalar4_ops'/>
	</abi-instr>
	<abi-instr address-size='64' path='module/zcommon/zfs_namecheck.c' language='LANG_C99'>
	<var-decl name='zfs_max_dataset_nesting' type-id='95e97e5e' mangled-name='zfs_max_dataset_nesting' visibility='default' elf-symbol-id='zfs_max_dataset_nesting'/>
	<function-decl name='get_dataset_depth' mangled-name='get_dataset_depth' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='get_dataset_depth'>
	<parameter type-id='80f4b756' name='path'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_component_namecheck' mangled-name='zfs_component_namecheck' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_component_namecheck'>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='053457bd' name='why'/>
	<parameter type-id='26a90f95' name='what'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='dataset_namecheck' mangled-name='dataset_namecheck' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='dataset_namecheck'>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='053457bd' name='why'/>
	<parameter type-id='26a90f95' name='what'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='bookmark_namecheck' mangled-name='bookmark_namecheck' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='bookmark_namecheck'>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='053457bd' name='why'/>
	<parameter type-id='26a90f95' name='what'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='snapshot_namecheck' mangled-name='snapshot_namecheck' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='snapshot_namecheck'>
	<parameter type-id='80f4b756' name='path'/>
	<parameter type-id='053457bd' name='why'/>
	<parameter type-id='26a90f95' name='what'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='module/zcommon/zfs_prop.c' language='LANG_C99'>
	<array-type-def dimensions='1' type-id='b99c00c9' size-in-bits='768' id='bcc77e38'>
	<subrange length='12' type-id='7359adad' id='84827bdc'/>
	</array-type-def>
	<pointer-type-def type-id='3eee3342' size-in-bits='64' id='73f8e240'/>
	<var-decl name='zfs_userquota_prop_prefixes' type-id='bcc77e38' mangled-name='zfs_userquota_prop_prefixes' visibility='default' elf-symbol-id='zfs_userquota_prop_prefixes'/>
	<function-decl name='zfs_mod_list_supported' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='80f4b756'/>
	<return type-id='73f8e240'/>
	</function-decl>
	<function-decl name='zfs_mod_list_supported_free' visibility='default' binding='global' size-in-bits='64'>
	<parameter type-id='73f8e240'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zprop_register_impl' mangled-name='zprop_register_impl' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_register_impl'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='31429eff'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='999701cc'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='c8bc397b'/>
	<parameter type-id='a3372543'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zprop_register_string' mangled-name='zprop_register_string' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_register_string'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='999701cc'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='a3372543'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zprop_register_number' mangled-name='zprop_register_number' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_register_number'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='999701cc'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='a3372543'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zprop_register_index' mangled-name='zprop_register_index' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_register_index'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='999701cc'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='c8bc397b'/>
	<parameter type-id='a3372543'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zprop_register_hidden' mangled-name='zprop_register_hidden' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_register_hidden'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='31429eff'/>
	<parameter type-id='999701cc'/>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='80f4b756'/>
	<parameter type-id='c19b74c3'/>
	<parameter type-id='a3372543'/>
	<return type-id='48b5725f'/>
	</function-decl>
	<function-decl name='zprop_index_to_string' mangled-name='zprop_index_to_string' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_index_to_string'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='7d3cd834'/>
	<parameter type-id='2e45de5d'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zprop_random_value' mangled-name='zprop_random_value' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_random_value'>
	<parameter type-id='95e97e5e'/>
	<parameter type-id='9c313c2d'/>
	<parameter type-id='2e45de5d'/>
	<return type-id='9c313c2d'/>
	</function-decl>
	<function-decl name='zprop_valid_char' mangled-name='zprop_valid_char' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zprop_valid_char'>
	<parameter type-id='a84c031d'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prop_string_to_index' mangled-name='zfs_prop_string_to_index' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_string_to_index'>
	<parameter type-id='58603c44' name='prop'/>
	<parameter type-id='80f4b756' name='string'/>
	<parameter type-id='5d6479ae' name='index'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prop_random_value' mangled-name='zfs_prop_random_value' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_random_value'>
	<parameter type-id='58603c44' name='prop'/>
	<parameter type-id='9c313c2d' name='seed'/>
	<return type-id='9c313c2d'/>
	</function-decl>
	<function-decl name='zfs_prop_visible' mangled-name='zfs_prop_visible' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_visible'>
	<parameter type-id='58603c44' name='prop'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='zfs_prop_values' mangled-name='zfs_prop_values' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_values'>
	<parameter type-id='58603c44' name='prop'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zfs_prop_is_string' mangled-name='zfs_prop_is_string' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_is_string'>
	<parameter type-id='58603c44' name='prop'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zfs_prop_column_name' mangled-name='zfs_prop_column_name' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_column_name'>
	<parameter type-id='58603c44' name='prop'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zfs_prop_align_right' mangled-name='zfs_prop_align_right' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_prop_align_right'>
	<parameter type-id='58603c44' name='prop'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='module/zcommon/zpool_prop.c' language='LANG_C99'>
	<function-decl name='zpool_prop_string_to_index' mangled-name='zpool_prop_string_to_index' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_string_to_index'>
	<parameter type-id='5d0c23fb' name='prop'/>
	<parameter type-id='80f4b756' name='string'/>
	<parameter type-id='5d6479ae' name='index'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='zpool_prop_random_value' mangled-name='zpool_prop_random_value' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_random_value'>
	<parameter type-id='5d0c23fb' name='prop'/>
	<parameter type-id='9c313c2d' name='seed'/>
	<return type-id='9c313c2d'/>
	</function-decl>
	<function-decl name='zpool_prop_values' mangled-name='zpool_prop_values' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_values'>
	<parameter type-id='5d0c23fb' name='prop'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zpool_prop_column_name' mangled-name='zpool_prop_column_name' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_column_name'>
	<parameter type-id='5d0c23fb' name='prop'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='zpool_prop_align_right' mangled-name='zpool_prop_align_right' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_align_right'>
	<parameter type-id='5d0c23fb' name='prop'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	<function-decl name='vdev_prop_get_table' mangled-name='vdev_prop_get_table' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='vdev_prop_get_table'>
	<return type-id='76c8174b'/>
	</function-decl>
	<function-decl name='vdev_prop_string_to_index' mangled-name='vdev_prop_string_to_index' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='vdev_prop_string_to_index'>
	<parameter type-id='5aa5c90c' name='prop'/>
	<parameter type-id='80f4b756' name='string'/>
	<parameter type-id='5d6479ae' name='index'/>
	<return type-id='95e97e5e'/>
	</function-decl>
	<function-decl name='vdev_prop_random_value' mangled-name='vdev_prop_random_value' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='vdev_prop_random_value'>
	<parameter type-id='5aa5c90c' name='prop'/>
	<parameter type-id='9c313c2d' name='seed'/>
	<return type-id='9c313c2d'/>
	</function-decl>
	<function-decl name='vdev_prop_values' mangled-name='vdev_prop_values' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='vdev_prop_values'>
	<parameter type-id='5aa5c90c' name='prop'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='vdev_prop_column_name' mangled-name='vdev_prop_column_name' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='vdev_prop_column_name'>
	<parameter type-id='5aa5c90c' name='prop'/>
	<return type-id='80f4b756'/>
	</function-decl>
	<function-decl name='vdev_prop_align_right' mangled-name='vdev_prop_align_right' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='vdev_prop_align_right'>
	<parameter type-id='5aa5c90c' name='prop'/>
	<return type-id='c19b74c3'/>
	</function-decl>
	</abi-instr>
	<abi-instr address-size='64' path='module/zcommon/zprop_common.c' language='LANG_C99'>
	<function-decl name='__ctype_tolower_loc' visibility='default' binding='global' size-in-bits='64'>
	<return type-id='24f95ba5'/>
	</function-decl>
	</abi-instr>
	</abi-corpus>
	diff --git a/sys/contrib/openzfs/lib/libzfs/libzfs_pool.c b/sys/contrib/openzfs/lib/libzfs/libzfs_pool.c
	index 2f9ccbc2ab57..e95b361da866 100644
	--- a/sys/contrib/openzfs/lib/libzfs/libzfs_pool.c
	+++ b/sys/contrib/openzfs/lib/libzfs/libzfs_pool.c
	@@ -1,5437 +1,5442 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright 2015 Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	- * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
	+ * Copyright (c) 2011, 2024 by Delphix. All rights reserved.
	* Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>
	* Copyright (c) 2018 Datto Inc.
	* Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
	* Copyright (c) 2017, Intel Corporation.
	* Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>
	* Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
	* Copyright (c) 2021, 2023, Klara Inc.
	*/

	#include <errno.h>
	#include <libintl.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <strings.h>
	#include <unistd.h>
	#include <libgen.h>
	#include <zone.h>
	#include <sys/stat.h>
	#include <sys/efi_partition.h>
	#include <sys/systeminfo.h>
	#include <sys/zfs_ioctl.h>
	#include <sys/zfs_sysfs.h>
	#include <sys/vdev_disk.h>
	#include <sys/types.h>
	#include <dlfcn.h>
	#include <libzutil.h>
	#include <fcntl.h>

	#include "zfs_namecheck.h"
	#include "zfs_prop.h"
	#include "libzfs_impl.h"
	#include "zfs_comutil.h"
	#include "zfeature_common.h"

	static boolean_t zpool_vdev_is_interior(const char *name);

	typedef struct prop_flags {
	unsigned int create:1; /* Validate property on creation */
	unsigned int import:1; /* Validate property on import */
	unsigned int vdevprop:1; /* Validate property as a VDEV property */
	} prop_flags_t;

	/*
	* ====================================================================
	* zpool property functions
	* ====================================================================
	*/

	static int
	zpool_get_all_props(zpool_handle_t *zhp)
	{
	zfs_cmd_t zc = {"\0"};
	libzfs_handle_t *hdl = zhp->zpool_hdl;

	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));

	zcmd_alloc_dst_nvlist(hdl, &zc, 0);

	while (zfs_ioctl(hdl, ZFS_IOC_POOL_GET_PROPS, &zc) != 0) {
	if (errno == ENOMEM)
	zcmd_expand_dst_nvlist(hdl, &zc);
	else {
	zcmd_free_nvlists(&zc);
	return (-1);
	}
	}

	if (zcmd_read_dst_nvlist(hdl, &zc, &zhp->zpool_props) != 0) {
	zcmd_free_nvlists(&zc);
	return (-1);
	}

	zcmd_free_nvlists(&zc);

	return (0);
	}

	int
	zpool_props_refresh(zpool_handle_t *zhp)
	{
	nvlist_t *old_props;

	old_props = zhp->zpool_props;

	if (zpool_get_all_props(zhp) != 0)
	return (-1);

	nvlist_free(old_props);
	return (0);
	}

	static const char *
	zpool_get_prop_string(zpool_handle_t *zhp, zpool_prop_t prop,
	zprop_source_t *src)
	{
	nvlist_t nv, nvl;
	const char *value;
	zprop_source_t source;

	nvl = zhp->zpool_props;
	if (nvlist_lookup_nvlist(nvl, zpool_prop_to_name(prop), &nv) == 0) {
	source = fnvlist_lookup_uint64(nv, ZPROP_SOURCE);
	value = fnvlist_lookup_string(nv, ZPROP_VALUE);
	} else {
	source = ZPROP_SRC_DEFAULT;
	if ((value = zpool_prop_default_string(prop)) == NULL)
	value = "-";
	}

	if (src)
	*src = source;

	return (value);
	}

	uint64_t
	zpool_get_prop_int(zpool_handle_t zhp, zpool_prop_t prop, zprop_source_t src)
	{
	nvlist_t nv, nvl;
	uint64_t value;
	zprop_source_t source;

	if (zhp->zpool_props == NULL && zpool_get_all_props(zhp)) {
	/*
	* zpool_get_all_props() has most likely failed because
	* the pool is faulted, but if all we need is the top level
	* vdev's guid then get it from the zhp config nvlist.
	*/
	if ((prop == ZPOOL_PROP_GUID) &&
	(nvlist_lookup_nvlist(zhp->zpool_config,
	ZPOOL_CONFIG_VDEV_TREE, &nv) == 0) &&
	(nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &value)
	== 0)) {
	return (value);
	}
	return (zpool_prop_default_numeric(prop));
	}

	nvl = zhp->zpool_props;
	if (nvlist_lookup_nvlist(nvl, zpool_prop_to_name(prop), &nv) == 0) {
	source = fnvlist_lookup_uint64(nv, ZPROP_SOURCE);
	value = fnvlist_lookup_uint64(nv, ZPROP_VALUE);
	} else {
	source = ZPROP_SRC_DEFAULT;
	value = zpool_prop_default_numeric(prop);
	}

	if (src)
	*src = source;

	return (value);
	}

	/*
	* Map VDEV STATE to printed strings.
	*/
	const char *
	zpool_state_to_name(vdev_state_t state, vdev_aux_t aux)
	{
	switch (state) {
	case VDEV_STATE_CLOSED:
	case VDEV_STATE_OFFLINE:
	return (gettext("OFFLINE"));
	case VDEV_STATE_REMOVED:
	return (gettext("REMOVED"));
	case VDEV_STATE_CANT_OPEN:
	if (aux == VDEV_AUX_CORRUPT_DATA \|\| aux == VDEV_AUX_BAD_LOG)
	return (gettext("FAULTED"));
	else if (aux == VDEV_AUX_SPLIT_POOL)
	return (gettext("SPLIT"));
	else
	return (gettext("UNAVAIL"));
	case VDEV_STATE_FAULTED:
	return (gettext("FAULTED"));
	case VDEV_STATE_DEGRADED:
	return (gettext("DEGRADED"));
	case VDEV_STATE_HEALTHY:
	return (gettext("ONLINE"));

	default:
	break;
	}

	return (gettext("UNKNOWN"));
	}

	/*
	* Map POOL STATE to printed strings.
	*/
	const char *
	zpool_pool_state_to_name(pool_state_t state)
	{
	switch (state) {
	default:
	break;
	case POOL_STATE_ACTIVE:
	return (gettext("ACTIVE"));
	case POOL_STATE_EXPORTED:
	return (gettext("EXPORTED"));
	case POOL_STATE_DESTROYED:
	return (gettext("DESTROYED"));
	case POOL_STATE_SPARE:
	return (gettext("SPARE"));
	case POOL_STATE_L2CACHE:
	return (gettext("L2CACHE"));
	case POOL_STATE_UNINITIALIZED:
	return (gettext("UNINITIALIZED"));
	case POOL_STATE_UNAVAIL:
	return (gettext("UNAVAIL"));
	case POOL_STATE_POTENTIALLY_ACTIVE:
	return (gettext("POTENTIALLY_ACTIVE"));
	}

	return (gettext("UNKNOWN"));
	}

	/*
	* Given a pool handle, return the pool health string ("ONLINE", "DEGRADED",
	* "SUSPENDED", etc).
	*/
	const char *
	zpool_get_state_str(zpool_handle_t *zhp)
	{
	zpool_errata_t errata;
	zpool_status_t status;
	const char *str;

	status = zpool_get_status(zhp, NULL, &errata);

	if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) {
	str = gettext("FAULTED");
	} else if (status == ZPOOL_STATUS_IO_FAILURE_WAIT \|\|
	status == ZPOOL_STATUS_IO_FAILURE_CONTINUE \|\|
	status == ZPOOL_STATUS_IO_FAILURE_MMP) {
	str = gettext("SUSPENDED");
	} else {
	nvlist_t *nvroot = fnvlist_lookup_nvlist(
	zpool_get_config(zhp, NULL), ZPOOL_CONFIG_VDEV_TREE);
	uint_t vsc;
	vdev_stat_t vs = (vdev_stat_t )fnvlist_lookup_uint64_array(
	nvroot, ZPOOL_CONFIG_VDEV_STATS, &vsc);
	str = zpool_state_to_name(vs->vs_state, vs->vs_aux);
	}
	return (str);
	}

	/*
	* Get a zpool property value for 'prop' and return the value in
	* a pre-allocated buffer.
	*/
	int
	zpool_get_prop(zpool_handle_t zhp, zpool_prop_t prop, char buf,
	size_t len, zprop_source_t *srctype, boolean_t literal)
	{
	uint64_t intval;
	const char *strval;
	zprop_source_t src = ZPROP_SRC_NONE;

	if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) {
	switch (prop) {
	case ZPOOL_PROP_NAME:
	(void) strlcpy(buf, zpool_get_name(zhp), len);
	break;

	case ZPOOL_PROP_HEALTH:
	(void) strlcpy(buf, zpool_get_state_str(zhp), len);
	break;

	case ZPOOL_PROP_GUID:
	intval = zpool_get_prop_int(zhp, prop, &src);
	(void) snprintf(buf, len, "%llu", (u_longlong_t)intval);
	break;

	case ZPOOL_PROP_ALTROOT:
	case ZPOOL_PROP_CACHEFILE:
	case ZPOOL_PROP_COMMENT:
	case ZPOOL_PROP_COMPATIBILITY:
	if (zhp->zpool_props != NULL \|\|
	zpool_get_all_props(zhp) == 0) {
	(void) strlcpy(buf,
	zpool_get_prop_string(zhp, prop, &src),
	len);
	break;
	}
	zfs_fallthrough;
	default:
	(void) strlcpy(buf, "-", len);
	break;
	}

	if (srctype != NULL)
	*srctype = src;
	return (0);
	}

	if (zhp->zpool_props == NULL && zpool_get_all_props(zhp) &&
	prop != ZPOOL_PROP_NAME)
	return (-1);

	switch (zpool_prop_get_type(prop)) {
	case PROP_TYPE_STRING:
	(void) strlcpy(buf, zpool_get_prop_string(zhp, prop, &src),
	len);
	break;

	case PROP_TYPE_NUMBER:
	intval = zpool_get_prop_int(zhp, prop, &src);

	switch (prop) {
	case ZPOOL_PROP_SIZE:
	case ZPOOL_PROP_ALLOCATED:
	case ZPOOL_PROP_FREE:
	case ZPOOL_PROP_FREEING:
	case ZPOOL_PROP_LEAKED:
	case ZPOOL_PROP_ASHIFT:
	case ZPOOL_PROP_MAXBLOCKSIZE:
	case ZPOOL_PROP_MAXDNODESIZE:
	case ZPOOL_PROP_BCLONESAVED:
	case ZPOOL_PROP_BCLONEUSED:
	if (literal)
	(void) snprintf(buf, len, "%llu",
	(u_longlong_t)intval);
	else
	(void) zfs_nicenum(intval, buf, len);
	break;

	case ZPOOL_PROP_EXPANDSZ:
	case ZPOOL_PROP_CHECKPOINT:
	if (intval == 0) {
	(void) strlcpy(buf, "-", len);
	} else if (literal) {
	(void) snprintf(buf, len, "%llu",
	(u_longlong_t)intval);
	} else {
	(void) zfs_nicebytes(intval, buf, len);
	}
	break;

	case ZPOOL_PROP_CAPACITY:
	if (literal) {
	(void) snprintf(buf, len, "%llu",
	(u_longlong_t)intval);
	} else {
	(void) snprintf(buf, len, "%llu%%",
	(u_longlong_t)intval);
	}
	break;

	case ZPOOL_PROP_FRAGMENTATION:
	if (intval == UINT64_MAX) {
	(void) strlcpy(buf, "-", len);
	} else if (literal) {
	(void) snprintf(buf, len, "%llu",
	(u_longlong_t)intval);
	} else {
	(void) snprintf(buf, len, "%llu%%",
	(u_longlong_t)intval);
	}
	break;

	case ZPOOL_PROP_BCLONERATIO:
	case ZPOOL_PROP_DEDUPRATIO:
	if (literal)
	(void) snprintf(buf, len, "%llu.%02llu",
	(u_longlong_t)(intval / 100),
	(u_longlong_t)(intval % 100));
	else
	(void) snprintf(buf, len, "%llu.%02llux",
	(u_longlong_t)(intval / 100),
	(u_longlong_t)(intval % 100));
	break;

	case ZPOOL_PROP_HEALTH:
	(void) strlcpy(buf, zpool_get_state_str(zhp), len);
	break;
	case ZPOOL_PROP_VERSION:
	if (intval >= SPA_VERSION_FEATURES) {
	(void) snprintf(buf, len, "-");
	break;
	}
	zfs_fallthrough;
	default:
	(void) snprintf(buf, len, "%llu", (u_longlong_t)intval);
	}
	break;

	case PROP_TYPE_INDEX:
	intval = zpool_get_prop_int(zhp, prop, &src);
	if (zpool_prop_index_to_string(prop, intval, &strval)
	!= 0)
	return (-1);
	(void) strlcpy(buf, strval, len);
	break;

	default:
	abort();
	}

	if (srctype)
	*srctype = src;

	return (0);
	}

	/*
	* Get a zpool property value for 'propname' and return the value in
	* a pre-allocated buffer.
	*/
	int
	zpool_get_userprop(zpool_handle_t zhp, const char propname, char *buf,
	size_t len, zprop_source_t *srctype)
	{
	nvlist_t nv, nvl;
	uint64_t ival;
	const char *value;
	zprop_source_t source = ZPROP_SRC_LOCAL;

	nvl = zhp->zpool_props;
	if (nvlist_lookup_nvlist(nvl, propname, &nv) == 0) {
	if (nvlist_lookup_uint64(nv, ZPROP_SOURCE, &ival) == 0)
	source = ival;
	verify(nvlist_lookup_string(nv, ZPROP_VALUE, &value) == 0);
	} else {
	source = ZPROP_SRC_DEFAULT;
	value = "-";
	}

	if (srctype)
	*srctype = source;

	(void) strlcpy(buf, value, len);

	return (0);
	}

	/*
	* Check if the bootfs name has the same pool name as it is set to.
	* Assuming bootfs is a valid dataset name.
	*/
	static boolean_t
	bootfs_name_valid(const char pool, const char bootfs)
	{
	int len = strlen(pool);
	if (bootfs[0] == '\0')
	return (B_TRUE);

	if (!zfs_name_valid(bootfs, ZFS_TYPE_FILESYSTEM\|ZFS_TYPE_SNAPSHOT))
	return (B_FALSE);

	if (strncmp(pool, bootfs, len) == 0 &&
	(bootfs[len] == '/' \|\| bootfs[len] == '\0'))
	return (B_TRUE);

	return (B_FALSE);
	}

	/*
	* Given an nvlist of zpool properties to be set, validate that they are
	* correct, and parse any numeric properties (index, boolean, etc) if they are
	* specified as strings.
	*/
	static nvlist_t *
	zpool_valid_proplist(libzfs_handle_t hdl, const char poolname,
	nvlist_t props, uint64_t version, prop_flags_t flags, char errbuf)
	{
	nvpair_t *elem;
	nvlist_t *retprops;
	zpool_prop_t prop;
	const char *strval;
	uint64_t intval;
	const char slash, check;
	struct stat64 statbuf;
	zpool_handle_t *zhp;
	char report[1024];

	if (nvlist_alloc(&retprops, NV_UNIQUE_NAME, 0) != 0) {
	(void) no_memory(hdl);
	return (NULL);
	}

	elem = NULL;
	while ((elem = nvlist_next_nvpair(props, elem)) != NULL) {
	const char *propname = nvpair_name(elem);

	if (flags.vdevprop && zpool_prop_vdev(propname)) {
	vdev_prop_t vprop = vdev_name_to_prop(propname);

	if (vdev_prop_readonly(vprop)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "'%s' "
	"is readonly"), propname);
	(void) zfs_error(hdl, EZFS_PROPREADONLY,
	errbuf);
	goto error;
	}

	if (zprop_parse_value(hdl, elem, vprop, ZFS_TYPE_VDEV,
	retprops, &strval, &intval, errbuf) != 0)
	goto error;

	continue;
	} else if (flags.vdevprop && vdev_prop_user(propname)) {
	if (nvlist_add_nvpair(retprops, elem) != 0) {
	(void) no_memory(hdl);
	goto error;
	}
	continue;
	} else if (flags.vdevprop) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"invalid property: '%s'"), propname);
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}

	prop = zpool_name_to_prop(propname);
	if (prop == ZPOOL_PROP_INVAL && zpool_prop_feature(propname)) {
	int err;
	char *fname = strchr(propname, '@') + 1;

	err = zfeature_lookup_name(fname, NULL);
	if (err != 0) {
	ASSERT3U(err, ==, ENOENT);
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"feature '%s' unsupported by kernel"),
	fname);
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}

	if (nvpair_type(elem) != DATA_TYPE_STRING) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"'%s' must be a string"), propname);
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}

	(void) nvpair_value_string(elem, &strval);
	if (strcmp(strval, ZFS_FEATURE_ENABLED) != 0 &&
	strcmp(strval, ZFS_FEATURE_DISABLED) != 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property '%s' can only be set to "
	"'enabled' or 'disabled'"), propname);
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}

	if (!flags.create &&
	strcmp(strval, ZFS_FEATURE_DISABLED) == 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property '%s' can only be set to "
	"'disabled' at creation time"), propname);
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}

	if (nvlist_add_uint64(retprops, propname, 0) != 0) {
	(void) no_memory(hdl);
	goto error;
	}
	continue;
	} else if (prop == ZPOOL_PROP_INVAL &&
	zfs_prop_user(propname)) {
	/*
	* This is a user property: make sure it's a
	* string, and that it's less than ZAP_MAXNAMELEN.
	*/
	if (nvpair_type(elem) != DATA_TYPE_STRING) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"'%s' must be a string"), propname);
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}

	if (strlen(nvpair_name(elem)) >= ZAP_MAXNAMELEN) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property name '%s' is too long"),
	propname);
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}

	(void) nvpair_value_string(elem, &strval);

	if (strlen(strval) >= ZFS_MAXPROPLEN) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property value '%s' is too long"),
	strval);
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}

	if (nvlist_add_string(retprops, propname,
	strval) != 0) {
	(void) no_memory(hdl);
	goto error;
	}

	continue;
	}

	/*
	* Make sure this property is valid and applies to this type.
	*/
	if (prop == ZPOOL_PROP_INVAL) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"invalid property '%s'"), propname);
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}

	if (zpool_prop_readonly(prop)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "'%s' "
	"is readonly"), propname);
	(void) zfs_error(hdl, EZFS_PROPREADONLY, errbuf);
	goto error;
	}

	if (!flags.create && zpool_prop_setonce(prop)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property '%s' can only be set at "
	"creation time"), propname);
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}

	if (zprop_parse_value(hdl, elem, prop, ZFS_TYPE_POOL, retprops,
	&strval, &intval, errbuf) != 0)
	goto error;

	/*
	* Perform additional checking for specific properties.
	*/
	switch (prop) {
	case ZPOOL_PROP_VERSION:
	if (intval < version \|\|
	!SPA_VERSION_IS_SUPPORTED(intval)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property '%s' number %llu is invalid."),
	propname, (unsigned long long)intval);
	(void) zfs_error(hdl, EZFS_BADVERSION, errbuf);
	goto error;
	}
	break;

	case ZPOOL_PROP_ASHIFT:
	if (intval != 0 &&
	(intval < ASHIFT_MIN \|\| intval > ASHIFT_MAX)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property '%s' number %llu is invalid, "
	"only values between %" PRId32 " and %"
	PRId32 " are allowed."),
	propname, (unsigned long long)intval,
	ASHIFT_MIN, ASHIFT_MAX);
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}
	break;

	case ZPOOL_PROP_BOOTFS:
	if (flags.create \|\| flags.import) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property '%s' cannot be set at creation "
	"or import time"), propname);
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}

	if (version < SPA_VERSION_BOOTFS) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"pool must be upgraded to support "
	"'%s' property"), propname);
	(void) zfs_error(hdl, EZFS_BADVERSION, errbuf);
	goto error;
	}

	/*
	* bootfs property value has to be a dataset name and
	* the dataset has to be in the same pool as it sets to.
	*/
	if (!bootfs_name_valid(poolname, strval)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "'%s' "
	"is an invalid name"), strval);
	(void) zfs_error(hdl, EZFS_INVALIDNAME, errbuf);
	goto error;
	}

	if ((zhp = zpool_open_canfail(hdl, poolname)) == NULL) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"could not open pool '%s'"), poolname);
	(void) zfs_error(hdl, EZFS_OPENFAILED, errbuf);
	goto error;
	}
	zpool_close(zhp);
	break;

	case ZPOOL_PROP_ALTROOT:
	if (!flags.create && !flags.import) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property '%s' can only be set during pool "
	"creation or import"), propname);
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}

	if (strval[0] != '/') {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"bad alternate root '%s'"), strval);
	(void) zfs_error(hdl, EZFS_BADPATH, errbuf);
	goto error;
	}
	break;

	case ZPOOL_PROP_CACHEFILE:
	if (strval[0] == '\0')
	break;

	if (strcmp(strval, "none") == 0)
	break;

	if (strval[0] != '/') {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property '%s' must be empty, an "
	"absolute path, or 'none'"), propname);
	(void) zfs_error(hdl, EZFS_BADPATH, errbuf);
	goto error;
	}

	slash = strrchr(strval, '/');

	if (slash[1] == '\0' \|\| strcmp(slash, "/.") == 0 \|\|
	strcmp(slash, "/..") == 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"'%s' is not a valid file"), strval);
	(void) zfs_error(hdl, EZFS_BADPATH, errbuf);
	goto error;
	}

	(char )slash = '\0';

	if (strval[0] != '\0' &&
	(stat64(strval, &statbuf) != 0 \|\|
	!S_ISDIR(statbuf.st_mode))) {
	(char )slash = '/';
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"'%s' is not a valid directory"),
	strval);
	(void) zfs_error(hdl, EZFS_BADPATH, errbuf);
	goto error;
	}

	(char )slash = '/';
	break;

	case ZPOOL_PROP_COMPATIBILITY:
	switch (zpool_load_compat(strval, NULL, report, 1024)) {
	case ZPOOL_COMPATIBILITY_OK:
	case ZPOOL_COMPATIBILITY_WARNTOKEN:
	break;
	case ZPOOL_COMPATIBILITY_BADFILE:
	case ZPOOL_COMPATIBILITY_BADTOKEN:
	case ZPOOL_COMPATIBILITY_NOFILES:
	zfs_error_aux(hdl, "%s", report);
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}
	break;

	case ZPOOL_PROP_COMMENT:
	for (check = strval; *check != '\0'; check++) {
	if (!isprint(*check)) {
	zfs_error_aux(hdl,
	dgettext(TEXT_DOMAIN,
	"comment may only have printable "
	"characters"));
	(void) zfs_error(hdl, EZFS_BADPROP,
	errbuf);
	goto error;
	}
	}
	if (strlen(strval) > ZPROP_MAX_COMMENT) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"comment must not exceed %d characters"),
	ZPROP_MAX_COMMENT);
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}
	break;
	case ZPOOL_PROP_READONLY:
	if (!flags.import) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property '%s' can only be set at "
	"import time"), propname);
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}
	break;
	case ZPOOL_PROP_MULTIHOST:
	if (get_system_hostid() == 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"requires a non-zero system hostid"));
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}
	break;
	case ZPOOL_PROP_DEDUPDITTO:
	printf("Note: property '%s' no longer has "
	"any effect\n", propname);
	break;

	default:
	break;
	}
	}

	return (retprops);
	error:
	nvlist_free(retprops);
	return (NULL);
	}

	/*
	* Set zpool property : propname=propval.
	*/
	int
	zpool_set_prop(zpool_handle_t zhp, const char propname, const char *propval)
	{
	zfs_cmd_t zc = {"\0"};
	int ret = -1;
	char errbuf[ERRBUFLEN];
	nvlist_t *nvl = NULL;
	nvlist_t *realprops;
	uint64_t version;
	prop_flags_t flags = { 0 };

	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot set property for '%s'"),
	zhp->zpool_name);

	if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0)
	return (no_memory(zhp->zpool_hdl));

	if (nvlist_add_string(nvl, propname, propval) != 0) {
	nvlist_free(nvl);
	return (no_memory(zhp->zpool_hdl));
	}

	version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
	if ((realprops = zpool_valid_proplist(zhp->zpool_hdl,
	zhp->zpool_name, nvl, version, flags, errbuf)) == NULL) {
	nvlist_free(nvl);
	return (-1);
	}

	nvlist_free(nvl);
	nvl = realprops;

	/*
	* Execute the corresponding ioctl() to set this property.
	*/
	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));

	zcmd_write_src_nvlist(zhp->zpool_hdl, &zc, nvl);

	ret = zfs_ioctl(zhp->zpool_hdl, ZFS_IOC_POOL_SET_PROPS, &zc);

	zcmd_free_nvlists(&zc);
	nvlist_free(nvl);

	if (ret)
	(void) zpool_standard_error(zhp->zpool_hdl, errno, errbuf);
	else
	(void) zpool_props_refresh(zhp);

	return (ret);
	}

	int
	zpool_expand_proplist(zpool_handle_t zhp, zprop_list_t *plp,
	zfs_type_t type, boolean_t literal)
	{
	libzfs_handle_t *hdl = zhp->zpool_hdl;
	zprop_list_t *entry;
	char buf[ZFS_MAXPROPLEN];
	nvlist_t *features = NULL;
	nvpair_t *nvp;
	zprop_list_t **last;
	boolean_t firstexpand = (NULL == *plp);
	int i;

	if (zprop_expand_list(hdl, plp, type) != 0)
	return (-1);

	if (type == ZFS_TYPE_VDEV)
	return (0);

	last = plp;
	while (*last != NULL)
	last = &(*last)->pl_next;

	if ((*plp)->pl_all)
	features = zpool_get_features(zhp);

	if ((*plp)->pl_all && firstexpand) {
	/* Handle userprops in the all properties case */
	if (zhp->zpool_props == NULL && zpool_props_refresh(zhp))
	return (-1);

	nvp = NULL;
	while ((nvp = nvlist_next_nvpair(zhp->zpool_props, nvp)) !=
	NULL) {
	const char *propname = nvpair_name(nvp);

	if (!zfs_prop_user(propname))
	continue;

	entry = zfs_alloc(hdl, sizeof (zprop_list_t));
	entry->pl_prop = ZPROP_USERPROP;
	entry->pl_user_prop = zfs_strdup(hdl, propname);
	entry->pl_width = strlen(entry->pl_user_prop);
	entry->pl_all = B_TRUE;

	*last = entry;
	last = &entry->pl_next;
	}

	for (i = 0; i < SPA_FEATURES; i++) {
	entry = zfs_alloc(hdl, sizeof (zprop_list_t));
	entry->pl_prop = ZPROP_USERPROP;
	entry->pl_user_prop = zfs_asprintf(hdl, "feature@%s",
	spa_feature_table[i].fi_uname);
	entry->pl_width = strlen(entry->pl_user_prop);
	entry->pl_all = B_TRUE;

	*last = entry;
	last = &entry->pl_next;
	}
	}

	/* add any unsupported features */
	for (nvp = nvlist_next_nvpair(features, NULL);
	nvp != NULL; nvp = nvlist_next_nvpair(features, nvp)) {
	char *propname;
	boolean_t found;

	if (zfeature_is_supported(nvpair_name(nvp)))
	continue;

	propname = zfs_asprintf(hdl, "unsupported@%s",
	nvpair_name(nvp));

	/*
	* Before adding the property to the list make sure that no
	* other pool already added the same property.
	*/
	found = B_FALSE;
	entry = *plp;
	while (entry != NULL) {
	if (entry->pl_user_prop != NULL &&
	strcmp(propname, entry->pl_user_prop) == 0) {
	found = B_TRUE;
	break;
	}
	entry = entry->pl_next;
	}
	if (found) {
	free(propname);
	continue;
	}

	entry = zfs_alloc(hdl, sizeof (zprop_list_t));
	entry->pl_prop = ZPROP_USERPROP;
	entry->pl_user_prop = propname;
	entry->pl_width = strlen(entry->pl_user_prop);
	entry->pl_all = B_TRUE;

	*last = entry;
	last = &entry->pl_next;
	}

	for (entry = *plp; entry != NULL; entry = entry->pl_next) {
	if (entry->pl_fixed && !literal)
	continue;

	if (entry->pl_prop != ZPROP_USERPROP &&
	zpool_get_prop(zhp, entry->pl_prop, buf, sizeof (buf),
	NULL, literal) == 0) {
	if (strlen(buf) > entry->pl_width)
	entry->pl_width = strlen(buf);
	} else if (entry->pl_prop == ZPROP_INVAL &&
	zfs_prop_user(entry->pl_user_prop) &&
	zpool_get_userprop(zhp, entry->pl_user_prop, buf,
	sizeof (buf), NULL) == 0) {
	if (strlen(buf) > entry->pl_width)
	entry->pl_width = strlen(buf);
	}
	}

	return (0);
	}

	int
	vdev_expand_proplist(zpool_handle_t zhp, const char vdevname,
	zprop_list_t **plp)
	{
	zprop_list_t *entry;
	char buf[ZFS_MAXPROPLEN];
	const char *strval = NULL;
	int err = 0;
	nvpair_t *elem = NULL;
	nvlist_t *vprops = NULL;
	nvlist_t *propval = NULL;
	const char *propname;
	vdev_prop_t prop;
	zprop_list_t **last;

	for (entry = *plp; entry != NULL; entry = entry->pl_next) {
	if (entry->pl_fixed)
	continue;

	if (zpool_get_vdev_prop(zhp, vdevname, entry->pl_prop,
	entry->pl_user_prop, buf, sizeof (buf), NULL,
	B_FALSE) == 0) {
	if (strlen(buf) > entry->pl_width)
	entry->pl_width = strlen(buf);
	}
	if (entry->pl_prop == VDEV_PROP_NAME &&
	strlen(vdevname) > entry->pl_width)
	entry->pl_width = strlen(vdevname);
	}

	/* Handle the all properties case */
	last = plp;
	if (last != NULL && (last)->pl_all == B_TRUE) {
	while (*last != NULL)
	last = &(*last)->pl_next;

	err = zpool_get_all_vdev_props(zhp, vdevname, &vprops);
	if (err != 0)
	return (err);

	while ((elem = nvlist_next_nvpair(vprops, elem)) != NULL) {
	propname = nvpair_name(elem);

	/* Skip properties that are not user defined */
	if ((prop = vdev_name_to_prop(propname)) !=
	VDEV_PROP_USERPROP)
	continue;

	if (nvpair_value_nvlist(elem, &propval) != 0)
	continue;

	strval = fnvlist_lookup_string(propval, ZPROP_VALUE);

	entry = zfs_alloc(zhp->zpool_hdl,
	sizeof (zprop_list_t));
	entry->pl_prop = prop;
	entry->pl_user_prop = zfs_strdup(zhp->zpool_hdl,
	propname);
	entry->pl_width = strlen(strval);
	entry->pl_all = B_TRUE;
	*last = entry;
	last = &entry->pl_next;
	}
	}

	return (0);
	}

	/*
	* Get the state for the given feature on the given ZFS pool.
	*/
	int
	zpool_prop_get_feature(zpool_handle_t zhp, const char propname, char *buf,
	size_t len)
	{
	uint64_t refcount;
	boolean_t found = B_FALSE;
	nvlist_t *features = zpool_get_features(zhp);
	boolean_t supported;
	const char *feature = strchr(propname, '@') + 1;

	supported = zpool_prop_feature(propname);
	ASSERT(supported \|\| zpool_prop_unsupported(propname));

	/*
	* Convert from feature name to feature guid. This conversion is
	* unnecessary for unsupported@... properties because they already
	* use guids.
	*/
	if (supported) {
	int ret;
	spa_feature_t fid;

	ret = zfeature_lookup_name(feature, &fid);
	if (ret != 0) {
	(void) strlcpy(buf, "-", len);
	return (ENOTSUP);
	}
	feature = spa_feature_table[fid].fi_guid;
	}

	if (nvlist_lookup_uint64(features, feature, &refcount) == 0)
	found = B_TRUE;

	if (supported) {
	if (!found) {
	(void) strlcpy(buf, ZFS_FEATURE_DISABLED, len);
	} else {
	if (refcount == 0)
	(void) strlcpy(buf, ZFS_FEATURE_ENABLED, len);
	else
	(void) strlcpy(buf, ZFS_FEATURE_ACTIVE, len);
	}
	} else {
	if (found) {
	if (refcount == 0) {
	(void) strcpy(buf, ZFS_UNSUPPORTED_INACTIVE);
	} else {
	(void) strcpy(buf, ZFS_UNSUPPORTED_READONLY);
	}
	} else {
	(void) strlcpy(buf, "-", len);
	return (ENOTSUP);
	}
	}

	return (0);
	}

	/*
	* Validate the given pool name, optionally putting an extended error message in
	* 'buf'.
	*/
	boolean_t
	zpool_name_valid(libzfs_handle_t hdl, boolean_t isopen, const char pool)
	{
	namecheck_err_t why;
	char what;
	int ret;

	ret = pool_namecheck(pool, &why, &what);

	/*
	* The rules for reserved pool names were extended at a later point.
	* But we need to support users with existing pools that may now be
	* invalid. So we only check for this expanded set of names during a
	* create (or import), and only in userland.
	*/
	if (ret == 0 && !isopen &&
	(strncmp(pool, "mirror", 6) == 0 \|\|
	strncmp(pool, "raidz", 5) == 0 \|\|
	strncmp(pool, "draid", 5) == 0 \|\|
	strncmp(pool, "spare", 5) == 0 \|\|
	strcmp(pool, "log") == 0)) {
	if (hdl != NULL)
	zfs_error_aux(hdl,
	dgettext(TEXT_DOMAIN, "name is reserved"));
	return (B_FALSE);
	}


	if (ret != 0) {
	if (hdl != NULL) {
	switch (why) {
	case NAME_ERR_TOOLONG:
	zfs_error_aux(hdl,
	dgettext(TEXT_DOMAIN, "name is too long"));
	break;

	case NAME_ERR_INVALCHAR:
	zfs_error_aux(hdl,
	dgettext(TEXT_DOMAIN, "invalid character "
	"'%c' in pool name"), what);
	break;

	case NAME_ERR_NOLETTER:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"name must begin with a letter"));
	break;

	case NAME_ERR_RESERVED:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"name is reserved"));
	break;

	case NAME_ERR_DISKLIKE:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"pool name is reserved"));
	break;

	case NAME_ERR_LEADING_SLASH:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"leading slash in name"));
	break;

	case NAME_ERR_EMPTY_COMPONENT:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"empty component in name"));
	break;

	case NAME_ERR_TRAILING_SLASH:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"trailing slash in name"));
	break;

	case NAME_ERR_MULTIPLE_DELIMITERS:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"multiple '@' and/or '#' delimiters in "
	"name"));
	break;

	case NAME_ERR_NO_AT:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"permission set is missing '@'"));
	break;

	default:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"(%d) not defined"), why);
	break;
	}
	}
	return (B_FALSE);
	}

	return (B_TRUE);
	}

	/*
	* Open a handle to the given pool, even if the pool is currently in the FAULTED
	* state.
	*/
	zpool_handle_t *
	zpool_open_canfail(libzfs_handle_t hdl, const char pool)
	{
	zpool_handle_t *zhp;
	boolean_t missing;

	/*
	* Make sure the pool name is valid.
	*/
	if (!zpool_name_valid(hdl, B_TRUE, pool)) {
	(void) zfs_error_fmt(hdl, EZFS_INVALIDNAME,
	dgettext(TEXT_DOMAIN, "cannot open '%s'"),
	pool);
	return (NULL);
	}

	zhp = zfs_alloc(hdl, sizeof (zpool_handle_t));

	zhp->zpool_hdl = hdl;
	(void) strlcpy(zhp->zpool_name, pool, sizeof (zhp->zpool_name));

	if (zpool_refresh_stats(zhp, &missing) != 0) {
	zpool_close(zhp);
	return (NULL);
	}

	if (missing) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "no such pool"));
	(void) zfs_error_fmt(hdl, EZFS_NOENT,
	dgettext(TEXT_DOMAIN, "cannot open '%s'"), pool);
	zpool_close(zhp);
	return (NULL);
	}

	return (zhp);
	}

	/*
	* Like the above, but silent on error. Used when iterating over pools (because
	* the configuration cache may be out of date).
	*/
	int
	zpool_open_silent(libzfs_handle_t hdl, const char pool, zpool_handle_t **ret)
	{
	zpool_handle_t *zhp;
	boolean_t missing;

	zhp = zfs_alloc(hdl, sizeof (zpool_handle_t));

	zhp->zpool_hdl = hdl;
	(void) strlcpy(zhp->zpool_name, pool, sizeof (zhp->zpool_name));

	if (zpool_refresh_stats(zhp, &missing) != 0) {
	zpool_close(zhp);
	return (-1);
	}

	if (missing) {
	zpool_close(zhp);
	*ret = NULL;
	return (0);
	}

	*ret = zhp;
	return (0);
	}

	/*
	* Similar to zpool_open_canfail(), but refuses to open pools in the faulted
	* state.
	*/
	zpool_handle_t *
	zpool_open(libzfs_handle_t hdl, const char pool)
	{
	zpool_handle_t *zhp;

	if ((zhp = zpool_open_canfail(hdl, pool)) == NULL)
	return (NULL);

	if (zhp->zpool_state == POOL_STATE_UNAVAIL) {
	(void) zfs_error_fmt(hdl, EZFS_POOLUNAVAIL,
	dgettext(TEXT_DOMAIN, "cannot open '%s'"), zhp->zpool_name);
	zpool_close(zhp);
	return (NULL);
	}

	return (zhp);
	}

	/*
	* Close the handle. Simply frees the memory associated with the handle.
	*/
	void
	zpool_close(zpool_handle_t *zhp)
	{
	nvlist_free(zhp->zpool_config);
	nvlist_free(zhp->zpool_old_config);
	nvlist_free(zhp->zpool_props);
	free(zhp);
	}

	/*
	* Return the name of the pool.
	*/
	const char *
	zpool_get_name(zpool_handle_t *zhp)
	{
	return (zhp->zpool_name);
	}


	/*
	* Return the state of the pool (ACTIVE or UNAVAILABLE)
	*/
	int
	zpool_get_state(zpool_handle_t *zhp)
	{
	return (zhp->zpool_state);
	}

	/*
	* Check if vdev list contains a special vdev
	*/
	static boolean_t
	zpool_has_special_vdev(nvlist_t *nvroot)
	{
	nvlist_t **child;
	uint_t children;

	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN, &child,
	&children) == 0) {
	for (uint_t c = 0; c < children; c++) {
	const char *bias;

	if (nvlist_lookup_string(child[c],
	ZPOOL_CONFIG_ALLOCATION_BIAS, &bias) == 0 &&
	strcmp(bias, VDEV_ALLOC_BIAS_SPECIAL) == 0) {
	return (B_TRUE);
	}
	}
	}
	return (B_FALSE);
	}

	/*
	* Check if vdev list contains a dRAID vdev
	*/
	static boolean_t
	zpool_has_draid_vdev(nvlist_t *nvroot)
	{
	nvlist_t **child;
	uint_t children;

	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
	&child, &children) == 0) {
	for (uint_t c = 0; c < children; c++) {
	const char *type;

	if (nvlist_lookup_string(child[c],
	ZPOOL_CONFIG_TYPE, &type) == 0 &&
	strcmp(type, VDEV_TYPE_DRAID) == 0) {
	return (B_TRUE);
	}
	}
	}
	return (B_FALSE);
	}

	/*
	* Output a dRAID top-level vdev name in to the provided buffer.
	*/
	static char *
	zpool_draid_name(char *name, int len, uint64_t data, uint64_t parity,
	uint64_t spares, uint64_t children)
	{
	snprintf(name, len, "%s%llu:%llud:%lluc:%llus",
	VDEV_TYPE_DRAID, (u_longlong_t)parity, (u_longlong_t)data,
	(u_longlong_t)children, (u_longlong_t)spares);

	return (name);
	}

	/*
	* Return B_TRUE if the provided name is a dRAID spare name.
	*/
	boolean_t
	zpool_is_draid_spare(const char *name)
	{
	uint64_t spare_id, parity, vdev_id;

	if (sscanf(name, VDEV_TYPE_DRAID "%llu-%llu-%llu",
	(u_longlong_t )&parity, (u_longlong_t )&vdev_id,
	(u_longlong_t *)&spare_id) == 3) {
	return (B_TRUE);
	}

	return (B_FALSE);
	}

	/*
	* Create the named pool, using the provided vdev list. It is assumed
	* that the consumer has already validated the contents of the nvlist, so we
	* don't have to worry about error semantics.
	*/
	int
	zpool_create(libzfs_handle_t hdl, const char pool, nvlist_t *nvroot,
	nvlist_t props, nvlist_t fsprops)
	{
	zfs_cmd_t zc = {"\0"};
	nvlist_t *zc_fsprops = NULL;
	nvlist_t *zc_props = NULL;
	nvlist_t *hidden_args = NULL;
	uint8_t *wkeydata = NULL;
	uint_t wkeylen = 0;
	char errbuf[ERRBUFLEN];
	int ret = -1;

	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot create '%s'"), pool);

	if (!zpool_name_valid(hdl, B_FALSE, pool))
	return (zfs_error(hdl, EZFS_INVALIDNAME, errbuf));

	zcmd_write_conf_nvlist(hdl, &zc, nvroot);

	if (props) {
	prop_flags_t flags = { .create = B_TRUE, .import = B_FALSE };

	if ((zc_props = zpool_valid_proplist(hdl, pool, props,
	SPA_VERSION_1, flags, errbuf)) == NULL) {
	goto create_failed;
	}
	}

	if (fsprops) {
	uint64_t zoned;
	const char *zonestr;

	zoned = ((nvlist_lookup_string(fsprops,
	zfs_prop_to_name(ZFS_PROP_ZONED), &zonestr) == 0) &&
	strcmp(zonestr, "on") == 0);

	if ((zc_fsprops = zfs_valid_proplist(hdl, ZFS_TYPE_FILESYSTEM,
	fsprops, zoned, NULL, NULL, B_TRUE, errbuf)) == NULL) {
	goto create_failed;
	}

	if (nvlist_exists(zc_fsprops,
	zfs_prop_to_name(ZFS_PROP_SPECIAL_SMALL_BLOCKS)) &&
	!zpool_has_special_vdev(nvroot)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"%s property requires a special vdev"),
	zfs_prop_to_name(ZFS_PROP_SPECIAL_SMALL_BLOCKS));
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto create_failed;
	}

	if (!zc_props &&
	(nvlist_alloc(&zc_props, NV_UNIQUE_NAME, 0) != 0)) {
	goto create_failed;
	}
	if (zfs_crypto_create(hdl, NULL, zc_fsprops, props, B_TRUE,
	&wkeydata, &wkeylen) != 0) {
	zfs_error(hdl, EZFS_CRYPTOFAILED, errbuf);
	goto create_failed;
	}
	if (nvlist_add_nvlist(zc_props,
	ZPOOL_ROOTFS_PROPS, zc_fsprops) != 0) {
	goto create_failed;
	}
	if (wkeydata != NULL) {
	if (nvlist_alloc(&hidden_args, NV_UNIQUE_NAME, 0) != 0)
	goto create_failed;

	if (nvlist_add_uint8_array(hidden_args, "wkeydata",
	wkeydata, wkeylen) != 0)
	goto create_failed;

	if (nvlist_add_nvlist(zc_props, ZPOOL_HIDDEN_ARGS,
	hidden_args) != 0)
	goto create_failed;
	}
	}

	if (zc_props)
	zcmd_write_src_nvlist(hdl, &zc, zc_props);

	(void) strlcpy(zc.zc_name, pool, sizeof (zc.zc_name));

	if ((ret = zfs_ioctl(hdl, ZFS_IOC_POOL_CREATE, &zc)) != 0) {

	zcmd_free_nvlists(&zc);
	nvlist_free(zc_props);
	nvlist_free(zc_fsprops);
	nvlist_free(hidden_args);
	if (wkeydata != NULL)
	free(wkeydata);

	switch (errno) {
	case EBUSY:
	/*
	* This can happen if the user has specified the same
	* device multiple times. We can't reliably detect this
	* until we try to add it and see we already have a
	* label. This can also happen under if the device is
	* part of an active md or lvm device.
	*/
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"one or more vdevs refer to the same device, or "
	"one of\nthe devices is part of an active md or "
	"lvm device"));
	return (zfs_error(hdl, EZFS_BADDEV, errbuf));

	case ERANGE:
	/*
	* This happens if the record size is smaller or larger
	* than the allowed size range, or not a power of 2.
	*
	* NOTE: although zfs_valid_proplist is called earlier,
	* this case may have slipped through since the
	* pool does not exist yet and it is therefore
	* impossible to read properties e.g. max blocksize
	* from the pool.
	*/
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"record size invalid"));
	return (zfs_error(hdl, EZFS_BADPROP, errbuf));

	case EOVERFLOW:
	/*
	* This occurs when one of the devices is below
	* SPA_MINDEVSIZE. Unfortunately, we can't detect which
	* device was the problem device since there's no
	* reliable way to determine device size from userland.
	*/
	{
	char buf[64];

	zfs_nicebytes(SPA_MINDEVSIZE, buf,
	sizeof (buf));

	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"one or more devices is less than the "
	"minimum size (%s)"), buf);
	}
	return (zfs_error(hdl, EZFS_BADDEV, errbuf));

	case ENOSPC:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"one or more devices is out of space"));
	return (zfs_error(hdl, EZFS_BADDEV, errbuf));

	case EINVAL:
	if (zpool_has_draid_vdev(nvroot) &&
	zfeature_lookup_name("draid", NULL) != 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"dRAID vdevs are unsupported by the "
	"kernel"));
	return (zfs_error(hdl, EZFS_BADDEV, errbuf));
	} else {
	return (zpool_standard_error(hdl, errno,
	errbuf));
	}

	default:
	return (zpool_standard_error(hdl, errno, errbuf));
	}
	}

	create_failed:
	zcmd_free_nvlists(&zc);
	nvlist_free(zc_props);
	nvlist_free(zc_fsprops);
	nvlist_free(hidden_args);
	if (wkeydata != NULL)
	free(wkeydata);
	return (ret);
	}

	/*
	* Destroy the given pool. It is up to the caller to ensure that there are no
	* datasets left in the pool.
	*/
	int
	zpool_destroy(zpool_handle_t zhp, const char log_str)
	{
	zfs_cmd_t zc = {"\0"};
	zfs_handle_t *zfp = NULL;
	libzfs_handle_t *hdl = zhp->zpool_hdl;
	char errbuf[ERRBUFLEN];

	if (zhp->zpool_state == POOL_STATE_ACTIVE &&
	(zfp = zfs_open(hdl, zhp->zpool_name, ZFS_TYPE_FILESYSTEM)) == NULL)
	return (-1);

	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	zc.zc_history = (uint64_t)(uintptr_t)log_str;

	if (zfs_ioctl(hdl, ZFS_IOC_POOL_DESTROY, &zc) != 0) {
	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot destroy '%s'"), zhp->zpool_name);

	if (errno == EROFS) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"one or more devices is read only"));
	(void) zfs_error(hdl, EZFS_BADDEV, errbuf);
	} else {
	(void) zpool_standard_error(hdl, errno, errbuf);
	}

	if (zfp)
	zfs_close(zfp);
	return (-1);
	}

	if (zfp) {
	remove_mountpoint(zfp);
	zfs_close(zfp);
	}

	return (0);
	}

	/*
	* Create a checkpoint in the given pool.
	*/
	int
	zpool_checkpoint(zpool_handle_t *zhp)
	{
	libzfs_handle_t *hdl = zhp->zpool_hdl;
	char errbuf[ERRBUFLEN];
	int error;

	error = lzc_pool_checkpoint(zhp->zpool_name);
	if (error != 0) {
	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot checkpoint '%s'"), zhp->zpool_name);
	(void) zpool_standard_error(hdl, error, errbuf);
	return (-1);
	}

	return (0);
	}

	/*
	* Discard the checkpoint from the given pool.
	*/
	int
	zpool_discard_checkpoint(zpool_handle_t *zhp)
	{
	libzfs_handle_t *hdl = zhp->zpool_hdl;
	char errbuf[ERRBUFLEN];
	int error;

	error = lzc_pool_checkpoint_discard(zhp->zpool_name);
	if (error != 0) {
	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot discard checkpoint in '%s'"), zhp->zpool_name);
	(void) zpool_standard_error(hdl, error, errbuf);
	return (-1);
	}

	return (0);
	}

	/*
	* Add the given vdevs to the pool. The caller must have already performed the
	* necessary verification to ensure that the vdev specification is well-formed.
	*/
	int
	-zpool_add(zpool_handle_t zhp, nvlist_t nvroot)
	+zpool_add(zpool_handle_t zhp, nvlist_t nvroot, boolean_t check_ashift)
	{
	zfs_cmd_t zc = {"\0"};
	int ret;
	libzfs_handle_t *hdl = zhp->zpool_hdl;
	char errbuf[ERRBUFLEN];
	nvlist_t spares, l2cache;
	uint_t nspares, nl2cache;

	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot add to '%s'"), zhp->zpool_name);

	if (zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL) <
	SPA_VERSION_SPARES &&
	nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
	&spares, &nspares) == 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "pool must be "
	"upgraded to add hot spares"));
	return (zfs_error(hdl, EZFS_BADVERSION, errbuf));
	}

	if (zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL) <
	SPA_VERSION_L2CACHE &&
	nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
	&l2cache, &nl2cache) == 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "pool must be "
	"upgraded to add cache devices"));
	return (zfs_error(hdl, EZFS_BADVERSION, errbuf));
	}

	zcmd_write_conf_nvlist(hdl, &zc, nvroot);
	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	+ zc.zc_flags = check_ashift;

	if (zfs_ioctl(hdl, ZFS_IOC_VDEV_ADD, &zc) != 0) {
	switch (errno) {
	case EBUSY:
	/*
	* This can happen if the user has specified the same
	* device multiple times. We can't reliably detect this
	* until we try to add it and see we already have a
	* label.
	*/
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"one or more vdevs refer to the same device"));
	(void) zfs_error(hdl, EZFS_BADDEV, errbuf);
	break;

	case EINVAL:

	if (zpool_has_draid_vdev(nvroot) &&
	zfeature_lookup_name("draid", NULL) != 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"dRAID vdevs are unsupported by the "
	"kernel"));
	} else {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"invalid config; a pool with removing/"
	"removed vdevs does not support adding "
	"raidz or dRAID vdevs"));
	}

	(void) zfs_error(hdl, EZFS_BADDEV, errbuf);
	break;

	case EOVERFLOW:
	/*
	* This occurs when one of the devices is below
	* SPA_MINDEVSIZE. Unfortunately, we can't detect which
	* device was the problem device since there's no
	* reliable way to determine device size from userland.
	*/
	{
	char buf[64];

	zfs_nicebytes(SPA_MINDEVSIZE, buf,
	sizeof (buf));

	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"device is less than the minimum "
	"size (%s)"), buf);
	}
	(void) zfs_error(hdl, EZFS_BADDEV, errbuf);
	break;

	case ENOTSUP:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"pool must be upgraded to add these vdevs"));
	(void) zfs_error(hdl, EZFS_BADVERSION, errbuf);
	break;

	default:
	(void) zpool_standard_error(hdl, errno, errbuf);
	}

	ret = -1;
	} else {
	ret = 0;
	}

	zcmd_free_nvlists(&zc);

	return (ret);
	}

	/*
	* Exports the pool from the system. The caller must ensure that there are no
	* mounted datasets in the pool.
	*/
	static int
	zpool_export_common(zpool_handle_t *zhp, boolean_t force, boolean_t hardforce,
	const char *log_str)
	{
	zfs_cmd_t zc = {"\0"};

	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	zc.zc_cookie = force;
	zc.zc_guid = hardforce;
	zc.zc_history = (uint64_t)(uintptr_t)log_str;

	if (zfs_ioctl(zhp->zpool_hdl, ZFS_IOC_POOL_EXPORT, &zc) != 0) {
	switch (errno) {
	case EXDEV:
	zfs_error_aux(zhp->zpool_hdl, dgettext(TEXT_DOMAIN,
	"use '-f' to override the following errors:\n"
	"'%s' has an active shared spare which could be"
	" used by other pools once '%s' is exported."),
	zhp->zpool_name, zhp->zpool_name);
	return (zfs_error_fmt(zhp->zpool_hdl, EZFS_ACTIVE_SPARE,
	dgettext(TEXT_DOMAIN, "cannot export '%s'"),
	zhp->zpool_name));
	default:
	return (zpool_standard_error_fmt(zhp->zpool_hdl, errno,
	dgettext(TEXT_DOMAIN, "cannot export '%s'"),
	zhp->zpool_name));
	}
	}

	return (0);
	}

	int
	zpool_export(zpool_handle_t zhp, boolean_t force, const char log_str)
	{
	return (zpool_export_common(zhp, force, B_FALSE, log_str));
	}

	int
	zpool_export_force(zpool_handle_t zhp, const char log_str)
	{
	return (zpool_export_common(zhp, B_TRUE, B_TRUE, log_str));
	}

	static void
	zpool_rewind_exclaim(libzfs_handle_t hdl, const char name, boolean_t dryrun,
	nvlist_t *config)
	{
	nvlist_t *nv = NULL;
	uint64_t rewindto;
	int64_t loss = -1;
	struct tm t;
	char timestr[128];

	if (!hdl->libzfs_printerr \|\| config == NULL)
	return;

	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, &nv) != 0 \|\|
	nvlist_lookup_nvlist(nv, ZPOOL_CONFIG_REWIND_INFO, &nv) != 0) {
	return;
	}

	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_LOAD_TIME, &rewindto) != 0)
	return;
	(void) nvlist_lookup_int64(nv, ZPOOL_CONFIG_REWIND_TIME, &loss);

	if (localtime_r((time_t *)&rewindto, &t) != NULL &&
	- strftime(timestr, 128, "%c", &t) != 0) {
	+ ctime_r((time_t *)&rewindto, timestr) != NULL) {
	+ timestr[24] = 0;
	if (dryrun) {
	(void) printf(dgettext(TEXT_DOMAIN,
	"Would be able to return %s "
	"to its state as of %s.\n"),
	name, timestr);
	} else {
	(void) printf(dgettext(TEXT_DOMAIN,
	"Pool %s returned to its state as of %s.\n"),
	name, timestr);
	}
	if (loss > 120) {
	(void) printf(dgettext(TEXT_DOMAIN,
	"%s approximately %lld "),
	dryrun ? "Would discard" : "Discarded",
	((longlong_t)loss + 30) / 60);
	(void) printf(dgettext(TEXT_DOMAIN,
	"minutes of transactions.\n"));
	} else if (loss > 0) {
	(void) printf(dgettext(TEXT_DOMAIN,
	"%s approximately %lld "),
	dryrun ? "Would discard" : "Discarded",
	(longlong_t)loss);
	(void) printf(dgettext(TEXT_DOMAIN,
	"seconds of transactions.\n"));
	}
	}
	}

	void
	zpool_explain_recover(libzfs_handle_t hdl, const char name, int reason,
	nvlist_t *config)
	{
	nvlist_t *nv = NULL;
	int64_t loss = -1;
	uint64_t edata = UINT64_MAX;
	uint64_t rewindto;
	struct tm t;
	char timestr[128];

	if (!hdl->libzfs_printerr)
	return;

	if (reason >= 0)
	(void) printf(dgettext(TEXT_DOMAIN, "action: "));
	else
	(void) printf(dgettext(TEXT_DOMAIN, "\t"));

	/* All attempted rewinds failed if ZPOOL_CONFIG_LOAD_TIME missing */
	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, &nv) != 0 \|\|
	nvlist_lookup_nvlist(nv, ZPOOL_CONFIG_REWIND_INFO, &nv) != 0 \|\|
	nvlist_lookup_uint64(nv, ZPOOL_CONFIG_LOAD_TIME, &rewindto) != 0)
	goto no_info;

	(void) nvlist_lookup_int64(nv, ZPOOL_CONFIG_REWIND_TIME, &loss);
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_LOAD_DATA_ERRORS,
	&edata);

	(void) printf(dgettext(TEXT_DOMAIN,
	"Recovery is possible, but will result in some data loss.\n"));

	if (localtime_r((time_t *)&rewindto, &t) != NULL &&
	- strftime(timestr, 128, "%c", &t) != 0) {
	+ ctime_r((time_t *)&rewindto, timestr) != NULL) {
	+ timestr[24] = 0;
	(void) printf(dgettext(TEXT_DOMAIN,
	"\tReturning the pool to its state as of %s\n"
	"\tshould correct the problem. "),
	timestr);
	} else {
	(void) printf(dgettext(TEXT_DOMAIN,
	"\tReverting the pool to an earlier state "
	"should correct the problem.\n\t"));
	}

	if (loss > 120) {
	(void) printf(dgettext(TEXT_DOMAIN,
	"Approximately %lld minutes of data\n"
	"\tmust be discarded, irreversibly. "),
	((longlong_t)loss + 30) / 60);
	} else if (loss > 0) {
	(void) printf(dgettext(TEXT_DOMAIN,
	"Approximately %lld seconds of data\n"
	"\tmust be discarded, irreversibly. "),
	(longlong_t)loss);
	}
	if (edata != 0 && edata != UINT64_MAX) {
	if (edata == 1) {
	(void) printf(dgettext(TEXT_DOMAIN,
	"After rewind, at least\n"
	"\tone persistent user-data error will remain. "));
	} else {
	(void) printf(dgettext(TEXT_DOMAIN,
	"After rewind, several\n"
	"\tpersistent user-data errors will remain. "));
	}
	}
	(void) printf(dgettext(TEXT_DOMAIN,
	"Recovery can be attempted\n\tby executing 'zpool %s -F %s'. "),
	reason >= 0 ? "clear" : "import", name);

	(void) printf(dgettext(TEXT_DOMAIN,
	"A scrub of the pool\n"
	"\tis strongly recommended after recovery.\n"));
	return;

	no_info:
	(void) printf(dgettext(TEXT_DOMAIN,
	"Destroy and re-create the pool from\n\ta backup source.\n"));
	}

	/*
	* zpool_import() is a contracted interface. Should be kept the same
	* if possible.
	*
	* Applications should use zpool_import_props() to import a pool with
	* new properties value to be set.
	*/
	int
	zpool_import(libzfs_handle_t hdl, nvlist_t config, const char *newname,
	char *altroot)
	{
	nvlist_t *props = NULL;
	int ret;

	if (altroot != NULL) {
	if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0) {
	return (zfs_error_fmt(hdl, EZFS_NOMEM,
	dgettext(TEXT_DOMAIN, "cannot import '%s'"),
	newname));
	}

	if (nvlist_add_string(props,
	zpool_prop_to_name(ZPOOL_PROP_ALTROOT), altroot) != 0 \|\|
	nvlist_add_string(props,
	zpool_prop_to_name(ZPOOL_PROP_CACHEFILE), "none") != 0) {
	nvlist_free(props);
	return (zfs_error_fmt(hdl, EZFS_NOMEM,
	dgettext(TEXT_DOMAIN, "cannot import '%s'"),
	newname));
	}
	}

	ret = zpool_import_props(hdl, config, newname, props,
	ZFS_IMPORT_NORMAL);
	nvlist_free(props);
	return (ret);
	}

	static void
	print_vdev_tree(libzfs_handle_t hdl, const char name, nvlist_t *nv,
	int indent)
	{
	nvlist_t **child;
	uint_t c, children;
	char *vname;
	uint64_t is_log = 0;

	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_IS_LOG,
	&is_log);

	if (name != NULL)
	(void) printf("\t%*s%s%s\n", indent, "", name,
	is_log ? " [log]" : "");

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
	&child, &children) != 0)
	return;

	for (c = 0; c < children; c++) {
	vname = zpool_vdev_name(hdl, NULL, child[c], VDEV_NAME_TYPE_ID);
	print_vdev_tree(hdl, vname, child[c], indent + 2);
	free(vname);
	}
	}

	void
	zpool_print_unsup_feat(nvlist_t *config)
	{
	nvlist_t nvinfo, unsup_feat;

	nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
	unsup_feat = fnvlist_lookup_nvlist(nvinfo, ZPOOL_CONFIG_UNSUP_FEAT);

	for (nvpair_t *nvp = nvlist_next_nvpair(unsup_feat, NULL);
	nvp != NULL; nvp = nvlist_next_nvpair(unsup_feat, nvp)) {
	const char *desc = fnvpair_value_string(nvp);
	if (strlen(desc) > 0)
	(void) printf("\t%s (%s)\n", nvpair_name(nvp), desc);
	else
	(void) printf("\t%s\n", nvpair_name(nvp));
	}
	}

	/*
	* Import the given pool using the known configuration and a list of
	* properties to be set. The configuration should have come from
	* zpool_find_import(). The 'newname' parameters control whether the pool
	* is imported with a different name.
	*/
	int
	zpool_import_props(libzfs_handle_t hdl, nvlist_t config, const char *newname,
	nvlist_t *props, int flags)
	{
	zfs_cmd_t zc = {"\0"};
	zpool_load_policy_t policy;
	nvlist_t *nv = NULL;
	nvlist_t *nvinfo = NULL;
	nvlist_t *missing = NULL;
	const char *thename;
	const char *origname;
	int ret;
	int error = 0;
	char errbuf[ERRBUFLEN];

	origname = fnvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME);

	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot import pool '%s'"), origname);

	if (newname != NULL) {
	if (!zpool_name_valid(hdl, B_FALSE, newname))
	return (zfs_error_fmt(hdl, EZFS_INVALIDNAME,
	dgettext(TEXT_DOMAIN, "cannot import '%s'"),
	newname));
	thename = newname;
	} else {
	thename = origname;
	}

	if (props != NULL) {
	uint64_t version;
	prop_flags_t flags = { .create = B_FALSE, .import = B_TRUE };

	version = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION);

	if ((props = zpool_valid_proplist(hdl, origname,
	props, version, flags, errbuf)) == NULL)
	return (-1);
	zcmd_write_src_nvlist(hdl, &zc, props);
	nvlist_free(props);
	}

	(void) strlcpy(zc.zc_name, thename, sizeof (zc.zc_name));

	zc.zc_guid = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID);

	zcmd_write_conf_nvlist(hdl, &zc, config);
	zcmd_alloc_dst_nvlist(hdl, &zc, zc.zc_nvlist_conf_size * 2);

	zc.zc_cookie = flags;
	while ((ret = zfs_ioctl(hdl, ZFS_IOC_POOL_IMPORT, &zc)) != 0 &&
	errno == ENOMEM)
	zcmd_expand_dst_nvlist(hdl, &zc);
	if (ret != 0)
	error = errno;

	(void) zcmd_read_dst_nvlist(hdl, &zc, &nv);

	zcmd_free_nvlists(&zc);

	zpool_get_load_policy(config, &policy);

	if (error) {
	char desc[1024];
	char aux[256];

	/*
	* Dry-run failed, but we print out what success
	* looks like if we found a best txg
	*/
	if (policy.zlp_rewind & ZPOOL_TRY_REWIND) {
	zpool_rewind_exclaim(hdl, newname ? origname : thename,
	B_TRUE, nv);
	nvlist_free(nv);
	return (-1);
	}

	if (newname == NULL)
	(void) snprintf(desc, sizeof (desc),
	dgettext(TEXT_DOMAIN, "cannot import '%s'"),
	thename);
	else
	(void) snprintf(desc, sizeof (desc),
	dgettext(TEXT_DOMAIN, "cannot import '%s' as '%s'"),
	origname, thename);

	switch (error) {
	case ENOTSUP:
	if (nv != NULL && nvlist_lookup_nvlist(nv,
	ZPOOL_CONFIG_LOAD_INFO, &nvinfo) == 0 &&
	nvlist_exists(nvinfo, ZPOOL_CONFIG_UNSUP_FEAT)) {
	(void) printf(dgettext(TEXT_DOMAIN, "This "
	"pool uses the following feature(s) not "
	"supported by this system:\n"));
	zpool_print_unsup_feat(nv);
	if (nvlist_exists(nvinfo,
	ZPOOL_CONFIG_CAN_RDONLY)) {
	(void) printf(dgettext(TEXT_DOMAIN,
	"All unsupported features are only "
	"required for writing to the pool."
	"\nThe pool can be imported using "
	"'-o readonly=on'.\n"));
	}
	}
	/*
	* Unsupported version.
	*/
	(void) zfs_error(hdl, EZFS_BADVERSION, desc);
	break;

	case EREMOTEIO:
	if (nv != NULL && nvlist_lookup_nvlist(nv,
	ZPOOL_CONFIG_LOAD_INFO, &nvinfo) == 0) {
	const char *hostname = "<unknown>";
	uint64_t hostid = 0;
	mmp_state_t mmp_state;

	mmp_state = fnvlist_lookup_uint64(nvinfo,
	ZPOOL_CONFIG_MMP_STATE);

	if (nvlist_exists(nvinfo,
	ZPOOL_CONFIG_MMP_HOSTNAME))
	hostname = fnvlist_lookup_string(nvinfo,
	ZPOOL_CONFIG_MMP_HOSTNAME);

	if (nvlist_exists(nvinfo,
	ZPOOL_CONFIG_MMP_HOSTID))
	hostid = fnvlist_lookup_uint64(nvinfo,
	ZPOOL_CONFIG_MMP_HOSTID);

	if (mmp_state == MMP_STATE_ACTIVE) {
	(void) snprintf(aux, sizeof (aux),
	dgettext(TEXT_DOMAIN, "pool is imp"
	"orted on host '%s' (hostid=%lx).\n"
	"Export the pool on the other "
	"system, then run 'zpool import'."),
	hostname, (unsigned long) hostid);
	} else if (mmp_state == MMP_STATE_NO_HOSTID) {
	(void) snprintf(aux, sizeof (aux),
	dgettext(TEXT_DOMAIN, "pool has "
	"the multihost property on and "
	"the\nsystem's hostid is not set. "
	"Set a unique system hostid with "
	"the zgenhostid(8) command.\n"));
	}

	(void) zfs_error_aux(hdl, "%s", aux);
	}
	(void) zfs_error(hdl, EZFS_ACTIVE_POOL, desc);
	break;

	case EINVAL:
	(void) zfs_error(hdl, EZFS_INVALCONFIG, desc);
	break;

	case EROFS:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"one or more devices is read only"));
	(void) zfs_error(hdl, EZFS_BADDEV, desc);
	break;

	case ENXIO:
	if (nv && nvlist_lookup_nvlist(nv,
	ZPOOL_CONFIG_LOAD_INFO, &nvinfo) == 0 &&
	nvlist_lookup_nvlist(nvinfo,
	ZPOOL_CONFIG_MISSING_DEVICES, &missing) == 0) {
	(void) printf(dgettext(TEXT_DOMAIN,
	"The devices below are missing or "
	"corrupted, use '-m' to import the pool "
	"anyway:\n"));
	print_vdev_tree(hdl, NULL, missing, 2);
	(void) printf("\n");
	}
	(void) zpool_standard_error(hdl, error, desc);
	break;

	case EEXIST:
	(void) zpool_standard_error(hdl, error, desc);
	break;

	case EBUSY:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"one or more devices are already in use\n"));
	(void) zfs_error(hdl, EZFS_BADDEV, desc);
	break;
	case ENAMETOOLONG:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"new name of at least one dataset is longer than "
	"the maximum allowable length"));
	(void) zfs_error(hdl, EZFS_NAMETOOLONG, desc);
	break;
	default:
	(void) zpool_standard_error(hdl, error, desc);
	zpool_explain_recover(hdl,
	newname ? origname : thename, -error, nv);
	break;
	}

	nvlist_free(nv);
	ret = -1;
	} else {
	zpool_handle_t *zhp;

	/*
	* This should never fail, but play it safe anyway.
	*/
	if (zpool_open_silent(hdl, thename, &zhp) != 0)
	ret = -1;
	else if (zhp != NULL)
	zpool_close(zhp);
	if (policy.zlp_rewind &
	(ZPOOL_DO_REWIND \| ZPOOL_TRY_REWIND)) {
	zpool_rewind_exclaim(hdl, newname ? origname : thename,
	((policy.zlp_rewind & ZPOOL_TRY_REWIND) != 0), nv);
	}
	nvlist_free(nv);
	}

	return (ret);
	}

	/*
	* Translate vdev names to guids. If a vdev_path is determined to be
	* unsuitable then a vd_errlist is allocated and the vdev path and errno
	* are added to it.
	*/
	static int
	zpool_translate_vdev_guids(zpool_handle_t zhp, nvlist_t vds,
	nvlist_t vdev_guids, nvlist_t guids_to_paths, nvlist_t **vd_errlist)
	{
	nvlist_t *errlist = NULL;
	int error = 0;

	for (nvpair_t *elem = nvlist_next_nvpair(vds, NULL); elem != NULL;
	elem = nvlist_next_nvpair(vds, elem)) {
	boolean_t spare, cache;

	const char *vd_path = nvpair_name(elem);
	nvlist_t *tgt = zpool_find_vdev(zhp, vd_path, &spare, &cache,
	NULL);

	if ((tgt == NULL) \|\| cache \|\| spare) {
	if (errlist == NULL) {
	errlist = fnvlist_alloc();
	error = EINVAL;
	}

	uint64_t err = (tgt == NULL) ? EZFS_NODEVICE :
	(spare ? EZFS_ISSPARE : EZFS_ISL2CACHE);
	fnvlist_add_int64(errlist, vd_path, err);
	continue;
	}

	uint64_t guid = fnvlist_lookup_uint64(tgt, ZPOOL_CONFIG_GUID);
	fnvlist_add_uint64(vdev_guids, vd_path, guid);

	char msg[MAXNAMELEN];
	(void) snprintf(msg, sizeof (msg), "%llu", (u_longlong_t)guid);
	fnvlist_add_string(guids_to_paths, msg, vd_path);
	}

	if (error != 0) {
	verify(errlist != NULL);
	if (vd_errlist != NULL)
	*vd_errlist = errlist;
	else
	fnvlist_free(errlist);
	}

	return (error);
	}

	static int
	xlate_init_err(int err)
	{
	switch (err) {
	case ENODEV:
	return (EZFS_NODEVICE);
	case EINVAL:
	case EROFS:
	return (EZFS_BADDEV);
	case EBUSY:
	return (EZFS_INITIALIZING);
	case ESRCH:
	return (EZFS_NO_INITIALIZE);
	}
	return (err);
	}

	/*
	* Begin, suspend, cancel, or uninit (clear) the initialization (initializing
	* of all free blocks) for the given vdevs in the given pool.
	*/
	static int
	zpool_initialize_impl(zpool_handle_t *zhp, pool_initialize_func_t cmd_type,
	nvlist_t *vds, boolean_t wait)
	{
	int err;

	nvlist_t *vdev_guids = fnvlist_alloc();
	nvlist_t *guids_to_paths = fnvlist_alloc();
	nvlist_t *vd_errlist = NULL;
	nvlist_t *errlist;
	nvpair_t *elem;

	err = zpool_translate_vdev_guids(zhp, vds, vdev_guids,
	guids_to_paths, &vd_errlist);

	if (err != 0) {
	verify(vd_errlist != NULL);
	goto list_errors;
	}

	err = lzc_initialize(zhp->zpool_name, cmd_type,
	vdev_guids, &errlist);

	if (err != 0) {
	if (errlist != NULL && nvlist_lookup_nvlist(errlist,
	ZPOOL_INITIALIZE_VDEVS, &vd_errlist) == 0) {
	goto list_errors;
	}

	if (err == EINVAL && cmd_type == POOL_INITIALIZE_UNINIT) {
	zfs_error_aux(zhp->zpool_hdl, dgettext(TEXT_DOMAIN,
	"uninitialize is not supported by kernel"));
	}

	(void) zpool_standard_error(zhp->zpool_hdl, err,
	dgettext(TEXT_DOMAIN, "operation failed"));
	goto out;
	}

	if (wait) {
	for (elem = nvlist_next_nvpair(vdev_guids, NULL); elem != NULL;
	elem = nvlist_next_nvpair(vdev_guids, elem)) {

	uint64_t guid = fnvpair_value_uint64(elem);

	err = lzc_wait_tag(zhp->zpool_name,
	ZPOOL_WAIT_INITIALIZE, guid, NULL);
	if (err != 0) {
	(void) zpool_standard_error_fmt(zhp->zpool_hdl,
	err, dgettext(TEXT_DOMAIN, "error "
	"waiting for '%s' to initialize"),
	nvpair_name(elem));

	goto out;
	}
	}
	}
	goto out;

	list_errors:
	for (elem = nvlist_next_nvpair(vd_errlist, NULL); elem != NULL;
	elem = nvlist_next_nvpair(vd_errlist, elem)) {
	int64_t vd_error = xlate_init_err(fnvpair_value_int64(elem));
	const char *path;

	if (nvlist_lookup_string(guids_to_paths, nvpair_name(elem),
	&path) != 0)
	path = nvpair_name(elem);

	(void) zfs_error_fmt(zhp->zpool_hdl, vd_error,
	"cannot initialize '%s'", path);
	}

	out:
	fnvlist_free(vdev_guids);
	fnvlist_free(guids_to_paths);

	if (vd_errlist != NULL)
	fnvlist_free(vd_errlist);

	return (err == 0 ? 0 : -1);
	}

	int
	zpool_initialize(zpool_handle_t *zhp, pool_initialize_func_t cmd_type,
	nvlist_t *vds)
	{
	return (zpool_initialize_impl(zhp, cmd_type, vds, B_FALSE));
	}

	int
	zpool_initialize_wait(zpool_handle_t *zhp, pool_initialize_func_t cmd_type,
	nvlist_t *vds)
	{
	return (zpool_initialize_impl(zhp, cmd_type, vds, B_TRUE));
	}

	static int
	xlate_trim_err(int err)
	{
	switch (err) {
	case ENODEV:
	return (EZFS_NODEVICE);
	case EINVAL:
	case EROFS:
	return (EZFS_BADDEV);
	case EBUSY:
	return (EZFS_TRIMMING);
	case ESRCH:
	return (EZFS_NO_TRIM);
	case EOPNOTSUPP:
	return (EZFS_TRIM_NOTSUP);
	}
	return (err);
	}

	static int
	zpool_trim_wait(zpool_handle_t zhp, nvlist_t vdev_guids)
	{
	int err;
	nvpair_t *elem;

	for (elem = nvlist_next_nvpair(vdev_guids, NULL); elem != NULL;
	elem = nvlist_next_nvpair(vdev_guids, elem)) {

	uint64_t guid = fnvpair_value_uint64(elem);

	err = lzc_wait_tag(zhp->zpool_name,
	ZPOOL_WAIT_TRIM, guid, NULL);
	if (err != 0) {
	(void) zpool_standard_error_fmt(zhp->zpool_hdl,
	err, dgettext(TEXT_DOMAIN, "error "
	"waiting to trim '%s'"), nvpair_name(elem));

	return (err);
	}
	}
	return (0);
	}

	/*
	* Check errlist and report any errors, omitting ones which should be
	* suppressed. Returns B_TRUE if any errors were reported.
	*/
	static boolean_t
	check_trim_errs(zpool_handle_t zhp, trimflags_t trim_flags,
	nvlist_t guids_to_paths, nvlist_t vds, nvlist_t *errlist)
	{
	nvpair_t *elem;
	boolean_t reported_errs = B_FALSE;
	int num_vds = 0;
	int num_suppressed_errs = 0;

	for (elem = nvlist_next_nvpair(vds, NULL);
	elem != NULL; elem = nvlist_next_nvpair(vds, elem)) {
	num_vds++;
	}

	for (elem = nvlist_next_nvpair(errlist, NULL);
	elem != NULL; elem = nvlist_next_nvpair(errlist, elem)) {
	int64_t vd_error = xlate_trim_err(fnvpair_value_int64(elem));
	const char *path;

	/*
	* If only the pool was specified, and it was not a secure
	* trim then suppress warnings for individual vdevs which
	* do not support trimming.
	*/
	if (vd_error == EZFS_TRIM_NOTSUP &&
	trim_flags->fullpool &&
	!trim_flags->secure) {
	num_suppressed_errs++;
	continue;
	}

	reported_errs = B_TRUE;
	if (nvlist_lookup_string(guids_to_paths, nvpair_name(elem),
	&path) != 0)
	path = nvpair_name(elem);

	(void) zfs_error_fmt(zhp->zpool_hdl, vd_error,
	"cannot trim '%s'", path);
	}

	if (num_suppressed_errs == num_vds) {
	(void) zfs_error_aux(zhp->zpool_hdl, dgettext(TEXT_DOMAIN,
	"no devices in pool support trim operations"));
	(void) (zfs_error(zhp->zpool_hdl, EZFS_TRIM_NOTSUP,
	dgettext(TEXT_DOMAIN, "cannot trim")));
	reported_errs = B_TRUE;
	}

	return (reported_errs);
	}

	/*
	* Begin, suspend, or cancel the TRIM (discarding of all free blocks) for
	* the given vdevs in the given pool.
	*/
	int
	zpool_trim(zpool_handle_t zhp, pool_trim_func_t cmd_type, nvlist_t vds,
	trimflags_t *trim_flags)
	{
	int err;
	int retval = 0;

	nvlist_t *vdev_guids = fnvlist_alloc();
	nvlist_t *guids_to_paths = fnvlist_alloc();
	nvlist_t *errlist = NULL;

	err = zpool_translate_vdev_guids(zhp, vds, vdev_guids,
	guids_to_paths, &errlist);
	if (err != 0) {
	check_trim_errs(zhp, trim_flags, guids_to_paths, vds, errlist);
	retval = -1;
	goto out;
	}

	err = lzc_trim(zhp->zpool_name, cmd_type, trim_flags->rate,
	trim_flags->secure, vdev_guids, &errlist);
	if (err != 0) {
	nvlist_t *vd_errlist;
	if (errlist != NULL && nvlist_lookup_nvlist(errlist,
	ZPOOL_TRIM_VDEVS, &vd_errlist) == 0) {
	if (check_trim_errs(zhp, trim_flags, guids_to_paths,
	vds, vd_errlist)) {
	retval = -1;
	goto out;
	}
	} else {
	char errbuf[ERRBUFLEN];

	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "operation failed"));
	zpool_standard_error(zhp->zpool_hdl, err, errbuf);
	retval = -1;
	goto out;
	}
	}


	if (trim_flags->wait)
	retval = zpool_trim_wait(zhp, vdev_guids);

	out:
	if (errlist != NULL)
	fnvlist_free(errlist);
	fnvlist_free(vdev_guids);
	fnvlist_free(guids_to_paths);
	return (retval);
	}

	/*
	* Scan the pool.
	*/
	int
	zpool_scan(zpool_handle_t *zhp, pool_scan_func_t func, pool_scrub_cmd_t cmd)
	{
	char errbuf[ERRBUFLEN];
	int err;
	libzfs_handle_t *hdl = zhp->zpool_hdl;

	nvlist_t *args = fnvlist_alloc();
	fnvlist_add_uint64(args, "scan_type", (uint64_t)func);
	fnvlist_add_uint64(args, "scan_command", (uint64_t)cmd);

	err = lzc_scrub(ZFS_IOC_POOL_SCRUB, zhp->zpool_name, args, NULL);
	fnvlist_free(args);

	if (err == 0) {
	return (0);
	} else if (err == ZFS_ERR_IOC_CMD_UNAVAIL) {
	zfs_cmd_t zc = {"\0"};
	(void) strlcpy(zc.zc_name, zhp->zpool_name,
	sizeof (zc.zc_name));
	zc.zc_cookie = func;
	zc.zc_flags = cmd;

	if (zfs_ioctl(hdl, ZFS_IOC_POOL_SCAN, &zc) == 0)
	return (0);
	}

	/*
	* An ECANCELED on a scrub means one of the following:
	* 1. we resumed a paused scrub.
	* 2. we resumed a paused error scrub.
	* 3. Error scrub is not run because of no error log.
	*/
	if (err == ECANCELED && (func == POOL_SCAN_SCRUB \|\|
	func == POOL_SCAN_ERRORSCRUB) && cmd == POOL_SCRUB_NORMAL)
	return (0);
	/*
	* The following cases have been handled here:
	* 1. Paused a scrub/error scrub if there is none in progress.
	*/
	if (err == ENOENT && func != POOL_SCAN_NONE && cmd ==
	POOL_SCRUB_PAUSE) {
	return (0);
	}

	ASSERT3U(func, >=, POOL_SCAN_NONE);
	ASSERT3U(func, <, POOL_SCAN_FUNCS);

	if (func == POOL_SCAN_SCRUB \|\| func == POOL_SCAN_ERRORSCRUB) {
	if (cmd == POOL_SCRUB_PAUSE) {
	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot pause scrubbing %s"),
	zhp->zpool_name);
	} else {
	assert(cmd == POOL_SCRUB_NORMAL);
	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot scrub %s"),
	zhp->zpool_name);
	}
	} else if (func == POOL_SCAN_RESILVER) {
	assert(cmd == POOL_SCRUB_NORMAL);
	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot restart resilver on %s"), zhp->zpool_name);
	} else if (func == POOL_SCAN_NONE) {
	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot cancel scrubbing %s"), zhp->zpool_name);
	} else {
	assert(!"unexpected result");
	}

	/*
	* With EBUSY, five cases are possible:
	*
	* Current state Requested
	* 1. Normal Scrub Running Normal Scrub or Error Scrub
	* 2. Normal Scrub Paused Error Scrub
	* 3. Normal Scrub Paused Pause Normal Scrub
	* 4. Error Scrub Running Normal Scrub or Error Scrub
	* 5. Error Scrub Paused Pause Error Scrub
	* 6. Resilvering Anything else
	*/
	if (err == EBUSY) {
	nvlist_t *nvroot;
	pool_scan_stat_t *ps = NULL;
	uint_t psc;

	nvroot = fnvlist_lookup_nvlist(zhp->zpool_config,
	ZPOOL_CONFIG_VDEV_TREE);
	(void) nvlist_lookup_uint64_array(nvroot,
	ZPOOL_CONFIG_SCAN_STATS, (uint64_t **)&ps, &psc);
	if (ps && ps->pss_func == POOL_SCAN_SCRUB &&
	ps->pss_state == DSS_SCANNING) {
	if (ps->pss_pass_scrub_pause == 0) {
	/* handles case 1 */
	assert(cmd == POOL_SCRUB_NORMAL);
	return (zfs_error(hdl, EZFS_SCRUBBING,
	errbuf));
	} else {
	if (func == POOL_SCAN_ERRORSCRUB) {
	/* handles case 2 */
	ASSERT3U(cmd, ==, POOL_SCRUB_NORMAL);
	return (zfs_error(hdl,
	EZFS_SCRUB_PAUSED_TO_CANCEL,
	errbuf));
	} else {
	/* handles case 3 */
	ASSERT3U(func, ==, POOL_SCAN_SCRUB);
	ASSERT3U(cmd, ==, POOL_SCRUB_PAUSE);
	return (zfs_error(hdl,
	EZFS_SCRUB_PAUSED, errbuf));
	}
	}
	} else if (ps &&
	ps->pss_error_scrub_func == POOL_SCAN_ERRORSCRUB &&
	ps->pss_error_scrub_state == DSS_ERRORSCRUBBING) {
	if (ps->pss_pass_error_scrub_pause == 0) {
	/* handles case 4 */
	ASSERT3U(cmd, ==, POOL_SCRUB_NORMAL);
	return (zfs_error(hdl, EZFS_ERRORSCRUBBING,
	errbuf));
	} else {
	/* handles case 5 */
	ASSERT3U(func, ==, POOL_SCAN_ERRORSCRUB);
	ASSERT3U(cmd, ==, POOL_SCRUB_PAUSE);
	return (zfs_error(hdl, EZFS_ERRORSCRUB_PAUSED,
	errbuf));
	}
	} else {
	/* handles case 6 */
	return (zfs_error(hdl, EZFS_RESILVERING, errbuf));
	}
	} else if (err == ENOENT) {
	return (zfs_error(hdl, EZFS_NO_SCRUB, errbuf));
	} else if (err == ENOTSUP && func == POOL_SCAN_RESILVER) {
	return (zfs_error(hdl, EZFS_NO_RESILVER_DEFER, errbuf));
	} else {
	return (zpool_standard_error(hdl, err, errbuf));
	}
	}

	/*
	* Find a vdev that matches the search criteria specified. We use the
	* the nvpair name to determine how we should look for the device.
	* 'avail_spare' is set to TRUE if the provided guid refers to an AVAIL
	* spare; but FALSE if its an INUSE spare.
	*/
	static nvlist_t *
	vdev_to_nvlist_iter(nvlist_t nv, nvlist_t search, boolean_t *avail_spare,
	boolean_t l2cache, boolean_t log)
	{
	uint_t c, children;
	nvlist_t **child;
	nvlist_t *ret;
	uint64_t is_log;
	const char *srchkey;
	nvpair_t *pair = nvlist_next_nvpair(search, NULL);

	/* Nothing to look for */
	if (search == NULL \|\| pair == NULL)
	return (NULL);

	/* Obtain the key we will use to search */
	srchkey = nvpair_name(pair);

	switch (nvpair_type(pair)) {
	case DATA_TYPE_UINT64:
	if (strcmp(srchkey, ZPOOL_CONFIG_GUID) == 0) {
	uint64_t srchval = fnvpair_value_uint64(pair);
	uint64_t theguid = fnvlist_lookup_uint64(nv,
	ZPOOL_CONFIG_GUID);
	if (theguid == srchval)
	return (nv);
	}
	break;

	case DATA_TYPE_STRING: {
	const char srchval, val;

	srchval = fnvpair_value_string(pair);
	if (nvlist_lookup_string(nv, srchkey, &val) != 0)
	break;

	/*
	* Search for the requested value. Special cases:
	*
	* - ZPOOL_CONFIG_PATH for whole disk entries. These end in
	* "-part1", or "p1". The suffix is hidden from the user,
	* but included in the string, so this matches around it.
	* - ZPOOL_CONFIG_PATH for short names zfs_strcmp_shortname()
	* is used to check all possible expanded paths.
	* - looking for a top-level vdev name (i.e. ZPOOL_CONFIG_TYPE).
	*
	* Otherwise, all other searches are simple string compares.
	*/
	if (strcmp(srchkey, ZPOOL_CONFIG_PATH) == 0) {
	uint64_t wholedisk = 0;

	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_WHOLE_DISK,
	&wholedisk);
	if (zfs_strcmp_pathname(srchval, val, wholedisk) == 0)
	return (nv);

	} else if (strcmp(srchkey, ZPOOL_CONFIG_TYPE) == 0) {
	char type, idx, end, p;
	uint64_t id, vdev_id;

	/*
	* Determine our vdev type, keeping in mind
	* that the srchval is composed of a type and
	* vdev id pair (i.e. mirror-4).
	*/
	if ((type = strdup(srchval)) == NULL)
	return (NULL);

	if ((p = strrchr(type, '-')) == NULL) {
	free(type);
	break;
	}
	idx = p + 1;
	*p = '\0';

	/*
	* If the types don't match then keep looking.
	*/
	if (strncmp(val, type, strlen(val)) != 0) {
	free(type);
	break;
	}

	verify(zpool_vdev_is_interior(type));

	id = fnvlist_lookup_uint64(nv, ZPOOL_CONFIG_ID);
	errno = 0;
	vdev_id = strtoull(idx, &end, 10);

	/*
	* If we are looking for a raidz and a parity is
	* specified, make sure it matches.
	*/
	int rzlen = strlen(VDEV_TYPE_RAIDZ);
	assert(rzlen == strlen(VDEV_TYPE_DRAID));
	int typlen = strlen(type);
	if ((strncmp(type, VDEV_TYPE_RAIDZ, rzlen) == 0 \|\|
	strncmp(type, VDEV_TYPE_DRAID, rzlen) == 0) &&
	typlen != rzlen) {
	uint64_t vdev_parity;
	int parity = *(type + rzlen) - '0';

	if (parity <= 0 \|\| parity > 3 \|\|
	(typlen - rzlen) != 1) {
	/*
	* Nonsense parity specified, can
	* never match
	*/
	free(type);
	return (NULL);
	}
	vdev_parity = fnvlist_lookup_uint64(nv,
	ZPOOL_CONFIG_NPARITY);
	if ((int)vdev_parity != parity) {
	free(type);
	break;
	}
	}

	free(type);
	if (errno != 0)
	return (NULL);

	/*
	* Now verify that we have the correct vdev id.
	*/
	if (vdev_id == id)
	return (nv);
	}

	/*
	* Common case
	*/
	if (strcmp(srchval, val) == 0)
	return (nv);
	break;
	}

	default:
	break;
	}

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
	&child, &children) != 0)
	return (NULL);

	for (c = 0; c < children; c++) {
	if ((ret = vdev_to_nvlist_iter(child[c], search,
	avail_spare, l2cache, NULL)) != NULL) {
	/*
	* The 'is_log' value is only set for the toplevel
	* vdev, not the leaf vdevs. So we always lookup the
	* log device from the root of the vdev tree (where
	* 'log' is non-NULL).
	*/
	if (log != NULL &&
	nvlist_lookup_uint64(child[c],
	ZPOOL_CONFIG_IS_LOG, &is_log) == 0 &&
	is_log) {
	*log = B_TRUE;
	}
	return (ret);
	}
	}

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
	&child, &children) == 0) {
	for (c = 0; c < children; c++) {
	if ((ret = vdev_to_nvlist_iter(child[c], search,
	avail_spare, l2cache, NULL)) != NULL) {
	*avail_spare = B_TRUE;
	return (ret);
	}
	}
	}

	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
	&child, &children) == 0) {
	for (c = 0; c < children; c++) {
	if ((ret = vdev_to_nvlist_iter(child[c], search,
	avail_spare, l2cache, NULL)) != NULL) {
	*l2cache = B_TRUE;
	return (ret);
	}
	}
	}

	return (NULL);
	}

	/*
	* Given a physical path or guid, find the associated vdev.
	*/
	nvlist_t *
	zpool_find_vdev_by_physpath(zpool_handle_t zhp, const char ppath,
	boolean_t avail_spare, boolean_t l2cache, boolean_t *log)
	{
	nvlist_t search, nvroot, *ret;
	uint64_t guid;
	char *end;

	search = fnvlist_alloc();

	guid = strtoull(ppath, &end, 0);
	if (guid != 0 && *end == '\0') {
	fnvlist_add_uint64(search, ZPOOL_CONFIG_GUID, guid);
	} else {
	fnvlist_add_string(search, ZPOOL_CONFIG_PHYS_PATH, ppath);
	}

	nvroot = fnvlist_lookup_nvlist(zhp->zpool_config,
	ZPOOL_CONFIG_VDEV_TREE);

	*avail_spare = B_FALSE;
	*l2cache = B_FALSE;
	if (log != NULL)
	*log = B_FALSE;
	ret = vdev_to_nvlist_iter(nvroot, search, avail_spare, l2cache, log);
	fnvlist_free(search);

	return (ret);
	}

	/*
	* Determine if we have an "interior" top-level vdev (i.e mirror/raidz).
	*/
	static boolean_t
	zpool_vdev_is_interior(const char *name)
	{
	if (strncmp(name, VDEV_TYPE_RAIDZ, strlen(VDEV_TYPE_RAIDZ)) == 0 \|\|
	strncmp(name, VDEV_TYPE_SPARE, strlen(VDEV_TYPE_SPARE)) == 0 \|\|
	strncmp(name,
	VDEV_TYPE_REPLACING, strlen(VDEV_TYPE_REPLACING)) == 0 \|\|
	strncmp(name, VDEV_TYPE_ROOT, strlen(VDEV_TYPE_ROOT)) == 0 \|\|
	strncmp(name, VDEV_TYPE_MIRROR, strlen(VDEV_TYPE_MIRROR)) == 0)
	return (B_TRUE);

	if (strncmp(name, VDEV_TYPE_DRAID, strlen(VDEV_TYPE_DRAID)) == 0 &&
	!zpool_is_draid_spare(name))
	return (B_TRUE);

	return (B_FALSE);
	}

	/*
	* Lookup the nvlist for a given vdev.
	*/
	nvlist_t *
	zpool_find_vdev(zpool_handle_t zhp, const char path, boolean_t *avail_spare,
	boolean_t l2cache, boolean_t log)
	{
	char *end;
	nvlist_t nvroot, search, *ret;
	uint64_t guid;
	boolean_t __avail_spare, __l2cache, __log;

	search = fnvlist_alloc();

	guid = strtoull(path, &end, 0);
	if (guid != 0 && *end == '\0') {
	fnvlist_add_uint64(search, ZPOOL_CONFIG_GUID, guid);
	} else if (zpool_vdev_is_interior(path)) {
	fnvlist_add_string(search, ZPOOL_CONFIG_TYPE, path);
	} else {
	fnvlist_add_string(search, ZPOOL_CONFIG_PATH, path);
	}

	nvroot = fnvlist_lookup_nvlist(zhp->zpool_config,
	ZPOOL_CONFIG_VDEV_TREE);

	/*
	* User can pass NULL for avail_spare, l2cache, and log, but
	* we still need to provide variables to vdev_to_nvlist_iter(), so
	* just point them to junk variables here.
	*/
	if (!avail_spare)
	avail_spare = &__avail_spare;
	if (!l2cache)
	l2cache = &__l2cache;
	if (!log)
	log = &__log;

	*avail_spare = B_FALSE;
	*l2cache = B_FALSE;
	if (log != NULL)
	*log = B_FALSE;
	ret = vdev_to_nvlist_iter(nvroot, search, avail_spare, l2cache, log);
	fnvlist_free(search);

	return (ret);
	}

	/*
	* Convert a vdev path to a GUID. Returns GUID or 0 on error.
	*
	* If is_spare, is_l2cache, or is_log is non-NULL, then store within it
	* if the VDEV is a spare, l2cache, or log device. If they're NULL then
	* ignore them.
	*/
	static uint64_t
	zpool_vdev_path_to_guid_impl(zpool_handle_t zhp, const char path,
	boolean_t is_spare, boolean_t is_l2cache, boolean_t *is_log)
	{
	boolean_t spare = B_FALSE, l2cache = B_FALSE, log = B_FALSE;
	nvlist_t *tgt;

	if ((tgt = zpool_find_vdev(zhp, path, &spare, &l2cache,
	&log)) == NULL)
	return (0);

	if (is_spare != NULL)
	*is_spare = spare;
	if (is_l2cache != NULL)
	*is_l2cache = l2cache;
	if (is_log != NULL)
	*is_log = log;

	return (fnvlist_lookup_uint64(tgt, ZPOOL_CONFIG_GUID));
	}

	/* Convert a vdev path to a GUID. Returns GUID or 0 on error. */
	uint64_t
	zpool_vdev_path_to_guid(zpool_handle_t zhp, const char path)
	{
	return (zpool_vdev_path_to_guid_impl(zhp, path, NULL, NULL, NULL));
	}

	/*
	* Bring the specified vdev online. The 'flags' parameter is a set of the
	* ZFS_ONLINE_* flags.
	*/
	int
	zpool_vdev_online(zpool_handle_t zhp, const char path, int flags,
	vdev_state_t *newstate)
	{
	zfs_cmd_t zc = {"\0"};
	char errbuf[ERRBUFLEN];
	nvlist_t *tgt;
	boolean_t avail_spare, l2cache, islog;
	libzfs_handle_t *hdl = zhp->zpool_hdl;

	if (flags & ZFS_ONLINE_EXPAND) {
	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot expand %s"), path);
	} else {
	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot online %s"), path);
	}

	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	if ((tgt = zpool_find_vdev(zhp, path, &avail_spare, &l2cache,
	&islog)) == NULL)
	return (zfs_error(hdl, EZFS_NODEVICE, errbuf));

	zc.zc_guid = fnvlist_lookup_uint64(tgt, ZPOOL_CONFIG_GUID);

	if (!(flags & ZFS_ONLINE_SPARE) && avail_spare)
	return (zfs_error(hdl, EZFS_ISSPARE, errbuf));

	#ifndef __FreeBSD__
	const char *pathname;
	if ((flags & ZFS_ONLINE_EXPAND \|\|
	zpool_get_prop_int(zhp, ZPOOL_PROP_AUTOEXPAND, NULL)) &&
	nvlist_lookup_string(tgt, ZPOOL_CONFIG_PATH, &pathname) == 0) {
	uint64_t wholedisk = 0;

	(void) nvlist_lookup_uint64(tgt, ZPOOL_CONFIG_WHOLE_DISK,
	&wholedisk);

	/*
	* XXX - L2ARC 1.0 devices can't support expansion.
	*/
	if (l2cache) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"cannot expand cache devices"));
	return (zfs_error(hdl, EZFS_VDEVNOTSUP, errbuf));
	}

	if (wholedisk) {
	const char *fullpath = path;
	char buf[MAXPATHLEN];
	int error;

	if (path[0] != '/') {
	error = zfs_resolve_shortname(path, buf,
	sizeof (buf));
	if (error != 0)
	return (zfs_error(hdl, EZFS_NODEVICE,
	errbuf));

	fullpath = buf;
	}

	error = zpool_relabel_disk(hdl, fullpath, errbuf);
	if (error != 0)
	return (error);
	}
	}
	#endif

	zc.zc_cookie = VDEV_STATE_ONLINE;
	zc.zc_obj = flags;

	if (zfs_ioctl(hdl, ZFS_IOC_VDEV_SET_STATE, &zc) != 0) {
	if (errno == EINVAL) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "was split "
	"from this pool into a new one. Use '%s' "
	"instead"), "zpool detach");
	return (zfs_error(hdl, EZFS_POSTSPLIT_ONLINE, errbuf));
	}
	return (zpool_standard_error(hdl, errno, errbuf));
	}

	*newstate = zc.zc_cookie;
	return (0);
	}

	/*
	* Take the specified vdev offline
	*/
	int
	zpool_vdev_offline(zpool_handle_t zhp, const char path, boolean_t istmp)
	{
	zfs_cmd_t zc = {"\0"};
	char errbuf[ERRBUFLEN];
	nvlist_t *tgt;
	boolean_t avail_spare, l2cache;
	libzfs_handle_t *hdl = zhp->zpool_hdl;

	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot offline %s"), path);

	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	if ((tgt = zpool_find_vdev(zhp, path, &avail_spare, &l2cache,
	NULL)) == NULL)
	return (zfs_error(hdl, EZFS_NODEVICE, errbuf));

	zc.zc_guid = fnvlist_lookup_uint64(tgt, ZPOOL_CONFIG_GUID);

	if (avail_spare)
	return (zfs_error(hdl, EZFS_ISSPARE, errbuf));

	zc.zc_cookie = VDEV_STATE_OFFLINE;
	zc.zc_obj = istmp ? ZFS_OFFLINE_TEMPORARY : 0;

	if (zfs_ioctl(hdl, ZFS_IOC_VDEV_SET_STATE, &zc) == 0)
	return (0);

	switch (errno) {
	case EBUSY:

	/*
	* There are no other replicas of this device.
	*/
	return (zfs_error(hdl, EZFS_NOREPLICAS, errbuf));

	case EEXIST:
	/*
	* The log device has unplayed logs
	*/
	return (zfs_error(hdl, EZFS_UNPLAYED_LOGS, errbuf));

	default:
	return (zpool_standard_error(hdl, errno, errbuf));
	}
	}

	/*
	* Remove the specified vdev asynchronously from the configuration, so
	* that it may come ONLINE if reinserted. This is called from zed on
	* Udev remove event.
	* Note: We also have a similar function zpool_vdev_remove() that
	* removes the vdev from the pool.
	*/
	int
	zpool_vdev_remove_wanted(zpool_handle_t zhp, const char path)
	{
	zfs_cmd_t zc = {"\0"};
	char errbuf[ERRBUFLEN];
	nvlist_t *tgt;
	boolean_t avail_spare, l2cache;
	libzfs_handle_t *hdl = zhp->zpool_hdl;

	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot remove %s"), path);

	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	if ((tgt = zpool_find_vdev(zhp, path, &avail_spare, &l2cache,
	NULL)) == NULL)
	return (zfs_error(hdl, EZFS_NODEVICE, errbuf));

	zc.zc_guid = fnvlist_lookup_uint64(tgt, ZPOOL_CONFIG_GUID);

	zc.zc_cookie = VDEV_STATE_REMOVED;

	if (zfs_ioctl(hdl, ZFS_IOC_VDEV_SET_STATE, &zc) == 0)
	return (0);

	return (zpool_standard_error(hdl, errno, errbuf));
	}

	/*
	* Mark the given vdev faulted.
	*/
	int
	zpool_vdev_fault(zpool_handle_t *zhp, uint64_t guid, vdev_aux_t aux)
	{
	zfs_cmd_t zc = {"\0"};
	char errbuf[ERRBUFLEN];
	libzfs_handle_t *hdl = zhp->zpool_hdl;

	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot fault %llu"), (u_longlong_t)guid);

	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	zc.zc_guid = guid;
	zc.zc_cookie = VDEV_STATE_FAULTED;
	zc.zc_obj = aux;

	if (zfs_ioctl(hdl, ZFS_IOC_VDEV_SET_STATE, &zc) == 0)
	return (0);

	switch (errno) {
	case EBUSY:

	/*
	* There are no other replicas of this device.
	*/
	return (zfs_error(hdl, EZFS_NOREPLICAS, errbuf));

	default:
	return (zpool_standard_error(hdl, errno, errbuf));
	}

	}

	/*
	* Generic set vdev state function
	*/
	static int
	zpool_vdev_set_state(zpool_handle_t *zhp, uint64_t guid, vdev_aux_t aux,
	vdev_state_t state)
	{
	zfs_cmd_t zc = {"\0"};
	char errbuf[ERRBUFLEN];
	libzfs_handle_t *hdl = zhp->zpool_hdl;

	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot set %s %llu"),
	zpool_state_to_name(state, aux), (u_longlong_t)guid);

	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	zc.zc_guid = guid;
	zc.zc_cookie = state;
	zc.zc_obj = aux;

	if (zfs_ioctl(hdl, ZFS_IOC_VDEV_SET_STATE, &zc) == 0)
	return (0);

	return (zpool_standard_error(hdl, errno, errbuf));
	}

	/*
	* Mark the given vdev degraded.
	*/
	int
	zpool_vdev_degrade(zpool_handle_t *zhp, uint64_t guid, vdev_aux_t aux)
	{
	return (zpool_vdev_set_state(zhp, guid, aux, VDEV_STATE_DEGRADED));
	}

	/*
	* Mark the given vdev as in a removed state (as if the device does not exist).
	*
	* This is different than zpool_vdev_remove() which does a removal of a device
	* from the pool (but the device does exist).
	*/
	int
	zpool_vdev_set_removed_state(zpool_handle_t *zhp, uint64_t guid, vdev_aux_t aux)
	{
	return (zpool_vdev_set_state(zhp, guid, aux, VDEV_STATE_REMOVED));
	}

	/*
	* Returns TRUE if the given nvlist is a vdev that was originally swapped in as
	* a hot spare.
	*/
	static boolean_t
	is_replacing_spare(nvlist_t search, nvlist_t tgt, int which)
	{
	nvlist_t **child;
	uint_t c, children;

	if (nvlist_lookup_nvlist_array(search, ZPOOL_CONFIG_CHILDREN, &child,
	&children) == 0) {
	const char *type = fnvlist_lookup_string(search,
	ZPOOL_CONFIG_TYPE);
	if ((strcmp(type, VDEV_TYPE_SPARE) == 0 \|\|
	strcmp(type, VDEV_TYPE_DRAID_SPARE) == 0) &&
	children == 2 && child[which] == tgt)
	return (B_TRUE);

	for (c = 0; c < children; c++)
	if (is_replacing_spare(child[c], tgt, which))
	return (B_TRUE);
	}

	return (B_FALSE);
	}

	/*
	* Attach new_disk (fully described by nvroot) to old_disk.
	* If 'replacing' is specified, the new disk will replace the old one.
	*/
	int
	zpool_vdev_attach(zpool_handle_t zhp, const char old_disk,
	const char new_disk, nvlist_t nvroot, int replacing, boolean_t rebuild)
	{
	zfs_cmd_t zc = {"\0"};
	char errbuf[ERRBUFLEN];
	int ret;
	nvlist_t *tgt;
	boolean_t avail_spare, l2cache, islog;
	uint64_t val;
	char *newname;
	nvlist_t **child;
	uint_t children;
	nvlist_t *config_root;
	libzfs_handle_t *hdl = zhp->zpool_hdl;

	if (replacing)
	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot replace %s with %s"), old_disk, new_disk);
	else
	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot attach %s to %s"), new_disk, old_disk);

	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	if ((tgt = zpool_find_vdev(zhp, old_disk, &avail_spare, &l2cache,
	&islog)) == NULL)
	return (zfs_error(hdl, EZFS_NODEVICE, errbuf));

	if (avail_spare)
	return (zfs_error(hdl, EZFS_ISSPARE, errbuf));

	if (l2cache)
	return (zfs_error(hdl, EZFS_ISL2CACHE, errbuf));

	zc.zc_guid = fnvlist_lookup_uint64(tgt, ZPOOL_CONFIG_GUID);
	zc.zc_cookie = replacing;
	zc.zc_simple = rebuild;

	if (rebuild &&
	zfeature_lookup_guid("org.openzfs:device_rebuild", NULL) != 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"the loaded zfs module doesn't support device rebuilds"));
	return (zfs_error(hdl, EZFS_POOL_NOTSUP, errbuf));
	}

	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
	&child, &children) != 0 \|\| children != 1) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"new device must be a single disk"));
	return (zfs_error(hdl, EZFS_INVALCONFIG, errbuf));
	}

	config_root = fnvlist_lookup_nvlist(zpool_get_config(zhp, NULL),
	ZPOOL_CONFIG_VDEV_TREE);

	if ((newname = zpool_vdev_name(NULL, NULL, child[0], 0)) == NULL)
	return (-1);

	/*
	* If the target is a hot spare that has been swapped in, we can only
	* replace it with another hot spare.
	*/
	if (replacing &&
	nvlist_lookup_uint64(tgt, ZPOOL_CONFIG_IS_SPARE, &val) == 0 &&
	(zpool_find_vdev(zhp, newname, &avail_spare, &l2cache,
	NULL) == NULL \|\| !avail_spare) &&
	is_replacing_spare(config_root, tgt, 1)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"can only be replaced by another hot spare"));
	free(newname);
	return (zfs_error(hdl, EZFS_BADTARGET, errbuf));
	}

	free(newname);

	zcmd_write_conf_nvlist(hdl, &zc, nvroot);

	ret = zfs_ioctl(hdl, ZFS_IOC_VDEV_ATTACH, &zc);

	zcmd_free_nvlists(&zc);

	if (ret == 0)
	return (0);

	switch (errno) {
	case ENOTSUP:
	/*
	* Can't attach to or replace this type of vdev.
	*/
	if (replacing) {
	uint64_t version = zpool_get_prop_int(zhp,
	ZPOOL_PROP_VERSION, NULL);

	if (islog) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"cannot replace a log with a spare"));
	} else if (rebuild) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"only mirror and dRAID vdevs support "
	"sequential reconstruction"));
	} else if (zpool_is_draid_spare(new_disk)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"dRAID spares can only replace child "
	"devices in their parent's dRAID vdev"));
	} else if (version >= SPA_VERSION_MULTI_REPLACE) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"already in replacing/spare config; wait "
	"for completion or use 'zpool detach'"));
	} else {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"cannot replace a replacing device"));
	}
	} else {
	char status[64] = {0};
	zpool_prop_get_feature(zhp,
	"feature@device_rebuild", status, 63);
	if (rebuild &&
	strncmp(status, ZFS_FEATURE_DISABLED, 64) == 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"device_rebuild feature must be enabled "
	"in order to use sequential "
	"reconstruction"));
	} else {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"can only attach to mirrors and top-level "
	"disks"));
	}
	}
	(void) zfs_error(hdl, EZFS_BADTARGET, errbuf);
	break;

	case EINVAL:
	/*
	* The new device must be a single disk.
	*/
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"new device must be a single disk"));
	(void) zfs_error(hdl, EZFS_INVALCONFIG, errbuf);
	break;

	case EBUSY:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "%s is busy, "
	"or device removal is in progress"),
	new_disk);
	(void) zfs_error(hdl, EZFS_BADDEV, errbuf);
	break;

	case EOVERFLOW:
	/*
	* The new device is too small.
	*/
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"device is too small"));
	(void) zfs_error(hdl, EZFS_BADDEV, errbuf);
	break;

	case EDOM:
	/*
	* The new device has a different optimal sector size.
	*/
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"new device has a different optimal sector size; use the "
	"option '-o ashift=N' to override the optimal size"));
	(void) zfs_error(hdl, EZFS_BADDEV, errbuf);
	break;

	case ENAMETOOLONG:
	/*
	* The resulting top-level vdev spec won't fit in the label.
	*/
	(void) zfs_error(hdl, EZFS_DEVOVERFLOW, errbuf);
	break;

	default:
	(void) zpool_standard_error(hdl, errno, errbuf);
	}

	return (-1);
	}

	/*
	* Detach the specified device.
	*/
	int
	zpool_vdev_detach(zpool_handle_t zhp, const char path)
	{
	zfs_cmd_t zc = {"\0"};
	char errbuf[ERRBUFLEN];
	nvlist_t *tgt;
	boolean_t avail_spare, l2cache;
	libzfs_handle_t *hdl = zhp->zpool_hdl;

	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot detach %s"), path);

	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	if ((tgt = zpool_find_vdev(zhp, path, &avail_spare, &l2cache,
	NULL)) == NULL)
	return (zfs_error(hdl, EZFS_NODEVICE, errbuf));

	if (avail_spare)
	return (zfs_error(hdl, EZFS_ISSPARE, errbuf));

	if (l2cache)
	return (zfs_error(hdl, EZFS_ISL2CACHE, errbuf));

	zc.zc_guid = fnvlist_lookup_uint64(tgt, ZPOOL_CONFIG_GUID);

	if (zfs_ioctl(hdl, ZFS_IOC_VDEV_DETACH, &zc) == 0)
	return (0);

	switch (errno) {

	case ENOTSUP:
	/*
	* Can't detach from this type of vdev.
	*/
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "only "
	"applicable to mirror and replacing vdevs"));
	(void) zfs_error(hdl, EZFS_BADTARGET, errbuf);
	break;

	case EBUSY:
	/*
	* There are no other replicas of this device.
	*/
	(void) zfs_error(hdl, EZFS_NOREPLICAS, errbuf);
	break;

	default:
	(void) zpool_standard_error(hdl, errno, errbuf);
	}

	return (-1);
	}

	/*
	* Find a mirror vdev in the source nvlist.
	*
	* The mchild array contains a list of disks in one of the top-level mirrors
	* of the source pool. The schild array contains a list of disks that the
	* user specified on the command line. We loop over the mchild array to
	* see if any entry in the schild array matches.
	*
	* If a disk in the mchild array is found in the schild array, we return
	* the index of that entry. Otherwise we return -1.
	*/
	static int
	find_vdev_entry(zpool_handle_t zhp, nvlist_t *mchild, uint_t mchildren,
	nvlist_t **schild, uint_t schildren)
	{
	uint_t mc;

	for (mc = 0; mc < mchildren; mc++) {
	uint_t sc;
	char *mpath = zpool_vdev_name(zhp->zpool_hdl, zhp,
	mchild[mc], 0);

	for (sc = 0; sc < schildren; sc++) {
	char *spath = zpool_vdev_name(zhp->zpool_hdl, zhp,
	schild[sc], 0);
	boolean_t result = (strcmp(mpath, spath) == 0);

	free(spath);
	if (result) {
	free(mpath);
	return (mc);
	}
	}

	free(mpath);
	}

	return (-1);
	}

	/*
	* Split a mirror pool. If newroot points to null, then a new nvlist
	* is generated and it is the responsibility of the caller to free it.
	*/
	int
	zpool_vdev_split(zpool_handle_t zhp, char newname, nvlist_t **newroot,
	nvlist_t *props, splitflags_t flags)
	{
	zfs_cmd_t zc = {"\0"};
	char errbuf[ERRBUFLEN];
	const char *bias;
	nvlist_t tree, config, child, newchild, *newconfig = NULL;
	nvlist_t *varray = NULL, zc_props = NULL;
	uint_t c, children, newchildren, lastlog = 0, vcount, found = 0;
	libzfs_handle_t *hdl = zhp->zpool_hdl;
	uint64_t vers, readonly = B_FALSE;
	boolean_t freelist = B_FALSE, memory_err = B_TRUE;
	int retval = 0;

	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "Unable to split %s"), zhp->zpool_name);

	if (!zpool_name_valid(hdl, B_FALSE, newname))
	return (zfs_error(hdl, EZFS_INVALIDNAME, errbuf));

	if ((config = zpool_get_config(zhp, NULL)) == NULL) {
	(void) fprintf(stderr, gettext("Internal error: unable to "
	"retrieve pool configuration\n"));
	return (-1);
	}

	tree = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
	vers = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION);

	if (props) {
	prop_flags_t flags = { .create = B_FALSE, .import = B_TRUE };
	if ((zc_props = zpool_valid_proplist(hdl, zhp->zpool_name,
	props, vers, flags, errbuf)) == NULL)
	return (-1);
	(void) nvlist_lookup_uint64(zc_props,
	zpool_prop_to_name(ZPOOL_PROP_READONLY), &readonly);
	if (readonly) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property %s can only be set at import time"),
	zpool_prop_to_name(ZPOOL_PROP_READONLY));
	return (-1);
	}
	}

	if (nvlist_lookup_nvlist_array(tree, ZPOOL_CONFIG_CHILDREN, &child,
	&children) != 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"Source pool is missing vdev tree"));
	nvlist_free(zc_props);
	return (-1);
	}

	varray = zfs_alloc(hdl, children * sizeof (nvlist_t *));
	vcount = 0;

	if (*newroot == NULL \|\|
	nvlist_lookup_nvlist_array(*newroot, ZPOOL_CONFIG_CHILDREN,
	&newchild, &newchildren) != 0)
	newchildren = 0;

	for (c = 0; c < children; c++) {
	uint64_t is_log = B_FALSE, is_hole = B_FALSE;
	boolean_t is_special = B_FALSE, is_dedup = B_FALSE;
	const char *type;
	nvlist_t *mchild, vdev;
	uint_t mchildren;
	int entry;

	/*
	* Unlike cache & spares, slogs are stored in the
	* ZPOOL_CONFIG_CHILDREN array. We filter them out here.
	*/
	(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
	&is_log);
	(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
	&is_hole);
	if (is_log \|\| is_hole) {
	/*
	* Create a hole vdev and put it in the config.
	*/
	if (nvlist_alloc(&vdev, NV_UNIQUE_NAME, 0) != 0)
	goto out;
	if (nvlist_add_string(vdev, ZPOOL_CONFIG_TYPE,
	VDEV_TYPE_HOLE) != 0)
	goto out;
	if (nvlist_add_uint64(vdev, ZPOOL_CONFIG_IS_HOLE,
	1) != 0)
	goto out;
	if (lastlog == 0)
	lastlog = vcount;
	varray[vcount++] = vdev;
	continue;
	}
	lastlog = 0;
	type = fnvlist_lookup_string(child[c], ZPOOL_CONFIG_TYPE);

	if (strcmp(type, VDEV_TYPE_INDIRECT) == 0) {
	vdev = child[c];
	if (nvlist_dup(vdev, &varray[vcount++], 0) != 0)
	goto out;
	continue;
	} else if (strcmp(type, VDEV_TYPE_MIRROR) != 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"Source pool must be composed only of mirrors\n"));
	retval = zfs_error(hdl, EZFS_INVALCONFIG, errbuf);
	goto out;
	}

	if (nvlist_lookup_string(child[c],
	ZPOOL_CONFIG_ALLOCATION_BIAS, &bias) == 0) {
	if (strcmp(bias, VDEV_ALLOC_BIAS_SPECIAL) == 0)
	is_special = B_TRUE;
	else if (strcmp(bias, VDEV_ALLOC_BIAS_DEDUP) == 0)
	is_dedup = B_TRUE;
	}
	verify(nvlist_lookup_nvlist_array(child[c],
	ZPOOL_CONFIG_CHILDREN, &mchild, &mchildren) == 0);

	/* find or add an entry for this top-level vdev */
	if (newchildren > 0 &&
	(entry = find_vdev_entry(zhp, mchild, mchildren,
	newchild, newchildren)) >= 0) {
	/* We found a disk that the user specified. */
	vdev = mchild[entry];
	++found;
	} else {
	/* User didn't specify a disk for this vdev. */
	vdev = mchild[mchildren - 1];
	}

	if (nvlist_dup(vdev, &varray[vcount++], 0) != 0)
	goto out;

	if (flags.dryrun != 0) {
	if (is_dedup == B_TRUE) {
	if (nvlist_add_string(varray[vcount - 1],
	ZPOOL_CONFIG_ALLOCATION_BIAS,
	VDEV_ALLOC_BIAS_DEDUP) != 0)
	goto out;
	} else if (is_special == B_TRUE) {
	if (nvlist_add_string(varray[vcount - 1],
	ZPOOL_CONFIG_ALLOCATION_BIAS,
	VDEV_ALLOC_BIAS_SPECIAL) != 0)
	goto out;
	}
	}
	}

	/* did we find every disk the user specified? */
	if (found != newchildren) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "Device list must "
	"include at most one disk from each mirror"));
	retval = zfs_error(hdl, EZFS_INVALCONFIG, errbuf);
	goto out;
	}

	/* Prepare the nvlist for populating. */
	if (*newroot == NULL) {
	if (nvlist_alloc(newroot, NV_UNIQUE_NAME, 0) != 0)
	goto out;
	freelist = B_TRUE;
	if (nvlist_add_string(*newroot, ZPOOL_CONFIG_TYPE,
	VDEV_TYPE_ROOT) != 0)
	goto out;
	} else {
	verify(nvlist_remove_all(*newroot, ZPOOL_CONFIG_CHILDREN) == 0);
	}

	/* Add all the children we found */
	if (nvlist_add_nvlist_array(*newroot, ZPOOL_CONFIG_CHILDREN,
	(const nvlist_t **)varray, lastlog == 0 ? vcount : lastlog) != 0)
	goto out;

	/*
	* If we're just doing a dry run, exit now with success.
	*/
	if (flags.dryrun) {
	memory_err = B_FALSE;
	freelist = B_FALSE;
	goto out;
	}

	/* now build up the config list & call the ioctl */
	if (nvlist_alloc(&newconfig, NV_UNIQUE_NAME, 0) != 0)
	goto out;

	if (nvlist_add_nvlist(newconfig,
	ZPOOL_CONFIG_VDEV_TREE, *newroot) != 0 \|\|
	nvlist_add_string(newconfig,
	ZPOOL_CONFIG_POOL_NAME, newname) != 0 \|\|
	nvlist_add_uint64(newconfig, ZPOOL_CONFIG_VERSION, vers) != 0)
	goto out;

	/*
	* The new pool is automatically part of the namespace unless we
	* explicitly export it.
	*/
	if (!flags.import)
	zc.zc_cookie = ZPOOL_EXPORT_AFTER_SPLIT;
	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	(void) strlcpy(zc.zc_string, newname, sizeof (zc.zc_string));
	zcmd_write_conf_nvlist(hdl, &zc, newconfig);
	if (zc_props != NULL)
	zcmd_write_src_nvlist(hdl, &zc, zc_props);

	if (zfs_ioctl(hdl, ZFS_IOC_VDEV_SPLIT, &zc) != 0) {
	retval = zpool_standard_error(hdl, errno, errbuf);
	goto out;
	}

	freelist = B_FALSE;
	memory_err = B_FALSE;

	out:
	if (varray != NULL) {
	int v;

	for (v = 0; v < vcount; v++)
	nvlist_free(varray[v]);
	free(varray);
	}
	zcmd_free_nvlists(&zc);
	nvlist_free(zc_props);
	nvlist_free(newconfig);
	if (freelist) {
	nvlist_free(*newroot);
	*newroot = NULL;
	}

	if (retval != 0)
	return (retval);

	if (memory_err)
	return (no_memory(hdl));

	return (0);
	}

	/*
	* Remove the given device.
	*/
	int
	zpool_vdev_remove(zpool_handle_t zhp, const char path)
	{
	zfs_cmd_t zc = {"\0"};
	char errbuf[ERRBUFLEN];
	nvlist_t *tgt;
	boolean_t avail_spare, l2cache, islog;
	libzfs_handle_t *hdl = zhp->zpool_hdl;
	uint64_t version;

	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot remove %s"), path);

	if (zpool_is_draid_spare(path)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"dRAID spares cannot be removed"));
	return (zfs_error(hdl, EZFS_NODEVICE, errbuf));
	}

	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	if ((tgt = zpool_find_vdev(zhp, path, &avail_spare, &l2cache,
	&islog)) == NULL)
	return (zfs_error(hdl, EZFS_NODEVICE, errbuf));

	version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
	if (islog && version < SPA_VERSION_HOLES) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"pool must be upgraded to support log removal"));
	return (zfs_error(hdl, EZFS_BADVERSION, errbuf));
	}

	zc.zc_guid = fnvlist_lookup_uint64(tgt, ZPOOL_CONFIG_GUID);

	if (zfs_ioctl(hdl, ZFS_IOC_VDEV_REMOVE, &zc) == 0)
	return (0);

	switch (errno) {

	case EALREADY:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"removal for this vdev is already in progress."));
	(void) zfs_error(hdl, EZFS_BUSY, errbuf);
	break;

	case EINVAL:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"invalid config; all top-level vdevs must "
	"have the same sector size and not be raidz."));
	(void) zfs_error(hdl, EZFS_INVALCONFIG, errbuf);
	break;

	case EBUSY:
	if (islog) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"Mount encrypted datasets to replay logs."));
	} else {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"Pool busy; removal may already be in progress"));
	}
	(void) zfs_error(hdl, EZFS_BUSY, errbuf);
	break;

	case EACCES:
	if (islog) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"Mount encrypted datasets to replay logs."));
	(void) zfs_error(hdl, EZFS_BUSY, errbuf);
	} else {
	(void) zpool_standard_error(hdl, errno, errbuf);
	}
	break;

	default:
	(void) zpool_standard_error(hdl, errno, errbuf);
	}
	return (-1);
	}

	int
	zpool_vdev_remove_cancel(zpool_handle_t *zhp)
	{
	zfs_cmd_t zc = {{0}};
	char errbuf[ERRBUFLEN];
	libzfs_handle_t *hdl = zhp->zpool_hdl;

	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot cancel removal"));

	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	zc.zc_cookie = 1;

	if (zfs_ioctl(hdl, ZFS_IOC_VDEV_REMOVE, &zc) == 0)
	return (0);

	return (zpool_standard_error(hdl, errno, errbuf));
	}

	int
	zpool_vdev_indirect_size(zpool_handle_t zhp, const char path,
	uint64_t *sizep)
	{
	char errbuf[ERRBUFLEN];
	nvlist_t *tgt;
	boolean_t avail_spare, l2cache, islog;
	libzfs_handle_t *hdl = zhp->zpool_hdl;

	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot determine indirect size of %s"),
	path);

	if ((tgt = zpool_find_vdev(zhp, path, &avail_spare, &l2cache,
	&islog)) == NULL)
	return (zfs_error(hdl, EZFS_NODEVICE, errbuf));

	if (avail_spare \|\| l2cache \|\| islog) {
	*sizep = 0;
	return (0);
	}

	if (nvlist_lookup_uint64(tgt, ZPOOL_CONFIG_INDIRECT_SIZE, sizep) != 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"indirect size not available"));
	return (zfs_error(hdl, EINVAL, errbuf));
	}
	return (0);
	}

	/*
	* Clear the errors for the pool, or the particular device if specified.
	*/
	int
	zpool_clear(zpool_handle_t zhp, const char path, nvlist_t *rewindnvl)
	{
	zfs_cmd_t zc = {"\0"};
	char errbuf[ERRBUFLEN];
	nvlist_t *tgt;
	zpool_load_policy_t policy;
	boolean_t avail_spare, l2cache;
	libzfs_handle_t *hdl = zhp->zpool_hdl;
	nvlist_t *nvi = NULL;
	int error;

	if (path)
	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot clear errors for %s"),
	path);
	else
	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot clear errors for %s"),
	zhp->zpool_name);

	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	if (path) {
	if ((tgt = zpool_find_vdev(zhp, path, &avail_spare,
	&l2cache, NULL)) == NULL)
	return (zfs_error(hdl, EZFS_NODEVICE, errbuf));

	/*
	* Don't allow error clearing for hot spares. Do allow
	* error clearing for l2cache devices.
	*/
	if (avail_spare)
	return (zfs_error(hdl, EZFS_ISSPARE, errbuf));

	zc.zc_guid = fnvlist_lookup_uint64(tgt, ZPOOL_CONFIG_GUID);
	}

	zpool_get_load_policy(rewindnvl, &policy);
	zc.zc_cookie = policy.zlp_rewind;

	zcmd_alloc_dst_nvlist(hdl, &zc, zhp->zpool_config_size * 2);
	zcmd_write_src_nvlist(hdl, &zc, rewindnvl);

	while ((error = zfs_ioctl(hdl, ZFS_IOC_CLEAR, &zc)) != 0 &&
	errno == ENOMEM)
	zcmd_expand_dst_nvlist(hdl, &zc);

	if (!error \|\| ((policy.zlp_rewind & ZPOOL_TRY_REWIND) &&
	errno != EPERM && errno != EACCES)) {
	if (policy.zlp_rewind &
	(ZPOOL_DO_REWIND \| ZPOOL_TRY_REWIND)) {
	(void) zcmd_read_dst_nvlist(hdl, &zc, &nvi);
	zpool_rewind_exclaim(hdl, zc.zc_name,
	((policy.zlp_rewind & ZPOOL_TRY_REWIND) != 0),
	nvi);
	nvlist_free(nvi);
	}
	zcmd_free_nvlists(&zc);
	return (0);
	}

	zcmd_free_nvlists(&zc);
	return (zpool_standard_error(hdl, errno, errbuf));
	}

	/*
	* Similar to zpool_clear(), but takes a GUID (used by fmd).
	*/
	int
	zpool_vdev_clear(zpool_handle_t *zhp, uint64_t guid)
	{
	zfs_cmd_t zc = {"\0"};
	char errbuf[ERRBUFLEN];
	libzfs_handle_t *hdl = zhp->zpool_hdl;

	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot clear errors for %llx"),
	(u_longlong_t)guid);

	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	zc.zc_guid = guid;
	zc.zc_cookie = ZPOOL_NO_REWIND;

	if (zfs_ioctl(hdl, ZFS_IOC_CLEAR, &zc) == 0)
	return (0);

	return (zpool_standard_error(hdl, errno, errbuf));
	}

	/*
	* Change the GUID for a pool.
	*/
	int
	zpool_reguid(zpool_handle_t *zhp)
	{
	char errbuf[ERRBUFLEN];
	libzfs_handle_t *hdl = zhp->zpool_hdl;
	zfs_cmd_t zc = {"\0"};

	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot reguid '%s'"), zhp->zpool_name);

	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	if (zfs_ioctl(hdl, ZFS_IOC_POOL_REGUID, &zc) == 0)
	return (0);

	return (zpool_standard_error(hdl, errno, errbuf));
	}

	/*
	* Reopen the pool.
	*/
	int
	zpool_reopen_one(zpool_handle_t zhp, void data)
	{
	libzfs_handle_t *hdl = zpool_get_handle(zhp);
	const char *pool_name = zpool_get_name(zhp);
	boolean_t *scrub_restart = data;
	int error;

	error = lzc_reopen(pool_name, *scrub_restart);
	if (error) {
	return (zpool_standard_error_fmt(hdl, error,
	dgettext(TEXT_DOMAIN, "cannot reopen '%s'"), pool_name));
	}

	return (0);
	}

	/* call into libzfs_core to execute the sync IOCTL per pool */
	int
	zpool_sync_one(zpool_handle_t zhp, void data)
	{
	int ret;
	libzfs_handle_t *hdl = zpool_get_handle(zhp);
	const char *pool_name = zpool_get_name(zhp);
	boolean_t *force = data;
	nvlist_t *innvl = fnvlist_alloc();

	fnvlist_add_boolean_value(innvl, "force", *force);
	if ((ret = lzc_sync(pool_name, innvl, NULL)) != 0) {
	nvlist_free(innvl);
	return (zpool_standard_error_fmt(hdl, ret,
	dgettext(TEXT_DOMAIN, "sync '%s' failed"), pool_name));
	}
	nvlist_free(innvl);

	return (0);
	}

	#define PATH_BUF_LEN 64

	/*
	* Given a vdev, return the name to display in iostat. If the vdev has a path,
	* we use that, stripping off any leading "/dev/dsk/"; if not, we use the type.
	* We also check if this is a whole disk, in which case we strip off the
	* trailing 's0' slice name.
	*
	* This routine is also responsible for identifying when disks have been
	* reconfigured in a new location. The kernel will have opened the device by
	* devid, but the path will still refer to the old location. To catch this, we
	* first do a path -> devid translation (which is fast for the common case). If
	* the devid matches, we're done. If not, we do a reverse devid -> path
	* translation and issue the appropriate ioctl() to update the path of the vdev.
	* If 'zhp' is NULL, then this is an exported pool, and we don't need to do any
	* of these checks.
	*/
	char *
	zpool_vdev_name(libzfs_handle_t hdl, zpool_handle_t zhp, nvlist_t *nv,
	int name_flags)
	{
	const char type, tpath;
	const char *path;
	uint64_t value;
	char buf[PATH_BUF_LEN];
	char tmpbuf[PATH_BUF_LEN * 2];

	/*
	* vdev_name will be "root"/"root-0" for the root vdev, but it is the
	* zpool name that will be displayed to the user.
	*/
	type = fnvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE);
	if (zhp != NULL && strcmp(type, "root") == 0)
	return (zfs_strdup(hdl, zpool_get_name(zhp)));

	if (libzfs_envvar_is_set("ZPOOL_VDEV_NAME_PATH"))
	name_flags \|= VDEV_NAME_PATH;
	if (libzfs_envvar_is_set("ZPOOL_VDEV_NAME_GUID"))
	name_flags \|= VDEV_NAME_GUID;
	if (libzfs_envvar_is_set("ZPOOL_VDEV_NAME_FOLLOW_LINKS"))
	name_flags \|= VDEV_NAME_FOLLOW_LINKS;

	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT, &value) == 0 \|\|
	name_flags & VDEV_NAME_GUID) {
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &value);
	(void) snprintf(buf, sizeof (buf), "%llu", (u_longlong_t)value);
	path = buf;
	} else if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &tpath) == 0) {
	path = tpath;

	if (name_flags & VDEV_NAME_FOLLOW_LINKS) {
	char *rp = realpath(path, NULL);
	if (rp) {
	strlcpy(buf, rp, sizeof (buf));
	path = buf;
	free(rp);
	}
	}

	/*
	* For a block device only use the name.
	*/
	if ((strcmp(type, VDEV_TYPE_DISK) == 0) &&
	!(name_flags & VDEV_NAME_PATH)) {
	path = zfs_strip_path(path);
	}

	/*
	* Remove the partition from the path if this is a whole disk.
	*/
	if (strcmp(type, VDEV_TYPE_DRAID_SPARE) != 0 &&
	nvlist_lookup_uint64(nv, ZPOOL_CONFIG_WHOLE_DISK, &value)
	== 0 && value && !(name_flags & VDEV_NAME_PATH)) {
	return (zfs_strip_partition(path));
	}
	} else {
	path = type;

	/*
	* If it's a raidz device, we need to stick in the parity level.
	*/
	if (strcmp(path, VDEV_TYPE_RAIDZ) == 0) {
	value = fnvlist_lookup_uint64(nv, ZPOOL_CONFIG_NPARITY);
	(void) snprintf(buf, sizeof (buf), "%s%llu", path,
	(u_longlong_t)value);
	path = buf;
	}

	/*
	* If it's a dRAID device, we add parity, groups, and spares.
	*/
	if (strcmp(path, VDEV_TYPE_DRAID) == 0) {
	uint64_t ndata, nparity, nspares;
	nvlist_t **child;
	uint_t children;

	verify(nvlist_lookup_nvlist_array(nv,
	ZPOOL_CONFIG_CHILDREN, &child, &children) == 0);
	nparity = fnvlist_lookup_uint64(nv,
	ZPOOL_CONFIG_NPARITY);
	ndata = fnvlist_lookup_uint64(nv,
	ZPOOL_CONFIG_DRAID_NDATA);
	nspares = fnvlist_lookup_uint64(nv,
	ZPOOL_CONFIG_DRAID_NSPARES);

	path = zpool_draid_name(buf, sizeof (buf), ndata,
	nparity, nspares, children);
	}

	/*
	* We identify each top-level vdev by using a <type-id>
	* naming convention.
	*/
	if (name_flags & VDEV_NAME_TYPE_ID) {
	uint64_t id = fnvlist_lookup_uint64(nv,
	ZPOOL_CONFIG_ID);
	(void) snprintf(tmpbuf, sizeof (tmpbuf), "%s-%llu",
	path, (u_longlong_t)id);
	path = tmpbuf;
	}
	}

	return (zfs_strdup(hdl, path));
	}

	static int
	zbookmark_mem_compare(const void a, const void b)
	{
	return (memcmp(a, b, sizeof (zbookmark_phys_t)));
	}

	/*
	* Retrieve the persistent error log, uniquify the members, and return to the
	* caller.
	*/
	int
	zpool_get_errlog(zpool_handle_t zhp, nvlist_t *nverrlistp)
	{
	zfs_cmd_t zc = {"\0"};
	libzfs_handle_t *hdl = zhp->zpool_hdl;
	zbookmark_phys_t *buf;
	uint64_t buflen = 10000; /* approx. 1MB of RAM */

	if (fnvlist_lookup_uint64(zhp->zpool_config,
	ZPOOL_CONFIG_ERRCOUNT) == 0)
	return (0);

	/*
	* Retrieve the raw error list from the kernel. If it doesn't fit,
	* allocate a larger buffer and retry.
	*/
	(void) strcpy(zc.zc_name, zhp->zpool_name);
	for (;;) {
	buf = zfs_alloc(zhp->zpool_hdl,
	buflen * sizeof (zbookmark_phys_t));
	zc.zc_nvlist_dst = (uintptr_t)buf;
	zc.zc_nvlist_dst_size = buflen;
	if (zfs_ioctl(zhp->zpool_hdl, ZFS_IOC_ERROR_LOG,
	&zc) != 0) {
	free(buf);
	if (errno == ENOMEM) {
	buflen *= 2;
	} else {
	return (zpool_standard_error_fmt(hdl, errno,
	dgettext(TEXT_DOMAIN, "errors: List of "
	"errors unavailable")));
	}
	} else {
	break;
	}
	}

	/*
	* Sort the resulting bookmarks. This is a little confusing due to the
	* implementation of ZFS_IOC_ERROR_LOG. The bookmarks are copied last
	* to first, and 'zc_nvlist_dst_size' indicates the number of bookmarks
	* _not_ copied as part of the process. So we point the start of our
	* array appropriate and decrement the total number of elements.
	*/
	zbookmark_phys_t *zb = buf + zc.zc_nvlist_dst_size;
	uint64_t zblen = buflen - zc.zc_nvlist_dst_size;

	qsort(zb, zblen, sizeof (zbookmark_phys_t), zbookmark_mem_compare);

	verify(nvlist_alloc(nverrlistp, 0, KM_SLEEP) == 0);

	/*
	* Fill in the nverrlistp with nvlist's of dataset and object numbers.
	*/
	for (uint64_t i = 0; i < zblen; i++) {
	nvlist_t *nv;

	/* ignoring zb_blkid and zb_level for now */
	if (i > 0 && zb[i-1].zb_objset == zb[i].zb_objset &&
	zb[i-1].zb_object == zb[i].zb_object)
	continue;

	if (nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_SLEEP) != 0)
	goto nomem;
	if (nvlist_add_uint64(nv, ZPOOL_ERR_DATASET,
	zb[i].zb_objset) != 0) {
	nvlist_free(nv);
	goto nomem;
	}
	if (nvlist_add_uint64(nv, ZPOOL_ERR_OBJECT,
	zb[i].zb_object) != 0) {
	nvlist_free(nv);
	goto nomem;
	}
	if (nvlist_add_nvlist(*nverrlistp, "ejk", nv) != 0) {
	nvlist_free(nv);
	goto nomem;
	}
	nvlist_free(nv);
	}

	free(buf);
	return (0);

	nomem:
	free(buf);
	return (no_memory(zhp->zpool_hdl));
	}

	/*
	* Upgrade a ZFS pool to the latest on-disk version.
	*/
	int
	zpool_upgrade(zpool_handle_t *zhp, uint64_t new_version)
	{
	zfs_cmd_t zc = {"\0"};
	libzfs_handle_t *hdl = zhp->zpool_hdl;

	(void) strcpy(zc.zc_name, zhp->zpool_name);
	zc.zc_cookie = new_version;

	if (zfs_ioctl(hdl, ZFS_IOC_POOL_UPGRADE, &zc) != 0)
	return (zpool_standard_error_fmt(hdl, errno,
	dgettext(TEXT_DOMAIN, "cannot upgrade '%s'"),
	zhp->zpool_name));
	return (0);
	}

	void
	zfs_save_arguments(int argc, char *argv, char string, int len)
	{
	int i;

	(void) strlcpy(string, zfs_basename(argv[0]), len);
	for (i = 1; i < argc; i++) {
	(void) strlcat(string, " ", len);
	(void) strlcat(string, argv[i], len);
	}
	}

	int
	zpool_log_history(libzfs_handle_t hdl, const char message)
	{
	zfs_cmd_t zc = {"\0"};
	nvlist_t *args;

	args = fnvlist_alloc();
	fnvlist_add_string(args, "message", message);
	zcmd_write_src_nvlist(hdl, &zc, args);
	int err = zfs_ioctl(hdl, ZFS_IOC_LOG_HISTORY, &zc);
	nvlist_free(args);
	zcmd_free_nvlists(&zc);
	return (err);
	}

	/*
	* Perform ioctl to get some command history of a pool.
	*
	* 'buf' is the buffer to fill up to 'len' bytes. 'off' is the
	* logical offset of the history buffer to start reading from.
	*
	* Upon return, 'off' is the next logical offset to read from and
	* 'len' is the actual amount of bytes read into 'buf'.
	*/
	static int
	get_history(zpool_handle_t zhp, char buf, uint64_t off, uint64_t len)
	{
	zfs_cmd_t zc = {"\0"};
	libzfs_handle_t *hdl = zhp->zpool_hdl;

	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));

	zc.zc_history = (uint64_t)(uintptr_t)buf;
	zc.zc_history_len = *len;
	zc.zc_history_offset = *off;

	if (zfs_ioctl(hdl, ZFS_IOC_POOL_GET_HISTORY, &zc) != 0) {
	switch (errno) {
	case EPERM:
	return (zfs_error_fmt(hdl, EZFS_PERM,
	dgettext(TEXT_DOMAIN,
	"cannot show history for pool '%s'"),
	zhp->zpool_name));
	case ENOENT:
	return (zfs_error_fmt(hdl, EZFS_NOHISTORY,
	dgettext(TEXT_DOMAIN, "cannot get history for pool "
	"'%s'"), zhp->zpool_name));
	case ENOTSUP:
	return (zfs_error_fmt(hdl, EZFS_BADVERSION,
	dgettext(TEXT_DOMAIN, "cannot get history for pool "
	"'%s', pool must be upgraded"), zhp->zpool_name));
	default:
	return (zpool_standard_error_fmt(hdl, errno,
	dgettext(TEXT_DOMAIN,
	"cannot get history for '%s'"), zhp->zpool_name));
	}
	}

	*len = zc.zc_history_len;
	*off = zc.zc_history_offset;

	return (0);
	}

	/*
	* Retrieve the command history of a pool.
	*/
	int
	zpool_get_history(zpool_handle_t zhp, nvlist_t nvhisp, uint64_t off,
	boolean_t *eof)
	{
	libzfs_handle_t *hdl = zhp->zpool_hdl;
	char *buf;
	int buflen = 128 * 1024;
	nvlist_t **records = NULL;
	uint_t numrecords = 0;
	int err = 0, i;
	uint64_t start = *off;

	buf = zfs_alloc(hdl, buflen);

	/* process about 1MiB a time */
	while (off - start < 1024 1024) {
	uint64_t bytes_read = buflen;
	uint64_t leftover;

	if ((err = get_history(zhp, buf, off, &bytes_read)) != 0)
	break;

	/* if nothing else was read in, we're at EOF, just return */
	if (!bytes_read) {
	*eof = B_TRUE;
	break;
	}

	if ((err = zpool_history_unpack(buf, bytes_read,
	&leftover, &records, &numrecords)) != 0) {
	zpool_standard_error_fmt(hdl, err,
	dgettext(TEXT_DOMAIN,
	"cannot get history for '%s'"), zhp->zpool_name);
	break;
	}
	*off -= leftover;
	if (leftover == bytes_read) {
	/*
	* no progress made, because buffer is not big enough
	* to hold this record; resize and retry.
	*/
	buflen *= 2;
	free(buf);
	buf = zfs_alloc(hdl, buflen);
	}
	}

	free(buf);

	if (!err) {
	*nvhisp = fnvlist_alloc();
	fnvlist_add_nvlist_array(*nvhisp, ZPOOL_HIST_RECORD,
	(const nvlist_t **)records, numrecords);
	}
	for (i = 0; i < numrecords; i++)
	nvlist_free(records[i]);
	free(records);

	return (err);
	}

	/*
	* Retrieve the next event given the passed 'zevent_fd' file descriptor.
	* If there is a new event available 'nvp' will contain a newly allocated
	* nvlist and 'dropped' will be set to the number of missed events since
	* the last call to this function. When 'nvp' is set to NULL it indicates
	* no new events are available. In either case the function returns 0 and
	* it is up to the caller to free 'nvp'. In the case of a fatal error the
	* function will return a non-zero value. When the function is called in
	* blocking mode (the default, unless the ZEVENT_NONBLOCK flag is passed),
	* it will not return until a new event is available.
	*/
	int
	zpool_events_next(libzfs_handle_t hdl, nvlist_t *nvp,
	int *dropped, unsigned flags, int zevent_fd)
	{
	zfs_cmd_t zc = {"\0"};
	int error = 0;

	*nvp = NULL;
	*dropped = 0;
	zc.zc_cleanup_fd = zevent_fd;

	if (flags & ZEVENT_NONBLOCK)
	zc.zc_guid = ZEVENT_NONBLOCK;

	zcmd_alloc_dst_nvlist(hdl, &zc, ZEVENT_SIZE);

	retry:
	if (zfs_ioctl(hdl, ZFS_IOC_EVENTS_NEXT, &zc) != 0) {
	switch (errno) {
	case ESHUTDOWN:
	error = zfs_error_fmt(hdl, EZFS_POOLUNAVAIL,
	dgettext(TEXT_DOMAIN, "zfs shutdown"));
	goto out;
	case ENOENT:
	/* Blocking error case should not occur */
	if (!(flags & ZEVENT_NONBLOCK))
	error = zpool_standard_error_fmt(hdl, errno,
	dgettext(TEXT_DOMAIN, "cannot get event"));

	goto out;
	case ENOMEM:
	zcmd_expand_dst_nvlist(hdl, &zc);
	goto retry;
	default:
	error = zpool_standard_error_fmt(hdl, errno,
	dgettext(TEXT_DOMAIN, "cannot get event"));
	goto out;
	}
	}

	error = zcmd_read_dst_nvlist(hdl, &zc, nvp);
	if (error != 0)
	goto out;

	*dropped = (int)zc.zc_cookie;
	out:
	zcmd_free_nvlists(&zc);

	return (error);
	}

	/*
	* Clear all events.
	*/
	int
	zpool_events_clear(libzfs_handle_t hdl, int count)
	{
	zfs_cmd_t zc = {"\0"};

	if (zfs_ioctl(hdl, ZFS_IOC_EVENTS_CLEAR, &zc) != 0)
	return (zpool_standard_error(hdl, errno,
	dgettext(TEXT_DOMAIN, "cannot clear events")));

	if (count != NULL)
	count = (int)zc.zc_cookie; / # of events cleared */

	return (0);
	}

	/*
	* Seek to a specific EID, ZEVENT_SEEK_START, or ZEVENT_SEEK_END for
	* the passed zevent_fd file handle. On success zero is returned,
	* otherwise -1 is returned and hdl->libzfs_error is set to the errno.
	*/
	int
	zpool_events_seek(libzfs_handle_t *hdl, uint64_t eid, int zevent_fd)
	{
	zfs_cmd_t zc = {"\0"};
	int error = 0;

	zc.zc_guid = eid;
	zc.zc_cleanup_fd = zevent_fd;

	if (zfs_ioctl(hdl, ZFS_IOC_EVENTS_SEEK, &zc) != 0) {
	switch (errno) {
	case ENOENT:
	error = zfs_error_fmt(hdl, EZFS_NOENT,
	dgettext(TEXT_DOMAIN, "cannot get event"));
	break;

	case ENOMEM:
	error = zfs_error_fmt(hdl, EZFS_NOMEM,
	dgettext(TEXT_DOMAIN, "cannot get event"));
	break;

	default:
	error = zpool_standard_error_fmt(hdl, errno,
	dgettext(TEXT_DOMAIN, "cannot get event"));
	break;
	}
	}

	return (error);
	}

	static void
	zpool_obj_to_path_impl(zpool_handle_t *zhp, uint64_t dsobj, uint64_t obj,
	char *pathname, size_t len, boolean_t always_unmounted)
	{
	zfs_cmd_t zc = {"\0"};
	boolean_t mounted = B_FALSE;
	char *mntpnt = NULL;
	char dsname[ZFS_MAX_DATASET_NAME_LEN];

	if (dsobj == 0) {
	/* special case for the MOS */
	(void) snprintf(pathname, len, "<metadata>:<0x%llx>",
	(longlong_t)obj);
	return;
	}

	/* get the dataset's name */
	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
	zc.zc_obj = dsobj;
	if (zfs_ioctl(zhp->zpool_hdl,
	ZFS_IOC_DSOBJ_TO_DSNAME, &zc) != 0) {
	/* just write out a path of two object numbers */
	(void) snprintf(pathname, len, "<0x%llx>:<0x%llx>",
	(longlong_t)dsobj, (longlong_t)obj);
	return;
	}
	(void) strlcpy(dsname, zc.zc_value, sizeof (dsname));

	/* find out if the dataset is mounted */
	mounted = !always_unmounted && is_mounted(zhp->zpool_hdl, dsname,
	&mntpnt);

	/* get the corrupted object's path */
	(void) strlcpy(zc.zc_name, dsname, sizeof (zc.zc_name));
	zc.zc_obj = obj;
	if (zfs_ioctl(zhp->zpool_hdl, ZFS_IOC_OBJ_TO_PATH,
	&zc) == 0) {
	if (mounted) {
	(void) snprintf(pathname, len, "%s%s", mntpnt,
	zc.zc_value);
	} else {
	(void) snprintf(pathname, len, "%s:%s",
	dsname, zc.zc_value);
	}
	} else {
	(void) snprintf(pathname, len, "%s:<0x%llx>", dsname,
	(longlong_t)obj);
	}
	free(mntpnt);
	}

	void
	zpool_obj_to_path(zpool_handle_t *zhp, uint64_t dsobj, uint64_t obj,
	char *pathname, size_t len)
	{
	zpool_obj_to_path_impl(zhp, dsobj, obj, pathname, len, B_FALSE);
	}

	void
	zpool_obj_to_path_ds(zpool_handle_t *zhp, uint64_t dsobj, uint64_t obj,
	char *pathname, size_t len)
	{
	zpool_obj_to_path_impl(zhp, dsobj, obj, pathname, len, B_TRUE);
	}
	/*
	* Wait while the specified activity is in progress in the pool.
	*/
	int
	zpool_wait(zpool_handle_t *zhp, zpool_wait_activity_t activity)
	{
	boolean_t missing;

	int error = zpool_wait_status(zhp, activity, &missing, NULL);

	if (missing) {
	(void) zpool_standard_error_fmt(zhp->zpool_hdl, ENOENT,
	dgettext(TEXT_DOMAIN, "error waiting in pool '%s'"),
	zhp->zpool_name);
	return (ENOENT);
	} else {
	return (error);
	}
	}

	/*
	* Wait for the given activity and return the status of the wait (whether or not
	* any waiting was done) in the 'waited' parameter. Non-existent pools are
	* reported via the 'missing' parameter, rather than by printing an error
	* message. This is convenient when this function is called in a loop over a
	* long period of time (as it is, for example, by zpool's wait cmd). In that
	* scenario, a pool being exported or destroyed should be considered a normal
	* event, so we don't want to print an error when we find that the pool doesn't
	* exist.
	*/
	int
	zpool_wait_status(zpool_handle_t *zhp, zpool_wait_activity_t activity,
	boolean_t missing, boolean_t waited)
	{
	int error = lzc_wait(zhp->zpool_name, activity, waited);
	*missing = (error == ENOENT);
	if (*missing)
	return (0);

	if (error != 0) {
	(void) zpool_standard_error_fmt(zhp->zpool_hdl, error,
	dgettext(TEXT_DOMAIN, "error waiting in pool '%s'"),
	zhp->zpool_name);
	}

	return (error);
	}

	int
	zpool_set_bootenv(zpool_handle_t zhp, const nvlist_t envmap)
	{
	int error = lzc_set_bootenv(zhp->zpool_name, envmap);
	if (error != 0) {
	(void) zpool_standard_error_fmt(zhp->zpool_hdl, error,
	dgettext(TEXT_DOMAIN,
	"error setting bootenv in pool '%s'"), zhp->zpool_name);
	}

	return (error);
	}

	int
	zpool_get_bootenv(zpool_handle_t zhp, nvlist_t *nvlp)
	{
	nvlist_t *nvl;
	int error;

	nvl = NULL;
	error = lzc_get_bootenv(zhp->zpool_name, &nvl);
	if (error != 0) {
	(void) zpool_standard_error_fmt(zhp->zpool_hdl, error,
	dgettext(TEXT_DOMAIN,
	"error getting bootenv in pool '%s'"), zhp->zpool_name);
	} else {
	*nvlp = nvl;
	}

	return (error);
	}

	/*
	* Attempt to read and parse feature file(s) (from "compatibility" property).
	* Files contain zpool feature names, comma or whitespace-separated.
	* Comments (# character to next newline) are discarded.
	*
	* Arguments:
	* compatibility : string containing feature filenames
	* features : either NULL or pointer to array of boolean
	* report : either NULL or pointer to string buffer
	* rlen : length of "report" buffer
	*
	* compatibility is NULL (unset), "", "off", "legacy", or list of
	* comma-separated filenames. filenames should either be absolute,
	* or relative to:
	* 1) ZPOOL_SYSCONF_COMPAT_D (eg: /etc/zfs/compatibility.d) or
	* 2) ZPOOL_DATA_COMPAT_D (eg: /usr/share/zfs/compatibility.d).
	* (Unset), "" or "off" => enable all features
	* "legacy" => disable all features
	*
	* Any feature names read from files which match unames in spa_feature_table
	* will have the corresponding boolean set in the features array (if non-NULL).
	* If more than one feature set specified, only features present in all of
	* them will be set.
	*
	* "report" if not NULL will be populated with a suitable status message.
	*
	* Return values:
	* ZPOOL_COMPATIBILITY_OK : files read and parsed ok
	* ZPOOL_COMPATIBILITY_BADFILE : file too big or not a text file
	* ZPOOL_COMPATIBILITY_BADTOKEN : SYSCONF file contains invalid feature name
	* ZPOOL_COMPATIBILITY_WARNTOKEN : DATA file contains invalid feature name
	* ZPOOL_COMPATIBILITY_NOFILES : no feature files found
	*/
	zpool_compat_status_t
	zpool_load_compat(const char compat, boolean_t features, char *report,
	size_t rlen)
	{
	int sdirfd, ddirfd, featfd;
	struct stat fs;
	char *fc;
	char ps, ls, *ws;
	char file, line, *word;

	char l_compat[ZFS_MAXPROPLEN];

	boolean_t ret_nofiles = B_TRUE;
	boolean_t ret_badfile = B_FALSE;
	boolean_t ret_badtoken = B_FALSE;
	boolean_t ret_warntoken = B_FALSE;

	/* special cases (unset), "" and "off" => enable all features */
	if (compat == NULL \|\| compat[0] == '\0' \|\|
	strcmp(compat, ZPOOL_COMPAT_OFF) == 0) {
	if (features != NULL)
	for (uint_t i = 0; i < SPA_FEATURES; i++)
	features[i] = B_TRUE;
	if (report != NULL)
	strlcpy(report, gettext("all features enabled"), rlen);
	return (ZPOOL_COMPATIBILITY_OK);
	}

	/* Final special case "legacy" => disable all features */
	if (strcmp(compat, ZPOOL_COMPAT_LEGACY) == 0) {
	if (features != NULL)
	for (uint_t i = 0; i < SPA_FEATURES; i++)
	features[i] = B_FALSE;
	if (report != NULL)
	strlcpy(report, gettext("all features disabled"), rlen);
	return (ZPOOL_COMPATIBILITY_OK);
	}

	/*
	* Start with all true; will be ANDed with results from each file
	*/
	if (features != NULL)
	for (uint_t i = 0; i < SPA_FEATURES; i++)
	features[i] = B_TRUE;

	char err_badfile[ZFS_MAXPROPLEN] = "";
	char err_badtoken[ZFS_MAXPROPLEN] = "";

	/*
	* We ignore errors from the directory open()
	* as they're only needed if the filename is relative
	* which will be checked during the openat().
	*/

	/* O_PATH safer than O_RDONLY if system allows it */
	#if defined(O_PATH)
	#define ZC_DIR_FLAGS (O_DIRECTORY \| O_CLOEXEC \| O_PATH)
	#else
	#define ZC_DIR_FLAGS (O_DIRECTORY \| O_CLOEXEC \| O_RDONLY)
	#endif

	sdirfd = open(ZPOOL_SYSCONF_COMPAT_D, ZC_DIR_FLAGS);
	ddirfd = open(ZPOOL_DATA_COMPAT_D, ZC_DIR_FLAGS);

	(void) strlcpy(l_compat, compat, ZFS_MAXPROPLEN);

	for (file = strtok_r(l_compat, ",", &ps);
	file != NULL;
	file = strtok_r(NULL, ",", &ps)) {

	boolean_t l_features[SPA_FEATURES];

	enum { Z_SYSCONF, Z_DATA } source;

	/* try sysconfdir first, then datadir */
	source = Z_SYSCONF;
	if ((featfd = openat(sdirfd, file, O_RDONLY \| O_CLOEXEC)) < 0) {
	featfd = openat(ddirfd, file, O_RDONLY \| O_CLOEXEC);
	source = Z_DATA;
	}

	/* File readable and correct size? */
	if (featfd < 0 \|\|
	fstat(featfd, &fs) < 0 \|\|
	fs.st_size < 1 \|\|
	fs.st_size > ZPOOL_COMPAT_MAXSIZE) {
	(void) close(featfd);
	strlcat(err_badfile, file, ZFS_MAXPROPLEN);
	strlcat(err_badfile, " ", ZFS_MAXPROPLEN);
	ret_badfile = B_TRUE;
	continue;
	}

	/* Prefault the file if system allows */
	#if defined(MAP_POPULATE)
	#define ZC_MMAP_FLAGS (MAP_PRIVATE \| MAP_POPULATE)
	#elif defined(MAP_PREFAULT_READ)
	#define ZC_MMAP_FLAGS (MAP_PRIVATE \| MAP_PREFAULT_READ)
	#else
	#define ZC_MMAP_FLAGS (MAP_PRIVATE)
	#endif

	/* private mmap() so we can strtok safely */
	fc = (char *)mmap(NULL, fs.st_size, PROT_READ \| PROT_WRITE,
	ZC_MMAP_FLAGS, featfd, 0);
	(void) close(featfd);

	/* map ok, and last character == newline? */
	if (fc == MAP_FAILED \|\| fc[fs.st_size - 1] != '\n') {
	(void) munmap((void *) fc, fs.st_size);
	strlcat(err_badfile, file, ZFS_MAXPROPLEN);
	strlcat(err_badfile, " ", ZFS_MAXPROPLEN);
	ret_badfile = B_TRUE;
	continue;
	}

	ret_nofiles = B_FALSE;

	for (uint_t i = 0; i < SPA_FEATURES; i++)
	l_features[i] = B_FALSE;

	/* replace final newline with NULL to ensure string ends */
	fc[fs.st_size - 1] = '\0';

	for (line = strtok_r(fc, "\n", &ls);
	line != NULL;
	line = strtok_r(NULL, "\n", &ls)) {
	/* discard comments */
	char *r = strchr(line, '#');
	if (r != NULL)
	*r = '\0';

	for (word = strtok_r(line, ", \t", &ws);
	word != NULL;
	word = strtok_r(NULL, ", \t", &ws)) {
	/* Find matching feature name */
	uint_t f;
	for (f = 0; f < SPA_FEATURES; f++) {
	zfeature_info_t *fi =
	&spa_feature_table[f];
	if (strcmp(word, fi->fi_uname) == 0) {
	l_features[f] = B_TRUE;
	break;
	}
	}
	if (f < SPA_FEATURES)
	continue;

	/* found an unrecognized word */
	/* lightly sanitize it */
	if (strlen(word) > 32)
	word[32] = '\0';
	for (char c = word; c != '\0'; c++)
	if (!isprint(*c))
	*c = '?';

	strlcat(err_badtoken, word, ZFS_MAXPROPLEN);
	strlcat(err_badtoken, " ", ZFS_MAXPROPLEN);
	if (source == Z_SYSCONF)
	ret_badtoken = B_TRUE;
	else
	ret_warntoken = B_TRUE;
	}
	}
	(void) munmap((void *) fc, fs.st_size);

	if (features != NULL)
	for (uint_t i = 0; i < SPA_FEATURES; i++)
	features[i] &= l_features[i];
	}
	(void) close(sdirfd);
	(void) close(ddirfd);

	/* Return the most serious error */
	if (ret_badfile) {
	if (report != NULL)
	snprintf(report, rlen, gettext("could not read/"
	"parse feature file(s): %s"), err_badfile);
	return (ZPOOL_COMPATIBILITY_BADFILE);
	}
	if (ret_nofiles) {
	if (report != NULL)
	strlcpy(report,
	gettext("no valid compatibility files specified"),
	rlen);
	return (ZPOOL_COMPATIBILITY_NOFILES);
	}
	if (ret_badtoken) {
	if (report != NULL)
	snprintf(report, rlen, gettext("invalid feature "
	"name(s) in local compatibility files: %s"),
	err_badtoken);
	return (ZPOOL_COMPATIBILITY_BADTOKEN);
	}
	if (ret_warntoken) {
	if (report != NULL)
	snprintf(report, rlen, gettext("unrecognized feature "
	"name(s) in distribution compatibility files: %s"),
	err_badtoken);
	return (ZPOOL_COMPATIBILITY_WARNTOKEN);
	}
	if (report != NULL)
	strlcpy(report, gettext("compatibility set ok"), rlen);
	return (ZPOOL_COMPATIBILITY_OK);
	}

	static int
	zpool_vdev_guid(zpool_handle_t zhp, const char vdevname, uint64_t *vdev_guid)
	{
	nvlist_t *tgt;
	boolean_t avail_spare, l2cache;

	verify(zhp != NULL);
	if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) {
	char errbuf[ERRBUFLEN];
	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "pool is in an unavailable state"));
	return (zfs_error(zhp->zpool_hdl, EZFS_POOLUNAVAIL, errbuf));
	}

	if ((tgt = zpool_find_vdev(zhp, vdevname, &avail_spare, &l2cache,
	NULL)) == NULL) {
	char errbuf[ERRBUFLEN];
	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "can not find %s in %s"),
	vdevname, zhp->zpool_name);
	return (zfs_error(zhp->zpool_hdl, EZFS_NODEVICE, errbuf));
	}

	*vdev_guid = fnvlist_lookup_uint64(tgt, ZPOOL_CONFIG_GUID);
	return (0);
	}

	/*
	* Get a vdev property value for 'prop' and return the value in
	* a pre-allocated buffer.
	*/
	int
	zpool_get_vdev_prop_value(nvlist_t nvprop, vdev_prop_t prop, char prop_name,
	char buf, size_t len, zprop_source_t srctype, boolean_t literal)
	{
	nvlist_t *nv;
	const char *strval;
	uint64_t intval;
	zprop_source_t src = ZPROP_SRC_NONE;

	if (prop == VDEV_PROP_USERPROP) {
	/* user property, prop_name must contain the property name */
	assert(prop_name != NULL);
	if (nvlist_lookup_nvlist(nvprop, prop_name, &nv) == 0) {
	src = fnvlist_lookup_uint64(nv, ZPROP_SOURCE);
	strval = fnvlist_lookup_string(nv, ZPROP_VALUE);
	} else {
	/* user prop not found */
	return (-1);
	}
	(void) strlcpy(buf, strval, len);
	if (srctype)
	*srctype = src;
	return (0);
	}

	if (prop_name == NULL)
	prop_name = (char *)vdev_prop_to_name(prop);

	switch (vdev_prop_get_type(prop)) {
	case PROP_TYPE_STRING:
	if (nvlist_lookup_nvlist(nvprop, prop_name, &nv) == 0) {
	src = fnvlist_lookup_uint64(nv, ZPROP_SOURCE);
	strval = fnvlist_lookup_string(nv, ZPROP_VALUE);
	} else {
	src = ZPROP_SRC_DEFAULT;
	if ((strval = vdev_prop_default_string(prop)) == NULL)
	strval = "-";
	}
	(void) strlcpy(buf, strval, len);
	break;

	case PROP_TYPE_NUMBER:
	if (nvlist_lookup_nvlist(nvprop, prop_name, &nv) == 0) {
	src = fnvlist_lookup_uint64(nv, ZPROP_SOURCE);
	intval = fnvlist_lookup_uint64(nv, ZPROP_VALUE);
	} else {
	src = ZPROP_SRC_DEFAULT;
	intval = vdev_prop_default_numeric(prop);
	}

	switch (prop) {
	case VDEV_PROP_ASIZE:
	case VDEV_PROP_PSIZE:
	case VDEV_PROP_SIZE:
	case VDEV_PROP_BOOTSIZE:
	case VDEV_PROP_ALLOCATED:
	case VDEV_PROP_FREE:
	case VDEV_PROP_READ_ERRORS:
	case VDEV_PROP_WRITE_ERRORS:
	case VDEV_PROP_CHECKSUM_ERRORS:
	case VDEV_PROP_INITIALIZE_ERRORS:
	case VDEV_PROP_OPS_NULL:
	case VDEV_PROP_OPS_READ:
	case VDEV_PROP_OPS_WRITE:
	case VDEV_PROP_OPS_FREE:
	case VDEV_PROP_OPS_CLAIM:
	case VDEV_PROP_OPS_TRIM:
	case VDEV_PROP_BYTES_NULL:
	case VDEV_PROP_BYTES_READ:
	case VDEV_PROP_BYTES_WRITE:
	case VDEV_PROP_BYTES_FREE:
	case VDEV_PROP_BYTES_CLAIM:
	case VDEV_PROP_BYTES_TRIM:
	if (literal) {
	(void) snprintf(buf, len, "%llu",
	(u_longlong_t)intval);
	} else {
	(void) zfs_nicenum(intval, buf, len);
	}
	break;
	case VDEV_PROP_EXPANDSZ:
	if (intval == 0) {
	(void) strlcpy(buf, "-", len);
	} else if (literal) {
	(void) snprintf(buf, len, "%llu",
	(u_longlong_t)intval);
	} else {
	(void) zfs_nicenum(intval, buf, len);
	}
	break;
	case VDEV_PROP_CAPACITY:
	if (literal) {
	(void) snprintf(buf, len, "%llu",
	(u_longlong_t)intval);
	} else {
	(void) snprintf(buf, len, "%llu%%",
	(u_longlong_t)intval);
	}
	break;
	case VDEV_PROP_CHECKSUM_N:
	case VDEV_PROP_CHECKSUM_T:
	case VDEV_PROP_IO_N:
	case VDEV_PROP_IO_T:
	+ case VDEV_PROP_SLOW_IO_N:
	+ case VDEV_PROP_SLOW_IO_T:
	if (intval == UINT64_MAX) {
	(void) strlcpy(buf, "-", len);
	} else {
	(void) snprintf(buf, len, "%llu",
	(u_longlong_t)intval);
	}
	break;
	case VDEV_PROP_FRAGMENTATION:
	if (intval == UINT64_MAX) {
	(void) strlcpy(buf, "-", len);
	} else {
	(void) snprintf(buf, len, "%llu%%",
	(u_longlong_t)intval);
	}
	break;
	case VDEV_PROP_STATE:
	if (literal) {
	(void) snprintf(buf, len, "%llu",
	(u_longlong_t)intval);
	} else {
	(void) strlcpy(buf, zpool_state_to_name(intval,
	VDEV_AUX_NONE), len);
	}
	break;
	default:
	(void) snprintf(buf, len, "%llu",
	(u_longlong_t)intval);
	}
	break;

	case PROP_TYPE_INDEX:
	if (nvlist_lookup_nvlist(nvprop, prop_name, &nv) == 0) {
	src = fnvlist_lookup_uint64(nv, ZPROP_SOURCE);
	intval = fnvlist_lookup_uint64(nv, ZPROP_VALUE);
	} else {
	src = ZPROP_SRC_DEFAULT;
	intval = vdev_prop_default_numeric(prop);
	}
	if (vdev_prop_index_to_string(prop, intval,
	(const char **)&strval) != 0)
	return (-1);
	(void) strlcpy(buf, strval, len);
	break;

	default:
	abort();
	}

	if (srctype)
	*srctype = src;

	return (0);
	}

	/*
	* Get a vdev property value for 'prop_name' and return the value in
	* a pre-allocated buffer.
	*/
	int
	zpool_get_vdev_prop(zpool_handle_t zhp, const char vdevname, vdev_prop_t prop,
	char prop_name, char buf, size_t len, zprop_source_t *srctype,
	boolean_t literal)
	{
	nvlist_t reqnvl, reqprops;
	nvlist_t *retprops = NULL;
	uint64_t vdev_guid = 0;
	int ret;

	if ((ret = zpool_vdev_guid(zhp, vdevname, &vdev_guid)) != 0)
	return (ret);

	if (nvlist_alloc(&reqnvl, NV_UNIQUE_NAME, 0) != 0)
	return (no_memory(zhp->zpool_hdl));
	if (nvlist_alloc(&reqprops, NV_UNIQUE_NAME, 0) != 0)
	return (no_memory(zhp->zpool_hdl));

	fnvlist_add_uint64(reqnvl, ZPOOL_VDEV_PROPS_GET_VDEV, vdev_guid);

	if (prop != VDEV_PROP_USERPROP) {
	/* prop_name overrides prop value */
	if (prop_name != NULL)
	prop = vdev_name_to_prop(prop_name);
	else
	prop_name = (char *)vdev_prop_to_name(prop);
	assert(prop < VDEV_NUM_PROPS);
	}

	assert(prop_name != NULL);
	if (nvlist_add_uint64(reqprops, prop_name, prop) != 0) {
	nvlist_free(reqnvl);
	nvlist_free(reqprops);
	return (no_memory(zhp->zpool_hdl));
	}

	fnvlist_add_nvlist(reqnvl, ZPOOL_VDEV_PROPS_GET_PROPS, reqprops);

	ret = lzc_get_vdev_prop(zhp->zpool_name, reqnvl, &retprops);

	if (ret == 0) {
	ret = zpool_get_vdev_prop_value(retprops, prop, prop_name, buf,
	len, srctype, literal);
	} else {
	char errbuf[ERRBUFLEN];
	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot get vdev property %s from"
	" %s in %s"), prop_name, vdevname, zhp->zpool_name);
	(void) zpool_standard_error(zhp->zpool_hdl, ret, errbuf);
	}

	nvlist_free(reqnvl);
	nvlist_free(reqprops);
	nvlist_free(retprops);

	return (ret);
	}

	/*
	* Get all vdev properties
	*/
	int
	zpool_get_all_vdev_props(zpool_handle_t zhp, const char vdevname,
	nvlist_t **outnvl)
	{
	nvlist_t *nvl = NULL;
	uint64_t vdev_guid = 0;
	int ret;

	if ((ret = zpool_vdev_guid(zhp, vdevname, &vdev_guid)) != 0)
	return (ret);

	if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0)
	return (no_memory(zhp->zpool_hdl));

	fnvlist_add_uint64(nvl, ZPOOL_VDEV_PROPS_GET_VDEV, vdev_guid);

	ret = lzc_get_vdev_prop(zhp->zpool_name, nvl, outnvl);

	nvlist_free(nvl);

	if (ret) {
	char errbuf[ERRBUFLEN];
	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot get vdev properties for"
	" %s in %s"), vdevname, zhp->zpool_name);
	(void) zpool_standard_error(zhp->zpool_hdl, errno, errbuf);
	}

	return (ret);
	}

	/*
	* Set vdev property
	*/
	int
	zpool_set_vdev_prop(zpool_handle_t zhp, const char vdevname,
	const char propname, const char propval)
	{
	int ret;
	nvlist_t *nvl = NULL;
	nvlist_t *outnvl = NULL;
	nvlist_t *props;
	nvlist_t *realprops;
	prop_flags_t flags = { 0 };
	uint64_t version;
	uint64_t vdev_guid;

	if ((ret = zpool_vdev_guid(zhp, vdevname, &vdev_guid)) != 0)
	return (ret);

	if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0)
	return (no_memory(zhp->zpool_hdl));
	if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0)
	return (no_memory(zhp->zpool_hdl));

	fnvlist_add_uint64(nvl, ZPOOL_VDEV_PROPS_SET_VDEV, vdev_guid);

	if (nvlist_add_string(props, propname, propval) != 0) {
	nvlist_free(props);
	return (no_memory(zhp->zpool_hdl));
	}

	char errbuf[ERRBUFLEN];
	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot set property %s for %s on %s"),
	propname, vdevname, zhp->zpool_name);

	flags.vdevprop = 1;
	version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
	if ((realprops = zpool_valid_proplist(zhp->zpool_hdl,
	zhp->zpool_name, props, version, flags, errbuf)) == NULL) {
	nvlist_free(props);
	nvlist_free(nvl);
	return (-1);
	}

	nvlist_free(props);
	props = realprops;

	fnvlist_add_nvlist(nvl, ZPOOL_VDEV_PROPS_SET_PROPS, props);

	ret = lzc_set_vdev_prop(zhp->zpool_name, nvl, &outnvl);

	nvlist_free(props);
	nvlist_free(nvl);
	nvlist_free(outnvl);

	if (ret)
	(void) zpool_standard_error(zhp->zpool_hdl, errno, errbuf);

	return (ret);
	}
	diff --git a/sys/contrib/openzfs/lib/libzfs/libzfs_sendrecv.c b/sys/contrib/openzfs/lib/libzfs/libzfs_sendrecv.c
	index e9bc78aa8d39..143aecb9459f 100644
	--- a/sys/contrib/openzfs/lib/libzfs/libzfs_sendrecv.c
	+++ b/sys/contrib/openzfs/lib/libzfs/libzfs_sendrecv.c
	@@ -1,5625 +1,5626 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
	* Copyright (c) 2012, Joyent, Inc. All rights reserved.
	* Copyright (c) 2012 Pawel Jakub Dawidek <pawel@dawidek.net>.
	* All rights reserved
	* Copyright (c) 2013 Steven Hartland. All rights reserved.
	* Copyright 2015, OmniTI Computer Consulting, Inc. All rights reserved.
	* Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>
	* Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
	* Copyright (c) 2019 Datto Inc.
	*/

	#include <assert.h>
	#include <ctype.h>
	#include <errno.h>
	#include <libintl.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <unistd.h>
	#include <stddef.h>
	#include <fcntl.h>
	#include <sys/mount.h>
	#include <sys/mntent.h>
	#include <sys/mnttab.h>
	#include <sys/avl.h>
	#include <sys/debug.h>
	#include <sys/stat.h>
	#include <pthread.h>
	#include <umem.h>
	#include <time.h>

	#include <libzfs.h>
	#include <libzfs_core.h>
	#include <libzutil.h>

	#include "zfs_namecheck.h"
	#include "zfs_prop.h"
	#include "zfs_fletcher.h"
	#include "libzfs_impl.h"
	#include <cityhash.h>
	#include <zlib.h>
	#include <sys/zio_checksum.h>
	#include <sys/dsl_crypt.h>
	#include <sys/ddt.h>
	#include <sys/socket.h>
	#include <sys/sha2.h>

	static int zfs_receive_impl(libzfs_handle_t , const char , const char *,
	recvflags_t , int, const char , nvlist_t , avl_tree_t , char **,
	const char , nvlist_t );
	static int guid_to_name_redact_snaps(libzfs_handle_t hdl, const char parent,
	uint64_t guid, boolean_t bookmark_ok, uint64_t *redact_snap_guids,
	uint64_t num_redact_snaps, char *name);
	static int guid_to_name(libzfs_handle_t , const char ,
	uint64_t, boolean_t, char *);

	typedef struct progress_arg {
	zfs_handle_t *pa_zhp;
	int pa_fd;
	boolean_t pa_parsable;
	boolean_t pa_estimate;
	int pa_verbosity;
	boolean_t pa_astitle;
	boolean_t pa_progress;
	uint64_t pa_size;
	} progress_arg_t;

	static int
	dump_record(dmu_replay_record_t drr, void payload, size_t payload_len,
	zio_cksum_t *zc, int outfd)
	{
	ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
	==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t));
	fletcher_4_incremental_native(drr,
	offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum), zc);
	if (drr->drr_type != DRR_BEGIN) {
	ASSERT(ZIO_CHECKSUM_IS_ZERO(&drr->drr_u.
	drr_checksum.drr_checksum));
	drr->drr_u.drr_checksum.drr_checksum = *zc;
	}
	fletcher_4_incremental_native(&drr->drr_u.drr_checksum.drr_checksum,
	sizeof (zio_cksum_t), zc);
	if (write(outfd, drr, sizeof (*drr)) == -1)
	return (errno);
	if (payload_len != 0) {
	fletcher_4_incremental_native(payload, payload_len, zc);
	if (write(outfd, payload, payload_len) == -1)
	return (errno);
	}
	return (0);
	}

	/*
	* Routines for dealing with the AVL tree of fs-nvlists
	*/
	typedef struct fsavl_node {
	avl_node_t fn_node;
	nvlist_t *fn_nvfs;
	const char *fn_snapname;
	uint64_t fn_guid;
	} fsavl_node_t;

	static int
	fsavl_compare(const void arg1, const void arg2)
	{
	const fsavl_node_t fn1 = (const fsavl_node_t )arg1;
	const fsavl_node_t fn2 = (const fsavl_node_t )arg2;

	return (TREE_CMP(fn1->fn_guid, fn2->fn_guid));
	}

	/*
	* Given the GUID of a snapshot, find its containing filesystem and
	* (optionally) name.
	*/
	static nvlist_t *
	fsavl_find(avl_tree_t avl, uint64_t snapguid, const char *snapname)
	{
	fsavl_node_t fn_find;
	fsavl_node_t *fn;

	fn_find.fn_guid = snapguid;

	fn = avl_find(avl, &fn_find, NULL);
	if (fn) {
	if (snapname)
	*snapname = fn->fn_snapname;
	return (fn->fn_nvfs);
	}
	return (NULL);
	}

	static void
	fsavl_destroy(avl_tree_t *avl)
	{
	fsavl_node_t *fn;
	void *cookie;

	if (avl == NULL)
	return;

	cookie = NULL;
	while ((fn = avl_destroy_nodes(avl, &cookie)) != NULL)
	free(fn);
	avl_destroy(avl);
	free(avl);
	}

	/*
	* Given an nvlist, produce an avl tree of snapshots, ordered by guid
	*/
	static avl_tree_t *
	fsavl_create(nvlist_t *fss)
	{
	avl_tree_t *fsavl;
	nvpair_t *fselem = NULL;

	if ((fsavl = malloc(sizeof (avl_tree_t))) == NULL)
	return (NULL);

	avl_create(fsavl, fsavl_compare, sizeof (fsavl_node_t),
	offsetof(fsavl_node_t, fn_node));

	while ((fselem = nvlist_next_nvpair(fss, fselem)) != NULL) {
	nvlist_t nvfs, snaps;
	nvpair_t *snapelem = NULL;

	nvfs = fnvpair_value_nvlist(fselem);
	snaps = fnvlist_lookup_nvlist(nvfs, "snaps");

	while ((snapelem =
	nvlist_next_nvpair(snaps, snapelem)) != NULL) {
	fsavl_node_t *fn;

	if ((fn = malloc(sizeof (fsavl_node_t))) == NULL) {
	fsavl_destroy(fsavl);
	return (NULL);
	}
	fn->fn_nvfs = nvfs;
	fn->fn_snapname = nvpair_name(snapelem);
	fn->fn_guid = fnvpair_value_uint64(snapelem);

	/*
	* Note: if there are multiple snaps with the
	* same GUID, we ignore all but one.
	*/
	avl_index_t where = 0;
	if (avl_find(fsavl, fn, &where) == NULL)
	avl_insert(fsavl, fn, where);
	else
	free(fn);
	}
	}

	return (fsavl);
	}

	/*
	* Routines for dealing with the giant nvlist of fs-nvlists, etc.
	*/
	typedef struct send_data {
	/*
	* assigned inside every recursive call,
	* restored from *_save on return:
	*
	* guid of fromsnap snapshot in parent dataset
	* txg of fromsnap snapshot in current dataset
	* txg of tosnap snapshot in current dataset
	*/

	uint64_t parent_fromsnap_guid;
	uint64_t fromsnap_txg;
	uint64_t tosnap_txg;

	/* the nvlists get accumulated during depth-first traversal */
	nvlist_t *parent_snaps;
	nvlist_t *fss;
	nvlist_t *snapprops;
	nvlist_t snapholds; / user holds */

	/* send-receive configuration, does not change during traversal */
	const char *fsname;
	const char *fromsnap;
	const char *tosnap;
	boolean_t recursive;
	boolean_t raw;
	boolean_t doall;
	boolean_t replicate;
	boolean_t skipmissing;
	boolean_t verbose;
	boolean_t backup;
	boolean_t seenfrom;
	boolean_t seento;
	boolean_t holds; /* were holds requested with send -h */
	boolean_t props;

	/*
	* The header nvlist is of the following format:
	* {
	* "tosnap" -> string
	* "fromsnap" -> string (if incremental)
	* "fss" -> {
	* id -> {
	*
	* "name" -> string (full name; for debugging)
	* "parentfromsnap" -> number (guid of fromsnap in parent)
	*
	* "props" -> { name -> value (only if set here) }
	* "snaps" -> { name (lastname) -> number (guid) }
	* "snapprops" -> { name (lastname) -> { name -> value } }
	* "snapholds" -> { name (lastname) -> { holdname -> crtime } }
	*
	* "origin" -> number (guid) (if clone)
	* "is_encroot" -> boolean
	* "sent" -> boolean (not on-disk)
	* }
	* }
	* }
	*
	*/
	} send_data_t;

	static void
	send_iterate_prop(zfs_handle_t zhp, boolean_t received_only, nvlist_t nv);

	/*
	* Collect guid, valid props, optionally holds, etc. of a snapshot.
	* This interface is intended for use as a zfs_iter_snapshots_v2_sorted visitor.
	*/
	static int
	send_iterate_snap(zfs_handle_t zhp, void arg)
	{
	send_data_t *sd = arg;
	uint64_t guid = zhp->zfs_dmustats.dds_guid;
	uint64_t txg = zhp->zfs_dmustats.dds_creation_txg;
	boolean_t isfromsnap, istosnap, istosnapwithnofrom;
	char *snapname;
	const char *from = sd->fromsnap;
	const char *to = sd->tosnap;

	snapname = strrchr(zhp->zfs_name, '@');
	assert(snapname != NULL);
	++snapname;

	isfromsnap = (from != NULL && strcmp(from, snapname) == 0);
	istosnap = (to != NULL && strcmp(to, snapname) == 0);
	istosnapwithnofrom = (istosnap && from == NULL);

	if (sd->tosnap_txg != 0 && txg > sd->tosnap_txg) {
	if (sd->verbose) {
	(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
	"skipping snapshot %s because it was created "
	"after the destination snapshot (%s)\n"),
	zhp->zfs_name, to);
	}
	zfs_close(zhp);
	return (0);
	}

	fnvlist_add_uint64(sd->parent_snaps, snapname, guid);

	/*
	* NB: if there is no fromsnap here (it's a newly created fs in
	* an incremental replication), we will substitute the tosnap.
	*/
	if (isfromsnap \|\| (sd->parent_fromsnap_guid == 0 && istosnap))
	sd->parent_fromsnap_guid = guid;

	if (!sd->recursive) {
	/*
	* To allow a doall stream to work properly
	* with a NULL fromsnap
	*/
	if (sd->doall && from == NULL && !sd->seenfrom)
	sd->seenfrom = B_TRUE;

	if (!sd->seenfrom && isfromsnap) {
	sd->seenfrom = B_TRUE;
	zfs_close(zhp);
	return (0);
	}

	if ((sd->seento \|\| !sd->seenfrom) && !istosnapwithnofrom) {
	zfs_close(zhp);
	return (0);
	}

	if (istosnap)
	sd->seento = B_TRUE;
	}

	nvlist_t *nv = fnvlist_alloc();
	send_iterate_prop(zhp, sd->backup, nv);
	fnvlist_add_nvlist(sd->snapprops, snapname, nv);
	fnvlist_free(nv);

	if (sd->holds) {
	nvlist_t *holds;
	if (lzc_get_holds(zhp->zfs_name, &holds) == 0) {
	fnvlist_add_nvlist(sd->snapholds, snapname, holds);
	fnvlist_free(holds);
	}
	}

	zfs_close(zhp);
	return (0);
	}

	/*
	* Collect all valid props from the handle snap into an nvlist.
	*/
	static void
	send_iterate_prop(zfs_handle_t zhp, boolean_t received_only, nvlist_t nv)
	{
	nvlist_t *props;

	if (received_only)
	props = zfs_get_recvd_props(zhp);
	else
	props = zhp->zfs_props;

	nvpair_t *elem = NULL;
	while ((elem = nvlist_next_nvpair(props, elem)) != NULL) {
	const char *propname = nvpair_name(elem);
	zfs_prop_t prop = zfs_name_to_prop(propname);

	if (!zfs_prop_user(propname)) {
	/*
	* Realistically, this should never happen. However,
	* we want the ability to add DSL properties without
	* needing to make incompatible version changes. We
	* need to ignore unknown properties to allow older
	* software to still send datasets containing these
	* properties, with the unknown properties elided.
	*/
	if (prop == ZPROP_INVAL)
	continue;

	if (zfs_prop_readonly(prop))
	continue;
	}

	nvlist_t *propnv = fnvpair_value_nvlist(elem);

	boolean_t isspacelimit = (prop == ZFS_PROP_QUOTA \|\|
	prop == ZFS_PROP_RESERVATION \|\|
	prop == ZFS_PROP_REFQUOTA \|\|
	prop == ZFS_PROP_REFRESERVATION);
	if (isspacelimit && zhp->zfs_type == ZFS_TYPE_SNAPSHOT)
	continue;

	const char *source;
	if (nvlist_lookup_string(propnv, ZPROP_SOURCE, &source) == 0) {
	if (strcmp(source, zhp->zfs_name) != 0 &&
	strcmp(source, ZPROP_SOURCE_VAL_RECVD) != 0)
	continue;
	} else {
	/*
	* May have no source before SPA_VERSION_RECVD_PROPS,
	* but is still modifiable.
	*/
	if (!isspacelimit)
	continue;
	}

	if (zfs_prop_user(propname) \|\|
	zfs_prop_get_type(prop) == PROP_TYPE_STRING) {
	const char *value;
	value = fnvlist_lookup_string(propnv, ZPROP_VALUE);
	fnvlist_add_string(nv, propname, value);
	} else {
	uint64_t value;
	value = fnvlist_lookup_uint64(propnv, ZPROP_VALUE);
	fnvlist_add_uint64(nv, propname, value);
	}
	}
	}

	/*
	* returns snapshot guid
	* and returns 0 if the snapshot does not exist
	*/
	static uint64_t
	get_snap_guid(libzfs_handle_t hdl, const char fs, const char *snap)
	{
	char name[MAXPATHLEN + 1];
	uint64_t guid = 0;

	if (fs == NULL \|\| fs[0] == '\0' \|\| snap == NULL \|\| snap[0] == '\0')
	return (guid);

	(void) snprintf(name, sizeof (name), "%s@%s", fs, snap);
	zfs_handle_t *zhp = zfs_open(hdl, name, ZFS_TYPE_SNAPSHOT);
	if (zhp != NULL) {
	guid = zfs_prop_get_int(zhp, ZFS_PROP_GUID);
	zfs_close(zhp);
	}

	return (guid);
	}

	/*
	* returns snapshot creation txg
	* and returns 0 if the snapshot does not exist
	*/
	static uint64_t
	get_snap_txg(libzfs_handle_t hdl, const char fs, const char *snap)
	{
	char name[ZFS_MAX_DATASET_NAME_LEN];
	uint64_t txg = 0;

	if (fs == NULL \|\| fs[0] == '\0' \|\| snap == NULL \|\| snap[0] == '\0')
	return (txg);

	(void) snprintf(name, sizeof (name), "%s@%s", fs, snap);
	if (zfs_dataset_exists(hdl, name, ZFS_TYPE_SNAPSHOT)) {
	zfs_handle_t *zhp = zfs_open(hdl, name, ZFS_TYPE_SNAPSHOT);
	if (zhp != NULL) {
	txg = zfs_prop_get_int(zhp, ZFS_PROP_CREATETXG);
	zfs_close(zhp);
	}
	}

	return (txg);
	}

	/*
	* Recursively generate nvlists describing datasets. See comment
	* for the data structure send_data_t above for description of contents
	* of the nvlist.
	*/
	static int
	send_iterate_fs(zfs_handle_t zhp, void arg)
	{
	send_data_t *sd = arg;
	nvlist_t nvfs = NULL, nv = NULL;
	int rv = 0;
	uint64_t min_txg = 0, max_txg = 0;
	uint64_t txg = zhp->zfs_dmustats.dds_creation_txg;
	uint64_t guid = zhp->zfs_dmustats.dds_guid;
	uint64_t fromsnap_txg, tosnap_txg;
	char guidstring[64];

	/* These fields are restored on return from a recursive call. */
	uint64_t parent_fromsnap_guid_save = sd->parent_fromsnap_guid;
	uint64_t fromsnap_txg_save = sd->fromsnap_txg;
	uint64_t tosnap_txg_save = sd->tosnap_txg;

	fromsnap_txg = get_snap_txg(zhp->zfs_hdl, zhp->zfs_name, sd->fromsnap);
	if (fromsnap_txg != 0)
	sd->fromsnap_txg = fromsnap_txg;

	tosnap_txg = get_snap_txg(zhp->zfs_hdl, zhp->zfs_name, sd->tosnap);
	if (tosnap_txg != 0)
	sd->tosnap_txg = tosnap_txg;

	/*
	* On the send side, if the current dataset does not have tosnap,
	* perform two additional checks:
	*
	* - Skip sending the current dataset if it was created later than
	* the parent tosnap.
	* - Return error if the current dataset was created earlier than
	* the parent tosnap, unless --skip-missing specified. Then
	* just print a warning.
	*/
	if (sd->tosnap != NULL && tosnap_txg == 0) {
	if (sd->tosnap_txg != 0 && txg > sd->tosnap_txg) {
	if (sd->verbose) {
	(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
	"skipping dataset %s: snapshot %s does "
	"not exist\n"), zhp->zfs_name, sd->tosnap);
	}
	} else if (sd->skipmissing) {
	(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
	"WARNING: skipping dataset %s and its children:"
	" snapshot %s does not exist\n"),
	zhp->zfs_name, sd->tosnap);
	} else {
	(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
	"cannot send %s@%s%s: snapshot %s@%s does not "
	"exist\n"), sd->fsname, sd->tosnap, sd->recursive ?
	dgettext(TEXT_DOMAIN, " recursively") : "",
	zhp->zfs_name, sd->tosnap);
	rv = EZFS_NOENT;
	}
	goto out;
	}

	nvfs = fnvlist_alloc();
	fnvlist_add_string(nvfs, "name", zhp->zfs_name);
	fnvlist_add_uint64(nvfs, "parentfromsnap", sd->parent_fromsnap_guid);

	if (zhp->zfs_dmustats.dds_origin[0] != '\0') {
	zfs_handle_t *origin = zfs_open(zhp->zfs_hdl,
	zhp->zfs_dmustats.dds_origin, ZFS_TYPE_SNAPSHOT);
	if (origin == NULL) {
	rv = -1;
	goto out;
	}
	fnvlist_add_uint64(nvfs, "origin",
	origin->zfs_dmustats.dds_guid);
	zfs_close(origin);
	}

	/* Iterate over props. */
	if (sd->props \|\| sd->backup \|\| sd->recursive) {
	nv = fnvlist_alloc();
	send_iterate_prop(zhp, sd->backup, nv);
	fnvlist_add_nvlist(nvfs, "props", nv);
	}
	if (zfs_prop_get_int(zhp, ZFS_PROP_ENCRYPTION) != ZIO_CRYPT_OFF) {
	boolean_t encroot;

	/* Determine if this dataset is an encryption root. */
	if (zfs_crypto_get_encryption_root(zhp, &encroot, NULL) != 0) {
	rv = -1;
	goto out;
	}

	if (encroot)
	fnvlist_add_boolean(nvfs, "is_encroot");

	/*
	* Encrypted datasets can only be sent with properties if
	* the raw flag is specified because the receive side doesn't
	* currently have a mechanism for recursively asking the user
	* for new encryption parameters.
	*/
	if (!sd->raw) {
	(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
	"cannot send %s@%s: encrypted dataset %s may not "
	"be sent with properties without the raw flag\n"),
	sd->fsname, sd->tosnap, zhp->zfs_name);
	rv = -1;
	goto out;
	}

	}

	/*
	* Iterate over snaps, and set sd->parent_fromsnap_guid.
	*
	* If this is a "doall" send, a replicate send or we're just trying
	* to gather a list of previous snapshots, iterate through all the
	* snaps in the txg range. Otherwise just look at the one we're
	* interested in.
	*/
	sd->parent_fromsnap_guid = 0;
	sd->parent_snaps = fnvlist_alloc();
	sd->snapprops = fnvlist_alloc();
	if (sd->holds)
	sd->snapholds = fnvlist_alloc();
	if (sd->doall \|\| sd->replicate \|\| sd->tosnap == NULL) {
	if (!sd->replicate && fromsnap_txg != 0)
	min_txg = fromsnap_txg;
	if (!sd->replicate && tosnap_txg != 0)
	max_txg = tosnap_txg;
	(void) zfs_iter_snapshots_sorted_v2(zhp, 0, send_iterate_snap,
	sd, min_txg, max_txg);
	} else {
	char snapname[MAXPATHLEN] = { 0 };
	zfs_handle_t *snap;

	(void) snprintf(snapname, sizeof (snapname), "%s@%s",
	zhp->zfs_name, sd->tosnap);
	if (sd->fromsnap != NULL)
	sd->seenfrom = B_TRUE;
	snap = zfs_open(zhp->zfs_hdl, snapname, ZFS_TYPE_SNAPSHOT);
	if (snap != NULL)
	(void) send_iterate_snap(snap, sd);
	}

	fnvlist_add_nvlist(nvfs, "snaps", sd->parent_snaps);
	fnvlist_free(sd->parent_snaps);
	fnvlist_add_nvlist(nvfs, "snapprops", sd->snapprops);
	fnvlist_free(sd->snapprops);
	if (sd->holds) {
	fnvlist_add_nvlist(nvfs, "snapholds", sd->snapholds);
	fnvlist_free(sd->snapholds);
	}

	/* Do not allow the size of the properties list to exceed the limit */
	if ((fnvlist_size(nvfs) + fnvlist_size(sd->fss)) >
	zhp->zfs_hdl->libzfs_max_nvlist) {
	(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
	"warning: cannot send %s@%s: the size of the list of "
	"snapshots and properties is too large to be received "
	"successfully.\n"
	"Select a smaller number of snapshots to send.\n"),
	zhp->zfs_name, sd->tosnap);
	rv = EZFS_NOSPC;
	goto out;
	}
	/* Add this fs to nvlist. */
	(void) snprintf(guidstring, sizeof (guidstring),
	"0x%llx", (longlong_t)guid);
	fnvlist_add_nvlist(sd->fss, guidstring, nvfs);

	/* Iterate over children. */
	if (sd->recursive)
	rv = zfs_iter_filesystems_v2(zhp, 0, send_iterate_fs, sd);

	out:
	/* Restore saved fields. */
	sd->parent_fromsnap_guid = parent_fromsnap_guid_save;
	sd->fromsnap_txg = fromsnap_txg_save;
	sd->tosnap_txg = tosnap_txg_save;

	fnvlist_free(nv);
	fnvlist_free(nvfs);

	zfs_close(zhp);
	return (rv);
	}

	static int
	gather_nvlist(libzfs_handle_t hdl, const char fsname, const char *fromsnap,
	const char *tosnap, boolean_t recursive, boolean_t raw, boolean_t doall,
	boolean_t replicate, boolean_t skipmissing, boolean_t verbose,
	boolean_t backup, boolean_t holds, boolean_t props, nvlist_t **nvlp,
	avl_tree_t **avlp)
	{
	zfs_handle_t *zhp;
	send_data_t sd = { 0 };
	int error;

	zhp = zfs_open(hdl, fsname, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME);
	if (zhp == NULL)
	return (EZFS_BADTYPE);

	sd.fss = fnvlist_alloc();
	sd.fsname = fsname;
	sd.fromsnap = fromsnap;
	sd.tosnap = tosnap;
	sd.recursive = recursive;
	sd.raw = raw;
	sd.doall = doall;
	sd.replicate = replicate;
	sd.skipmissing = skipmissing;
	sd.verbose = verbose;
	sd.backup = backup;
	sd.holds = holds;
	sd.props = props;

	if ((error = send_iterate_fs(zhp, &sd)) != 0) {
	fnvlist_free(sd.fss);
	if (avlp != NULL)
	*avlp = NULL;
	*nvlp = NULL;
	return (error);
	}

	if (avlp != NULL && (*avlp = fsavl_create(sd.fss)) == NULL) {
	fnvlist_free(sd.fss);
	*nvlp = NULL;
	return (EZFS_NOMEM);
	}

	*nvlp = sd.fss;
	return (0);
	}

	/*
	* Routines specific to "zfs send"
	*/
	typedef struct send_dump_data {
	/* these are all just the short snapname (the part after the @) */
	const char *fromsnap;
	const char *tosnap;
	char prevsnap[ZFS_MAX_DATASET_NAME_LEN];
	uint64_t prevsnap_obj;
	boolean_t seenfrom, seento, replicate, doall, fromorigin;
	boolean_t dryrun, parsable, progress, embed_data, std_out;
	boolean_t large_block, compress, raw, holds;
	boolean_t progressastitle;
	int outfd;
	boolean_t err;
	nvlist_t *fss;
	nvlist_t *snapholds;
	avl_tree_t *fsavl;
	snapfilter_cb_t *filter_cb;
	void *filter_cb_arg;
	nvlist_t *debugnv;
	char holdtag[ZFS_MAX_DATASET_NAME_LEN];
	int cleanup_fd;
	int verbosity;
	uint64_t size;
	} send_dump_data_t;

	static int
	zfs_send_space(zfs_handle_t zhp, const char snapname, const char *from,
	enum lzc_send_flags flags, uint64_t *spacep)
	{
	assert(snapname != NULL);

	int error = lzc_send_space(snapname, from, flags, spacep);
	if (error == 0)
	return (0);

	char errbuf[ERRBUFLEN];
	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"warning: cannot estimate space for '%s'"), snapname);

	libzfs_handle_t *hdl = zhp->zfs_hdl;
	switch (error) {
	case EXDEV:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"not an earlier snapshot from the same fs"));
	return (zfs_error(hdl, EZFS_CROSSTARGET, errbuf));

	case ENOENT:
	if (zfs_dataset_exists(hdl, snapname,
	ZFS_TYPE_SNAPSHOT)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"incremental source (%s) does not exist"),
	snapname);
	}
	return (zfs_error(hdl, EZFS_NOENT, errbuf));

	case EDQUOT:
	case EFBIG:
	case EIO:
	case ENOLINK:
	case ENOSPC:
	case ENOSTR:
	case ENXIO:
	case EPIPE:
	case ERANGE:
	case EFAULT:
	case EROFS:
	case EINVAL:
	zfs_error_aux(hdl, "%s", strerror(error));
	return (zfs_error(hdl, EZFS_BADBACKUP, errbuf));

	default:
	return (zfs_standard_error(hdl, error, errbuf));
	}
	}

	/*
	* Dumps a backup of the given snapshot (incremental from fromsnap if it's not
	* NULL) to the file descriptor specified by outfd.
	*/
	static int
	dump_ioctl(zfs_handle_t zhp, const char fromsnap, uint64_t fromsnap_obj,
	boolean_t fromorigin, int outfd, enum lzc_send_flags flags,
	nvlist_t *debugnv)
	{
	zfs_cmd_t zc = {"\0"};
	libzfs_handle_t *hdl = zhp->zfs_hdl;
	nvlist_t *thisdbg;

	assert(zhp->zfs_type == ZFS_TYPE_SNAPSHOT);
	assert(fromsnap_obj == 0 \|\| !fromorigin);

	(void) strlcpy(zc.zc_name, zhp->zfs_name, sizeof (zc.zc_name));
	zc.zc_cookie = outfd;
	zc.zc_obj = fromorigin;
	zc.zc_sendobj = zfs_prop_get_int(zhp, ZFS_PROP_OBJSETID);
	zc.zc_fromobj = fromsnap_obj;
	zc.zc_flags = flags;

	if (debugnv != NULL) {
	thisdbg = fnvlist_alloc();
	if (fromsnap != NULL && fromsnap[0] != '\0')
	fnvlist_add_string(thisdbg, "fromsnap", fromsnap);
	}

	if (zfs_ioctl(zhp->zfs_hdl, ZFS_IOC_SEND, &zc) != 0) {
	char errbuf[ERRBUFLEN];
	int error = errno;

	(void) snprintf(errbuf, sizeof (errbuf), "%s '%s'",
	dgettext(TEXT_DOMAIN, "warning: cannot send"),
	zhp->zfs_name);

	if (debugnv != NULL) {
	fnvlist_add_uint64(thisdbg, "error", error);
	fnvlist_add_nvlist(debugnv, zhp->zfs_name, thisdbg);
	fnvlist_free(thisdbg);
	}

	switch (error) {
	case EXDEV:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"not an earlier snapshot from the same fs"));
	return (zfs_error(hdl, EZFS_CROSSTARGET, errbuf));

	case EACCES:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"source key must be loaded"));
	return (zfs_error(hdl, EZFS_CRYPTOFAILED, errbuf));

	case ENOENT:
	if (zfs_dataset_exists(hdl, zc.zc_name,
	ZFS_TYPE_SNAPSHOT)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"incremental source (@%s) does not exist"),
	zc.zc_value);
	}
	return (zfs_error(hdl, EZFS_NOENT, errbuf));

	case EDQUOT:
	case EFBIG:
	case EIO:
	case ENOLINK:
	case ENOSPC:
	case ENOSTR:
	case ENXIO:
	case EPIPE:
	case ERANGE:
	case EFAULT:
	case EROFS:
	case EINVAL:
	zfs_error_aux(hdl, "%s", strerror(errno));
	return (zfs_error(hdl, EZFS_BADBACKUP, errbuf));

	default:
	return (zfs_standard_error(hdl, errno, errbuf));
	}
	}

	if (debugnv != NULL) {
	fnvlist_add_nvlist(debugnv, zhp->zfs_name, thisdbg);
	fnvlist_free(thisdbg);
	}

	return (0);
	}

	static void
	gather_holds(zfs_handle_t zhp, send_dump_data_t sdd)
	{
	assert(zhp->zfs_type == ZFS_TYPE_SNAPSHOT);

	/*
	* zfs_send() only sets snapholds for sends that need them,
	* e.g. replication and doall.
	*/
	if (sdd->snapholds == NULL)
	return;

	fnvlist_add_string(sdd->snapholds, zhp->zfs_name, sdd->holdtag);
	}

	int
	zfs_send_progress(zfs_handle_t zhp, int fd, uint64_t bytes_written,
	uint64_t *blocks_visited)
	{
	zfs_cmd_t zc = {"\0"};

	if (bytes_written != NULL)
	*bytes_written = 0;
	if (blocks_visited != NULL)
	*blocks_visited = 0;
	(void) strlcpy(zc.zc_name, zhp->zfs_name, sizeof (zc.zc_name));
	zc.zc_cookie = fd;
	if (zfs_ioctl(zhp->zfs_hdl, ZFS_IOC_SEND_PROGRESS, &zc) != 0)
	return (errno);
	if (bytes_written != NULL)
	*bytes_written = zc.zc_cookie;
	if (blocks_visited != NULL)
	*blocks_visited = zc.zc_objset_type;
	return (0);
	}

	static volatile boolean_t send_progress_thread_signal_duetotimer;
	static void
	send_progress_thread_act(int sig, siginfo_t info, void ucontext)
	{
	(void) sig, (void) ucontext;
	send_progress_thread_signal_duetotimer = info->si_code == SI_TIMER;
	}

	struct timer_desirability {
	timer_t timer;
	boolean_t desired;
	};
	static void
	timer_delete_cleanup(void *timer)
	{
	struct timer_desirability *td = timer;
	if (td->desired)
	timer_delete(td->timer);
	}

	#ifdef SIGINFO
	#define SEND_PROGRESS_THREAD_PARENT_BLOCK_SIGINFO sigaddset(&new, SIGINFO)
	#else
	#define SEND_PROGRESS_THREAD_PARENT_BLOCK_SIGINFO
	#endif
	#define SEND_PROGRESS_THREAD_PARENT_BLOCK(old) { \
	sigset_t new; \
	sigemptyset(&new); \
	sigaddset(&new, SIGUSR1); \
	SEND_PROGRESS_THREAD_PARENT_BLOCK_SIGINFO; \
	pthread_sigmask(SIG_BLOCK, &new, old); \
	}

	static void *
	send_progress_thread(void *arg)
	{
	progress_arg_t *pa = arg;
	zfs_handle_t *zhp = pa->pa_zhp;
	uint64_t bytes;
	uint64_t blocks;
	uint64_t total = pa->pa_size / 100;
	char buf[16];
	time_t t;
	struct tm tm;
	int err;

	const struct sigaction signal_action =
	{.sa_sigaction = send_progress_thread_act, .sa_flags = SA_SIGINFO};
	struct sigevent timer_cfg =
	{.sigev_notify = SIGEV_SIGNAL, .sigev_signo = SIGUSR1};
	const struct itimerspec timer_time =
	{.it_value = {.tv_sec = 1}, .it_interval = {.tv_sec = 1}};
	struct timer_desirability timer = {};

	sigaction(SIGUSR1, &signal_action, NULL);
	#ifdef SIGINFO
	sigaction(SIGINFO, &signal_action, NULL);
	#endif

	if ((timer.desired = pa->pa_progress \|\| pa->pa_astitle)) {
	if (timer_create(CLOCK_MONOTONIC, &timer_cfg, &timer.timer))
	return ((void *)(uintptr_t)errno);
	(void) timer_settime(timer.timer, 0, &timer_time, NULL);
	}
	pthread_cleanup_push(timer_delete_cleanup, &timer);

	if (!pa->pa_parsable && pa->pa_progress) {
	(void) fprintf(stderr,
	"TIME %s %sSNAPSHOT %s\n",
	pa->pa_estimate ? "BYTES" : " SENT",
	pa->pa_verbosity >= 2 ? " BLOCKS " : "",
	zhp->zfs_name);
	}

	/*
	* Print the progress from ZFS_IOC_SEND_PROGRESS every second.
	*/
	for (;;) {
	pause();
	if ((err = zfs_send_progress(zhp, pa->pa_fd, &bytes,
	&blocks)) != 0) {
	if (err == EINTR \|\| err == ENOENT)
	err = 0;
	pthread_exit(((void *)(uintptr_t)err));
	}

	(void) time(&t);
	localtime_r(&t, &tm);

	if (pa->pa_astitle) {
	char buf_bytes[16];
	char buf_size[16];
	int pct;
	zfs_nicenum(bytes, buf_bytes, sizeof (buf_bytes));
	zfs_nicenum(pa->pa_size, buf_size, sizeof (buf_size));
	pct = (total > 0) ? bytes / total : 100;
	zfs_setproctitle("sending %s (%d%%: %s/%s)",
	zhp->zfs_name, MIN(pct, 100), buf_bytes, buf_size);
	}

	if (pa->pa_verbosity >= 2 && pa->pa_parsable) {
	(void) fprintf(stderr,
	"%02d:%02d:%02d\t%llu\t%llu\t%s\n",
	tm.tm_hour, tm.tm_min, tm.tm_sec,
	(u_longlong_t)bytes, (u_longlong_t)blocks,
	zhp->zfs_name);
	} else if (pa->pa_verbosity >= 2) {
	zfs_nicenum(bytes, buf, sizeof (buf));
	(void) fprintf(stderr,
	"%02d:%02d:%02d %5s %8llu %s\n",
	tm.tm_hour, tm.tm_min, tm.tm_sec,
	buf, (u_longlong_t)blocks, zhp->zfs_name);
	} else if (pa->pa_parsable) {
	(void) fprintf(stderr, "%02d:%02d:%02d\t%llu\t%s\n",
	tm.tm_hour, tm.tm_min, tm.tm_sec,
	(u_longlong_t)bytes, zhp->zfs_name);
	} else if (pa->pa_progress \|\|
	!send_progress_thread_signal_duetotimer) {
	zfs_nicebytes(bytes, buf, sizeof (buf));
	(void) fprintf(stderr, "%02d:%02d:%02d %5s %s\n",
	tm.tm_hour, tm.tm_min, tm.tm_sec,
	buf, zhp->zfs_name);
	}
	}
	pthread_cleanup_pop(B_TRUE);
	+ return (NULL);
	}

	static boolean_t
	send_progress_thread_exit(
	libzfs_handle_t hdl, pthread_t ptid, sigset_t oldmask)
	{
	void *status = NULL;
	(void) pthread_cancel(ptid);
	(void) pthread_join(ptid, &status);
	pthread_sigmask(SIG_SETMASK, oldmask, NULL);
	int error = (int)(uintptr_t)status;
	if (error != 0 && status != PTHREAD_CANCELED)
	return (zfs_standard_error(hdl, error,
	dgettext(TEXT_DOMAIN, "progress thread exited nonzero")));
	else
	return (B_FALSE);
	}

	static void
	send_print_verbose(FILE fout, const char tosnap, const char *fromsnap,
	uint64_t size, boolean_t parsable)
	{
	if (parsable) {
	if (fromsnap != NULL) {
	(void) fprintf(fout, dgettext(TEXT_DOMAIN,
	"incremental\t%s\t%s"), fromsnap, tosnap);
	} else {
	/*
	* Workaround for GCC 12+ with UBSan enabled deficencies.
	*
	* GCC 12+ invoked with -fsanitize=undefined incorrectly reports the code
	* below as violating -Wformat-overflow.
	*/
	#if defined(__GNUC__) && !defined(__clang__) && \
	defined(ZFS_UBSAN_ENABLED) && defined(HAVE_FORMAT_OVERFLOW)
	#pragma GCC diagnostic push
	#pragma GCC diagnostic ignored "-Wformat-overflow"
	#endif
	(void) fprintf(fout, dgettext(TEXT_DOMAIN,
	"full\t%s"), tosnap);
	#if defined(__GNUC__) && !defined(__clang__) && \
	defined(ZFS_UBSAN_ENABLED) && defined(HAVE_FORMAT_OVERFLOW)
	#pragma GCC diagnostic pop
	#endif
	}
	(void) fprintf(fout, "\t%llu", (longlong_t)size);
	} else {
	if (fromsnap != NULL) {
	if (strchr(fromsnap, '@') == NULL &&
	strchr(fromsnap, '#') == NULL) {
	(void) fprintf(fout, dgettext(TEXT_DOMAIN,
	"send from @%s to %s"), fromsnap, tosnap);
	} else {
	(void) fprintf(fout, dgettext(TEXT_DOMAIN,
	"send from %s to %s"), fromsnap, tosnap);
	}
	} else {
	(void) fprintf(fout, dgettext(TEXT_DOMAIN,
	"full send of %s"), tosnap);
	}
	if (size != 0) {
	char buf[16];
	zfs_nicebytes(size, buf, sizeof (buf));
	/*
	* Workaround for GCC 12+ with UBSan enabled deficencies.
	*
	* GCC 12+ invoked with -fsanitize=undefined incorrectly reports the code
	* below as violating -Wformat-overflow.
	*/
	#if defined(__GNUC__) && !defined(__clang__) && \
	defined(ZFS_UBSAN_ENABLED) && defined(HAVE_FORMAT_OVERFLOW)
	#pragma GCC diagnostic push
	#pragma GCC diagnostic ignored "-Wformat-overflow"
	#endif
	(void) fprintf(fout, dgettext(TEXT_DOMAIN,
	" estimated size is %s"), buf);
	#if defined(__GNUC__) && !defined(__clang__) && \
	defined(ZFS_UBSAN_ENABLED) && defined(HAVE_FORMAT_OVERFLOW)
	#pragma GCC diagnostic pop
	#endif
	}
	}
	(void) fprintf(fout, "\n");
	}

	/*
	* Send a single filesystem snapshot, updating the send dump data.
	* This interface is intended for use as a zfs_iter_snapshots_v2_sorted visitor.
	*/
	static int
	dump_snapshot(zfs_handle_t zhp, void arg)
	{
	send_dump_data_t *sdd = arg;
	progress_arg_t pa = { 0 };
	pthread_t tid;
	char *thissnap;
	enum lzc_send_flags flags = 0;
	int err;
	boolean_t isfromsnap, istosnap, fromorigin;
	boolean_t exclude = B_FALSE;
	FILE *fout = sdd->std_out ? stdout : stderr;

	err = 0;
	thissnap = strchr(zhp->zfs_name, '@') + 1;
	isfromsnap = (sdd->fromsnap != NULL &&
	strcmp(sdd->fromsnap, thissnap) == 0);

	if (!sdd->seenfrom && isfromsnap) {
	gather_holds(zhp, sdd);
	sdd->seenfrom = B_TRUE;
	(void) strlcpy(sdd->prevsnap, thissnap, sizeof (sdd->prevsnap));
	sdd->prevsnap_obj = zfs_prop_get_int(zhp, ZFS_PROP_OBJSETID);
	zfs_close(zhp);
	return (0);
	}

	if (sdd->seento \|\| !sdd->seenfrom) {
	zfs_close(zhp);
	return (0);
	}

	istosnap = (strcmp(sdd->tosnap, thissnap) == 0);
	if (istosnap)
	sdd->seento = B_TRUE;

	if (sdd->large_block)
	flags \|= LZC_SEND_FLAG_LARGE_BLOCK;
	if (sdd->embed_data)
	flags \|= LZC_SEND_FLAG_EMBED_DATA;
	if (sdd->compress)
	flags \|= LZC_SEND_FLAG_COMPRESS;
	if (sdd->raw)
	flags \|= LZC_SEND_FLAG_RAW;

	if (!sdd->doall && !isfromsnap && !istosnap) {
	if (sdd->replicate) {
	const char *snapname;
	nvlist_t *snapprops;
	/*
	* Filter out all intermediate snapshots except origin
	* snapshots needed to replicate clones.
	*/
	nvlist_t *nvfs = fsavl_find(sdd->fsavl,
	zhp->zfs_dmustats.dds_guid, &snapname);

	if (nvfs != NULL) {
	snapprops = fnvlist_lookup_nvlist(nvfs,
	"snapprops");
	snapprops = fnvlist_lookup_nvlist(snapprops,
	thissnap);
	exclude = !nvlist_exists(snapprops,
	"is_clone_origin");
	}
	} else {
	exclude = B_TRUE;
	}
	}

	/*
	* If a filter function exists, call it to determine whether
	* this snapshot will be sent.
	*/
	if (exclude \|\| (sdd->filter_cb != NULL &&
	sdd->filter_cb(zhp, sdd->filter_cb_arg) == B_FALSE)) {
	/*
	* This snapshot is filtered out. Don't send it, and don't
	* set prevsnap_obj, so it will be as if this snapshot didn't
	* exist, and the next accepted snapshot will be sent as
	* an incremental from the last accepted one, or as the
	* first (and full) snapshot in the case of a replication,
	* non-incremental send.
	*/
	zfs_close(zhp);
	return (0);
	}

	gather_holds(zhp, sdd);
	fromorigin = sdd->prevsnap[0] == '\0' &&
	(sdd->fromorigin \|\| sdd->replicate);

	if (sdd->verbosity != 0) {
	uint64_t size = 0;
	char fromds[ZFS_MAX_DATASET_NAME_LEN];

	if (sdd->prevsnap[0] != '\0') {
	(void) strlcpy(fromds, zhp->zfs_name, sizeof (fromds));
	*(strchr(fromds, '@') + 1) = '\0';
	(void) strlcat(fromds, sdd->prevsnap, sizeof (fromds));
	}
	if (zfs_send_space(zhp, zhp->zfs_name,
	sdd->prevsnap[0] ? fromds : NULL, flags, &size) == 0) {
	send_print_verbose(fout, zhp->zfs_name,
	sdd->prevsnap[0] ? sdd->prevsnap : NULL,
	size, sdd->parsable);
	sdd->size += size;
	}
	}

	if (!sdd->dryrun) {
	/*
	* If progress reporting is requested, spawn a new thread to
	* poll ZFS_IOC_SEND_PROGRESS at a regular interval.
	*/
	sigset_t oldmask;
	{
	pa.pa_zhp = zhp;
	pa.pa_fd = sdd->outfd;
	pa.pa_parsable = sdd->parsable;
	pa.pa_estimate = B_FALSE;
	pa.pa_verbosity = sdd->verbosity;
	pa.pa_size = sdd->size;
	pa.pa_astitle = sdd->progressastitle;
	pa.pa_progress = sdd->progress;

	if ((err = pthread_create(&tid, NULL,
	send_progress_thread, &pa)) != 0) {
	zfs_close(zhp);
	return (err);
	}
	SEND_PROGRESS_THREAD_PARENT_BLOCK(&oldmask);
	}

	err = dump_ioctl(zhp, sdd->prevsnap, sdd->prevsnap_obj,
	fromorigin, sdd->outfd, flags, sdd->debugnv);

	if (send_progress_thread_exit(zhp->zfs_hdl, tid, &oldmask))
	return (-1);
	}

	(void) strlcpy(sdd->prevsnap, thissnap, sizeof (sdd->prevsnap));
	sdd->prevsnap_obj = zfs_prop_get_int(zhp, ZFS_PROP_OBJSETID);
	zfs_close(zhp);
	return (err);
	}

	/*
	* Send all snapshots for a filesystem, updating the send dump data.
	*/
	static int
	dump_filesystem(zfs_handle_t zhp, send_dump_data_t sdd)
	{
	int rv = 0;
	boolean_t missingfrom = B_FALSE;
	zfs_cmd_t zc = {"\0"};
	uint64_t min_txg = 0, max_txg = 0;

	/*
	* Make sure the tosnap exists.
	*/
	(void) snprintf(zc.zc_name, sizeof (zc.zc_name), "%s@%s",
	zhp->zfs_name, sdd->tosnap);
	if (zfs_ioctl(zhp->zfs_hdl, ZFS_IOC_OBJSET_STATS, &zc) != 0) {
	(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
	"WARNING: could not send %s@%s: does not exist\n"),
	zhp->zfs_name, sdd->tosnap);
	sdd->err = B_TRUE;
	return (0);
	}

	/*
	* If this fs does not have fromsnap, and we're doing
	* recursive, we need to send a full stream from the
	* beginning (or an incremental from the origin if this
	* is a clone). If we're doing non-recursive, then let
	* them get the error.
	*/
	if (sdd->replicate && sdd->fromsnap) {
	/*
	* Make sure the fromsnap exists.
	*/
	(void) snprintf(zc.zc_name, sizeof (zc.zc_name), "%s@%s",
	zhp->zfs_name, sdd->fromsnap);
	if (zfs_ioctl(zhp->zfs_hdl, ZFS_IOC_OBJSET_STATS, &zc) != 0)
	missingfrom = B_TRUE;
	}

	sdd->seenfrom = sdd->seento = B_FALSE;
	sdd->prevsnap[0] = '\0';
	sdd->prevsnap_obj = 0;
	if (sdd->fromsnap == NULL \|\| missingfrom)
	sdd->seenfrom = B_TRUE;

	/*
	* Iterate through all snapshots and process the ones we will be
	* sending. If we only have a "from" and "to" snapshot to deal
	* with, we can avoid iterating through all the other snapshots.
	*/
	if (sdd->doall \|\| sdd->replicate \|\| sdd->tosnap == NULL) {
	if (!sdd->replicate) {
	if (sdd->fromsnap != NULL) {
	min_txg = get_snap_txg(zhp->zfs_hdl,
	zhp->zfs_name, sdd->fromsnap);
	}
	if (sdd->tosnap != NULL) {
	max_txg = get_snap_txg(zhp->zfs_hdl,
	zhp->zfs_name, sdd->tosnap);
	}
	}
	rv = zfs_iter_snapshots_sorted_v2(zhp, 0, dump_snapshot, sdd,
	min_txg, max_txg);
	} else {
	char snapname[MAXPATHLEN] = { 0 };
	zfs_handle_t *snap;

	/* Dump fromsnap. */
	if (!sdd->seenfrom) {
	(void) snprintf(snapname, sizeof (snapname),
	"%s@%s", zhp->zfs_name, sdd->fromsnap);
	snap = zfs_open(zhp->zfs_hdl, snapname,
	ZFS_TYPE_SNAPSHOT);
	if (snap != NULL)
	rv = dump_snapshot(snap, sdd);
	else
	rv = errno;
	}

	/* Dump tosnap. */
	if (rv == 0) {
	(void) snprintf(snapname, sizeof (snapname),
	"%s@%s", zhp->zfs_name, sdd->tosnap);
	snap = zfs_open(zhp->zfs_hdl, snapname,
	ZFS_TYPE_SNAPSHOT);
	if (snap != NULL)
	rv = dump_snapshot(snap, sdd);
	else
	rv = errno;
	}
	}

	if (!sdd->seenfrom) {
	(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
	"WARNING: could not send %s@%s:\n"
	"incremental source (%s@%s) does not exist\n"),
	zhp->zfs_name, sdd->tosnap,
	zhp->zfs_name, sdd->fromsnap);
	sdd->err = B_TRUE;
	} else if (!sdd->seento) {
	if (sdd->fromsnap) {
	(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
	"WARNING: could not send %s@%s:\n"
	"incremental source (%s@%s) "
	"is not earlier than it\n"),
	zhp->zfs_name, sdd->tosnap,
	zhp->zfs_name, sdd->fromsnap);
	} else {
	(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
	"WARNING: "
	"could not send %s@%s: does not exist\n"),
	zhp->zfs_name, sdd->tosnap);
	}
	sdd->err = B_TRUE;
	}

	return (rv);
	}

	/*
	* Send all snapshots for all filesystems in sdd.
	*/
	static int
	dump_filesystems(zfs_handle_t rzhp, send_dump_data_t sdd)
	{
	nvpair_t *fspair;
	boolean_t needagain, progress;

	if (!sdd->replicate)
	return (dump_filesystem(rzhp, sdd));

	/* Mark the clone origin snapshots. */
	for (fspair = nvlist_next_nvpair(sdd->fss, NULL); fspair;
	fspair = nvlist_next_nvpair(sdd->fss, fspair)) {
	nvlist_t *nvfs;
	uint64_t origin_guid = 0;

	nvfs = fnvpair_value_nvlist(fspair);
	(void) nvlist_lookup_uint64(nvfs, "origin", &origin_guid);
	if (origin_guid != 0) {
	const char *snapname;
	nvlist_t *origin_nv = fsavl_find(sdd->fsavl,
	origin_guid, &snapname);
	if (origin_nv != NULL) {
	nvlist_t *snapprops;
	snapprops = fnvlist_lookup_nvlist(origin_nv,
	"snapprops");
	snapprops = fnvlist_lookup_nvlist(snapprops,
	snapname);
	fnvlist_add_boolean(snapprops,
	"is_clone_origin");
	}
	}
	}
	again:
	needagain = progress = B_FALSE;
	for (fspair = nvlist_next_nvpair(sdd->fss, NULL); fspair;
	fspair = nvlist_next_nvpair(sdd->fss, fspair)) {
	nvlist_t fslist, parent_nv;
	const char *fsname;
	zfs_handle_t *zhp;
	int err;
	uint64_t origin_guid = 0;
	uint64_t parent_guid = 0;

	fslist = fnvpair_value_nvlist(fspair);
	if (nvlist_lookup_boolean(fslist, "sent") == 0)
	continue;

	fsname = fnvlist_lookup_string(fslist, "name");
	(void) nvlist_lookup_uint64(fslist, "origin", &origin_guid);
	(void) nvlist_lookup_uint64(fslist, "parentfromsnap",
	&parent_guid);

	if (parent_guid != 0) {
	parent_nv = fsavl_find(sdd->fsavl, parent_guid, NULL);
	if (!nvlist_exists(parent_nv, "sent")) {
	/* Parent has not been sent; skip this one. */
	needagain = B_TRUE;
	continue;
	}
	}

	if (origin_guid != 0) {
	nvlist_t *origin_nv = fsavl_find(sdd->fsavl,
	origin_guid, NULL);
	if (origin_nv != NULL &&
	!nvlist_exists(origin_nv, "sent")) {
	/*
	* Origin has not been sent yet;
	* skip this clone.
	*/
	needagain = B_TRUE;
	continue;
	}
	}

	zhp = zfs_open(rzhp->zfs_hdl, fsname, ZFS_TYPE_DATASET);
	if (zhp == NULL)
	return (-1);
	err = dump_filesystem(zhp, sdd);
	fnvlist_add_boolean(fslist, "sent");
	progress = B_TRUE;
	zfs_close(zhp);
	if (err)
	return (err);
	}
	if (needagain) {
	assert(progress);
	goto again;
	}

	/* Clean out the sent flags in case we reuse this fss. */
	for (fspair = nvlist_next_nvpair(sdd->fss, NULL); fspair;
	fspair = nvlist_next_nvpair(sdd->fss, fspair)) {
	nvlist_t *fslist;

	fslist = fnvpair_value_nvlist(fspair);
	(void) nvlist_remove_all(fslist, "sent");
	}

	return (0);
	}

	nvlist_t *
	zfs_send_resume_token_to_nvlist(libzfs_handle_t hdl, const char token)
	{
	unsigned int version;
	int nread, i;
	unsigned long long checksum, packed_len;

	/*
	* Decode token header, which is:
	* <token version>-<checksum of payload>-<uncompressed payload length>
	* Note that the only supported token version is 1.
	*/
	nread = sscanf(token, "%u-%llx-%llx-",
	&version, &checksum, &packed_len);
	if (nread != 3) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"resume token is corrupt (invalid format)"));
	return (NULL);
	}

	if (version != ZFS_SEND_RESUME_TOKEN_VERSION) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"resume token is corrupt (invalid version %u)"),
	version);
	return (NULL);
	}

	/* Convert hexadecimal representation to binary. */
	token = strrchr(token, '-') + 1;
	int len = strlen(token) / 2;
	unsigned char *compressed = zfs_alloc(hdl, len);
	for (i = 0; i < len; i++) {
	nread = sscanf(token + i * 2, "%2hhx", compressed + i);
	if (nread != 1) {
	free(compressed);
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"resume token is corrupt "
	"(payload is not hex-encoded)"));
	return (NULL);
	}
	}

	/* Verify checksum. */
	zio_cksum_t cksum;
	fletcher_4_native_varsize(compressed, len, &cksum);
	if (cksum.zc_word[0] != checksum) {
	free(compressed);
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"resume token is corrupt (incorrect checksum)"));
	return (NULL);
	}

	/* Uncompress. */
	void *packed = zfs_alloc(hdl, packed_len);
	uLongf packed_len_long = packed_len;
	if (uncompress(packed, &packed_len_long, compressed, len) != Z_OK \|\|
	packed_len_long != packed_len) {
	free(packed);
	free(compressed);
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"resume token is corrupt (decompression failed)"));
	return (NULL);
	}

	/* Unpack nvlist. */
	nvlist_t *nv;
	int error = nvlist_unpack(packed, packed_len, &nv, KM_SLEEP);
	free(packed);
	free(compressed);
	if (error != 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"resume token is corrupt (nvlist_unpack failed)"));
	return (NULL);
	}
	return (nv);
	}

	static enum lzc_send_flags
	lzc_flags_from_sendflags(const sendflags_t *flags)
	{
	enum lzc_send_flags lzc_flags = 0;

	if (flags->largeblock)
	lzc_flags \|= LZC_SEND_FLAG_LARGE_BLOCK;
	if (flags->embed_data)
	lzc_flags \|= LZC_SEND_FLAG_EMBED_DATA;
	if (flags->compress)
	lzc_flags \|= LZC_SEND_FLAG_COMPRESS;
	if (flags->raw)
	lzc_flags \|= LZC_SEND_FLAG_RAW;
	if (flags->saved)
	lzc_flags \|= LZC_SEND_FLAG_SAVED;

	return (lzc_flags);
	}

	static int
	estimate_size(zfs_handle_t zhp, const char from, int fd, sendflags_t *flags,
	uint64_t resumeobj, uint64_t resumeoff, uint64_t bytes,
	const char redactbook, char errbuf, uint64_t *sizep)
	{
	uint64_t size;
	FILE *fout = flags->dryrun ? stdout : stderr;
	progress_arg_t pa = { 0 };
	int err = 0;
	pthread_t ptid;
	sigset_t oldmask;

	{
	pa.pa_zhp = zhp;
	pa.pa_fd = fd;
	pa.pa_parsable = flags->parsable;
	pa.pa_estimate = B_TRUE;
	pa.pa_verbosity = flags->verbosity;

	err = pthread_create(&ptid, NULL,
	send_progress_thread, &pa);
	if (err != 0) {
	zfs_error_aux(zhp->zfs_hdl, "%s", strerror(errno));
	return (zfs_error(zhp->zfs_hdl,
	EZFS_THREADCREATEFAILED, errbuf));
	}
	SEND_PROGRESS_THREAD_PARENT_BLOCK(&oldmask);
	}

	err = lzc_send_space_resume_redacted(zhp->zfs_name, from,
	lzc_flags_from_sendflags(flags), resumeobj, resumeoff, bytes,
	redactbook, fd, &size);
	*sizep = size;

	if (send_progress_thread_exit(zhp->zfs_hdl, ptid, &oldmask))
	return (-1);

	if (!flags->progress && !flags->parsable)
	return (err);

	if (err != 0) {
	zfs_error_aux(zhp->zfs_hdl, "%s", strerror(err));
	return (zfs_error(zhp->zfs_hdl, EZFS_BADBACKUP,
	errbuf));
	}
	send_print_verbose(fout, zhp->zfs_name, from, size,
	flags->parsable);

	if (flags->parsable) {
	(void) fprintf(fout, "size\t%llu\n", (longlong_t)size);
	} else {
	char buf[16];
	zfs_nicenum(size, buf, sizeof (buf));
	(void) fprintf(fout, dgettext(TEXT_DOMAIN,
	"total estimated size is %s\n"), buf);
	}
	return (0);
	}

	static boolean_t
	redact_snaps_contains(const uint64_t *snaps, uint64_t num_snaps, uint64_t guid)
	{
	for (int i = 0; i < num_snaps; i++) {
	if (snaps[i] == guid)
	return (B_TRUE);
	}
	return (B_FALSE);
	}

	static boolean_t
	redact_snaps_equal(const uint64_t *snaps1, uint64_t num_snaps1,
	const uint64_t *snaps2, uint64_t num_snaps2)
	{
	if (num_snaps1 != num_snaps2)
	return (B_FALSE);
	for (int i = 0; i < num_snaps1; i++) {
	if (!redact_snaps_contains(snaps2, num_snaps2, snaps1[i]))
	return (B_FALSE);
	}
	return (B_TRUE);
	}

	static int
	get_bookmarks(const char path, nvlist_t *bmarksp)
	{
	nvlist_t *props = fnvlist_alloc();
	int error;

	fnvlist_add_boolean(props, "redact_complete");
	fnvlist_add_boolean(props, zfs_prop_to_name(ZFS_PROP_REDACT_SNAPS));
	error = lzc_get_bookmarks(path, props, bmarksp);
	fnvlist_free(props);
	return (error);
	}

	static nvpair_t *
	find_redact_pair(nvlist_t bmarks, const uint64_t redact_snap_guids,
	int num_redact_snaps)
	{
	nvpair_t *pair;

	for (pair = nvlist_next_nvpair(bmarks, NULL); pair;
	pair = nvlist_next_nvpair(bmarks, pair)) {

	nvlist_t *bmark = fnvpair_value_nvlist(pair);
	nvlist_t *vallist = fnvlist_lookup_nvlist(bmark,
	zfs_prop_to_name(ZFS_PROP_REDACT_SNAPS));
	uint_t len = 0;
	uint64_t *bmarksnaps = fnvlist_lookup_uint64_array(vallist,
	ZPROP_VALUE, &len);
	if (redact_snaps_equal(redact_snap_guids,
	num_redact_snaps, bmarksnaps, len)) {
	break;
	}
	}
	return (pair);
	}

	static boolean_t
	get_redact_complete(nvpair_t *pair)
	{
	nvlist_t *bmark = fnvpair_value_nvlist(pair);
	nvlist_t *vallist = fnvlist_lookup_nvlist(bmark, "redact_complete");
	boolean_t complete = fnvlist_lookup_boolean_value(vallist,
	ZPROP_VALUE);

	return (complete);
	}

	/*
	* Check that the list of redaction snapshots in the bookmark matches the send
	* we're resuming, and return whether or not it's complete.
	*
	* Note that the caller needs to free the contents of *bookname with free() if
	* this function returns successfully.
	*/
	static int
	find_redact_book(libzfs_handle_t hdl, const char path,
	const uint64_t *redact_snap_guids, int num_redact_snaps,
	char **bookname)
	{
	char errbuf[ERRBUFLEN];
	nvlist_t *bmarks;

	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot resume send"));

	int error = get_bookmarks(path, &bmarks);
	if (error != 0) {
	if (error == ESRCH) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"nonexistent redaction bookmark provided"));
	} else if (error == ENOENT) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"dataset to be sent no longer exists"));
	} else {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"unknown error: %s"), strerror(error));
	}
	return (zfs_error(hdl, EZFS_BADPROP, errbuf));
	}
	nvpair_t *pair = find_redact_pair(bmarks, redact_snap_guids,
	num_redact_snaps);
	if (pair == NULL) {
	fnvlist_free(bmarks);
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"no appropriate redaction bookmark exists"));
	return (zfs_error(hdl, EZFS_BADPROP, errbuf));
	}
	boolean_t complete = get_redact_complete(pair);
	if (!complete) {
	fnvlist_free(bmarks);
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"incomplete redaction bookmark provided"));
	return (zfs_error(hdl, EZFS_BADPROP, errbuf));
	}
	*bookname = strndup(nvpair_name(pair), ZFS_MAX_DATASET_NAME_LEN);
	ASSERT3P(*bookname, !=, NULL);
	fnvlist_free(bmarks);
	return (0);
	}

	static enum lzc_send_flags
	lzc_flags_from_resume_nvl(nvlist_t *resume_nvl)
	{
	enum lzc_send_flags lzc_flags = 0;

	if (nvlist_exists(resume_nvl, "largeblockok"))
	lzc_flags \|= LZC_SEND_FLAG_LARGE_BLOCK;
	if (nvlist_exists(resume_nvl, "embedok"))
	lzc_flags \|= LZC_SEND_FLAG_EMBED_DATA;
	if (nvlist_exists(resume_nvl, "compressok"))
	lzc_flags \|= LZC_SEND_FLAG_COMPRESS;
	if (nvlist_exists(resume_nvl, "rawok"))
	lzc_flags \|= LZC_SEND_FLAG_RAW;
	if (nvlist_exists(resume_nvl, "savedok"))
	lzc_flags \|= LZC_SEND_FLAG_SAVED;

	return (lzc_flags);
	}

	static int
	zfs_send_resume_impl_cb_impl(libzfs_handle_t hdl, sendflags_t flags,
	int outfd, nvlist_t *resume_nvl)
	{
	char errbuf[ERRBUFLEN];
	const char *toname;
	const char *fromname = NULL;
	uint64_t resumeobj, resumeoff, toguid, fromguid, bytes;
	zfs_handle_t *zhp;
	int error = 0;
	char name[ZFS_MAX_DATASET_NAME_LEN];
	FILE *fout = (flags->verbosity > 0 && flags->dryrun) ? stdout : stderr;
	uint64_t *redact_snap_guids = NULL;
	int num_redact_snaps = 0;
	char *redact_book = NULL;
	uint64_t size = 0;

	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot resume send"));

	if (flags->verbosity != 0) {
	(void) fprintf(fout, dgettext(TEXT_DOMAIN,
	"resume token contents:\n"));
	nvlist_print(fout, resume_nvl);
	}

	if (nvlist_lookup_string(resume_nvl, "toname", &toname) != 0 \|\|
	nvlist_lookup_uint64(resume_nvl, "object", &resumeobj) != 0 \|\|
	nvlist_lookup_uint64(resume_nvl, "offset", &resumeoff) != 0 \|\|
	nvlist_lookup_uint64(resume_nvl, "bytes", &bytes) != 0 \|\|
	nvlist_lookup_uint64(resume_nvl, "toguid", &toguid) != 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"resume token is corrupt"));
	return (zfs_error(hdl, EZFS_FAULT, errbuf));
	}
	fromguid = 0;
	(void) nvlist_lookup_uint64(resume_nvl, "fromguid", &fromguid);

	if (flags->saved) {
	(void) strlcpy(name, toname, sizeof (name));
	} else {
	error = guid_to_name(hdl, toname, toguid, B_FALSE, name);
	if (error != 0) {
	if (zfs_dataset_exists(hdl, toname, ZFS_TYPE_DATASET)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"'%s' is no longer the same snapshot "
	"used in the initial send"), toname);
	} else {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"'%s' used in the initial send no "
	"longer exists"), toname);
	}
	return (zfs_error(hdl, EZFS_BADPATH, errbuf));
	}
	}

	zhp = zfs_open(hdl, name, ZFS_TYPE_DATASET);
	if (zhp == NULL) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"unable to access '%s'"), name);
	return (zfs_error(hdl, EZFS_BADPATH, errbuf));
	}

	if (nvlist_lookup_uint64_array(resume_nvl, "book_redact_snaps",
	&redact_snap_guids, (uint_t *)&num_redact_snaps) != 0) {
	num_redact_snaps = -1;
	}

	if (fromguid != 0) {
	if (guid_to_name_redact_snaps(hdl, toname, fromguid, B_TRUE,
	redact_snap_guids, num_redact_snaps, name) != 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"incremental source %#llx no longer exists"),
	(longlong_t)fromguid);
	return (zfs_error(hdl, EZFS_BADPATH, errbuf));
	}
	fromname = name;
	}

	redact_snap_guids = NULL;

	if (nvlist_lookup_uint64_array(resume_nvl,
	zfs_prop_to_name(ZFS_PROP_REDACT_SNAPS), &redact_snap_guids,
	(uint_t *)&num_redact_snaps) == 0) {
	char path[ZFS_MAX_DATASET_NAME_LEN];

	(void) strlcpy(path, toname, sizeof (path));
	char *at = strchr(path, '@');
	ASSERT3P(at, !=, NULL);

	*at = '\0';

	if ((error = find_redact_book(hdl, path, redact_snap_guids,
	num_redact_snaps, &redact_book)) != 0) {
	return (error);
	}
	}

	enum lzc_send_flags lzc_flags = lzc_flags_from_sendflags(flags) \|
	lzc_flags_from_resume_nvl(resume_nvl);

	if (flags->verbosity != 0 \|\| flags->progressastitle) {
	/*
	* Some of these may have come from the resume token, set them
	* here for size estimate purposes.
	*/
	sendflags_t tmpflags = *flags;
	if (lzc_flags & LZC_SEND_FLAG_LARGE_BLOCK)
	tmpflags.largeblock = B_TRUE;
	if (lzc_flags & LZC_SEND_FLAG_COMPRESS)
	tmpflags.compress = B_TRUE;
	if (lzc_flags & LZC_SEND_FLAG_EMBED_DATA)
	tmpflags.embed_data = B_TRUE;
	if (lzc_flags & LZC_SEND_FLAG_RAW)
	tmpflags.raw = B_TRUE;
	if (lzc_flags & LZC_SEND_FLAG_SAVED)
	tmpflags.saved = B_TRUE;
	error = estimate_size(zhp, fromname, outfd, &tmpflags,
	resumeobj, resumeoff, bytes, redact_book, errbuf, &size);
	}

	if (!flags->dryrun) {
	progress_arg_t pa = { 0 };
	pthread_t tid;
	sigset_t oldmask;
	/*
	* If progress reporting is requested, spawn a new thread to
	* poll ZFS_IOC_SEND_PROGRESS at a regular interval.
	*/
	{
	pa.pa_zhp = zhp;
	pa.pa_fd = outfd;
	pa.pa_parsable = flags->parsable;
	pa.pa_estimate = B_FALSE;
	pa.pa_verbosity = flags->verbosity;
	pa.pa_size = size;
	pa.pa_astitle = flags->progressastitle;
	pa.pa_progress = flags->progress;

	error = pthread_create(&tid, NULL,
	send_progress_thread, &pa);
	if (error != 0) {
	if (redact_book != NULL)
	free(redact_book);
	zfs_close(zhp);
	return (error);
	}
	SEND_PROGRESS_THREAD_PARENT_BLOCK(&oldmask);
	}

	error = lzc_send_resume_redacted(zhp->zfs_name, fromname, outfd,
	lzc_flags, resumeobj, resumeoff, redact_book);
	if (redact_book != NULL)
	free(redact_book);

	if (send_progress_thread_exit(hdl, tid, &oldmask)) {
	zfs_close(zhp);
	return (-1);
	}

	char errbuf[ERRBUFLEN];
	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"warning: cannot send '%s'"), zhp->zfs_name);

	zfs_close(zhp);

	switch (error) {
	case 0:
	return (0);
	case EACCES:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"source key must be loaded"));
	return (zfs_error(hdl, EZFS_CRYPTOFAILED, errbuf));
	case ESRCH:
	if (lzc_exists(zhp->zfs_name)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"incremental source could not be found"));
	}
	return (zfs_error(hdl, EZFS_NOENT, errbuf));

	case EXDEV:
	case ENOENT:
	case EDQUOT:
	case EFBIG:
	case EIO:
	case ENOLINK:
	case ENOSPC:
	case ENOSTR:
	case ENXIO:
	case EPIPE:
	case ERANGE:
	case EFAULT:
	case EROFS:
	zfs_error_aux(hdl, "%s", strerror(errno));
	return (zfs_error(hdl, EZFS_BADBACKUP, errbuf));

	default:
	return (zfs_standard_error(hdl, errno, errbuf));
	}
	} else {
	if (redact_book != NULL)
	free(redact_book);
	}

	zfs_close(zhp);

	return (error);
	}

	struct zfs_send_resume_impl {
	libzfs_handle_t *hdl;
	sendflags_t *flags;
	nvlist_t *resume_nvl;
	};

	static int
	zfs_send_resume_impl_cb(int outfd, void *arg)
	{
	struct zfs_send_resume_impl *zsri = arg;
	return (zfs_send_resume_impl_cb_impl(zsri->hdl, zsri->flags, outfd,
	zsri->resume_nvl));
	}

	static int
	zfs_send_resume_impl(libzfs_handle_t hdl, sendflags_t flags, int outfd,
	nvlist_t *resume_nvl)
	{
	struct zfs_send_resume_impl zsri = {
	.hdl = hdl,
	.flags = flags,
	.resume_nvl = resume_nvl,
	};
	return (lzc_send_wrapper(zfs_send_resume_impl_cb, outfd, &zsri));
	}

	int
	zfs_send_resume(libzfs_handle_t hdl, sendflags_t flags, int outfd,
	const char *resume_token)
	{
	int ret;
	char errbuf[ERRBUFLEN];
	nvlist_t *resume_nvl;

	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot resume send"));

	resume_nvl = zfs_send_resume_token_to_nvlist(hdl, resume_token);
	if (resume_nvl == NULL) {
	/*
	* zfs_error_aux has already been set by
	* zfs_send_resume_token_to_nvlist()
	*/
	return (zfs_error(hdl, EZFS_FAULT, errbuf));
	}

	ret = zfs_send_resume_impl(hdl, flags, outfd, resume_nvl);
	fnvlist_free(resume_nvl);

	return (ret);
	}

	int
	zfs_send_saved(zfs_handle_t zhp, sendflags_t flags, int outfd,
	const char *resume_token)
	{
	int ret;
	libzfs_handle_t *hdl = zhp->zfs_hdl;
	nvlist_t saved_nvl = NULL, resume_nvl = NULL;
	uint64_t saved_guid = 0, resume_guid = 0;
	uint64_t obj = 0, off = 0, bytes = 0;
	char token_buf[ZFS_MAXPROPLEN];
	char errbuf[ERRBUFLEN];

	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"saved send failed"));

	ret = zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN,
	token_buf, sizeof (token_buf), NULL, NULL, 0, B_TRUE);
	if (ret != 0)
	goto out;

	saved_nvl = zfs_send_resume_token_to_nvlist(hdl, token_buf);
	if (saved_nvl == NULL) {
	/*
	* zfs_error_aux has already been set by
	* zfs_send_resume_token_to_nvlist()
	*/
	ret = zfs_error(hdl, EZFS_FAULT, errbuf);
	goto out;
	}

	/*
	* If a resume token is provided we use the object and offset
	* from that instead of the default, which starts from the
	* beginning.
	*/
	if (resume_token != NULL) {
	resume_nvl = zfs_send_resume_token_to_nvlist(hdl,
	resume_token);
	if (resume_nvl == NULL) {
	ret = zfs_error(hdl, EZFS_FAULT, errbuf);
	goto out;
	}

	if (nvlist_lookup_uint64(resume_nvl, "object", &obj) != 0 \|\|
	nvlist_lookup_uint64(resume_nvl, "offset", &off) != 0 \|\|
	nvlist_lookup_uint64(resume_nvl, "bytes", &bytes) != 0 \|\|
	nvlist_lookup_uint64(resume_nvl, "toguid",
	&resume_guid) != 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"provided resume token is corrupt"));
	ret = zfs_error(hdl, EZFS_FAULT, errbuf);
	goto out;
	}

	if (nvlist_lookup_uint64(saved_nvl, "toguid",
	&saved_guid)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"dataset's resume token is corrupt"));
	ret = zfs_error(hdl, EZFS_FAULT, errbuf);
	goto out;
	}

	if (resume_guid != saved_guid) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"provided resume token does not match dataset"));
	ret = zfs_error(hdl, EZFS_BADBACKUP, errbuf);
	goto out;
	}
	}

	(void) nvlist_remove_all(saved_nvl, "object");
	fnvlist_add_uint64(saved_nvl, "object", obj);

	(void) nvlist_remove_all(saved_nvl, "offset");
	fnvlist_add_uint64(saved_nvl, "offset", off);

	(void) nvlist_remove_all(saved_nvl, "bytes");
	fnvlist_add_uint64(saved_nvl, "bytes", bytes);

	(void) nvlist_remove_all(saved_nvl, "toname");
	fnvlist_add_string(saved_nvl, "toname", zhp->zfs_name);

	ret = zfs_send_resume_impl(hdl, flags, outfd, saved_nvl);

	out:
	fnvlist_free(saved_nvl);
	fnvlist_free(resume_nvl);
	return (ret);
	}

	/*
	* This function informs the target system that the recursive send is complete.
	* The record is also expected in the case of a send -p.
	*/
	static int
	send_conclusion_record(int fd, zio_cksum_t *zc)
	{
	dmu_replay_record_t drr = { 0 };
	drr.drr_type = DRR_END;
	if (zc != NULL)
	drr.drr_u.drr_end.drr_checksum = *zc;
	if (write(fd, &drr, sizeof (drr)) == -1) {
	return (errno);
	}
	return (0);
	}

	/*
	* This function is responsible for sending the records that contain the
	* necessary information for the target system's libzfs to be able to set the
	* properties of the filesystem being received, or to be able to prepare for
	* a recursive receive.
	*
	* The "zhp" argument is the handle of the snapshot we are sending
	* (the "tosnap"). The "from" argument is the short snapshot name (the part
	* after the @) of the incremental source.
	*/
	static int
	send_prelim_records(zfs_handle_t zhp, const char from, int fd,
	boolean_t gather_props, boolean_t recursive, boolean_t verbose,
	boolean_t dryrun, boolean_t raw, boolean_t replicate, boolean_t skipmissing,
	boolean_t backup, boolean_t holds, boolean_t props, boolean_t doall,
	nvlist_t fssp, avl_tree_t fsavlp)
	{
	int err = 0;
	char *packbuf = NULL;
	size_t buflen = 0;
	zio_cksum_t zc = { {0} };
	int featureflags = 0;
	/* name of filesystem/volume that contains snapshot we are sending */
	char tofs[ZFS_MAX_DATASET_NAME_LEN];
	/* short name of snap we are sending */
	const char *tosnap = "";

	char errbuf[ERRBUFLEN];
	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"warning: cannot send '%s'"), zhp->zfs_name);
	if (zhp->zfs_type == ZFS_TYPE_FILESYSTEM && zfs_prop_get_int(zhp,
	ZFS_PROP_VERSION) >= ZPL_VERSION_SA) {
	featureflags \|= DMU_BACKUP_FEATURE_SA_SPILL;
	}

	if (holds)
	featureflags \|= DMU_BACKUP_FEATURE_HOLDS;

	(void) strlcpy(tofs, zhp->zfs_name, ZFS_MAX_DATASET_NAME_LEN);
	char *at = strchr(tofs, '@');
	if (at != NULL) {
	*at = '\0';
	tosnap = at + 1;
	}

	if (gather_props) {
	nvlist_t *hdrnv = fnvlist_alloc();
	nvlist_t *fss = NULL;

	if (from != NULL)
	fnvlist_add_string(hdrnv, "fromsnap", from);
	fnvlist_add_string(hdrnv, "tosnap", tosnap);
	if (!recursive)
	fnvlist_add_boolean(hdrnv, "not_recursive");

	if (raw) {
	fnvlist_add_boolean(hdrnv, "raw");
	}

	if (gather_nvlist(zhp->zfs_hdl, tofs,
	from, tosnap, recursive, raw, doall, replicate, skipmissing,
	verbose, backup, holds, props, &fss, fsavlp) != 0) {
	return (zfs_error(zhp->zfs_hdl, EZFS_BADBACKUP,
	errbuf));
	}
	/*
	* Do not allow the size of the properties list to exceed
	* the limit
	*/
	if ((fnvlist_size(fss) + fnvlist_size(hdrnv)) >
	zhp->zfs_hdl->libzfs_max_nvlist) {
	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "warning: cannot send '%s': "
	"the size of the list of snapshots and properties "
	"is too large to be received successfully.\n"
	"Select a smaller number of snapshots to send.\n"),
	zhp->zfs_name);
	return (zfs_error(zhp->zfs_hdl, EZFS_NOSPC,
	errbuf));
	}
	fnvlist_add_nvlist(hdrnv, "fss", fss);
	VERIFY0(nvlist_pack(hdrnv, &packbuf, &buflen, NV_ENCODE_XDR,
	0));
	if (fssp != NULL) {
	*fssp = fss;
	} else {
	fnvlist_free(fss);
	}
	fnvlist_free(hdrnv);
	}

	if (!dryrun) {
	dmu_replay_record_t drr = { 0 };
	/* write first begin record */
	drr.drr_type = DRR_BEGIN;
	drr.drr_u.drr_begin.drr_magic = DMU_BACKUP_MAGIC;
	DMU_SET_STREAM_HDRTYPE(drr.drr_u.drr_begin.
	drr_versioninfo, DMU_COMPOUNDSTREAM);
	DMU_SET_FEATUREFLAGS(drr.drr_u.drr_begin.
	drr_versioninfo, featureflags);
	if (snprintf(drr.drr_u.drr_begin.drr_toname,
	sizeof (drr.drr_u.drr_begin.drr_toname), "%s@%s", tofs,
	tosnap) >= sizeof (drr.drr_u.drr_begin.drr_toname)) {
	return (zfs_error(zhp->zfs_hdl, EZFS_BADBACKUP,
	errbuf));
	}
	drr.drr_payloadlen = buflen;

	err = dump_record(&drr, packbuf, buflen, &zc, fd);
	free(packbuf);
	if (err != 0) {
	zfs_error_aux(zhp->zfs_hdl, "%s", strerror(err));
	return (zfs_error(zhp->zfs_hdl, EZFS_BADBACKUP,
	errbuf));
	}
	err = send_conclusion_record(fd, &zc);
	if (err != 0) {
	zfs_error_aux(zhp->zfs_hdl, "%s", strerror(err));
	return (zfs_error(zhp->zfs_hdl, EZFS_BADBACKUP,
	errbuf));
	}
	}
	return (0);
	}

	/*
	* Generate a send stream. The "zhp" argument is the filesystem/volume
	* that contains the snapshot to send. The "fromsnap" argument is the
	* short name (the part after the '@') of the snapshot that is the
	* incremental source to send from (if non-NULL). The "tosnap" argument
	* is the short name of the snapshot to send.
	*
	* The content of the send stream is the snapshot identified by
	* 'tosnap'. Incremental streams are requested in two ways:
	* - from the snapshot identified by "fromsnap" (if non-null) or
	* - from the origin of the dataset identified by zhp, which must
	* be a clone. In this case, "fromsnap" is null and "fromorigin"
	* is TRUE.
	*
	* The send stream is recursive (i.e. dumps a hierarchy of snapshots) and
	* uses a special header (with a hdrtype field of DMU_COMPOUNDSTREAM)
	* if "replicate" is set. If "doall" is set, dump all the intermediate
	* snapshots. The DMU_COMPOUNDSTREAM header is used in the "doall"
	* case too. If "props" is set, send properties.
	*
	* Pre-wrapped (cf. lzc_send_wrapper()).
	*/
	static int
	zfs_send_cb_impl(zfs_handle_t zhp, const char fromsnap, const char *tosnap,
	sendflags_t *flags, int outfd, snapfilter_cb_t filter_func,
	void cb_arg, nvlist_t *debugnvp)
	{
	char errbuf[ERRBUFLEN];
	send_dump_data_t sdd = { 0 };
	int err = 0;
	nvlist_t *fss = NULL;
	avl_tree_t *fsavl = NULL;
	static uint64_t holdseq;
	int spa_version;
	FILE *fout;

	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot send '%s'"), zhp->zfs_name);

	if (fromsnap && fromsnap[0] == '\0') {
	zfs_error_aux(zhp->zfs_hdl, dgettext(TEXT_DOMAIN,
	"zero-length incremental source"));
	return (zfs_error(zhp->zfs_hdl, EZFS_NOENT, errbuf));
	}

	if (fromsnap) {
	char full_fromsnap_name[ZFS_MAX_DATASET_NAME_LEN];
	if (snprintf(full_fromsnap_name, sizeof (full_fromsnap_name),
	"%s@%s", zhp->zfs_name, fromsnap) >=
	sizeof (full_fromsnap_name)) {
	err = EINVAL;
	goto stderr_out;
	}
	zfs_handle_t *fromsnapn = zfs_open(zhp->zfs_hdl,
	full_fromsnap_name, ZFS_TYPE_SNAPSHOT);
	if (fromsnapn == NULL) {
	err = -1;
	goto err_out;
	}
	zfs_close(fromsnapn);
	}

	if (flags->replicate \|\| flags->doall \|\| flags->props \|\|
	flags->holds \|\| flags->backup) {
	char full_tosnap_name[ZFS_MAX_DATASET_NAME_LEN];
	if (snprintf(full_tosnap_name, sizeof (full_tosnap_name),
	"%s@%s", zhp->zfs_name, tosnap) >=
	sizeof (full_tosnap_name)) {
	err = EINVAL;
	goto stderr_out;
	}
	zfs_handle_t *tosnap = zfs_open(zhp->zfs_hdl,
	full_tosnap_name, ZFS_TYPE_SNAPSHOT);
	if (tosnap == NULL) {
	err = -1;
	goto err_out;
	}
	err = send_prelim_records(tosnap, fromsnap, outfd,
	flags->replicate \|\| flags->props \|\| flags->holds,
	flags->replicate, flags->verbosity > 0, flags->dryrun,
	flags->raw, flags->replicate, flags->skipmissing,
	flags->backup, flags->holds, flags->props, flags->doall,
	&fss, &fsavl);
	zfs_close(tosnap);
	if (err != 0)
	goto err_out;
	}

	/* dump each stream */
	sdd.fromsnap = fromsnap;
	sdd.tosnap = tosnap;
	sdd.outfd = outfd;
	sdd.replicate = flags->replicate;
	sdd.doall = flags->doall;
	sdd.fromorigin = flags->fromorigin;
	sdd.fss = fss;
	sdd.fsavl = fsavl;
	sdd.verbosity = flags->verbosity;
	sdd.parsable = flags->parsable;
	sdd.progress = flags->progress;
	sdd.progressastitle = flags->progressastitle;
	sdd.dryrun = flags->dryrun;
	sdd.large_block = flags->largeblock;
	sdd.embed_data = flags->embed_data;
	sdd.compress = flags->compress;
	sdd.raw = flags->raw;
	sdd.holds = flags->holds;
	sdd.filter_cb = filter_func;
	sdd.filter_cb_arg = cb_arg;
	if (debugnvp)
	sdd.debugnv = *debugnvp;
	if (sdd.verbosity != 0 && sdd.dryrun)
	sdd.std_out = B_TRUE;
	fout = sdd.std_out ? stdout : stderr;

	/*
	* Some flags require that we place user holds on the datasets that are
	* being sent so they don't get destroyed during the send. We can skip
	* this step if the pool is imported read-only since the datasets cannot
	* be destroyed.
	*/
	if (!flags->dryrun && !zpool_get_prop_int(zfs_get_pool_handle(zhp),
	ZPOOL_PROP_READONLY, NULL) &&
	zfs_spa_version(zhp, &spa_version) == 0 &&
	spa_version >= SPA_VERSION_USERREFS &&
	(flags->doall \|\| flags->replicate)) {
	++holdseq;
	(void) snprintf(sdd.holdtag, sizeof (sdd.holdtag),
	".send-%d-%llu", getpid(), (u_longlong_t)holdseq);
	sdd.cleanup_fd = open(ZFS_DEV, O_RDWR \| O_CLOEXEC);
	if (sdd.cleanup_fd < 0) {
	err = errno;
	goto stderr_out;
	}
	sdd.snapholds = fnvlist_alloc();
	} else {
	sdd.cleanup_fd = -1;
	sdd.snapholds = NULL;
	}

	if (flags->verbosity != 0 \|\| sdd.snapholds != NULL) {
	/*
	* Do a verbose no-op dry run to get all the verbose output
	* or to gather snapshot hold's before generating any data,
	* then do a non-verbose real run to generate the streams.
	*/
	sdd.dryrun = B_TRUE;
	err = dump_filesystems(zhp, &sdd);

	if (err != 0)
	goto stderr_out;

	if (flags->verbosity != 0) {
	if (flags->parsable) {
	(void) fprintf(fout, "size\t%llu\n",
	(longlong_t)sdd.size);
	} else {
	char buf[16];
	zfs_nicebytes(sdd.size, buf, sizeof (buf));
	(void) fprintf(fout, dgettext(TEXT_DOMAIN,
	"total estimated size is %s\n"), buf);
	}
	}

	/* Ensure no snaps found is treated as an error. */
	if (!sdd.seento) {
	err = ENOENT;
	goto err_out;
	}

	/* Skip the second run if dryrun was requested. */
	if (flags->dryrun)
	goto err_out;

	if (sdd.snapholds != NULL) {
	err = zfs_hold_nvl(zhp, sdd.cleanup_fd, sdd.snapholds);
	if (err != 0)
	goto stderr_out;

	fnvlist_free(sdd.snapholds);
	sdd.snapholds = NULL;
	}

	sdd.dryrun = B_FALSE;
	sdd.verbosity = 0;
	}

	err = dump_filesystems(zhp, &sdd);
	fsavl_destroy(fsavl);
	fnvlist_free(fss);

	/* Ensure no snaps found is treated as an error. */
	if (err == 0 && !sdd.seento)
	err = ENOENT;

	if (sdd.cleanup_fd != -1) {
	VERIFY(0 == close(sdd.cleanup_fd));
	sdd.cleanup_fd = -1;
	}

	if (!flags->dryrun && (flags->replicate \|\| flags->doall \|\|
	flags->props \|\| flags->backup \|\| flags->holds)) {
	/*
	* write final end record. NB: want to do this even if
	* there was some error, because it might not be totally
	* failed.
	*/
	int err2 = send_conclusion_record(outfd, NULL);
	if (err2 != 0)
	return (zfs_standard_error(zhp->zfs_hdl, err2, errbuf));
	}

	return (err \|\| sdd.err);

	stderr_out:
	err = zfs_standard_error(zhp->zfs_hdl, err, errbuf);
	err_out:
	fsavl_destroy(fsavl);
	fnvlist_free(fss);
	fnvlist_free(sdd.snapholds);

	if (sdd.cleanup_fd != -1)
	VERIFY(0 == close(sdd.cleanup_fd));
	return (err);
	}

	struct zfs_send {
	zfs_handle_t *zhp;
	const char *fromsnap;
	const char *tosnap;
	sendflags_t *flags;
	snapfilter_cb_t *filter_func;
	void *cb_arg;
	nvlist_t **debugnvp;
	};

	static int
	zfs_send_cb(int outfd, void *arg)
	{
	struct zfs_send *zs = arg;
	return (zfs_send_cb_impl(zs->zhp, zs->fromsnap, zs->tosnap, zs->flags,
	outfd, zs->filter_func, zs->cb_arg, zs->debugnvp));
	}

	int
	zfs_send(zfs_handle_t zhp, const char fromsnap, const char *tosnap,
	sendflags_t *flags, int outfd, snapfilter_cb_t filter_func,
	void cb_arg, nvlist_t *debugnvp)
	{
	struct zfs_send arg = {
	.zhp = zhp,
	.fromsnap = fromsnap,
	.tosnap = tosnap,
	.flags = flags,
	.filter_func = filter_func,
	.cb_arg = cb_arg,
	.debugnvp = debugnvp,
	};
	return (lzc_send_wrapper(zfs_send_cb, outfd, &arg));
	}


	static zfs_handle_t *
	name_to_dir_handle(libzfs_handle_t hdl, const char snapname)
	{
	char dirname[ZFS_MAX_DATASET_NAME_LEN];
	(void) strlcpy(dirname, snapname, ZFS_MAX_DATASET_NAME_LEN);
	char *c = strchr(dirname, '@');
	if (c != NULL)
	*c = '\0';
	return (zfs_open(hdl, dirname, ZFS_TYPE_DATASET));
	}

	/*
	* Returns B_TRUE if earlier is an earlier snapshot in later's timeline; either
	* an earlier snapshot in the same filesystem, or a snapshot before later's
	* origin, or it's origin's origin, etc.
	*/
	static boolean_t
	snapshot_is_before(zfs_handle_t earlier, zfs_handle_t later)
	{
	boolean_t ret;
	uint64_t later_txg =
	(later->zfs_type == ZFS_TYPE_FILESYSTEM \|\|
	later->zfs_type == ZFS_TYPE_VOLUME ?
	UINT64_MAX : zfs_prop_get_int(later, ZFS_PROP_CREATETXG));
	uint64_t earlier_txg = zfs_prop_get_int(earlier, ZFS_PROP_CREATETXG);

	if (earlier_txg >= later_txg)
	return (B_FALSE);

	zfs_handle_t *earlier_dir = name_to_dir_handle(earlier->zfs_hdl,
	earlier->zfs_name);
	zfs_handle_t *later_dir = name_to_dir_handle(later->zfs_hdl,
	later->zfs_name);

	if (strcmp(earlier_dir->zfs_name, later_dir->zfs_name) == 0) {
	zfs_close(earlier_dir);
	zfs_close(later_dir);
	return (B_TRUE);
	}

	char clonename[ZFS_MAX_DATASET_NAME_LEN];
	if (zfs_prop_get(later_dir, ZFS_PROP_ORIGIN, clonename,
	ZFS_MAX_DATASET_NAME_LEN, NULL, NULL, 0, B_TRUE) != 0) {
	zfs_close(earlier_dir);
	zfs_close(later_dir);
	return (B_FALSE);
	}

	zfs_handle_t *origin = zfs_open(earlier->zfs_hdl, clonename,
	ZFS_TYPE_DATASET);
	uint64_t origin_txg = zfs_prop_get_int(origin, ZFS_PROP_CREATETXG);

	/*
	* If "earlier" is exactly the origin, then
	* snapshot_is_before(earlier, origin) will return false (because
	* they're the same).
	*/
	if (origin_txg == earlier_txg &&
	strcmp(origin->zfs_name, earlier->zfs_name) == 0) {
	zfs_close(earlier_dir);
	zfs_close(later_dir);
	zfs_close(origin);
	return (B_TRUE);
	}
	zfs_close(earlier_dir);
	zfs_close(later_dir);

	ret = snapshot_is_before(earlier, origin);
	zfs_close(origin);
	return (ret);
	}

	/*
	* The "zhp" argument is the handle of the dataset to send (typically a
	* snapshot). The "from" argument is the full name of the snapshot or
	* bookmark that is the incremental source.
	*
	* Pre-wrapped (cf. lzc_send_wrapper()).
	*/
	static int
	zfs_send_one_cb_impl(zfs_handle_t zhp, const char from, int fd,
	sendflags_t flags, const char redactbook)
	{
	int err;
	libzfs_handle_t *hdl = zhp->zfs_hdl;
	char *name = zhp->zfs_name;
	pthread_t ptid;
	progress_arg_t pa = { 0 };
	uint64_t size = 0;

	char errbuf[ERRBUFLEN];
	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"warning: cannot send '%s'"), name);

	if (from != NULL && strchr(from, '@')) {
	zfs_handle_t *from_zhp = zfs_open(hdl, from,
	ZFS_TYPE_DATASET);
	if (from_zhp == NULL)
	return (-1);
	if (!snapshot_is_before(from_zhp, zhp)) {
	zfs_close(from_zhp);
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"not an earlier snapshot from the same fs"));
	return (zfs_error(hdl, EZFS_CROSSTARGET, errbuf));
	}
	zfs_close(from_zhp);
	}

	if (redactbook != NULL) {
	char bookname[ZFS_MAX_DATASET_NAME_LEN];
	nvlist_t *redact_snaps;
	zfs_handle_t *book_zhp;
	char at, pound;
	int dsnamelen;

	pound = strchr(redactbook, '#');
	if (pound != NULL)
	redactbook = pound + 1;
	at = strchr(name, '@');
	if (at == NULL) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"cannot do a redacted send to a filesystem"));
	return (zfs_error(hdl, EZFS_BADTYPE, errbuf));
	}
	dsnamelen = at - name;
	if (snprintf(bookname, sizeof (bookname), "%.*s#%s",
	dsnamelen, name, redactbook)
	>= sizeof (bookname)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"invalid bookmark name"));
	return (zfs_error(hdl, EZFS_INVALIDNAME, errbuf));
	}
	book_zhp = zfs_open(hdl, bookname, ZFS_TYPE_BOOKMARK);
	if (book_zhp == NULL)
	return (-1);
	if (nvlist_lookup_nvlist(book_zhp->zfs_props,
	zfs_prop_to_name(ZFS_PROP_REDACT_SNAPS),
	&redact_snaps) != 0 \|\| redact_snaps == NULL) {
	zfs_close(book_zhp);
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"not a redaction bookmark"));
	return (zfs_error(hdl, EZFS_BADTYPE, errbuf));
	}
	zfs_close(book_zhp);
	}

	/*
	* Send fs properties
	*/
	if (flags->props \|\| flags->holds \|\| flags->backup) {
	/*
	* Note: the header generated by send_prelim_records()
	* assumes that the incremental source is in the same
	* filesystem/volume as the target (which is a requirement
	* when doing "zfs send -R"). But that isn't always the
	* case here (e.g. send from snap in origin, or send from
	* bookmark). We pass from=NULL, which will omit this
	* information from the prelim records; it isn't used
	* when receiving this type of stream.
	*/
	err = send_prelim_records(zhp, NULL, fd, B_TRUE, B_FALSE,
	flags->verbosity > 0, flags->dryrun, flags->raw,
	flags->replicate, B_FALSE, flags->backup, flags->holds,
	flags->props, flags->doall, NULL, NULL);
	if (err != 0)
	return (err);
	}

	/*
	* Perform size estimate if verbose was specified.
	*/
	if (flags->verbosity != 0 \|\| flags->progressastitle) {
	err = estimate_size(zhp, from, fd, flags, 0, 0, 0, redactbook,
	errbuf, &size);
	if (err != 0)
	return (err);
	}

	if (flags->dryrun)
	return (0);

	/*
	* If progress reporting is requested, spawn a new thread to poll
	* ZFS_IOC_SEND_PROGRESS at a regular interval.
	*/
	sigset_t oldmask;
	{
	pa.pa_zhp = zhp;
	pa.pa_fd = fd;
	pa.pa_parsable = flags->parsable;
	pa.pa_estimate = B_FALSE;
	pa.pa_verbosity = flags->verbosity;
	pa.pa_size = size;
	pa.pa_astitle = flags->progressastitle;
	pa.pa_progress = flags->progress;

	err = pthread_create(&ptid, NULL,
	send_progress_thread, &pa);
	if (err != 0) {
	zfs_error_aux(zhp->zfs_hdl, "%s", strerror(errno));
	return (zfs_error(zhp->zfs_hdl,
	EZFS_THREADCREATEFAILED, errbuf));
	}
	SEND_PROGRESS_THREAD_PARENT_BLOCK(&oldmask);
	}

	err = lzc_send_redacted(name, from, fd,
	lzc_flags_from_sendflags(flags), redactbook);

	if (send_progress_thread_exit(hdl, ptid, &oldmask))
	return (-1);

	if (err == 0 && (flags->props \|\| flags->holds \|\| flags->backup)) {
	/* Write the final end record. */
	err = send_conclusion_record(fd, NULL);
	if (err != 0)
	return (zfs_standard_error(hdl, err, errbuf));
	}
	if (err != 0) {
	switch (errno) {
	case EXDEV:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"not an earlier snapshot from the same fs"));
	return (zfs_error(hdl, EZFS_CROSSTARGET, errbuf));

	case ENOENT:
	case ESRCH:
	if (lzc_exists(name)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"incremental source (%s) does not exist"),
	from);
	}
	return (zfs_error(hdl, EZFS_NOENT, errbuf));

	case EACCES:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"dataset key must be loaded"));
	return (zfs_error(hdl, EZFS_CRYPTOFAILED, errbuf));

	case EBUSY:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"target is busy; if a filesystem, "
	"it must not be mounted"));
	return (zfs_error(hdl, EZFS_BUSY, errbuf));

	case EDQUOT:
	case EFAULT:
	case EFBIG:
	case EINVAL:
	case EIO:
	case ENOLINK:
	case ENOSPC:
	case ENOSTR:
	case ENXIO:
	case EPIPE:
	case ERANGE:
	case EROFS:
	zfs_error_aux(hdl, "%s", strerror(errno));
	return (zfs_error(hdl, EZFS_BADBACKUP, errbuf));

	default:
	return (zfs_standard_error(hdl, errno, errbuf));
	}
	}
	return (err != 0);
	}

	struct zfs_send_one {
	zfs_handle_t *zhp;
	const char *from;
	sendflags_t *flags;
	const char *redactbook;
	};

	static int
	zfs_send_one_cb(int fd, void *arg)
	{
	struct zfs_send_one *zso = arg;
	return (zfs_send_one_cb_impl(zso->zhp, zso->from, fd, zso->flags,
	zso->redactbook));
	}

	int
	zfs_send_one(zfs_handle_t zhp, const char from, int fd, sendflags_t *flags,
	const char *redactbook)
	{
	struct zfs_send_one zso = {
	.zhp = zhp,
	.from = from,
	.flags = flags,
	.redactbook = redactbook,
	};
	return (lzc_send_wrapper(zfs_send_one_cb, fd, &zso));
	}

	/*
	* Routines specific to "zfs recv"
	*/

	static int
	recv_read(libzfs_handle_t hdl, int fd, void buf, int ilen,
	boolean_t byteswap, zio_cksum_t *zc)
	{
	char *cp = buf;
	int rv;
	int len = ilen;

	do {
	rv = read(fd, cp, len);
	cp += rv;
	len -= rv;
	} while (rv > 0);

	if (rv < 0 \|\| len != 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"failed to read from stream"));
	return (zfs_error(hdl, EZFS_BADSTREAM, dgettext(TEXT_DOMAIN,
	"cannot receive")));
	}

	if (zc) {
	if (byteswap)
	fletcher_4_incremental_byteswap(buf, ilen, zc);
	else
	fletcher_4_incremental_native(buf, ilen, zc);
	}
	return (0);
	}

	static int
	recv_read_nvlist(libzfs_handle_t hdl, int fd, int len, nvlist_t *nvp,
	boolean_t byteswap, zio_cksum_t *zc)
	{
	char *buf;
	int err;

	buf = zfs_alloc(hdl, len);

	if (len > hdl->libzfs_max_nvlist) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "nvlist too large"));
	free(buf);
	return (ENOMEM);
	}

	err = recv_read(hdl, fd, buf, len, byteswap, zc);
	if (err != 0) {
	free(buf);
	return (err);
	}

	err = nvlist_unpack(buf, len, nvp, 0);
	free(buf);
	if (err != 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid "
	"stream (malformed nvlist)"));
	return (EINVAL);
	}
	return (0);
	}

	/*
	* Returns the grand origin (origin of origin of origin...) of a given handle.
	* If this dataset is not a clone, it simply returns a copy of the original
	* handle.
	*/
	static zfs_handle_t *
	recv_open_grand_origin(zfs_handle_t *zhp)
	{
	char origin[ZFS_MAX_DATASET_NAME_LEN];
	zprop_source_t src;
	zfs_handle_t *ozhp = zfs_handle_dup(zhp);

	while (ozhp != NULL) {
	if (zfs_prop_get(ozhp, ZFS_PROP_ORIGIN, origin,
	sizeof (origin), &src, NULL, 0, B_FALSE) != 0)
	break;

	(void) zfs_close(ozhp);
	ozhp = zfs_open(zhp->zfs_hdl, origin, ZFS_TYPE_FILESYSTEM);
	}

	return (ozhp);
	}

	static int
	recv_rename_impl(zfs_handle_t zhp, const char name, const char *newname)
	{
	int err;
	zfs_handle_t *ozhp = NULL;

	/*
	* Attempt to rename the dataset. If it fails with EACCES we have
	* attempted to rename the dataset outside of its encryption root.
	* Force the dataset to become an encryption root and try again.
	*/
	err = lzc_rename(name, newname);
	if (err == EACCES) {
	ozhp = recv_open_grand_origin(zhp);
	if (ozhp == NULL) {
	err = ENOENT;
	goto out;
	}

	err = lzc_change_key(ozhp->zfs_name, DCP_CMD_FORCE_NEW_KEY,
	NULL, NULL, 0);
	if (err != 0)
	goto out;

	err = lzc_rename(name, newname);
	}

	out:
	if (ozhp != NULL)
	zfs_close(ozhp);
	return (err);
	}

	static int
	recv_rename(libzfs_handle_t hdl, const char name, const char *tryname,
	int baselen, char newname, recvflags_t flags)
	{
	static int seq;
	int err;
	prop_changelist_t *clp = NULL;
	zfs_handle_t *zhp = NULL;

	zhp = zfs_open(hdl, name, ZFS_TYPE_DATASET);
	if (zhp == NULL) {
	err = -1;
	goto out;
	}
	clp = changelist_gather(zhp, ZFS_PROP_NAME, 0,
	flags->force ? MS_FORCE : 0);
	if (clp == NULL) {
	err = -1;
	goto out;
	}
	err = changelist_prefix(clp);
	if (err)
	goto out;

	if (tryname) {
	(void) strlcpy(newname, tryname, ZFS_MAX_DATASET_NAME_LEN);
	if (flags->verbose) {
	(void) printf("attempting rename %s to %s\n",
	name, newname);
	}
	err = recv_rename_impl(zhp, name, newname);
	if (err == 0)
	changelist_rename(clp, name, tryname);
	} else {
	err = ENOENT;
	}

	if (err != 0 && strncmp(name + baselen, "recv-", 5) != 0) {
	seq++;

	(void) snprintf(newname, ZFS_MAX_DATASET_NAME_LEN,
	"%.*srecv-%u-%u", baselen, name, getpid(), seq);

	if (flags->verbose) {
	(void) printf("failed - trying rename %s to %s\n",
	name, newname);
	}
	err = recv_rename_impl(zhp, name, newname);
	if (err == 0)
	changelist_rename(clp, name, newname);
	if (err && flags->verbose) {
	(void) printf("failed (%u) - "
	"will try again on next pass\n", errno);
	}
	err = EAGAIN;
	} else if (flags->verbose) {
	if (err == 0)
	(void) printf("success\n");
	else
	(void) printf("failed (%u)\n", errno);
	}

	(void) changelist_postfix(clp);

	out:
	if (clp != NULL)
	changelist_free(clp);
	if (zhp != NULL)
	zfs_close(zhp);

	return (err);
	}

	static int
	recv_promote(libzfs_handle_t hdl, const char fsname,
	const char origin_fsname, recvflags_t flags)
	{
	int err;
	zfs_cmd_t zc = {"\0"};
	zfs_handle_t zhp = NULL, ozhp = NULL;

	if (flags->verbose)
	(void) printf("promoting %s\n", fsname);

	(void) strlcpy(zc.zc_value, origin_fsname, sizeof (zc.zc_value));
	(void) strlcpy(zc.zc_name, fsname, sizeof (zc.zc_name));

	/*
	* Attempt to promote the dataset. If it fails with EACCES the
	* promotion would cause this dataset to leave its encryption root.
	* Force the origin to become an encryption root and try again.
	*/
	err = zfs_ioctl(hdl, ZFS_IOC_PROMOTE, &zc);
	if (err == EACCES) {
	zhp = zfs_open(hdl, fsname, ZFS_TYPE_DATASET);
	if (zhp == NULL) {
	err = -1;
	goto out;
	}

	ozhp = recv_open_grand_origin(zhp);
	if (ozhp == NULL) {
	err = -1;
	goto out;
	}

	err = lzc_change_key(ozhp->zfs_name, DCP_CMD_FORCE_NEW_KEY,
	NULL, NULL, 0);
	if (err != 0)
	goto out;

	err = zfs_ioctl(hdl, ZFS_IOC_PROMOTE, &zc);
	}

	out:
	if (zhp != NULL)
	zfs_close(zhp);
	if (ozhp != NULL)
	zfs_close(ozhp);

	return (err);
	}

	static int
	recv_destroy(libzfs_handle_t hdl, const char name, int baselen,
	char newname, recvflags_t flags)
	{
	int err = 0;
	prop_changelist_t *clp;
	zfs_handle_t *zhp;
	boolean_t defer = B_FALSE;
	int spa_version;

	zhp = zfs_open(hdl, name, ZFS_TYPE_DATASET);
	if (zhp == NULL)
	return (-1);
	zfs_type_t type = zfs_get_type(zhp);
	if (type == ZFS_TYPE_SNAPSHOT &&
	zfs_spa_version(zhp, &spa_version) == 0 &&
	spa_version >= SPA_VERSION_USERREFS)
	defer = B_TRUE;
	clp = changelist_gather(zhp, ZFS_PROP_NAME, 0,
	flags->force ? MS_FORCE : 0);
	zfs_close(zhp);
	if (clp == NULL)
	return (-1);

	err = changelist_prefix(clp);
	if (err)
	return (err);

	if (flags->verbose)
	(void) printf("attempting destroy %s\n", name);
	if (type == ZFS_TYPE_SNAPSHOT) {
	nvlist_t *nv = fnvlist_alloc();
	fnvlist_add_boolean(nv, name);
	err = lzc_destroy_snaps(nv, defer, NULL);
	fnvlist_free(nv);
	} else {
	err = lzc_destroy(name);
	}
	if (err == 0) {
	if (flags->verbose)
	(void) printf("success\n");
	changelist_remove(clp, name);
	}

	(void) changelist_postfix(clp);
	changelist_free(clp);

	/*
	* Deferred destroy might destroy the snapshot or only mark it to be
	* destroyed later, and it returns success in either case.
	*/
	if (err != 0 \|\| (defer && zfs_dataset_exists(hdl, name,
	ZFS_TYPE_SNAPSHOT))) {
	err = recv_rename(hdl, name, NULL, baselen, newname, flags);
	}

	return (err);
	}

	typedef struct guid_to_name_data {
	uint64_t guid;
	boolean_t bookmark_ok;
	char *name;
	char *skip;
	uint64_t *redact_snap_guids;
	uint64_t num_redact_snaps;
	} guid_to_name_data_t;

	static boolean_t
	redact_snaps_match(zfs_handle_t zhp, guid_to_name_data_t gtnd)
	{
	uint64_t *bmark_snaps;
	uint_t bmark_num_snaps;
	nvlist_t *nvl;
	if (zhp->zfs_type != ZFS_TYPE_BOOKMARK)
	return (B_FALSE);

	nvl = fnvlist_lookup_nvlist(zhp->zfs_props,
	zfs_prop_to_name(ZFS_PROP_REDACT_SNAPS));
	bmark_snaps = fnvlist_lookup_uint64_array(nvl, ZPROP_VALUE,
	&bmark_num_snaps);
	if (bmark_num_snaps != gtnd->num_redact_snaps)
	return (B_FALSE);
	int i = 0;
	for (; i < bmark_num_snaps; i++) {
	int j = 0;
	for (; j < bmark_num_snaps; j++) {
	if (bmark_snaps[i] == gtnd->redact_snap_guids[j])
	break;
	}
	if (j == bmark_num_snaps)
	break;
	}
	return (i == bmark_num_snaps);
	}

	static int
	guid_to_name_cb(zfs_handle_t zhp, void arg)
	{
	guid_to_name_data_t *gtnd = arg;
	const char *slash;
	int err;

	if (gtnd->skip != NULL &&
	(slash = strrchr(zhp->zfs_name, '/')) != NULL &&
	strcmp(slash + 1, gtnd->skip) == 0) {
	zfs_close(zhp);
	return (0);
	}

	if (zfs_prop_get_int(zhp, ZFS_PROP_GUID) == gtnd->guid &&
	(gtnd->num_redact_snaps == -1 \|\| redact_snaps_match(zhp, gtnd))) {
	(void) strcpy(gtnd->name, zhp->zfs_name);
	zfs_close(zhp);
	return (EEXIST);
	}

	err = zfs_iter_children_v2(zhp, 0, guid_to_name_cb, gtnd);
	if (err != EEXIST && gtnd->bookmark_ok)
	err = zfs_iter_bookmarks_v2(zhp, 0, guid_to_name_cb, gtnd);
	zfs_close(zhp);
	return (err);
	}

	/*
	* Attempt to find the local dataset associated with this guid. In the case of
	* multiple matches, we attempt to find the "best" match by searching
	* progressively larger portions of the hierarchy. This allows one to send a
	* tree of datasets individually and guarantee that we will find the source
	* guid within that hierarchy, even if there are multiple matches elsewhere.
	*
	* If num_redact_snaps is not -1, we attempt to find a redaction bookmark with
	* the specified number of redaction snapshots. If num_redact_snaps isn't 0 or
	* -1, then redact_snap_guids will be an array of the guids of the snapshots the
	* redaction bookmark was created with. If num_redact_snaps is -1, then we will
	* attempt to find a snapshot or bookmark (if bookmark_ok is passed) with the
	* given guid. Note that a redaction bookmark can be returned if
	* num_redact_snaps == -1.
	*/
	static int
	guid_to_name_redact_snaps(libzfs_handle_t hdl, const char parent,
	uint64_t guid, boolean_t bookmark_ok, uint64_t *redact_snap_guids,
	uint64_t num_redact_snaps, char *name)
	{
	char pname[ZFS_MAX_DATASET_NAME_LEN];
	guid_to_name_data_t gtnd;

	gtnd.guid = guid;
	gtnd.bookmark_ok = bookmark_ok;
	gtnd.name = name;
	gtnd.skip = NULL;
	gtnd.redact_snap_guids = redact_snap_guids;
	gtnd.num_redact_snaps = num_redact_snaps;

	/*
	* Search progressively larger portions of the hierarchy, starting
	* with the filesystem specified by 'parent'. This will
	* select the "most local" version of the origin snapshot in the case
	* that there are multiple matching snapshots in the system.
	*/
	(void) strlcpy(pname, parent, sizeof (pname));
	char *cp = strrchr(pname, '@');
	if (cp == NULL)
	cp = strchr(pname, '\0');
	for (; cp != NULL; cp = strrchr(pname, '/')) {
	/* Chop off the last component and open the parent */
	*cp = '\0';
	zfs_handle_t *zhp = make_dataset_handle(hdl, pname);

	if (zhp == NULL)
	continue;
	int err = guid_to_name_cb(zfs_handle_dup(zhp), &gtnd);
	if (err != EEXIST)
	err = zfs_iter_children_v2(zhp, 0, guid_to_name_cb,
	&gtnd);
	if (err != EEXIST && bookmark_ok)
	err = zfs_iter_bookmarks_v2(zhp, 0, guid_to_name_cb,
	&gtnd);
	zfs_close(zhp);
	if (err == EEXIST)
	return (0);

	/*
	* Remember the last portion of the dataset so we skip it next
	* time through (as we've already searched that portion of the
	* hierarchy).
	*/
	gtnd.skip = strrchr(pname, '/') + 1;
	}

	return (ENOENT);
	}

	static int
	guid_to_name(libzfs_handle_t hdl, const char parent, uint64_t guid,
	boolean_t bookmark_ok, char *name)
	{
	return (guid_to_name_redact_snaps(hdl, parent, guid, bookmark_ok, NULL,
	-1, name));
	}

	/*
	* Return +1 if guid1 is before guid2, 0 if they are the same, and -1 if
	* guid1 is after guid2.
	*/
	static int
	created_before(libzfs_handle_t hdl, avl_tree_t avl,
	uint64_t guid1, uint64_t guid2)
	{
	nvlist_t *nvfs;
	const char fsname = NULL, snapname = NULL;
	char buf[ZFS_MAX_DATASET_NAME_LEN];
	int rv;
	zfs_handle_t guid1hdl, guid2hdl;
	uint64_t create1, create2;

	if (guid2 == 0)
	return (0);
	if (guid1 == 0)
	return (1);

	nvfs = fsavl_find(avl, guid1, &snapname);
	fsname = fnvlist_lookup_string(nvfs, "name");
	(void) snprintf(buf, sizeof (buf), "%s@%s", fsname, snapname);
	guid1hdl = zfs_open(hdl, buf, ZFS_TYPE_SNAPSHOT);
	if (guid1hdl == NULL)
	return (-1);

	nvfs = fsavl_find(avl, guid2, &snapname);
	fsname = fnvlist_lookup_string(nvfs, "name");
	(void) snprintf(buf, sizeof (buf), "%s@%s", fsname, snapname);
	guid2hdl = zfs_open(hdl, buf, ZFS_TYPE_SNAPSHOT);
	if (guid2hdl == NULL) {
	zfs_close(guid1hdl);
	return (-1);
	}

	create1 = zfs_prop_get_int(guid1hdl, ZFS_PROP_CREATETXG);
	create2 = zfs_prop_get_int(guid2hdl, ZFS_PROP_CREATETXG);

	if (create1 < create2)
	rv = -1;
	else if (create1 > create2)
	rv = +1;
	else
	rv = 0;

	zfs_close(guid1hdl);
	zfs_close(guid2hdl);

	return (rv);
	}

	/*
	* This function reestablishes the hierarchy of encryption roots after a
	* recursive incremental receive has completed. This must be done after the
	* second call to recv_incremental_replication() has renamed and promoted all
	* sent datasets to their final locations in the dataset hierarchy.
	*/
	static int
	recv_fix_encryption_hierarchy(libzfs_handle_t hdl, const char top_zfs,
	nvlist_t *stream_nv)
	{
	int err;
	nvpair_t *fselem = NULL;
	nvlist_t *stream_fss;

	stream_fss = fnvlist_lookup_nvlist(stream_nv, "fss");

	while ((fselem = nvlist_next_nvpair(stream_fss, fselem)) != NULL) {
	zfs_handle_t *zhp = NULL;
	uint64_t crypt;
	nvlist_t snaps, props, *stream_nvfs = NULL;
	nvpair_t *snapel = NULL;
	boolean_t is_encroot, is_clone, stream_encroot;
	char *cp;
	const char *stream_keylocation = NULL;
	char keylocation[MAXNAMELEN];
	char fsname[ZFS_MAX_DATASET_NAME_LEN];

	keylocation[0] = '\0';
	stream_nvfs = fnvpair_value_nvlist(fselem);
	snaps = fnvlist_lookup_nvlist(stream_nvfs, "snaps");
	props = fnvlist_lookup_nvlist(stream_nvfs, "props");
	stream_encroot = nvlist_exists(stream_nvfs, "is_encroot");

	/* find a snapshot from the stream that exists locally */
	err = ENOENT;
	while ((snapel = nvlist_next_nvpair(snaps, snapel)) != NULL) {
	uint64_t guid;

	guid = fnvpair_value_uint64(snapel);
	err = guid_to_name(hdl, top_zfs, guid, B_FALSE,
	fsname);
	if (err == 0)
	break;
	}

	if (err != 0)
	continue;

	cp = strchr(fsname, '@');
	if (cp != NULL)
	*cp = '\0';

	zhp = zfs_open(hdl, fsname, ZFS_TYPE_DATASET);
	if (zhp == NULL) {
	err = ENOENT;
	goto error;
	}

	crypt = zfs_prop_get_int(zhp, ZFS_PROP_ENCRYPTION);
	is_clone = zhp->zfs_dmustats.dds_origin[0] != '\0';
	(void) zfs_crypto_get_encryption_root(zhp, &is_encroot, NULL);

	/* we don't need to do anything for unencrypted datasets */
	if (crypt == ZIO_CRYPT_OFF) {
	zfs_close(zhp);
	continue;
	}

	/*
	* If the dataset is flagged as an encryption root, was not
	* received as a clone and is not currently an encryption root,
	* force it to become one. Fixup the keylocation if necessary.
	*/
	if (stream_encroot) {
	if (!is_clone && !is_encroot) {
	err = lzc_change_key(fsname,
	DCP_CMD_FORCE_NEW_KEY, NULL, NULL, 0);
	if (err != 0) {
	zfs_close(zhp);
	goto error;
	}
	}

	stream_keylocation = fnvlist_lookup_string(props,
	zfs_prop_to_name(ZFS_PROP_KEYLOCATION));

	/*
	* Refresh the properties in case the call to
	* lzc_change_key() changed the value.
	*/
	zfs_refresh_properties(zhp);
	err = zfs_prop_get(zhp, ZFS_PROP_KEYLOCATION,
	keylocation, sizeof (keylocation), NULL, NULL,
	0, B_TRUE);
	if (err != 0) {
	zfs_close(zhp);
	goto error;
	}

	if (strcmp(keylocation, stream_keylocation) != 0) {
	err = zfs_prop_set(zhp,
	zfs_prop_to_name(ZFS_PROP_KEYLOCATION),
	stream_keylocation);
	if (err != 0) {
	zfs_close(zhp);
	goto error;
	}
	}
	}

	/*
	* If the dataset is not flagged as an encryption root and is
	* currently an encryption root, force it to inherit from its
	* parent. The root of a raw send should never be
	* force-inherited.
	*/
	if (!stream_encroot && is_encroot &&
	strcmp(top_zfs, fsname) != 0) {
	err = lzc_change_key(fsname, DCP_CMD_FORCE_INHERIT,
	NULL, NULL, 0);
	if (err != 0) {
	zfs_close(zhp);
	goto error;
	}
	}

	zfs_close(zhp);
	}

	return (0);

	error:
	return (err);
	}

	static int
	recv_incremental_replication(libzfs_handle_t hdl, const char tofs,
	recvflags_t flags, nvlist_t stream_nv, avl_tree_t *stream_avl,
	nvlist_t *renamed)
	{
	nvlist_t local_nv, deleted = NULL;
	avl_tree_t *local_avl;
	nvpair_t fselem, nextfselem;
	const char *fromsnap;
	char newname[ZFS_MAX_DATASET_NAME_LEN];
	char guidname[32];
	int error;
	boolean_t needagain, progress, recursive;
	const char s1, s2;

	fromsnap = fnvlist_lookup_string(stream_nv, "fromsnap");

	recursive = (nvlist_lookup_boolean(stream_nv, "not_recursive") ==
	ENOENT);

	if (flags->dryrun)
	return (0);

	again:
	needagain = progress = B_FALSE;

	deleted = fnvlist_alloc();

	if ((error = gather_nvlist(hdl, tofs, fromsnap, NULL,
	recursive, B_TRUE, B_FALSE, recursive, B_FALSE, B_FALSE, B_FALSE,
	B_FALSE, B_TRUE, &local_nv, &local_avl)) != 0)
	return (error);

	/*
	* Process deletes and renames
	*/
	for (fselem = nvlist_next_nvpair(local_nv, NULL);
	fselem; fselem = nextfselem) {
	nvlist_t nvfs, snaps;
	nvlist_t *stream_nvfs = NULL;
	nvpair_t snapelem, nextsnapelem;
	uint64_t fromguid = 0;
	uint64_t originguid = 0;
	uint64_t stream_originguid = 0;
	uint64_t parent_fromsnap_guid, stream_parent_fromsnap_guid;
	const char fsname, stream_fsname;

	nextfselem = nvlist_next_nvpair(local_nv, fselem);

	nvfs = fnvpair_value_nvlist(fselem);
	snaps = fnvlist_lookup_nvlist(nvfs, "snaps");
	fsname = fnvlist_lookup_string(nvfs, "name");
	parent_fromsnap_guid = fnvlist_lookup_uint64(nvfs,
	"parentfromsnap");
	(void) nvlist_lookup_uint64(nvfs, "origin", &originguid);

	/*
	* First find the stream's fs, so we can check for
	* a different origin (due to "zfs promote")
	*/
	for (snapelem = nvlist_next_nvpair(snaps, NULL);
	snapelem; snapelem = nvlist_next_nvpair(snaps, snapelem)) {
	uint64_t thisguid;

	thisguid = fnvpair_value_uint64(snapelem);
	stream_nvfs = fsavl_find(stream_avl, thisguid, NULL);

	if (stream_nvfs != NULL)
	break;
	}

	/* check for promote */
	(void) nvlist_lookup_uint64(stream_nvfs, "origin",
	&stream_originguid);
	if (stream_nvfs && originguid != stream_originguid) {
	switch (created_before(hdl, local_avl,
	stream_originguid, originguid)) {
	case 1: {
	/* promote it! */
	nvlist_t *origin_nvfs;
	const char *origin_fsname;

	origin_nvfs = fsavl_find(local_avl, originguid,
	NULL);
	origin_fsname = fnvlist_lookup_string(
	origin_nvfs, "name");
	error = recv_promote(hdl, fsname, origin_fsname,
	flags);
	if (error == 0)
	progress = B_TRUE;
	break;
	}
	default:
	break;
	case -1:
	fsavl_destroy(local_avl);
	fnvlist_free(local_nv);
	return (-1);
	}
	/*
	* We had/have the wrong origin, therefore our
	* list of snapshots is wrong. Need to handle
	* them on the next pass.
	*/
	needagain = B_TRUE;
	continue;
	}

	for (snapelem = nvlist_next_nvpair(snaps, NULL);
	snapelem; snapelem = nextsnapelem) {
	uint64_t thisguid;
	const char *stream_snapname;
	nvlist_t found, props;

	nextsnapelem = nvlist_next_nvpair(snaps, snapelem);

	thisguid = fnvpair_value_uint64(snapelem);
	found = fsavl_find(stream_avl, thisguid,
	&stream_snapname);

	/* check for delete */
	if (found == NULL) {
	char name[ZFS_MAX_DATASET_NAME_LEN];

	if (!flags->force)
	continue;

	(void) snprintf(name, sizeof (name), "%s@%s",
	fsname, nvpair_name(snapelem));

	error = recv_destroy(hdl, name,
	strlen(fsname)+1, newname, flags);
	if (error)
	needagain = B_TRUE;
	else
	progress = B_TRUE;
	sprintf(guidname, "%llu",
	(u_longlong_t)thisguid);
	nvlist_add_boolean(deleted, guidname);
	continue;
	}

	stream_nvfs = found;

	if (0 == nvlist_lookup_nvlist(stream_nvfs, "snapprops",
	&props) && 0 == nvlist_lookup_nvlist(props,
	stream_snapname, &props)) {
	zfs_cmd_t zc = {"\0"};

	zc.zc_cookie = B_TRUE; /* received */
	(void) snprintf(zc.zc_name, sizeof (zc.zc_name),
	"%s@%s", fsname, nvpair_name(snapelem));
	zcmd_write_src_nvlist(hdl, &zc, props);
	(void) zfs_ioctl(hdl,
	ZFS_IOC_SET_PROP, &zc);
	zcmd_free_nvlists(&zc);
	}

	/* check for different snapname */
	if (strcmp(nvpair_name(snapelem),
	stream_snapname) != 0) {
	char name[ZFS_MAX_DATASET_NAME_LEN];
	char tryname[ZFS_MAX_DATASET_NAME_LEN];

	(void) snprintf(name, sizeof (name), "%s@%s",
	fsname, nvpair_name(snapelem));
	(void) snprintf(tryname, sizeof (name), "%s@%s",
	fsname, stream_snapname);

	error = recv_rename(hdl, name, tryname,
	strlen(fsname)+1, newname, flags);
	if (error)
	needagain = B_TRUE;
	else
	progress = B_TRUE;
	}

	if (strcmp(stream_snapname, fromsnap) == 0)
	fromguid = thisguid;
	}

	/* check for delete */
	if (stream_nvfs == NULL) {
	if (!flags->force)
	continue;

	error = recv_destroy(hdl, fsname, strlen(tofs)+1,
	newname, flags);
	if (error)
	needagain = B_TRUE;
	else
	progress = B_TRUE;
	sprintf(guidname, "%llu",
	(u_longlong_t)parent_fromsnap_guid);
	nvlist_add_boolean(deleted, guidname);
	continue;
	}

	if (fromguid == 0) {
	if (flags->verbose) {
	(void) printf("local fs %s does not have "
	"fromsnap (%s in stream); must have "
	"been deleted locally; ignoring\n",
	fsname, fromsnap);
	}
	continue;
	}

	stream_fsname = fnvlist_lookup_string(stream_nvfs, "name");
	stream_parent_fromsnap_guid = fnvlist_lookup_uint64(
	stream_nvfs, "parentfromsnap");

	s1 = strrchr(fsname, '/');
	s2 = strrchr(stream_fsname, '/');

	/*
	* Check if we're going to rename based on parent guid change
	* and the current parent guid was also deleted. If it was then
	* rename will fail and is likely unneeded, so avoid this and
	* force an early retry to determine the new
	* parent_fromsnap_guid.
	*/
	if (stream_parent_fromsnap_guid != 0 &&
	parent_fromsnap_guid != 0 &&
	stream_parent_fromsnap_guid != parent_fromsnap_guid) {
	sprintf(guidname, "%llu",
	(u_longlong_t)parent_fromsnap_guid);
	if (nvlist_exists(deleted, guidname)) {
	progress = B_TRUE;
	needagain = B_TRUE;
	goto doagain;
	}
	}

	/*
	* Check for rename. If the exact receive path is specified, it
	* does not count as a rename, but we still need to check the
	* datasets beneath it.
	*/
	if ((stream_parent_fromsnap_guid != 0 &&
	parent_fromsnap_guid != 0 &&
	stream_parent_fromsnap_guid != parent_fromsnap_guid) \|\|
	((flags->isprefix \|\| strcmp(tofs, fsname) != 0) &&
	(s1 != NULL) && (s2 != NULL) && strcmp(s1, s2) != 0)) {
	nvlist_t *parent;
	char tryname[ZFS_MAX_DATASET_NAME_LEN];

	parent = fsavl_find(local_avl,
	stream_parent_fromsnap_guid, NULL);
	/*
	* NB: parent might not be found if we used the
	* tosnap for stream_parent_fromsnap_guid,
	* because the parent is a newly-created fs;
	* we'll be able to rename it after we recv the
	* new fs.
	*/
	if (parent != NULL) {
	const char *pname;

	pname = fnvlist_lookup_string(parent, "name");
	(void) snprintf(tryname, sizeof (tryname),
	"%s%s", pname, strrchr(stream_fsname, '/'));
	} else {
	tryname[0] = '\0';
	if (flags->verbose) {
	(void) printf("local fs %s new parent "
	"not found\n", fsname);
	}
	}

	newname[0] = '\0';

	error = recv_rename(hdl, fsname, tryname,
	strlen(tofs)+1, newname, flags);

	if (renamed != NULL && newname[0] != '\0') {
	fnvlist_add_boolean(renamed, newname);
	}

	if (error)
	needagain = B_TRUE;
	else
	progress = B_TRUE;
	}
	}

	doagain:
	fsavl_destroy(local_avl);
	fnvlist_free(local_nv);
	fnvlist_free(deleted);

	if (needagain && progress) {
	/* do another pass to fix up temporary names */
	if (flags->verbose)
	(void) printf("another pass:\n");
	goto again;
	}

	return (needagain \|\| error != 0);
	}

	static int
	zfs_receive_package(libzfs_handle_t hdl, int fd, const char destname,
	recvflags_t flags, dmu_replay_record_t drr, zio_cksum_t *zc,
	char *top_zfs, nvlist_t cmdprops)
	{
	nvlist_t *stream_nv = NULL;
	avl_tree_t *stream_avl = NULL;
	const char *fromsnap = NULL;
	const char *sendsnap = NULL;
	char *cp;
	char tofs[ZFS_MAX_DATASET_NAME_LEN];
	char sendfs[ZFS_MAX_DATASET_NAME_LEN];
	char errbuf[ERRBUFLEN];
	dmu_replay_record_t drre;
	int error;
	boolean_t anyerr = B_FALSE;
	boolean_t softerr = B_FALSE;
	boolean_t recursive, raw;

	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot receive"));

	assert(drr->drr_type == DRR_BEGIN);
	assert(drr->drr_u.drr_begin.drr_magic == DMU_BACKUP_MAGIC);
	assert(DMU_GET_STREAM_HDRTYPE(drr->drr_u.drr_begin.drr_versioninfo) ==
	DMU_COMPOUNDSTREAM);

	/*
	* Read in the nvlist from the stream.
	*/
	if (drr->drr_payloadlen != 0) {
	error = recv_read_nvlist(hdl, fd, drr->drr_payloadlen,
	&stream_nv, flags->byteswap, zc);
	if (error) {
	error = zfs_error(hdl, EZFS_BADSTREAM, errbuf);
	goto out;
	}
	}

	recursive = (nvlist_lookup_boolean(stream_nv, "not_recursive") ==
	ENOENT);
	raw = (nvlist_lookup_boolean(stream_nv, "raw") == 0);

	if (recursive && strchr(destname, '@')) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"cannot specify snapshot name for multi-snapshot stream"));
	error = zfs_error(hdl, EZFS_BADSTREAM, errbuf);
	goto out;
	}

	/*
	* Read in the end record and verify checksum.
	*/
	if (0 != (error = recv_read(hdl, fd, &drre, sizeof (drre),
	flags->byteswap, NULL)))
	goto out;
	if (flags->byteswap) {
	drre.drr_type = BSWAP_32(drre.drr_type);
	drre.drr_u.drr_end.drr_checksum.zc_word[0] =
	BSWAP_64(drre.drr_u.drr_end.drr_checksum.zc_word[0]);
	drre.drr_u.drr_end.drr_checksum.zc_word[1] =
	BSWAP_64(drre.drr_u.drr_end.drr_checksum.zc_word[1]);
	drre.drr_u.drr_end.drr_checksum.zc_word[2] =
	BSWAP_64(drre.drr_u.drr_end.drr_checksum.zc_word[2]);
	drre.drr_u.drr_end.drr_checksum.zc_word[3] =
	BSWAP_64(drre.drr_u.drr_end.drr_checksum.zc_word[3]);
	}
	if (drre.drr_type != DRR_END) {
	error = zfs_error(hdl, EZFS_BADSTREAM, errbuf);
	goto out;
	}
	if (!ZIO_CHECKSUM_EQUAL(drre.drr_u.drr_end.drr_checksum, *zc)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"incorrect header checksum"));
	error = zfs_error(hdl, EZFS_BADSTREAM, errbuf);
	goto out;
	}

	(void) nvlist_lookup_string(stream_nv, "fromsnap", &fromsnap);

	if (drr->drr_payloadlen != 0) {
	nvlist_t *stream_fss;

	stream_fss = fnvlist_lookup_nvlist(stream_nv, "fss");
	if ((stream_avl = fsavl_create(stream_fss)) == NULL) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"couldn't allocate avl tree"));
	error = zfs_error(hdl, EZFS_NOMEM, errbuf);
	goto out;
	}

	if (fromsnap != NULL && recursive) {
	nvlist_t *renamed = NULL;
	nvpair_t *pair = NULL;

	(void) strlcpy(tofs, destname, sizeof (tofs));
	if (flags->isprefix) {
	struct drr_begin *drrb = &drr->drr_u.drr_begin;
	int i;

	if (flags->istail) {
	cp = strrchr(drrb->drr_toname, '/');
	if (cp == NULL) {
	(void) strlcat(tofs, "/",
	sizeof (tofs));
	i = 0;
	} else {
	i = (cp - drrb->drr_toname);
	}
	} else {
	i = strcspn(drrb->drr_toname, "/@");
	}
	/* zfs_receive_one() will create_parents() */
	(void) strlcat(tofs, &drrb->drr_toname[i],
	sizeof (tofs));
	*strchr(tofs, '@') = '\0';
	}

	if (!flags->dryrun && !flags->nomount) {
	renamed = fnvlist_alloc();
	}

	softerr = recv_incremental_replication(hdl, tofs, flags,
	stream_nv, stream_avl, renamed);

	/* Unmount renamed filesystems before receiving. */
	while ((pair = nvlist_next_nvpair(renamed,
	pair)) != NULL) {
	zfs_handle_t *zhp;
	prop_changelist_t *clp = NULL;

	zhp = zfs_open(hdl, nvpair_name(pair),
	ZFS_TYPE_FILESYSTEM);
	if (zhp != NULL) {
	clp = changelist_gather(zhp,
	ZFS_PROP_MOUNTPOINT, 0,
	flags->forceunmount ? MS_FORCE : 0);
	zfs_close(zhp);
	if (clp != NULL) {
	softerr \|=
	changelist_prefix(clp);
	changelist_free(clp);
	}
	}
	}

	fnvlist_free(renamed);
	}
	}

	/*
	* Get the fs specified by the first path in the stream (the top level
	* specified by 'zfs send') and pass it to each invocation of
	* zfs_receive_one().
	*/
	(void) strlcpy(sendfs, drr->drr_u.drr_begin.drr_toname,
	sizeof (sendfs));
	if ((cp = strchr(sendfs, '@')) != NULL) {
	*cp = '\0';
	/*
	* Find the "sendsnap", the final snapshot in a replication
	* stream. zfs_receive_one() handles certain errors
	* differently, depending on if the contained stream is the
	* last one or not.
	*/
	sendsnap = (cp + 1);
	}

	/* Finally, receive each contained stream */
	do {
	/*
	* we should figure out if it has a recoverable
	* error, in which case do a recv_skip() and drive on.
	* Note, if we fail due to already having this guid,
	* zfs_receive_one() will take care of it (ie,
	* recv_skip() and return 0).
	*/
	error = zfs_receive_impl(hdl, destname, NULL, flags, fd,
	sendfs, stream_nv, stream_avl, top_zfs, sendsnap, cmdprops);
	if (error == ENODATA) {
	error = 0;
	break;
	}
	anyerr \|= error;
	} while (error == 0);

	if (drr->drr_payloadlen != 0 && recursive && fromsnap != NULL) {
	/*
	* Now that we have the fs's they sent us, try the
	* renames again.
	*/
	softerr = recv_incremental_replication(hdl, tofs, flags,
	stream_nv, stream_avl, NULL);
	}

	if (raw && softerr == 0 && *top_zfs != NULL) {
	softerr = recv_fix_encryption_hierarchy(hdl, *top_zfs,
	stream_nv);
	}

	out:
	fsavl_destroy(stream_avl);
	fnvlist_free(stream_nv);
	if (softerr)
	error = -2;
	if (anyerr)
	error = -1;
	return (error);
	}

	static void
	trunc_prop_errs(int truncated)
	{
	ASSERT(truncated != 0);

	if (truncated == 1)
	(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
	"1 more property could not be set\n"));
	else
	(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
	"%d more properties could not be set\n"), truncated);
	}

	static int
	recv_skip(libzfs_handle_t *hdl, int fd, boolean_t byteswap)
	{
	dmu_replay_record_t *drr;
	void *buf = zfs_alloc(hdl, SPA_MAXBLOCKSIZE);
	uint64_t payload_size;
	char errbuf[ERRBUFLEN];

	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot receive"));

	/* XXX would be great to use lseek if possible... */
	drr = buf;

	while (recv_read(hdl, fd, drr, sizeof (dmu_replay_record_t),
	byteswap, NULL) == 0) {
	if (byteswap)
	drr->drr_type = BSWAP_32(drr->drr_type);

	switch (drr->drr_type) {
	case DRR_BEGIN:
	if (drr->drr_payloadlen != 0) {
	(void) recv_read(hdl, fd, buf,
	drr->drr_payloadlen, B_FALSE, NULL);
	}
	break;

	case DRR_END:
	free(buf);
	return (0);

	case DRR_OBJECT:
	if (byteswap) {
	drr->drr_u.drr_object.drr_bonuslen =
	BSWAP_32(drr->drr_u.drr_object.
	drr_bonuslen);
	drr->drr_u.drr_object.drr_raw_bonuslen =
	BSWAP_32(drr->drr_u.drr_object.
	drr_raw_bonuslen);
	}

	payload_size =
	DRR_OBJECT_PAYLOAD_SIZE(&drr->drr_u.drr_object);
	(void) recv_read(hdl, fd, buf, payload_size,
	B_FALSE, NULL);
	break;

	case DRR_WRITE:
	if (byteswap) {
	drr->drr_u.drr_write.drr_logical_size =
	BSWAP_64(
	drr->drr_u.drr_write.drr_logical_size);
	drr->drr_u.drr_write.drr_compressed_size =
	BSWAP_64(
	drr->drr_u.drr_write.drr_compressed_size);
	}
	payload_size =
	DRR_WRITE_PAYLOAD_SIZE(&drr->drr_u.drr_write);
	assert(payload_size <= SPA_MAXBLOCKSIZE);
	(void) recv_read(hdl, fd, buf,
	payload_size, B_FALSE, NULL);
	break;
	case DRR_SPILL:
	if (byteswap) {
	drr->drr_u.drr_spill.drr_length =
	BSWAP_64(drr->drr_u.drr_spill.drr_length);
	drr->drr_u.drr_spill.drr_compressed_size =
	BSWAP_64(drr->drr_u.drr_spill.
	drr_compressed_size);
	}

	payload_size =
	DRR_SPILL_PAYLOAD_SIZE(&drr->drr_u.drr_spill);
	(void) recv_read(hdl, fd, buf, payload_size,
	B_FALSE, NULL);
	break;
	case DRR_WRITE_EMBEDDED:
	if (byteswap) {
	drr->drr_u.drr_write_embedded.drr_psize =
	BSWAP_32(drr->drr_u.drr_write_embedded.
	drr_psize);
	}
	(void) recv_read(hdl, fd, buf,
	P2ROUNDUP(drr->drr_u.drr_write_embedded.drr_psize,
	8), B_FALSE, NULL);
	break;
	case DRR_OBJECT_RANGE:
	case DRR_WRITE_BYREF:
	case DRR_FREEOBJECTS:
	case DRR_FREE:
	break;

	default:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"invalid record type"));
	free(buf);
	return (zfs_error(hdl, EZFS_BADSTREAM, errbuf));
	}
	}

	free(buf);
	return (-1);
	}

	static void
	recv_ecksum_set_aux(libzfs_handle_t hdl, const char target_snap,
	boolean_t resumable, boolean_t checksum)
	{
	char target_fs[ZFS_MAX_DATASET_NAME_LEN];

	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, (checksum ?
	"checksum mismatch" : "incomplete stream")));

	if (!resumable)
	return;
	(void) strlcpy(target_fs, target_snap, sizeof (target_fs));
	*strchr(target_fs, '@') = '\0';
	zfs_handle_t *zhp = zfs_open(hdl, target_fs,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME);
	if (zhp == NULL)
	return;

	char token_buf[ZFS_MAXPROPLEN];
	int error = zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN,
	token_buf, sizeof (token_buf),
	NULL, NULL, 0, B_TRUE);
	if (error == 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"checksum mismatch or incomplete stream.\n"
	"Partially received snapshot is saved.\n"
	"A resuming stream can be generated on the sending "
	"system by running:\n"
	" zfs send -t %s"),
	token_buf);
	}
	zfs_close(zhp);
	}

	/*
	* Prepare a new nvlist of properties that are to override (-o) or be excluded
	* (-x) from the received dataset
	* recvprops: received properties from the send stream
	* cmdprops: raw input properties from command line
	* origprops: properties, both locally-set and received, currently set on the
	* target dataset if it exists, NULL otherwise.
	* oxprops: valid output override (-o) and excluded (-x) properties
	*/
	static int
	zfs_setup_cmdline_props(libzfs_handle_t *hdl, zfs_type_t type,
	char *fsname, boolean_t zoned, boolean_t recursive, boolean_t newfs,
	boolean_t raw, boolean_t toplevel, nvlist_t recvprops, nvlist_t cmdprops,
	nvlist_t origprops, nvlist_t oxprops, uint8_t *wkeydata_out,
	uint_t wkeylen_out, const char errbuf)
	{
	nvpair_t *nvp;
	nvlist_t oprops, voprops;
	zfs_handle_t *zhp = NULL;
	zpool_handle_t *zpool_hdl = NULL;
	char *cp;
	int ret = 0;
	char namebuf[ZFS_MAX_DATASET_NAME_LEN];

	if (nvlist_empty(cmdprops))
	return (0); /* No properties to override or exclude */

	*oxprops = fnvlist_alloc();
	oprops = fnvlist_alloc();

	strlcpy(namebuf, fsname, ZFS_MAX_DATASET_NAME_LEN);

	/*
	* Get our dataset handle. The target dataset may not exist yet.
	*/
	if (zfs_dataset_exists(hdl, namebuf, ZFS_TYPE_DATASET)) {
	zhp = zfs_open(hdl, namebuf, ZFS_TYPE_DATASET);
	if (zhp == NULL) {
	ret = -1;
	goto error;
	}
	}

	/* open the zpool handle */
	cp = strchr(namebuf, '/');
	if (cp != NULL)
	*cp = '\0';
	zpool_hdl = zpool_open(hdl, namebuf);
	if (zpool_hdl == NULL) {
	ret = -1;
	goto error;
	}

	/* restore namebuf to match fsname for later use */
	if (cp != NULL)
	*cp = '/';

	/*
	* first iteration: process excluded (-x) properties now and gather
	* added (-o) properties to be later processed by zfs_valid_proplist()
	*/
	nvp = NULL;
	while ((nvp = nvlist_next_nvpair(cmdprops, nvp)) != NULL) {
	const char *name = nvpair_name(nvp);
	zfs_prop_t prop = zfs_name_to_prop(name);

	/*
	* It turns out, if we don't normalize "aliased" names
	* e.g. compress= against the "real" names (e.g. compression)
	* here, then setting/excluding them does not work as
	* intended.
	*
	* But since user-defined properties wouldn't have a valid
	* mapping here, we do this conditional dance.
	*/
	const char *newname = name;
	if (prop >= ZFS_PROP_TYPE)
	newname = zfs_prop_to_name(prop);

	/* "origin" is processed separately, don't handle it here */
	if (prop == ZFS_PROP_ORIGIN)
	continue;

	/* raw streams can't override encryption properties */
	if ((zfs_prop_encryption_key_param(prop) \|\|
	prop == ZFS_PROP_ENCRYPTION) && raw) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"encryption property '%s' cannot "
	"be set or excluded for raw streams."), name);
	ret = zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}

	/*
	* For plain replicated send, we can ignore encryption
	* properties other than first stream
	*/
	if ((zfs_prop_encryption_key_param(prop) \|\| prop ==
	ZFS_PROP_ENCRYPTION) && !newfs && recursive && !raw) {
	continue;
	}

	/* incremental streams can only exclude encryption properties */
	if ((zfs_prop_encryption_key_param(prop) \|\|
	prop == ZFS_PROP_ENCRYPTION) && !newfs &&
	nvpair_type(nvp) != DATA_TYPE_BOOLEAN) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"encryption property '%s' cannot "
	"be set for incremental streams."), name);
	ret = zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}

	switch (nvpair_type(nvp)) {
	case DATA_TYPE_BOOLEAN: /* -x property */
	/*
	* DATA_TYPE_BOOLEAN is the way we're asked to "exclude"
	* a property: this is done by forcing an explicit
	* inherit on the destination so the effective value is
	* not the one we received from the send stream.
	*/
	if (!zfs_prop_valid_for_type(prop, type, B_FALSE) &&
	!zfs_prop_user(name)) {
	(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
	"Warning: %s: property '%s' does not "
	"apply to datasets of this type\n"),
	fsname, name);
	continue;
	}
	/*
	* We do this only if the property is not already
	* locally-set, in which case its value will take
	* priority over the received anyway.
	*/
	if (nvlist_exists(origprops, newname)) {
	nvlist_t *attrs;
	const char *source = NULL;

	attrs = fnvlist_lookup_nvlist(origprops,
	newname);
	if (nvlist_lookup_string(attrs,
	ZPROP_SOURCE, &source) == 0 &&
	strcmp(source, ZPROP_SOURCE_VAL_RECVD) != 0)
	continue;
	}
	/*
	* We can't force an explicit inherit on non-inheritable
	* properties: if we're asked to exclude this kind of
	* values we remove them from "recvprops" input nvlist.
	*/
	if (!zfs_prop_user(name) && /* can be inherited too */
	!zfs_prop_inheritable(prop) &&
	nvlist_exists(recvprops, newname))
	fnvlist_remove(recvprops, newname);
	else
	fnvlist_add_boolean(*oxprops, newname);
	break;
	case DATA_TYPE_STRING: /* -o property=value */
	/*
	* we're trying to override a property that does not
	* make sense for this type of dataset, but we don't
	* want to fail if the receive is recursive: this comes
	* in handy when the send stream contains, for
	* instance, a child ZVOL and we're trying to receive
	* it with "-o atime=on"
	*/
	if (!zfs_prop_valid_for_type(prop, type, B_FALSE) &&
	!zfs_prop_user(name)) {
	if (recursive)
	continue;
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property '%s' does not apply to datasets "
	"of this type"), name);
	ret = zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}
	fnvlist_add_string(oprops, newname,
	fnvpair_value_string(nvp));
	break;
	default:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property '%s' must be a string or boolean"), name);
	ret = zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}
	}

	if (toplevel) {
	/* convert override strings properties to native */
	if ((voprops = zfs_valid_proplist(hdl, ZFS_TYPE_DATASET,
	oprops, zoned, zhp, zpool_hdl, B_FALSE, errbuf)) == NULL) {
	ret = zfs_error(hdl, EZFS_BADPROP, errbuf);
	goto error;
	}

	/*
	* zfs_crypto_create() requires the parent name. Get it
	* by truncating the fsname copy stored in namebuf.
	*/
	cp = strrchr(namebuf, '/');
	if (cp != NULL)
	*cp = '\0';

	if (!raw && !(!newfs && recursive) &&
	zfs_crypto_create(hdl, namebuf, voprops, NULL,
	B_FALSE, wkeydata_out, wkeylen_out) != 0) {
	fnvlist_free(voprops);
	ret = zfs_error(hdl, EZFS_CRYPTOFAILED, errbuf);
	goto error;
	}

	/* second pass: process "-o" properties */
	fnvlist_merge(*oxprops, voprops);
	fnvlist_free(voprops);
	} else {
	/* override props on child dataset are inherited */
	nvp = NULL;
	while ((nvp = nvlist_next_nvpair(oprops, nvp)) != NULL) {
	const char *name = nvpair_name(nvp);
	fnvlist_add_boolean(*oxprops, name);
	}
	}

	error:
	if (zhp != NULL)
	zfs_close(zhp);
	if (zpool_hdl != NULL)
	zpool_close(zpool_hdl);
	fnvlist_free(oprops);
	return (ret);
	}

	/*
	* Restores a backup of tosnap from the file descriptor specified by infd.
	*/
	static int
	zfs_receive_one(libzfs_handle_t hdl, int infd, const char tosnap,
	const char originsnap, recvflags_t flags, dmu_replay_record_t *drr,
	dmu_replay_record_t drr_noswap, const char sendfs, nvlist_t *stream_nv,
	avl_tree_t stream_avl, char *top_zfs,
	const char finalsnap, nvlist_t cmdprops)
	{
	struct timespec begin_time;
	int ioctl_err, ioctl_errno, err;
	char *cp;
	struct drr_begin *drrb = &drr->drr_u.drr_begin;
	char errbuf[ERRBUFLEN];
	const char *chopprefix;
	boolean_t newfs = B_FALSE;
	boolean_t stream_wantsnewfs, stream_resumingnewfs;
	boolean_t newprops = B_FALSE;
	uint64_t read_bytes = 0;
	uint64_t errflags = 0;
	uint64_t parent_snapguid = 0;
	prop_changelist_t *clp = NULL;
	nvlist_t *snapprops_nvlist = NULL;
	nvlist_t *snapholds_nvlist = NULL;
	zprop_errflags_t prop_errflags;
	nvlist_t *prop_errors = NULL;
	boolean_t recursive;
	const char *snapname = NULL;
	char destsnap[MAXPATHLEN * 2];
	char origin[MAXNAMELEN] = {0};
	char name[MAXPATHLEN];
	char tmp_keylocation[MAXNAMELEN] = {0};
	nvlist_t rcvprops = NULL; / props received from the send stream */
	nvlist_t oxprops = NULL; / override (-o) and exclude (-x) props */
	nvlist_t origprops = NULL; / original props (if destination exists) */
	zfs_type_t type = ZFS_TYPE_INVALID;
	boolean_t toplevel = B_FALSE;
	boolean_t zoned = B_FALSE;
	boolean_t hastoken = B_FALSE;
	boolean_t redacted;
	uint8_t *wkeydata = NULL;
	uint_t wkeylen = 0;

	#ifndef CLOCK_MONOTONIC_RAW
	#define CLOCK_MONOTONIC_RAW CLOCK_MONOTONIC
	#endif
	clock_gettime(CLOCK_MONOTONIC_RAW, &begin_time);

	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot receive"));

	recursive = (nvlist_lookup_boolean(stream_nv, "not_recursive") ==
	ENOENT);

	/* Did the user request holds be skipped via zfs recv -k? */
	boolean_t holds = flags->holds && !flags->skipholds;

	if (stream_avl != NULL) {
	const char *keylocation = NULL;
	nvlist_t *lookup = NULL;
	nvlist_t *fs = fsavl_find(stream_avl, drrb->drr_toguid,
	&snapname);

	(void) nvlist_lookup_uint64(fs, "parentfromsnap",
	&parent_snapguid);
	err = nvlist_lookup_nvlist(fs, "props", &rcvprops);
	if (err) {
	rcvprops = fnvlist_alloc();
	newprops = B_TRUE;
	}

	/*
	* The keylocation property may only be set on encryption roots,
	* but this dataset might not become an encryption root until
	* recv_fix_encryption_hierarchy() is called. That function
	* will fixup the keylocation anyway, so we temporarily unset
	* the keylocation for now to avoid any errors from the receive
	* ioctl.
	*/
	err = nvlist_lookup_string(rcvprops,
	zfs_prop_to_name(ZFS_PROP_KEYLOCATION), &keylocation);
	if (err == 0) {
	strlcpy(tmp_keylocation, keylocation, MAXNAMELEN);
	(void) nvlist_remove_all(rcvprops,
	zfs_prop_to_name(ZFS_PROP_KEYLOCATION));
	}

	if (flags->canmountoff) {
	fnvlist_add_uint64(rcvprops,
	zfs_prop_to_name(ZFS_PROP_CANMOUNT), 0);
	} else if (newprops) { /* nothing in rcvprops, eliminate it */
	fnvlist_free(rcvprops);
	rcvprops = NULL;
	newprops = B_FALSE;
	}
	if (0 == nvlist_lookup_nvlist(fs, "snapprops", &lookup)) {
	snapprops_nvlist = fnvlist_lookup_nvlist(lookup,
	snapname);
	}
	if (holds) {
	if (0 == nvlist_lookup_nvlist(fs, "snapholds",
	&lookup)) {
	snapholds_nvlist = fnvlist_lookup_nvlist(
	lookup, snapname);
	}
	}
	}

	cp = NULL;

	/*
	* Determine how much of the snapshot name stored in the stream
	* we are going to tack on to the name they specified on the
	* command line, and how much we are going to chop off.
	*
	* If they specified a snapshot, chop the entire name stored in
	* the stream.
	*/
	if (flags->istail) {
	/*
	* A filesystem was specified with -e. We want to tack on only
	* the tail of the sent snapshot path.
	*/
	if (strchr(tosnap, '@')) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid "
	"argument - snapshot not allowed with -e"));
	err = zfs_error(hdl, EZFS_INVALIDNAME, errbuf);
	goto out;
	}

	chopprefix = strrchr(sendfs, '/');

	if (chopprefix == NULL) {
	/*
	* The tail is the poolname, so we need to
	* prepend a path separator.
	*/
	int len = strlen(drrb->drr_toname);
	cp = umem_alloc(len + 2, UMEM_NOFAIL);
	cp[0] = '/';
	(void) strcpy(&cp[1], drrb->drr_toname);
	chopprefix = cp;
	} else {
	chopprefix = drrb->drr_toname + (chopprefix - sendfs);
	}
	} else if (flags->isprefix) {
	/*
	* A filesystem was specified with -d. We want to tack on
	* everything but the first element of the sent snapshot path
	* (all but the pool name).
	*/
	if (strchr(tosnap, '@')) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid "
	"argument - snapshot not allowed with -d"));
	err = zfs_error(hdl, EZFS_INVALIDNAME, errbuf);
	goto out;
	}

	chopprefix = strchr(drrb->drr_toname, '/');
	if (chopprefix == NULL)
	chopprefix = strchr(drrb->drr_toname, '@');
	} else if (strchr(tosnap, '@') == NULL) {
	/*
	* If a filesystem was specified without -d or -e, we want to
	* tack on everything after the fs specified by 'zfs send'.
	*/
	chopprefix = drrb->drr_toname + strlen(sendfs);
	} else {
	/* A snapshot was specified as an exact path (no -d or -e). */
	if (recursive) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"cannot specify snapshot name for multi-snapshot "
	"stream"));
	err = zfs_error(hdl, EZFS_BADSTREAM, errbuf);
	goto out;
	}
	chopprefix = drrb->drr_toname + strlen(drrb->drr_toname);
	}

	ASSERT(strstr(drrb->drr_toname, sendfs) == drrb->drr_toname);
	ASSERT(chopprefix > drrb->drr_toname \|\| strchr(sendfs, '/') == NULL);
	ASSERT(chopprefix <= drrb->drr_toname + strlen(drrb->drr_toname) \|\|
	strchr(sendfs, '/') == NULL);
	ASSERT(chopprefix[0] == '/' \|\| chopprefix[0] == '@' \|\|
	chopprefix[0] == '\0');

	/*
	* Determine name of destination snapshot.
	*/
	(void) strlcpy(destsnap, tosnap, sizeof (destsnap));
	(void) strlcat(destsnap, chopprefix, sizeof (destsnap));
	if (cp != NULL)
	umem_free(cp, strlen(cp) + 1);
	if (!zfs_name_valid(destsnap, ZFS_TYPE_SNAPSHOT)) {
	err = zfs_error(hdl, EZFS_INVALIDNAME, errbuf);
	goto out;
	}

	/*
	* Determine the name of the origin snapshot.
	*/
	if (originsnap) {
	(void) strlcpy(origin, originsnap, sizeof (origin));
	if (flags->verbose)
	(void) printf("using provided clone origin %s\n",
	origin);
	} else if (drrb->drr_flags & DRR_FLAG_CLONE) {
	if (guid_to_name(hdl, destsnap,
	drrb->drr_fromguid, B_FALSE, origin) != 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"local origin for clone %s does not exist"),
	destsnap);
	err = zfs_error(hdl, EZFS_NOENT, errbuf);
	goto out;
	}
	if (flags->verbose)
	(void) printf("found clone origin %s\n", origin);
	}

	if ((DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
	DMU_BACKUP_FEATURE_DEDUP)) {
	(void) fprintf(stderr,
	gettext("ERROR: \"zfs receive\" no longer supports "
	"deduplicated send streams. Use\n"
	"the \"zstream redup\" command to convert this stream "
	"to a regular,\n"
	"non-deduplicated stream.\n"));
	err = zfs_error(hdl, EZFS_NOTSUP, errbuf);
	goto out;
	}

	boolean_t resuming = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
	DMU_BACKUP_FEATURE_RESUMING;
	boolean_t raw = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
	DMU_BACKUP_FEATURE_RAW;
	boolean_t embedded = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
	DMU_BACKUP_FEATURE_EMBED_DATA;
	stream_wantsnewfs = (drrb->drr_fromguid == 0 \|\|
	(drrb->drr_flags & DRR_FLAG_CLONE) \|\| originsnap) && !resuming;
	stream_resumingnewfs = (drrb->drr_fromguid == 0 \|\|
	(drrb->drr_flags & DRR_FLAG_CLONE) \|\| originsnap) && resuming;

	if (stream_wantsnewfs) {
	/*
	* if the parent fs does not exist, look for it based on
	* the parent snap GUID
	*/
	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot receive new filesystem stream"));

	(void) strlcpy(name, destsnap, sizeof (name));
	cp = strrchr(name, '/');
	if (cp)
	*cp = '\0';
	if (cp &&
	!zfs_dataset_exists(hdl, name, ZFS_TYPE_DATASET)) {
	char suffix[ZFS_MAX_DATASET_NAME_LEN];
	(void) strlcpy(suffix, strrchr(destsnap, '/'),
	sizeof (suffix));
	if (guid_to_name(hdl, name, parent_snapguid,
	B_FALSE, destsnap) == 0) {
	*strchr(destsnap, '@') = '\0';
	(void) strlcat(destsnap, suffix,
	sizeof (destsnap));
	}
	}
	} else {
	/*
	* If the fs does not exist, look for it based on the
	* fromsnap GUID.
	*/
	if (resuming) {
	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN,
	"cannot receive resume stream"));
	} else {
	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN,
	"cannot receive incremental stream"));
	}

	(void) strlcpy(name, destsnap, sizeof (name));
	*strchr(name, '@') = '\0';

	/*
	* If the exact receive path was specified and this is the
	* topmost path in the stream, then if the fs does not exist we
	* should look no further.
	*/
	if ((flags->isprefix \|\| (*(chopprefix = drrb->drr_toname +
	strlen(sendfs)) != '\0' && *chopprefix != '@')) &&
	!zfs_dataset_exists(hdl, name, ZFS_TYPE_DATASET)) {
	char snap[ZFS_MAX_DATASET_NAME_LEN];
	(void) strlcpy(snap, strchr(destsnap, '@'),
	sizeof (snap));
	if (guid_to_name(hdl, name, drrb->drr_fromguid,
	B_FALSE, destsnap) == 0) {
	*strchr(destsnap, '@') = '\0';
	(void) strlcat(destsnap, snap,
	sizeof (destsnap));
	}
	}
	}

	(void) strlcpy(name, destsnap, sizeof (name));
	*strchr(name, '@') = '\0';

	redacted = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
	DMU_BACKUP_FEATURE_REDACTED;

	if (flags->heal) {
	if (flags->isprefix \|\| flags->istail \|\| flags->force \|\|
	flags->canmountoff \|\| flags->resumable \|\| flags->nomount \|\|
	flags->skipholds) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"corrective recv can not be used when combined with"
	" this flag"));
	err = zfs_error(hdl, EZFS_INVALIDNAME, errbuf);
	goto out;
	}
	uint64_t guid =
	get_snap_guid(hdl, name, strchr(destsnap, '@') + 1);
	if (guid == 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"corrective recv must specify an existing snapshot"
	" to heal"));
	err = zfs_error(hdl, EZFS_INVALIDNAME, errbuf);
	goto out;
	} else if (guid != drrb->drr_toguid) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"local snapshot doesn't match the snapshot"
	" in the provided stream"));
	err = zfs_error(hdl, EZFS_WRONG_PARENT, errbuf);
	goto out;
	}
	} else if (zfs_dataset_exists(hdl, name, ZFS_TYPE_DATASET)) {
	zfs_cmd_t zc = {"\0"};
	zfs_handle_t *zhp = NULL;
	boolean_t encrypted;

	(void) strcpy(zc.zc_name, name);

	/*
	* Destination fs exists. It must be one of these cases:
	* - an incremental send stream
	* - the stream specifies a new fs (full stream or clone)
	* and they want us to blow away the existing fs (and
	* have therefore specified -F and removed any snapshots)
	* - we are resuming a failed receive.
	*/
	if (stream_wantsnewfs) {
	boolean_t is_volume = drrb->drr_type == DMU_OST_ZVOL;
	if (!flags->force) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"destination '%s' exists\n"
	"must specify -F to overwrite it"), name);
	err = zfs_error(hdl, EZFS_EXISTS, errbuf);
	goto out;
	}
	if (zfs_ioctl(hdl, ZFS_IOC_SNAPSHOT_LIST_NEXT,
	&zc) == 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"destination has snapshots (eg. %s)\n"
	"must destroy them to overwrite it"),
	zc.zc_name);
	err = zfs_error(hdl, EZFS_EXISTS, errbuf);
	goto out;
	}
	if (is_volume && strrchr(name, '/') == NULL) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"destination %s is the root dataset\n"
	"cannot overwrite with a ZVOL"),
	name);
	err = zfs_error(hdl, EZFS_EXISTS, errbuf);
	goto out;
	}
	if (is_volume &&
	zfs_ioctl(hdl, ZFS_IOC_DATASET_LIST_NEXT,
	&zc) == 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"destination has children (eg. %s)\n"
	"cannot overwrite with a ZVOL"),
	zc.zc_name);
	err = zfs_error(hdl, EZFS_WRONG_PARENT, errbuf);
	goto out;
	}
	}

	if ((zhp = zfs_open(hdl, name,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME)) == NULL) {
	err = -1;
	goto out;
	}

	/*
	* When receiving full/newfs on existing dataset, then it
	* should be done with "-F" flag. Its enforced for initial
	* receive in previous checks in this function.
	* Similarly, on resuming full/newfs recv on existing dataset,
	* it should be done with "-F" flag.
	*
	* When dataset doesn't exist, then full/newfs recv is done on
	* newly created dataset and it's marked INCONSISTENT. But
	* When receiving on existing dataset, recv is first done on
	* %recv and its marked INCONSISTENT. Existing dataset is not
	* marked INCONSISTENT.
	* Resume of full/newfs receive with dataset not INCONSISTENT
	* indicates that its resuming newfs on existing dataset. So,
	* enforce "-F" flag in this case.
	*/
	if (stream_resumingnewfs &&
	!zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT) &&
	!flags->force) {
	zfs_close(zhp);
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"Resuming recv on existing destination '%s'\n"
	"must specify -F to overwrite it"), name);
	err = zfs_error(hdl, EZFS_RESUME_EXISTS, errbuf);
	goto out;
	}

	if (stream_wantsnewfs &&
	zhp->zfs_dmustats.dds_origin[0]) {
	zfs_close(zhp);
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"destination '%s' is a clone\n"
	"must destroy it to overwrite it"), name);
	err = zfs_error(hdl, EZFS_EXISTS, errbuf);
	goto out;
	}

	/*
	* Raw sends can not be performed as an incremental on top
	* of existing unencrypted datasets. zfs recv -F can't be
	* used to blow away an existing encrypted filesystem. This
	* is because it would require the dsl dir to point to the
	* new key (or lack of a key) and the old key at the same
	* time. The -F flag may still be used for deleting
	* intermediate snapshots that would otherwise prevent the
	* receive from working.
	*/
	encrypted = zfs_prop_get_int(zhp, ZFS_PROP_ENCRYPTION) !=
	ZIO_CRYPT_OFF;
	if (!stream_wantsnewfs && !encrypted && raw) {
	zfs_close(zhp);
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"cannot perform raw receive on top of "
	"existing unencrypted dataset"));
	err = zfs_error(hdl, EZFS_BADRESTORE, errbuf);
	goto out;
	}

	if (stream_wantsnewfs && flags->force &&
	((raw && !encrypted) \|\| encrypted)) {
	zfs_close(zhp);
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"zfs receive -F cannot be used to destroy an "
	"encrypted filesystem or overwrite an "
	"unencrypted one with an encrypted one"));
	err = zfs_error(hdl, EZFS_BADRESTORE, errbuf);
	goto out;
	}

	if (!flags->dryrun && zhp->zfs_type == ZFS_TYPE_FILESYSTEM &&
	(stream_wantsnewfs \|\| stream_resumingnewfs)) {
	/* We can't do online recv in this case */
	clp = changelist_gather(zhp, ZFS_PROP_NAME, 0,
	flags->forceunmount ? MS_FORCE : 0);
	if (clp == NULL) {
	zfs_close(zhp);
	err = -1;
	goto out;
	}
	if (changelist_prefix(clp) != 0) {
	changelist_free(clp);
	zfs_close(zhp);
	err = -1;
	goto out;
	}
	}

	/*
	* If we are resuming a newfs, set newfs here so that we will
	* mount it if the recv succeeds this time. We can tell
	* that it was a newfs on the first recv because the fs
	* itself will be inconsistent (if the fs existed when we
	* did the first recv, we would have received it into
	* .../%recv).
	*/
	if (resuming && zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT))
	newfs = B_TRUE;

	/* we want to know if we're zoned when validating -o\|-x props */
	zoned = zfs_prop_get_int(zhp, ZFS_PROP_ZONED);

	/* may need this info later, get it now we have zhp around */
	if (zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN, NULL, 0,
	NULL, NULL, 0, B_TRUE) == 0)
	hastoken = B_TRUE;

	/* gather existing properties on destination */
	origprops = fnvlist_alloc();
	fnvlist_merge(origprops, zhp->zfs_props);
	fnvlist_merge(origprops, zhp->zfs_user_props);

	zfs_close(zhp);
	} else {
	zfs_handle_t *zhp;

	/*
	* Destination filesystem does not exist. Therefore we better
	* be creating a new filesystem (either from a full backup, or
	* a clone). It would therefore be invalid if the user
	* specified only the pool name (i.e. if the destination name
	* contained no slash character).
	*/
	cp = strrchr(name, '/');

	if (!stream_wantsnewfs \|\| cp == NULL) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"destination '%s' does not exist"), name);
	err = zfs_error(hdl, EZFS_NOENT, errbuf);
	goto out;
	}

	/*
	* Trim off the final dataset component so we perform the
	* recvbackup ioctl to the filesystems's parent.
	*/
	*cp = '\0';

	if (flags->isprefix && !flags->istail && !flags->dryrun &&
	create_parents(hdl, destsnap, strlen(tosnap)) != 0) {
	err = zfs_error(hdl, EZFS_BADRESTORE, errbuf);
	goto out;
	}

	/* validate parent */
	zhp = zfs_open(hdl, name, ZFS_TYPE_DATASET);
	if (zhp == NULL) {
	err = zfs_error(hdl, EZFS_BADRESTORE, errbuf);
	goto out;
	}
	if (zfs_get_type(zhp) != ZFS_TYPE_FILESYSTEM) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"parent '%s' is not a filesystem"), name);
	err = zfs_error(hdl, EZFS_WRONG_PARENT, errbuf);
	zfs_close(zhp);
	goto out;
	}

	zfs_close(zhp);

	newfs = B_TRUE;
	*cp = '/';
	}

	if (flags->verbose) {
	(void) printf("%s %s%s stream of %s into %s\n",
	flags->dryrun ? "would receive" : "receiving",
	flags->heal ? " corrective" : "",
	drrb->drr_fromguid ? "incremental" : "full",
	drrb->drr_toname, destsnap);
	(void) fflush(stdout);
	}

	/*
	* If this is the top-level dataset, record it so we can use it
	* for recursive operations later.
	*/
	if (top_zfs != NULL &&
	(top_zfs == NULL \|\| strcmp(top_zfs, name) == 0)) {
	toplevel = B_TRUE;
	if (*top_zfs == NULL)
	*top_zfs = zfs_strdup(hdl, name);
	}

	if (drrb->drr_type == DMU_OST_ZVOL) {
	type = ZFS_TYPE_VOLUME;
	} else if (drrb->drr_type == DMU_OST_ZFS) {
	type = ZFS_TYPE_FILESYSTEM;
	} else {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"invalid record type: 0x%d"), drrb->drr_type);
	err = zfs_error(hdl, EZFS_BADSTREAM, errbuf);
	goto out;
	}
	if ((err = zfs_setup_cmdline_props(hdl, type, name, zoned, recursive,
	stream_wantsnewfs, raw, toplevel, rcvprops, cmdprops, origprops,
	&oxprops, &wkeydata, &wkeylen, errbuf)) != 0)
	goto out;

	/*
	* When sending with properties (zfs send -p), the encryption property
	* is not included because it is a SETONCE property and therefore
	* treated as read only. However, we are always able to determine its
	* value because raw sends will include it in the DRR_BDEGIN payload
	* and non-raw sends with properties are not allowed for encrypted
	* datasets. Therefore, if this is a non-raw properties stream, we can
	* infer that the value should be ZIO_CRYPT_OFF and manually add that
	* to the received properties.
	*/
	if (stream_wantsnewfs && !raw && rcvprops != NULL &&
	!nvlist_exists(cmdprops, zfs_prop_to_name(ZFS_PROP_ENCRYPTION))) {
	if (oxprops == NULL)
	oxprops = fnvlist_alloc();
	fnvlist_add_uint64(oxprops,
	zfs_prop_to_name(ZFS_PROP_ENCRYPTION), ZIO_CRYPT_OFF);
	}

	if (flags->dryrun) {
	void *buf = zfs_alloc(hdl, SPA_MAXBLOCKSIZE);

	/*
	* We have read the DRR_BEGIN record, but we have
	* not yet read the payload. For non-dryrun sends
	* this will be done by the kernel, so we must
	* emulate that here, before attempting to read
	* more records.
	*/
	err = recv_read(hdl, infd, buf, drr->drr_payloadlen,
	flags->byteswap, NULL);
	free(buf);
	if (err != 0)
	goto out;

	err = recv_skip(hdl, infd, flags->byteswap);
	goto out;
	}

	if (flags->heal) {
	err = ioctl_err = lzc_receive_with_heal(destsnap, rcvprops,
	oxprops, wkeydata, wkeylen, origin, flags->force,
	flags->heal, flags->resumable, raw, infd, drr_noswap, -1,
	&read_bytes, &errflags, NULL, &prop_errors);
	} else {
	err = ioctl_err = lzc_receive_with_cmdprops(destsnap, rcvprops,
	oxprops, wkeydata, wkeylen, origin, flags->force,
	flags->resumable, raw, infd, drr_noswap, -1, &read_bytes,
	&errflags, NULL, &prop_errors);
	}
	ioctl_errno = ioctl_err;
	prop_errflags = errflags;

	if (err == 0) {
	nvpair_t *prop_err = NULL;

	while ((prop_err = nvlist_next_nvpair(prop_errors,
	prop_err)) != NULL) {
	char tbuf[1024];
	zfs_prop_t prop;
	int intval;

	prop = zfs_name_to_prop(nvpair_name(prop_err));
	(void) nvpair_value_int32(prop_err, &intval);
	if (strcmp(nvpair_name(prop_err),
	ZPROP_N_MORE_ERRORS) == 0) {
	trunc_prop_errs(intval);
	break;
	} else if (snapname == NULL \|\| finalsnap == NULL \|\|
	strcmp(finalsnap, snapname) == 0 \|\|
	strcmp(nvpair_name(prop_err),
	zfs_prop_to_name(ZFS_PROP_REFQUOTA)) != 0) {
	/*
	* Skip the special case of, for example,
	* "refquota", errors on intermediate
	* snapshots leading up to a final one.
	* That's why we have all of the checks above.
	*
	* See zfs_ioctl.c's extract_delay_props() for
	* a list of props which can fail on
	* intermediate snapshots, but shouldn't
	* affect the overall receive.
	*/
	(void) snprintf(tbuf, sizeof (tbuf),
	dgettext(TEXT_DOMAIN,
	"cannot receive %s property on %s"),
	nvpair_name(prop_err), name);
	zfs_setprop_error(hdl, prop, intval, tbuf);
	}
	}
	}

	if (err == 0 && snapprops_nvlist) {
	zfs_cmd_t zc = {"\0"};

	(void) strlcpy(zc.zc_name, destsnap, sizeof (zc.zc_name));
	zc.zc_cookie = B_TRUE; /* received */
	zcmd_write_src_nvlist(hdl, &zc, snapprops_nvlist);
	(void) zfs_ioctl(hdl, ZFS_IOC_SET_PROP, &zc);
	zcmd_free_nvlists(&zc);
	}
	if (err == 0 && snapholds_nvlist) {
	nvpair_t *pair;
	nvlist_t holds, errors = NULL;
	int cleanup_fd = -1;

	VERIFY(0 == nvlist_alloc(&holds, 0, KM_SLEEP));
	for (pair = nvlist_next_nvpair(snapholds_nvlist, NULL);
	pair != NULL;
	pair = nvlist_next_nvpair(snapholds_nvlist, pair)) {
	fnvlist_add_string(holds, destsnap, nvpair_name(pair));
	}
	(void) lzc_hold(holds, cleanup_fd, &errors);
	fnvlist_free(snapholds_nvlist);
	fnvlist_free(holds);
	}

	if (err && (ioctl_errno == ENOENT \|\| ioctl_errno == EEXIST)) {
	/*
	* It may be that this snapshot already exists,
	* in which case we want to consume & ignore it
	* rather than failing.
	*/
	avl_tree_t *local_avl;
	nvlist_t local_nv, fs;
	cp = strchr(destsnap, '@');

	/*
	* XXX Do this faster by just iterating over snaps in
	* this fs. Also if zc_value does not exist, we will
	* get a strange "does not exist" error message.
	*/
	*cp = '\0';
	if (gather_nvlist(hdl, destsnap, NULL, NULL, B_FALSE, B_TRUE,
	B_FALSE, B_FALSE, B_FALSE, B_FALSE, B_FALSE, B_FALSE,
	B_TRUE, &local_nv, &local_avl) == 0) {
	*cp = '@';
	fs = fsavl_find(local_avl, drrb->drr_toguid, NULL);
	fsavl_destroy(local_avl);
	fnvlist_free(local_nv);

	if (fs != NULL) {
	if (flags->verbose) {
	(void) printf("snap %s already exists; "
	"ignoring\n", destsnap);
	}
	err = ioctl_err = recv_skip(hdl, infd,
	flags->byteswap);
	}
	}
	*cp = '@';
	}

	if (ioctl_err != 0) {
	switch (ioctl_errno) {
	case ENODEV:
	cp = strchr(destsnap, '@');
	*cp = '\0';
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"most recent snapshot of %s does not\n"
	"match incremental source"), destsnap);
	(void) zfs_error(hdl, EZFS_BADRESTORE, errbuf);
	*cp = '@';
	break;
	case ETXTBSY:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"destination %s has been modified\n"
	"since most recent snapshot"), name);
	(void) zfs_error(hdl, EZFS_BADRESTORE, errbuf);
	break;
	case EACCES:
	if (flags->heal) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"key must be loaded to do a non-raw "
	"corrective recv on an encrypted "
	"dataset."));
	} else if (raw && stream_wantsnewfs) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"failed to create encryption key"));
	} else if (raw && !stream_wantsnewfs) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"encryption key does not match "
	"existing key"));
	} else {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"inherited key must be loaded"));
	}
	(void) zfs_error(hdl, EZFS_CRYPTOFAILED, errbuf);
	break;
	case EEXIST:
	cp = strchr(destsnap, '@');
	if (newfs) {
	/* it's the containing fs that exists */
	*cp = '\0';
	}
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"destination already exists"));
	(void) zfs_error_fmt(hdl, EZFS_EXISTS,
	dgettext(TEXT_DOMAIN, "cannot restore to %s"),
	destsnap);
	*cp = '@';
	break;
	case EINVAL:
	if (embedded && !raw) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"incompatible embedded data stream "
	"feature with encrypted receive."));
	} else if (flags->resumable) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"kernel modules must be upgraded to "
	"receive this stream."));
	}
	(void) zfs_error(hdl, EZFS_BADSTREAM, errbuf);
	break;
	case ECKSUM:
	case ZFS_ERR_STREAM_TRUNCATED:
	if (flags->heal)
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"corrective receive was not able to "
	"reconstruct the data needed for "
	"healing."));
	else
	recv_ecksum_set_aux(hdl, destsnap,
	flags->resumable, ioctl_err == ECKSUM);
	(void) zfs_error(hdl, EZFS_BADSTREAM, errbuf);
	break;
	case ZFS_ERR_STREAM_LARGE_BLOCK_MISMATCH:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"incremental send stream requires -L "
	"(--large-block), to match previous receive."));
	(void) zfs_error(hdl, EZFS_BADSTREAM, errbuf);
	break;
	case ENOTSUP:
	if (flags->heal)
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"stream is not compatible with the "
	"data in the pool."));
	else
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"pool must be upgraded to receive this "
	"stream."));
	(void) zfs_error(hdl, EZFS_BADVERSION, errbuf);
	break;
	case ZFS_ERR_CRYPTO_NOTSUP:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"stream uses crypto parameters not compatible with "
	"this pool"));
	(void) zfs_error(hdl, EZFS_BADSTREAM, errbuf);
	break;
	case EDQUOT:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"destination %s space quota exceeded."), name);
	(void) zfs_error(hdl, EZFS_NOSPC, errbuf);
	break;
	case ZFS_ERR_FROM_IVSET_GUID_MISSING:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"IV set guid missing. See errata %u at "
	"https://openzfs.github.io/openzfs-docs/msg/"
	"ZFS-8000-ER."),
	ZPOOL_ERRATA_ZOL_8308_ENCRYPTION);
	(void) zfs_error(hdl, EZFS_BADSTREAM, errbuf);
	break;
	case ZFS_ERR_FROM_IVSET_GUID_MISMATCH:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"IV set guid mismatch. See the 'zfs receive' "
	"man page section\n discussing the limitations "
	"of raw encrypted send streams."));
	(void) zfs_error(hdl, EZFS_BADSTREAM, errbuf);
	break;
	case ZFS_ERR_SPILL_BLOCK_FLAG_MISSING:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"Spill block flag missing for raw send.\n"
	"The zfs software on the sending system must "
	"be updated."));
	(void) zfs_error(hdl, EZFS_BADSTREAM, errbuf);
	break;
	case ZFS_ERR_RESUME_EXISTS:
	cp = strchr(destsnap, '@');
	if (newfs) {
	/* it's the containing fs that exists */
	*cp = '\0';
	}
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"Resuming recv on existing dataset without force"));
	(void) zfs_error_fmt(hdl, EZFS_RESUME_EXISTS,
	dgettext(TEXT_DOMAIN, "cannot resume recv %s"),
	destsnap);
	*cp = '@';
	break;
	case E2BIG:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"zfs receive required kernel memory allocation "
	"larger than the system can support. Please file "
	"an issue at the OpenZFS issue tracker:\n"
	"https://github.com/openzfs/zfs/issues/new"));
	(void) zfs_error(hdl, EZFS_BADSTREAM, errbuf);
	break;
	case EBUSY:
	if (hastoken) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"destination %s contains "
	"partially-complete state from "
	"\"zfs receive -s\"."), name);
	(void) zfs_error(hdl, EZFS_BUSY, errbuf);
	break;
	}
	zfs_fallthrough;
	default:
	(void) zfs_standard_error(hdl, ioctl_errno, errbuf);
	}
	}

	/*
	* Mount the target filesystem (if created). Also mount any
	* children of the target filesystem if we did a replication
	* receive (indicated by stream_avl being non-NULL).
	*/
	if (clp) {
	if (!flags->nomount)
	err \|= changelist_postfix(clp);
	changelist_free(clp);
	}

	if ((newfs \|\| stream_avl) && type == ZFS_TYPE_FILESYSTEM && !redacted)
	flags->domount = B_TRUE;

	if (prop_errflags & ZPROP_ERR_NOCLEAR) {
	(void) fprintf(stderr, dgettext(TEXT_DOMAIN, "Warning: "
	"failed to clear unreceived properties on %s"), name);
	(void) fprintf(stderr, "\n");
	}
	if (prop_errflags & ZPROP_ERR_NORESTORE) {
	(void) fprintf(stderr, dgettext(TEXT_DOMAIN, "Warning: "
	"failed to restore original properties on %s"), name);
	(void) fprintf(stderr, "\n");
	}

	if (err \|\| ioctl_err) {
	err = -1;
	goto out;
	}

	if (flags->verbose) {
	char buf1[64];
	char buf2[64];
	uint64_t bytes = read_bytes;
	struct timespec delta;
	clock_gettime(CLOCK_MONOTONIC_RAW, &delta);
	if (begin_time.tv_nsec > delta.tv_nsec) {
	delta.tv_nsec =
	1000000000 + delta.tv_nsec - begin_time.tv_nsec;
	delta.tv_sec -= 1;
	} else
	delta.tv_nsec -= begin_time.tv_nsec;
	delta.tv_sec -= begin_time.tv_sec;
	if (delta.tv_sec == 0 && delta.tv_nsec == 0)
	delta.tv_nsec = 1;
	double delta_f = delta.tv_sec + (delta.tv_nsec / 1e9);
	zfs_nicebytes(bytes, buf1, sizeof (buf1));
	zfs_nicebytes(bytes / delta_f, buf2, sizeof (buf2));

	(void) printf("received %s stream in %.2f seconds (%s/sec)\n",
	buf1, delta_f, buf2);
	}

	err = 0;
	out:
	if (prop_errors != NULL)
	fnvlist_free(prop_errors);

	if (tmp_keylocation[0] != '\0') {
	fnvlist_add_string(rcvprops,
	zfs_prop_to_name(ZFS_PROP_KEYLOCATION), tmp_keylocation);
	}

	if (newprops)
	fnvlist_free(rcvprops);

	fnvlist_free(oxprops);
	fnvlist_free(origprops);

	return (err);
	}

	/*
	* Check properties we were asked to override (both -o\|-x)
	*/
	static boolean_t
	zfs_receive_checkprops(libzfs_handle_t hdl, nvlist_t props,
	const char *errbuf)
	{
	nvpair_t *nvp = NULL;
	zfs_prop_t prop;
	const char *name;

	while ((nvp = nvlist_next_nvpair(props, nvp)) != NULL) {
	name = nvpair_name(nvp);
	prop = zfs_name_to_prop(name);

	if (prop == ZPROP_USERPROP) {
	if (!zfs_prop_user(name)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"%s: invalid property '%s'"), errbuf, name);
	return (B_FALSE);
	}
	continue;
	}
	/*
	* "origin" is readonly but is used to receive datasets as
	* clones so we don't raise an error here
	*/
	if (prop == ZFS_PROP_ORIGIN)
	continue;

	/* encryption params have their own verification later */
	if (prop == ZFS_PROP_ENCRYPTION \|\|
	zfs_prop_encryption_key_param(prop))
	continue;

	/*
	* cannot override readonly, set-once and other specific
	* settable properties
	*/
	if (zfs_prop_readonly(prop) \|\| prop == ZFS_PROP_VERSION \|\|
	prop == ZFS_PROP_VOLSIZE) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"%s: invalid property '%s'"), errbuf, name);
	return (B_FALSE);
	}
	}

	return (B_TRUE);
	}

	static int
	zfs_receive_impl(libzfs_handle_t hdl, const char tosnap,
	const char originsnap, recvflags_t flags, int infd, const char *sendfs,
	nvlist_t stream_nv, avl_tree_t stream_avl, char **top_zfs,
	const char finalsnap, nvlist_t cmdprops)
	{
	int err;
	dmu_replay_record_t drr, drr_noswap;
	struct drr_begin *drrb = &drr.drr_u.drr_begin;
	char errbuf[ERRBUFLEN];
	zio_cksum_t zcksum = { { 0 } };
	uint64_t featureflags;
	int hdrtype;

	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
	"cannot receive"));

	/* check cmdline props, raise an error if they cannot be received */
	if (!zfs_receive_checkprops(hdl, cmdprops, errbuf))
	return (zfs_error(hdl, EZFS_BADPROP, errbuf));

	if (flags->isprefix &&
	!zfs_dataset_exists(hdl, tosnap, ZFS_TYPE_DATASET)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "specified fs "
	"(%s) does not exist"), tosnap);
	return (zfs_error(hdl, EZFS_NOENT, errbuf));
	}
	if (originsnap &&
	!zfs_dataset_exists(hdl, originsnap, ZFS_TYPE_DATASET)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "specified origin fs "
	"(%s) does not exist"), originsnap);
	return (zfs_error(hdl, EZFS_NOENT, errbuf));
	}

	/* read in the BEGIN record */
	if (0 != (err = recv_read(hdl, infd, &drr, sizeof (drr), B_FALSE,
	&zcksum)))
	return (err);

	if (drr.drr_type == DRR_END \|\| drr.drr_type == BSWAP_32(DRR_END)) {
	/* It's the double end record at the end of a package */
	return (ENODATA);
	}

	/* the kernel needs the non-byteswapped begin record */
	drr_noswap = drr;

	flags->byteswap = B_FALSE;
	if (drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC)) {
	/*
	* We computed the checksum in the wrong byteorder in
	* recv_read() above; do it again correctly.
	*/
	memset(&zcksum, 0, sizeof (zio_cksum_t));
	fletcher_4_incremental_byteswap(&drr, sizeof (drr), &zcksum);
	flags->byteswap = B_TRUE;

	drr.drr_type = BSWAP_32(drr.drr_type);
	drr.drr_payloadlen = BSWAP_32(drr.drr_payloadlen);
	drrb->drr_magic = BSWAP_64(drrb->drr_magic);
	drrb->drr_versioninfo = BSWAP_64(drrb->drr_versioninfo);
	drrb->drr_creation_time = BSWAP_64(drrb->drr_creation_time);
	drrb->drr_type = BSWAP_32(drrb->drr_type);
	drrb->drr_flags = BSWAP_32(drrb->drr_flags);
	drrb->drr_toguid = BSWAP_64(drrb->drr_toguid);
	drrb->drr_fromguid = BSWAP_64(drrb->drr_fromguid);
	}

	if (drrb->drr_magic != DMU_BACKUP_MAGIC \|\| drr.drr_type != DRR_BEGIN) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid "
	"stream (bad magic number)"));
	return (zfs_error(hdl, EZFS_BADSTREAM, errbuf));
	}

	featureflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
	hdrtype = DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo);

	if (!DMU_STREAM_SUPPORTED(featureflags) \|\|
	(hdrtype != DMU_SUBSTREAM && hdrtype != DMU_COMPOUNDSTREAM)) {
	/*
	* Let's be explicit about this one, since rather than
	* being a new feature we can't know, it's an old
	* feature we dropped.
	*/
	if (featureflags & DMU_BACKUP_FEATURE_DEDUP) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"stream has deprecated feature: dedup, try "
	"'zstream redup [send in a file] \| zfs recv "
	"[...]'"));
	} else {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"stream has unsupported feature, feature flags = "
	"%llx (unknown flags = %llx)"),
	(u_longlong_t)featureflags,
	(u_longlong_t)((featureflags) &
	~DMU_BACKUP_FEATURE_MASK));
	}
	return (zfs_error(hdl, EZFS_BADSTREAM, errbuf));
	}

	/* Holds feature is set once in the compound stream header. */
	if (featureflags & DMU_BACKUP_FEATURE_HOLDS)
	flags->holds = B_TRUE;

	if (strchr(drrb->drr_toname, '@') == NULL) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid "
	"stream (bad snapshot name)"));
	return (zfs_error(hdl, EZFS_BADSTREAM, errbuf));
	}

	if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) == DMU_SUBSTREAM) {
	char nonpackage_sendfs[ZFS_MAX_DATASET_NAME_LEN];
	if (sendfs == NULL) {
	/*
	* We were not called from zfs_receive_package(). Get
	* the fs specified by 'zfs send'.
	*/
	char *cp;
	(void) strlcpy(nonpackage_sendfs,
	drr.drr_u.drr_begin.drr_toname,
	sizeof (nonpackage_sendfs));
	if ((cp = strchr(nonpackage_sendfs, '@')) != NULL)
	*cp = '\0';
	sendfs = nonpackage_sendfs;
	VERIFY(finalsnap == NULL);
	}
	return (zfs_receive_one(hdl, infd, tosnap, originsnap, flags,
	&drr, &drr_noswap, sendfs, stream_nv, stream_avl, top_zfs,
	finalsnap, cmdprops));
	} else {
	assert(DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
	DMU_COMPOUNDSTREAM);
	return (zfs_receive_package(hdl, infd, tosnap, flags, &drr,
	&zcksum, top_zfs, cmdprops));
	}
	}

	/*
	* Restores a backup of tosnap from the file descriptor specified by infd.
	* Return 0 on total success, -2 if some things couldn't be
	* destroyed/renamed/promoted, -1 if some things couldn't be received.
	* (-1 will override -2, if -1 and the resumable flag was specified the
	* transfer can be resumed if the sending side supports it).
	*/
	int
	zfs_receive(libzfs_handle_t hdl, const char tosnap, nvlist_t *props,
	recvflags_t flags, int infd, avl_tree_t stream_avl)
	{
	char *top_zfs = NULL;
	int err;
	struct stat sb;
	const char *originsnap = NULL;

	/*
	* The only way fstat can fail is if we do not have a valid file
	* descriptor.
	*/
	if (fstat(infd, &sb) == -1) {
	perror("fstat");
	return (-2);
	}

	if (props) {
	err = nvlist_lookup_string(props, "origin", &originsnap);
	if (err && err != ENOENT)
	return (err);
	}

	err = zfs_receive_impl(hdl, tosnap, originsnap, flags, infd, NULL, NULL,
	stream_avl, &top_zfs, NULL, props);

	if (err == 0 && !flags->nomount && flags->domount && top_zfs) {
	zfs_handle_t *zhp = NULL;
	prop_changelist_t *clp = NULL;

	zhp = zfs_open(hdl, top_zfs,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME);
	if (zhp == NULL) {
	err = -1;
	goto out;
	} else {
	if (zhp->zfs_type == ZFS_TYPE_VOLUME) {
	zfs_close(zhp);
	goto out;
	}

	clp = changelist_gather(zhp, ZFS_PROP_MOUNTPOINT,
	CL_GATHER_MOUNT_ALWAYS,
	flags->forceunmount ? MS_FORCE : 0);
	zfs_close(zhp);
	if (clp == NULL) {
	err = -1;
	goto out;
	}

	/* mount and share received datasets */
	err = changelist_postfix(clp);
	changelist_free(clp);
	if (err != 0)
	err = -1;
	}
	}

	out:
	if (top_zfs)
	free(top_zfs);

	return (err);
	}
	diff --git a/sys/contrib/openzfs/lib/libzfs/libzfs_util.c b/sys/contrib/openzfs/lib/libzfs/libzfs_util.c
	index fdd1975fa677..60e9262f6b71 100644
	--- a/sys/contrib/openzfs/lib/libzfs/libzfs_util.c
	+++ b/sys/contrib/openzfs/lib/libzfs/libzfs_util.c
	@@ -1,2266 +1,2274 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright 2020 Joyent, Inc. All rights reserved.
	- * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
	+ * Copyright (c) 2011, 2024 by Delphix. All rights reserved.
	* Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>
	* Copyright (c) 2017 Datto Inc.
	* Copyright (c) 2020 The FreeBSD Foundation
	*
	* Portions of this software were developed by Allan Jude
	* under sponsorship from the FreeBSD Foundation.
	*/

	/*
	* Internal utility routines for the ZFS library.
	*/

	#include <errno.h>
	#include <fcntl.h>
	#include <libintl.h>
	#include <stdarg.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <strings.h>
	#include <unistd.h>
	#include <math.h>
	#if LIBFETCH_DYNAMIC
	#include <dlfcn.h>
	#endif
	#include <sys/stat.h>
	#include <sys/mnttab.h>
	#include <sys/mntent.h>
	#include <sys/types.h>
	#include <sys/wait.h>

	#include <libzfs.h>
	#include <libzfs_core.h>

	#include "libzfs_impl.h"
	#include "zfs_prop.h"
	#include "zfeature_common.h"
	#include <zfs_fletcher.h>
	#include <libzutil.h>

	/*
	* We only care about the scheme in order to match the scheme
	* with the handler. Each handler should validate the full URI
	* as necessary.
	*/
	#define URI_REGEX "^\$[A-Za-z][A-Za-z0-9+.\\-]*\$:"

	int
	libzfs_errno(libzfs_handle_t *hdl)
	{
	return (hdl->libzfs_error);
	}

	const char *
	libzfs_error_action(libzfs_handle_t *hdl)
	{
	return (hdl->libzfs_action);
	}

	const char *
	libzfs_error_description(libzfs_handle_t *hdl)
	{
	if (hdl->libzfs_desc[0] != '\0')
	return (hdl->libzfs_desc);

	switch (hdl->libzfs_error) {
	case EZFS_NOMEM:
	return (dgettext(TEXT_DOMAIN, "out of memory"));
	case EZFS_BADPROP:
	return (dgettext(TEXT_DOMAIN, "invalid property value"));
	case EZFS_PROPREADONLY:
	return (dgettext(TEXT_DOMAIN, "read-only property"));
	case EZFS_PROPTYPE:
	return (dgettext(TEXT_DOMAIN, "property doesn't apply to "
	"datasets of this type"));
	case EZFS_PROPNONINHERIT:
	return (dgettext(TEXT_DOMAIN, "property cannot be inherited"));
	case EZFS_PROPSPACE:
	return (dgettext(TEXT_DOMAIN, "invalid quota or reservation"));
	case EZFS_BADTYPE:
	return (dgettext(TEXT_DOMAIN, "operation not applicable to "
	"datasets of this type"));
	case EZFS_BUSY:
	return (dgettext(TEXT_DOMAIN, "pool or dataset is busy"));
	case EZFS_EXISTS:
	return (dgettext(TEXT_DOMAIN, "pool or dataset exists"));
	case EZFS_NOENT:
	return (dgettext(TEXT_DOMAIN, "no such pool or dataset"));
	case EZFS_BADSTREAM:
	return (dgettext(TEXT_DOMAIN, "invalid backup stream"));
	case EZFS_DSREADONLY:
	return (dgettext(TEXT_DOMAIN, "dataset is read-only"));
	case EZFS_VOLTOOBIG:
	return (dgettext(TEXT_DOMAIN, "volume size exceeds limit for "
	"this system"));
	case EZFS_INVALIDNAME:
	return (dgettext(TEXT_DOMAIN, "invalid name"));
	case EZFS_BADRESTORE:
	return (dgettext(TEXT_DOMAIN, "unable to restore to "
	"destination"));
	case EZFS_BADBACKUP:
	return (dgettext(TEXT_DOMAIN, "backup failed"));
	case EZFS_BADTARGET:
	return (dgettext(TEXT_DOMAIN, "invalid target vdev"));
	case EZFS_NODEVICE:
	return (dgettext(TEXT_DOMAIN, "no such device in pool"));
	case EZFS_BADDEV:
	return (dgettext(TEXT_DOMAIN, "invalid device"));
	case EZFS_NOREPLICAS:
	return (dgettext(TEXT_DOMAIN, "no valid replicas"));
	case EZFS_RESILVERING:
	return (dgettext(TEXT_DOMAIN, "currently resilvering"));
	case EZFS_BADVERSION:
	return (dgettext(TEXT_DOMAIN, "unsupported version or "
	"feature"));
	case EZFS_POOLUNAVAIL:
	return (dgettext(TEXT_DOMAIN, "pool is unavailable"));
	case EZFS_DEVOVERFLOW:
	return (dgettext(TEXT_DOMAIN, "too many devices in one vdev"));
	case EZFS_BADPATH:
	return (dgettext(TEXT_DOMAIN, "must be an absolute path"));
	case EZFS_CROSSTARGET:
	return (dgettext(TEXT_DOMAIN, "operation crosses datasets or "
	"pools"));
	case EZFS_ZONED:
	return (dgettext(TEXT_DOMAIN, "dataset in use by local zone"));
	case EZFS_MOUNTFAILED:
	return (dgettext(TEXT_DOMAIN, "mount failed"));
	case EZFS_UMOUNTFAILED:
	return (dgettext(TEXT_DOMAIN, "unmount failed"));
	case EZFS_UNSHARENFSFAILED:
	return (dgettext(TEXT_DOMAIN, "NFS share removal failed"));
	case EZFS_SHARENFSFAILED:
	return (dgettext(TEXT_DOMAIN, "NFS share creation failed"));
	case EZFS_UNSHARESMBFAILED:
	return (dgettext(TEXT_DOMAIN, "SMB share removal failed"));
	case EZFS_SHARESMBFAILED:
	return (dgettext(TEXT_DOMAIN, "SMB share creation failed"));
	case EZFS_PERM:
	return (dgettext(TEXT_DOMAIN, "permission denied"));
	case EZFS_NOSPC:
	return (dgettext(TEXT_DOMAIN, "out of space"));
	case EZFS_FAULT:
	return (dgettext(TEXT_DOMAIN, "bad address"));
	case EZFS_IO:
	return (dgettext(TEXT_DOMAIN, "I/O error"));
	case EZFS_INTR:
	return (dgettext(TEXT_DOMAIN, "signal received"));
	case EZFS_CKSUM:
	return (dgettext(TEXT_DOMAIN, "insufficient replicas"));
	case EZFS_ISSPARE:
	return (dgettext(TEXT_DOMAIN, "device is reserved as a hot "
	"spare"));
	case EZFS_INVALCONFIG:
	return (dgettext(TEXT_DOMAIN, "invalid vdev configuration"));
	case EZFS_RECURSIVE:
	return (dgettext(TEXT_DOMAIN, "recursive dataset dependency"));
	case EZFS_NOHISTORY:
	return (dgettext(TEXT_DOMAIN, "no history available"));
	case EZFS_POOLPROPS:
	return (dgettext(TEXT_DOMAIN, "failed to retrieve "
	"pool properties"));
	case EZFS_POOL_NOTSUP:
	return (dgettext(TEXT_DOMAIN, "operation not supported "
	"on this type of pool"));
	case EZFS_POOL_INVALARG:
	return (dgettext(TEXT_DOMAIN, "invalid argument for "
	"this pool operation"));
	case EZFS_NAMETOOLONG:
	return (dgettext(TEXT_DOMAIN, "dataset name is too long"));
	case EZFS_OPENFAILED:
	return (dgettext(TEXT_DOMAIN, "open failed"));
	case EZFS_NOCAP:
	return (dgettext(TEXT_DOMAIN,
	"disk capacity information could not be retrieved"));
	case EZFS_LABELFAILED:
	return (dgettext(TEXT_DOMAIN, "write of label failed"));
	case EZFS_BADWHO:
	return (dgettext(TEXT_DOMAIN, "invalid user/group"));
	case EZFS_BADPERM:
	return (dgettext(TEXT_DOMAIN, "invalid permission"));
	case EZFS_BADPERMSET:
	return (dgettext(TEXT_DOMAIN, "invalid permission set name"));
	case EZFS_NODELEGATION:
	return (dgettext(TEXT_DOMAIN, "delegated administration is "
	"disabled on pool"));
	case EZFS_BADCACHE:
	return (dgettext(TEXT_DOMAIN, "invalid or missing cache file"));
	case EZFS_ISL2CACHE:
	return (dgettext(TEXT_DOMAIN, "device is in use as a cache"));
	case EZFS_VDEVNOTSUP:
	return (dgettext(TEXT_DOMAIN, "vdev specification is not "
	"supported"));
	case EZFS_NOTSUP:
	return (dgettext(TEXT_DOMAIN, "operation not supported "
	"on this dataset"));
	case EZFS_IOC_NOTSUPPORTED:
	return (dgettext(TEXT_DOMAIN, "operation not supported by "
	"zfs kernel module"));
	case EZFS_ACTIVE_SPARE:
	return (dgettext(TEXT_DOMAIN, "pool has active shared spare "
	"device"));
	case EZFS_UNPLAYED_LOGS:
	return (dgettext(TEXT_DOMAIN, "log device has unplayed intent "
	"logs"));
	case EZFS_REFTAG_RELE:
	return (dgettext(TEXT_DOMAIN, "no such tag on this dataset"));
	case EZFS_REFTAG_HOLD:
	return (dgettext(TEXT_DOMAIN, "tag already exists on this "
	"dataset"));
	case EZFS_TAGTOOLONG:
	return (dgettext(TEXT_DOMAIN, "tag too long"));
	case EZFS_PIPEFAILED:
	return (dgettext(TEXT_DOMAIN, "pipe create failed"));
	case EZFS_THREADCREATEFAILED:
	return (dgettext(TEXT_DOMAIN, "thread create failed"));
	case EZFS_POSTSPLIT_ONLINE:
	return (dgettext(TEXT_DOMAIN, "disk was split from this pool "
	"into a new one"));
	case EZFS_SCRUB_PAUSED:
	return (dgettext(TEXT_DOMAIN, "scrub is paused; "
	"use 'zpool scrub' to resume scrub"));
	case EZFS_SCRUB_PAUSED_TO_CANCEL:
	return (dgettext(TEXT_DOMAIN, "scrub is paused; "
	"use 'zpool scrub' to resume or 'zpool scrub -s' to "
	"cancel scrub"));
	case EZFS_SCRUBBING:
	return (dgettext(TEXT_DOMAIN, "currently scrubbing; "
	"use 'zpool scrub -s' to cancel scrub"));
	case EZFS_ERRORSCRUBBING:
	return (dgettext(TEXT_DOMAIN, "currently error scrubbing; "
	"use 'zpool scrub -s' to cancel error scrub"));
	case EZFS_ERRORSCRUB_PAUSED:
	return (dgettext(TEXT_DOMAIN, "error scrub is paused; "
	"use 'zpool scrub -e' to resume error scrub"));
	case EZFS_NO_SCRUB:
	return (dgettext(TEXT_DOMAIN, "there is no active scrub"));
	case EZFS_DIFF:
	return (dgettext(TEXT_DOMAIN, "unable to generate diffs"));
	case EZFS_DIFFDATA:
	return (dgettext(TEXT_DOMAIN, "invalid diff data"));
	case EZFS_POOLREADONLY:
	return (dgettext(TEXT_DOMAIN, "pool is read-only"));
	case EZFS_NO_PENDING:
	return (dgettext(TEXT_DOMAIN, "operation is not "
	"in progress"));
	case EZFS_CHECKPOINT_EXISTS:
	return (dgettext(TEXT_DOMAIN, "checkpoint exists"));
	case EZFS_DISCARDING_CHECKPOINT:
	return (dgettext(TEXT_DOMAIN, "currently discarding "
	"checkpoint"));
	case EZFS_NO_CHECKPOINT:
	return (dgettext(TEXT_DOMAIN, "checkpoint does not exist"));
	case EZFS_DEVRM_IN_PROGRESS:
	return (dgettext(TEXT_DOMAIN, "device removal in progress"));
	case EZFS_VDEV_TOO_BIG:
	return (dgettext(TEXT_DOMAIN, "device exceeds supported size"));
	case EZFS_ACTIVE_POOL:
	return (dgettext(TEXT_DOMAIN, "pool is imported on a "
	"different host"));
	case EZFS_CRYPTOFAILED:
	return (dgettext(TEXT_DOMAIN, "encryption failure"));
	case EZFS_TOOMANY:
	return (dgettext(TEXT_DOMAIN, "argument list too long"));
	case EZFS_INITIALIZING:
	return (dgettext(TEXT_DOMAIN, "currently initializing"));
	case EZFS_NO_INITIALIZE:
	return (dgettext(TEXT_DOMAIN, "there is no active "
	"initialization"));
	case EZFS_WRONG_PARENT:
	return (dgettext(TEXT_DOMAIN, "invalid parent dataset"));
	case EZFS_TRIMMING:
	return (dgettext(TEXT_DOMAIN, "currently trimming"));
	case EZFS_NO_TRIM:
	return (dgettext(TEXT_DOMAIN, "there is no active trim"));
	case EZFS_TRIM_NOTSUP:
	return (dgettext(TEXT_DOMAIN, "trim operations are not "
	"supported by this device"));
	case EZFS_NO_RESILVER_DEFER:
	return (dgettext(TEXT_DOMAIN, "this action requires the "
	"resilver_defer feature"));
	case EZFS_EXPORT_IN_PROGRESS:
	return (dgettext(TEXT_DOMAIN, "pool export in progress"));
	case EZFS_REBUILDING:
	return (dgettext(TEXT_DOMAIN, "currently sequentially "
	"resilvering"));
	case EZFS_VDEV_NOTSUP:
	return (dgettext(TEXT_DOMAIN, "operation not supported "
	"on this type of vdev"));
	case EZFS_NOT_USER_NAMESPACE:
	return (dgettext(TEXT_DOMAIN, "the provided file "
	"was not a user namespace file"));
	case EZFS_RESUME_EXISTS:
	return (dgettext(TEXT_DOMAIN, "Resuming recv on existing "
	"dataset without force"));
	+ case EZFS_ASHIFT_MISMATCH:
	+ return (dgettext(TEXT_DOMAIN, "adding devices with "
	+ "different physical sector sizes is not allowed"));
	case EZFS_UNKNOWN:
	return (dgettext(TEXT_DOMAIN, "unknown error"));
	default:
	assert(hdl->libzfs_error == 0);
	return (dgettext(TEXT_DOMAIN, "no error"));
	}
	}

	void
	zfs_error_aux(libzfs_handle_t hdl, const char fmt, ...)
	{
	va_list ap;

	va_start(ap, fmt);

	(void) vsnprintf(hdl->libzfs_desc, sizeof (hdl->libzfs_desc),
	fmt, ap);
	hdl->libzfs_desc_active = 1;

	va_end(ap);
	}

	static void
	zfs_verror(libzfs_handle_t hdl, int error, const char fmt, va_list ap)
	{
	(void) vsnprintf(hdl->libzfs_action, sizeof (hdl->libzfs_action),
	fmt, ap);
	hdl->libzfs_error = error;

	if (hdl->libzfs_desc_active)
	hdl->libzfs_desc_active = 0;
	else
	hdl->libzfs_desc[0] = '\0';

	if (hdl->libzfs_printerr) {
	if (error == EZFS_UNKNOWN) {
	(void) fprintf(stderr, dgettext(TEXT_DOMAIN, "internal "
	"error: %s: %s\n"), hdl->libzfs_action,
	libzfs_error_description(hdl));
	abort();
	}

	(void) fprintf(stderr, "%s: %s\n", hdl->libzfs_action,
	libzfs_error_description(hdl));
	if (error == EZFS_NOMEM)
	exit(1);
	}
	}

	int
	zfs_error(libzfs_handle_t hdl, int error, const char msg)
	{
	return (zfs_error_fmt(hdl, error, "%s", msg));
	}

	int
	zfs_error_fmt(libzfs_handle_t hdl, int error, const char fmt, ...)
	{
	va_list ap;

	va_start(ap, fmt);

	zfs_verror(hdl, error, fmt, ap);

	va_end(ap);

	return (-1);
	}

	static int
	zfs_common_error(libzfs_handle_t hdl, int error, const char fmt,
	va_list ap)
	{
	switch (error) {
	case EPERM:
	case EACCES:
	zfs_verror(hdl, EZFS_PERM, fmt, ap);
	return (-1);

	case ECANCELED:
	zfs_verror(hdl, EZFS_NODELEGATION, fmt, ap);
	return (-1);

	case EIO:
	zfs_verror(hdl, EZFS_IO, fmt, ap);
	return (-1);

	case EFAULT:
	zfs_verror(hdl, EZFS_FAULT, fmt, ap);
	return (-1);

	case EINTR:
	zfs_verror(hdl, EZFS_INTR, fmt, ap);
	return (-1);

	case ECKSUM:
	zfs_verror(hdl, EZFS_CKSUM, fmt, ap);
	return (-1);
	}

	return (0);
	}

	int
	zfs_standard_error(libzfs_handle_t hdl, int error, const char msg)
	{
	return (zfs_standard_error_fmt(hdl, error, "%s", msg));
	}

	int
	zfs_standard_error_fmt(libzfs_handle_t hdl, int error, const char fmt, ...)
	{
	va_list ap;

	va_start(ap, fmt);

	if (zfs_common_error(hdl, error, fmt, ap) != 0) {
	va_end(ap);
	return (-1);
	}

	switch (error) {
	case ENXIO:
	case ENODEV:
	case EPIPE:
	zfs_verror(hdl, EZFS_IO, fmt, ap);
	break;

	case ENOENT:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"dataset does not exist"));
	zfs_verror(hdl, EZFS_NOENT, fmt, ap);
	break;

	case ENOSPC:
	case EDQUOT:
	zfs_verror(hdl, EZFS_NOSPC, fmt, ap);
	break;

	case EEXIST:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"dataset already exists"));
	zfs_verror(hdl, EZFS_EXISTS, fmt, ap);
	break;

	case EBUSY:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"dataset is busy"));
	zfs_verror(hdl, EZFS_BUSY, fmt, ap);
	break;
	case EROFS:
	zfs_verror(hdl, EZFS_POOLREADONLY, fmt, ap);
	break;
	case ENAMETOOLONG:
	zfs_verror(hdl, EZFS_NAMETOOLONG, fmt, ap);
	break;
	case ENOTSUP:
	zfs_verror(hdl, EZFS_BADVERSION, fmt, ap);
	break;
	case EAGAIN:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"pool I/O is currently suspended"));
	zfs_verror(hdl, EZFS_POOLUNAVAIL, fmt, ap);
	break;
	case EREMOTEIO:
	zfs_verror(hdl, EZFS_ACTIVE_POOL, fmt, ap);
	break;
	case ZFS_ERR_UNKNOWN_SEND_STREAM_FEATURE:
	case ZFS_ERR_IOC_CMD_UNAVAIL:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "the loaded zfs "
	"module does not support this operation. A reboot may "
	"be required to enable this operation."));
	zfs_verror(hdl, EZFS_IOC_NOTSUPPORTED, fmt, ap);
	break;
	case ZFS_ERR_IOC_ARG_UNAVAIL:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "the loaded zfs "
	"module does not support an option for this operation. "
	"A reboot may be required to enable this option."));
	zfs_verror(hdl, EZFS_IOC_NOTSUPPORTED, fmt, ap);
	break;
	case ZFS_ERR_IOC_ARG_REQUIRED:
	case ZFS_ERR_IOC_ARG_BADTYPE:
	zfs_verror(hdl, EZFS_IOC_NOTSUPPORTED, fmt, ap);
	break;
	case ZFS_ERR_WRONG_PARENT:
	zfs_verror(hdl, EZFS_WRONG_PARENT, fmt, ap);
	break;
	case ZFS_ERR_BADPROP:
	zfs_verror(hdl, EZFS_BADPROP, fmt, ap);
	break;
	case ZFS_ERR_NOT_USER_NAMESPACE:
	zfs_verror(hdl, EZFS_NOT_USER_NAMESPACE, fmt, ap);
	break;
	default:
	zfs_error_aux(hdl, "%s", strerror(error));
	zfs_verror(hdl, EZFS_UNKNOWN, fmt, ap);
	break;
	}

	va_end(ap);
	return (-1);
	}

	void
	zfs_setprop_error(libzfs_handle_t *hdl, zfs_prop_t prop, int err,
	char *errbuf)
	{
	switch (err) {

	case ENOSPC:
	/*
	* For quotas and reservations, ENOSPC indicates
	* something different; setting a quota or reservation
	* doesn't use any disk space.
	*/
	switch (prop) {
	case ZFS_PROP_QUOTA:
	case ZFS_PROP_REFQUOTA:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"size is less than current used or "
	"reserved space"));
	(void) zfs_error(hdl, EZFS_PROPSPACE, errbuf);
	break;

	case ZFS_PROP_RESERVATION:
	case ZFS_PROP_REFRESERVATION:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"size is greater than available space"));
	(void) zfs_error(hdl, EZFS_PROPSPACE, errbuf);
	break;

	default:
	(void) zfs_standard_error(hdl, err, errbuf);
	break;
	}
	break;

	case EBUSY:
	(void) zfs_standard_error(hdl, EBUSY, errbuf);
	break;

	case EROFS:
	(void) zfs_error(hdl, EZFS_DSREADONLY, errbuf);
	break;

	case E2BIG:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property value too long"));
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	break;

	case ENOTSUP:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"pool and or dataset must be upgraded to set this "
	"property or value"));
	(void) zfs_error(hdl, EZFS_BADVERSION, errbuf);
	break;

	case ERANGE:
	if (prop == ZFS_PROP_COMPRESSION \|\|
	prop == ZFS_PROP_DNODESIZE \|\|
	prop == ZFS_PROP_RECORDSIZE) {
	(void) zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property setting is not allowed on "
	"bootable datasets"));
	(void) zfs_error(hdl, EZFS_NOTSUP, errbuf);
	} else if (prop == ZFS_PROP_CHECKSUM \|\|
	prop == ZFS_PROP_DEDUP) {
	(void) zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"property setting is not allowed on "
	"root pools"));
	(void) zfs_error(hdl, EZFS_NOTSUP, errbuf);
	} else {
	(void) zfs_standard_error(hdl, err, errbuf);
	}
	break;

	case EINVAL:
	if (prop == ZPROP_INVAL) {
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	} else {
	(void) zfs_standard_error(hdl, err, errbuf);
	}
	break;

	case ZFS_ERR_BADPROP:
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	break;

	case EACCES:
	if (prop == ZFS_PROP_KEYLOCATION) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"keylocation may only be set on encryption roots"));
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	} else {
	(void) zfs_standard_error(hdl, err, errbuf);
	}
	break;

	case EOVERFLOW:
	/*
	* This platform can't address a volume this big.
	*/
	#ifdef _ILP32
	if (prop == ZFS_PROP_VOLSIZE) {
	(void) zfs_error(hdl, EZFS_VOLTOOBIG, errbuf);
	break;
	}
	zfs_fallthrough;
	#endif
	default:
	(void) zfs_standard_error(hdl, err, errbuf);
	}
	}

	int
	zpool_standard_error(libzfs_handle_t hdl, int error, const char msg)
	{
	return (zpool_standard_error_fmt(hdl, error, "%s", msg));
	}

	int
	zpool_standard_error_fmt(libzfs_handle_t hdl, int error, const char fmt, ...)
	{
	va_list ap;

	va_start(ap, fmt);

	if (zfs_common_error(hdl, error, fmt, ap) != 0) {
	va_end(ap);
	return (-1);
	}

	switch (error) {
	case ENODEV:
	zfs_verror(hdl, EZFS_NODEVICE, fmt, ap);
	break;

	case ENOENT:
	zfs_error_aux(hdl,
	dgettext(TEXT_DOMAIN, "no such pool or dataset"));
	zfs_verror(hdl, EZFS_NOENT, fmt, ap);
	break;

	case EEXIST:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"pool already exists"));
	zfs_verror(hdl, EZFS_EXISTS, fmt, ap);
	break;

	case EBUSY:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "pool is busy"));
	zfs_verror(hdl, EZFS_BUSY, fmt, ap);
	break;

	/* There is no pending operation to cancel */
	case ENOTACTIVE:
	zfs_verror(hdl, EZFS_NO_PENDING, fmt, ap);
	break;

	case ENXIO:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"one or more devices is currently unavailable"));
	zfs_verror(hdl, EZFS_BADDEV, fmt, ap);
	break;

	case ENAMETOOLONG:
	zfs_verror(hdl, EZFS_DEVOVERFLOW, fmt, ap);
	break;

	case ENOTSUP:
	zfs_verror(hdl, EZFS_POOL_NOTSUP, fmt, ap);
	break;

	case EINVAL:
	zfs_verror(hdl, EZFS_POOL_INVALARG, fmt, ap);
	break;

	case ENOSPC:
	case EDQUOT:
	zfs_verror(hdl, EZFS_NOSPC, fmt, ap);
	break;

	case EAGAIN:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"pool I/O is currently suspended"));
	zfs_verror(hdl, EZFS_POOLUNAVAIL, fmt, ap);
	break;

	case EROFS:
	zfs_verror(hdl, EZFS_POOLREADONLY, fmt, ap);
	break;
	case EDOM:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"block size out of range or does not match"));
	zfs_verror(hdl, EZFS_BADPROP, fmt, ap);
	break;
	case EREMOTEIO:
	zfs_verror(hdl, EZFS_ACTIVE_POOL, fmt, ap);
	break;
	case ZFS_ERR_CHECKPOINT_EXISTS:
	zfs_verror(hdl, EZFS_CHECKPOINT_EXISTS, fmt, ap);
	break;
	case ZFS_ERR_DISCARDING_CHECKPOINT:
	zfs_verror(hdl, EZFS_DISCARDING_CHECKPOINT, fmt, ap);
	break;
	case ZFS_ERR_NO_CHECKPOINT:
	zfs_verror(hdl, EZFS_NO_CHECKPOINT, fmt, ap);
	break;
	case ZFS_ERR_DEVRM_IN_PROGRESS:
	zfs_verror(hdl, EZFS_DEVRM_IN_PROGRESS, fmt, ap);
	break;
	case ZFS_ERR_VDEV_TOO_BIG:
	zfs_verror(hdl, EZFS_VDEV_TOO_BIG, fmt, ap);
	break;
	case ZFS_ERR_EXPORT_IN_PROGRESS:
	zfs_verror(hdl, EZFS_EXPORT_IN_PROGRESS, fmt, ap);
	break;
	case ZFS_ERR_RESILVER_IN_PROGRESS:
	zfs_verror(hdl, EZFS_RESILVERING, fmt, ap);
	break;
	case ZFS_ERR_REBUILD_IN_PROGRESS:
	zfs_verror(hdl, EZFS_REBUILDING, fmt, ap);
	break;
	case ZFS_ERR_BADPROP:
	zfs_verror(hdl, EZFS_BADPROP, fmt, ap);
	break;
	case ZFS_ERR_VDEV_NOTSUP:
	zfs_verror(hdl, EZFS_VDEV_NOTSUP, fmt, ap);
	break;
	case ZFS_ERR_IOC_CMD_UNAVAIL:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "the loaded zfs "
	"module does not support this operation. A reboot may "
	"be required to enable this operation."));
	zfs_verror(hdl, EZFS_IOC_NOTSUPPORTED, fmt, ap);
	break;
	case ZFS_ERR_IOC_ARG_UNAVAIL:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "the loaded zfs "
	"module does not support an option for this operation. "
	"A reboot may be required to enable this option."));
	zfs_verror(hdl, EZFS_IOC_NOTSUPPORTED, fmt, ap);
	break;
	case ZFS_ERR_IOC_ARG_REQUIRED:
	case ZFS_ERR_IOC_ARG_BADTYPE:
	zfs_verror(hdl, EZFS_IOC_NOTSUPPORTED, fmt, ap);
	break;
	+ case ZFS_ERR_ASHIFT_MISMATCH:
	+ zfs_verror(hdl, EZFS_ASHIFT_MISMATCH, fmt, ap);
	+ break;
	default:
	zfs_error_aux(hdl, "%s", strerror(error));
	zfs_verror(hdl, EZFS_UNKNOWN, fmt, ap);
	}

	va_end(ap);
	return (-1);
	}

	/*
	* Display an out of memory error message and abort the current program.
	*/
	int
	no_memory(libzfs_handle_t *hdl)
	{
	return (zfs_error(hdl, EZFS_NOMEM, "internal error"));
	}

	/*
	* A safe form of malloc() which will die if the allocation fails.
	*/
	void *
	zfs_alloc(libzfs_handle_t *hdl, size_t size)
	{
	void *data;

	if ((data = calloc(1, size)) == NULL)
	(void) no_memory(hdl);

	return (data);
	}

	/*
	* A safe form of asprintf() which will die if the allocation fails.
	*/
	char *
	zfs_asprintf(libzfs_handle_t hdl, const char fmt, ...)
	{
	va_list ap;
	char *ret;
	int err;

	va_start(ap, fmt);

	err = vasprintf(&ret, fmt, ap);

	va_end(ap);

	if (err < 0) {
	(void) no_memory(hdl);
	ret = NULL;
	}

	return (ret);
	}

	/*
	* A safe form of realloc(), which also zeroes newly allocated space.
	*/
	void *
	zfs_realloc(libzfs_handle_t hdl, void ptr, size_t oldsize, size_t newsize)
	{
	void *ret;

	if ((ret = realloc(ptr, newsize)) == NULL) {
	(void) no_memory(hdl);
	return (NULL);
	}

	memset((char *)ret + oldsize, 0, newsize - oldsize);
	return (ret);
	}

	/*
	* A safe form of strdup() which will die if the allocation fails.
	*/
	char *
	zfs_strdup(libzfs_handle_t hdl, const char str)
	{
	char *ret;

	if ((ret = strdup(str)) == NULL)
	(void) no_memory(hdl);

	return (ret);
	}

	void
	libzfs_print_on_error(libzfs_handle_t *hdl, boolean_t printerr)
	{
	hdl->libzfs_printerr = printerr;
	}

	/*
	* Read lines from an open file descriptor and store them in an array of
	* strings until EOF. lines[] will be allocated and populated with all the
	* lines read. All newlines are replaced with NULL terminators for
	* convenience. lines[] must be freed after use with libzfs_free_str_array().
	*
	* Returns the number of lines read.
	*/
	static int
	libzfs_read_stdout_from_fd(int fd, char **lines[])
	{

	FILE *fp;
	int lines_cnt = 0;
	size_t len = 0;
	char *line = NULL;
	char tmp_lines = NULL, tmp;

	fp = fdopen(fd, "r");
	if (fp == NULL) {
	close(fd);
	return (0);
	}
	while (getline(&line, &len, fp) != -1) {
	tmp = realloc(tmp_lines, sizeof (tmp_lines) (lines_cnt + 1));
	if (tmp == NULL) {
	/* Return the lines we were able to process */
	break;
	}
	tmp_lines = tmp;

	/* Remove newline if not EOF */
	if (line[strlen(line) - 1] == '\n')
	line[strlen(line) - 1] = '\0';

	tmp_lines[lines_cnt] = strdup(line);
	if (tmp_lines[lines_cnt] == NULL)
	break;
	++lines_cnt;
	}
	free(line);
	fclose(fp);
	*lines = tmp_lines;
	return (lines_cnt);
	}

	static int
	libzfs_run_process_impl(const char path, char argv[], char *env[], int flags,
	char *lines[], int lines_cnt)
	{
	pid_t pid;
	int error, devnull_fd;
	int link[2];

	/*
	* Setup a pipe between our child and parent process if we're
	* reading stdout.
	*/
	if (lines != NULL && pipe2(link, O_NONBLOCK \| O_CLOEXEC) == -1)
	return (-EPIPE);

	pid = fork();
	if (pid == 0) {
	/* Child process */
	devnull_fd = open("/dev/null", O_WRONLY \| O_CLOEXEC);

	if (devnull_fd < 0)
	_exit(-1);

	if (!(flags & STDOUT_VERBOSE) && (lines == NULL))
	(void) dup2(devnull_fd, STDOUT_FILENO);
	else if (lines != NULL) {
	/* Save the output to lines[] */
	dup2(link[1], STDOUT_FILENO);
	}

	if (!(flags & STDERR_VERBOSE))
	(void) dup2(devnull_fd, STDERR_FILENO);

	if (flags & NO_DEFAULT_PATH) {
	if (env == NULL)
	execv(path, argv);
	else
	execve(path, argv, env);
	} else {
	if (env == NULL)
	execvp(path, argv);
	else
	execvpe(path, argv, env);
	}

	_exit(-1);
	} else if (pid > 0) {
	/* Parent process */
	int status;

	while ((error = waitpid(pid, &status, 0)) == -1 &&
	errno == EINTR)
	;
	if (error < 0 \|\| !WIFEXITED(status))
	return (-1);

	if (lines != NULL) {
	close(link[1]);
	*lines_cnt = libzfs_read_stdout_from_fd(link[0], lines);
	}
	return (WEXITSTATUS(status));
	}

	return (-1);
	}

	int
	libzfs_run_process(const char path, char argv[], int flags)
	{
	return (libzfs_run_process_impl(path, argv, NULL, flags, NULL, NULL));
	}

	/*
	* Run a command and store its stdout lines in an array of strings (lines[]).
	* lines[] is allocated and populated for you, and the number of lines is set in
	* lines_cnt. lines[] must be freed after use with libzfs_free_str_array().
	* All newlines (\n) in lines[] are terminated for convenience.
	*/
	int
	libzfs_run_process_get_stdout(const char path, char argv[], char *env[],
	char *lines[], int lines_cnt)
	{
	return (libzfs_run_process_impl(path, argv, env, 0, lines, lines_cnt));
	}

	/*
	* Same as libzfs_run_process_get_stdout(), but run without $PATH set. This
	* means that *path needs to be the full path to the executable.
	*/
	int
	libzfs_run_process_get_stdout_nopath(const char path, char argv[],
	char env[], char lines[], int lines_cnt)
	{
	return (libzfs_run_process_impl(path, argv, env, NO_DEFAULT_PATH,
	lines, lines_cnt));
	}

	/*
	* Free an array of strings. Free both the strings contained in the array and
	* the array itself.
	*/
	void
	libzfs_free_str_array(char **strs, int count)
	{
	while (--count >= 0)
	free(strs[count]);

	free(strs);
	}

	/*
	* Returns 1 if environment variable is set to "YES", "yes", "ON", "on", or
	* a non-zero number.
	*
	* Returns 0 otherwise.
	*/
	boolean_t
	libzfs_envvar_is_set(const char *envvar)
	{
	char *env = getenv(envvar);
	return (env && (strtoul(env, NULL, 0) > 0 \|\|
	(!strncasecmp(env, "YES", 3) && strnlen(env, 4) == 3) \|\|
	(!strncasecmp(env, "ON", 2) && strnlen(env, 3) == 2)));
	}

	libzfs_handle_t *
	libzfs_init(void)
	{
	libzfs_handle_t *hdl;
	int error;
	char *env;

	if ((error = libzfs_load_module()) != 0) {
	errno = error;
	return (NULL);
	}

	if ((hdl = calloc(1, sizeof (libzfs_handle_t))) == NULL) {
	return (NULL);
	}

	if (regcomp(&hdl->libzfs_urire, URI_REGEX, 0) != 0) {
	free(hdl);
	return (NULL);
	}

	if ((hdl->libzfs_fd = open(ZFS_DEV, O_RDWR\|O_EXCL\|O_CLOEXEC)) < 0) {
	free(hdl);
	return (NULL);
	}

	if (libzfs_core_init() != 0) {
	(void) close(hdl->libzfs_fd);
	free(hdl);
	return (NULL);
	}

	zfs_prop_init();
	zpool_prop_init();
	zpool_feature_init();
	vdev_prop_init();
	libzfs_mnttab_init(hdl);
	fletcher_4_init();

	if (getenv("ZFS_PROP_DEBUG") != NULL) {
	hdl->libzfs_prop_debug = B_TRUE;
	}
	if ((env = getenv("ZFS_SENDRECV_MAX_NVLIST")) != NULL) {
	if ((error = zfs_nicestrtonum(hdl, env,
	&hdl->libzfs_max_nvlist))) {
	errno = error;
	(void) close(hdl->libzfs_fd);
	free(hdl);
	return (NULL);
	}
	} else {
	hdl->libzfs_max_nvlist = (SPA_MAXBLOCKSIZE * 4);
	}

	/*
	* For testing, remove some settable properties and features
	*/
	if (libzfs_envvar_is_set("ZFS_SYSFS_PROP_SUPPORT_TEST")) {
	zprop_desc_t *proptbl;

	proptbl = zpool_prop_get_table();
	proptbl[ZPOOL_PROP_COMMENT].pd_zfs_mod_supported = B_FALSE;

	proptbl = zfs_prop_get_table();
	proptbl[ZFS_PROP_DNODESIZE].pd_zfs_mod_supported = B_FALSE;

	zfeature_info_t *ftbl = spa_feature_table;
	ftbl[SPA_FEATURE_LARGE_BLOCKS].fi_zfs_mod_supported = B_FALSE;
	}

	return (hdl);
	}

	void
	libzfs_fini(libzfs_handle_t *hdl)
	{
	(void) close(hdl->libzfs_fd);
	zpool_free_handles(hdl);
	namespace_clear(hdl);
	libzfs_mnttab_fini(hdl);
	libzfs_core_fini();
	regfree(&hdl->libzfs_urire);
	fletcher_4_fini();
	#if LIBFETCH_DYNAMIC
	if (hdl->libfetch != (void *)-1 && hdl->libfetch != NULL)
	(void) dlclose(hdl->libfetch);
	free(hdl->libfetch_load_error);
	#endif
	free(hdl);
	}

	libzfs_handle_t *
	zpool_get_handle(zpool_handle_t *zhp)
	{
	return (zhp->zpool_hdl);
	}

	libzfs_handle_t *
	zfs_get_handle(zfs_handle_t *zhp)
	{
	return (zhp->zfs_hdl);
	}

	zpool_handle_t *
	zfs_get_pool_handle(const zfs_handle_t *zhp)
	{
	return (zhp->zpool_hdl);
	}

	/*
	* Given a name, determine whether or not it's a valid path
	* (starts with '/' or "./"). If so, walk the mnttab trying
	* to match the device number. If not, treat the path as an
	* fs/vol/snap/bkmark name.
	*/
	zfs_handle_t *
	zfs_path_to_zhandle(libzfs_handle_t hdl, const char path, zfs_type_t argtype)
	{
	struct stat64 statbuf;
	struct extmnttab entry;

	if (path[0] != '/' && strncmp(path, "./", strlen("./")) != 0) {
	/*
	* It's not a valid path, assume it's a name of type 'argtype'.
	*/
	return (zfs_open(hdl, path, argtype));
	}

	if (getextmntent(path, &entry, &statbuf) != 0)
	return (NULL);

	if (strcmp(entry.mnt_fstype, MNTTYPE_ZFS) != 0) {
	(void) fprintf(stderr, gettext("'%s': not a ZFS filesystem\n"),
	path);
	return (NULL);
	}

	return (zfs_open(hdl, entry.mnt_special, ZFS_TYPE_FILESYSTEM));
	}

	/*
	* Initialize the zc_nvlist_dst member to prepare for receiving an nvlist from
	* an ioctl().
	*/
	void
	zcmd_alloc_dst_nvlist(libzfs_handle_t hdl, zfs_cmd_t zc, size_t len)
	{
	if (len == 0)
	len = 256 * 1024;
	zc->zc_nvlist_dst_size = len;
	zc->zc_nvlist_dst =
	(uint64_t)(uintptr_t)zfs_alloc(hdl, zc->zc_nvlist_dst_size);
	}

	/*
	* Called when an ioctl() which returns an nvlist fails with ENOMEM. This will
	* expand the nvlist to the size specified in 'zc_nvlist_dst_size', which was
	* filled in by the kernel to indicate the actual required size.
	*/
	void
	zcmd_expand_dst_nvlist(libzfs_handle_t hdl, zfs_cmd_t zc)
	{
	free((void *)(uintptr_t)zc->zc_nvlist_dst);
	zc->zc_nvlist_dst =
	(uint64_t)(uintptr_t)zfs_alloc(hdl, zc->zc_nvlist_dst_size);
	}

	/*
	* Called to free the src and dst nvlists stored in the command structure.
	*/
	void
	zcmd_free_nvlists(zfs_cmd_t *zc)
	{
	free((void *)(uintptr_t)zc->zc_nvlist_conf);
	free((void *)(uintptr_t)zc->zc_nvlist_src);
	free((void *)(uintptr_t)zc->zc_nvlist_dst);
	zc->zc_nvlist_conf = 0;
	zc->zc_nvlist_src = 0;
	zc->zc_nvlist_dst = 0;
	}

	static void
	zcmd_write_nvlist_com(libzfs_handle_t hdl, uint64_t outnv, uint64_t *outlen,
	nvlist_t *nvl)
	{
	char *packed;

	size_t len = fnvlist_size(nvl);
	packed = zfs_alloc(hdl, len);

	verify(nvlist_pack(nvl, &packed, &len, NV_ENCODE_NATIVE, 0) == 0);

	*outnv = (uint64_t)(uintptr_t)packed;
	*outlen = len;
	}

	void
	zcmd_write_conf_nvlist(libzfs_handle_t hdl, zfs_cmd_t zc, nvlist_t *nvl)
	{
	zcmd_write_nvlist_com(hdl, &zc->zc_nvlist_conf,
	&zc->zc_nvlist_conf_size, nvl);
	}

	void
	zcmd_write_src_nvlist(libzfs_handle_t hdl, zfs_cmd_t zc, nvlist_t *nvl)
	{
	zcmd_write_nvlist_com(hdl, &zc->zc_nvlist_src,
	&zc->zc_nvlist_src_size, nvl);
	}

	/*
	* Unpacks an nvlist from the ZFS ioctl command structure.
	*/
	int
	zcmd_read_dst_nvlist(libzfs_handle_t hdl, zfs_cmd_t zc, nvlist_t **nvlp)
	{
	if (nvlist_unpack((void *)(uintptr_t)zc->zc_nvlist_dst,
	zc->zc_nvlist_dst_size, nvlp, 0) != 0)
	return (no_memory(hdl));

	return (0);
	}

	/*
	* ================================================================
	* API shared by zfs and zpool property management
	* ================================================================
	*/

	static void
	zprop_print_headers(zprop_get_cbdata_t *cbp, zfs_type_t type)
	{
	zprop_list_t *pl;
	int i;
	char *title;
	size_t len;

	cbp->cb_first = B_FALSE;
	if (cbp->cb_scripted)
	return;

	/*
	* Start with the length of the column headers.
	*/
	cbp->cb_colwidths[GET_COL_NAME] = strlen(dgettext(TEXT_DOMAIN, "NAME"));
	cbp->cb_colwidths[GET_COL_PROPERTY] = strlen(dgettext(TEXT_DOMAIN,
	"PROPERTY"));
	cbp->cb_colwidths[GET_COL_VALUE] = strlen(dgettext(TEXT_DOMAIN,
	"VALUE"));
	cbp->cb_colwidths[GET_COL_RECVD] = strlen(dgettext(TEXT_DOMAIN,
	"RECEIVED"));
	cbp->cb_colwidths[GET_COL_SOURCE] = strlen(dgettext(TEXT_DOMAIN,
	"SOURCE"));

	/* first property is always NAME */
	assert(cbp->cb_proplist->pl_prop ==
	((type == ZFS_TYPE_POOL) ? ZPOOL_PROP_NAME :
	((type == ZFS_TYPE_VDEV) ? VDEV_PROP_NAME : ZFS_PROP_NAME)));

	/*
	* Go through and calculate the widths for each column. For the
	* 'source' column, we kludge it up by taking the worst-case scenario of
	* inheriting from the longest name. This is acceptable because in the
	* majority of cases 'SOURCE' is the last column displayed, and we don't
	* use the width anyway. Note that the 'VALUE' column can be oversized,
	* if the name of the property is much longer than any values we find.
	*/
	for (pl = cbp->cb_proplist; pl != NULL; pl = pl->pl_next) {
	/*
	* 'PROPERTY' column
	*/
	if (pl->pl_prop != ZPROP_USERPROP) {
	const char *propname = (type == ZFS_TYPE_POOL) ?
	zpool_prop_to_name(pl->pl_prop) :
	((type == ZFS_TYPE_VDEV) ?
	vdev_prop_to_name(pl->pl_prop) :
	zfs_prop_to_name(pl->pl_prop));

	assert(propname != NULL);
	len = strlen(propname);
	if (len > cbp->cb_colwidths[GET_COL_PROPERTY])
	cbp->cb_colwidths[GET_COL_PROPERTY] = len;
	} else {
	assert(pl->pl_user_prop != NULL);
	len = strlen(pl->pl_user_prop);
	if (len > cbp->cb_colwidths[GET_COL_PROPERTY])
	cbp->cb_colwidths[GET_COL_PROPERTY] = len;
	}

	/*
	* 'VALUE' column. The first property is always the 'name'
	* property that was tacked on either by /sbin/zfs's
	* zfs_do_get() or when calling zprop_expand_list(), so we
	* ignore its width. If the user specified the name property
	* to display, then it will be later in the list in any case.
	*/
	if (pl != cbp->cb_proplist &&
	pl->pl_width > cbp->cb_colwidths[GET_COL_VALUE])
	cbp->cb_colwidths[GET_COL_VALUE] = pl->pl_width;

	/* 'RECEIVED' column. */
	if (pl != cbp->cb_proplist &&
	pl->pl_recvd_width > cbp->cb_colwidths[GET_COL_RECVD])
	cbp->cb_colwidths[GET_COL_RECVD] = pl->pl_recvd_width;

	/*
	* 'NAME' and 'SOURCE' columns
	*/
	if (pl->pl_prop == ((type == ZFS_TYPE_POOL) ? ZPOOL_PROP_NAME :
	((type == ZFS_TYPE_VDEV) ? VDEV_PROP_NAME :
	ZFS_PROP_NAME)) && pl->pl_width >
	cbp->cb_colwidths[GET_COL_NAME]) {
	cbp->cb_colwidths[GET_COL_NAME] = pl->pl_width;
	cbp->cb_colwidths[GET_COL_SOURCE] = pl->pl_width +
	strlen(dgettext(TEXT_DOMAIN, "inherited from"));
	}
	}

	/*
	* Now go through and print the headers.
	*/
	for (i = 0; i < ZFS_GET_NCOLS; i++) {
	switch (cbp->cb_columns[i]) {
	case GET_COL_NAME:
	title = dgettext(TEXT_DOMAIN, "NAME");
	break;
	case GET_COL_PROPERTY:
	title = dgettext(TEXT_DOMAIN, "PROPERTY");
	break;
	case GET_COL_VALUE:
	title = dgettext(TEXT_DOMAIN, "VALUE");
	break;
	case GET_COL_RECVD:
	title = dgettext(TEXT_DOMAIN, "RECEIVED");
	break;
	case GET_COL_SOURCE:
	title = dgettext(TEXT_DOMAIN, "SOURCE");
	break;
	default:
	title = NULL;
	}

	if (title != NULL) {
	if (i == (ZFS_GET_NCOLS - 1) \|\|
	cbp->cb_columns[i + 1] == GET_COL_NONE)
	(void) printf("%s", title);
	else
	(void) printf("%-*s ",
	cbp->cb_colwidths[cbp->cb_columns[i]],
	title);
	}
	}
	(void) printf("\n");
	}

	/*
	* Display a single line of output, according to the settings in the callback
	* structure.
	*/
	void
	zprop_print_one_property(const char name, zprop_get_cbdata_t cbp,
	const char propname, const char value, zprop_source_t sourcetype,
	const char source, const char recvd_value)
	{
	int i;
	const char *str = NULL;
	char buf[128];

	/*
	* Ignore those source types that the user has chosen to ignore.
	*/
	if ((sourcetype & cbp->cb_sources) == 0)
	return;

	if (cbp->cb_first)
	zprop_print_headers(cbp, cbp->cb_type);

	for (i = 0; i < ZFS_GET_NCOLS; i++) {
	switch (cbp->cb_columns[i]) {
	case GET_COL_NAME:
	str = name;
	break;

	case GET_COL_PROPERTY:
	str = propname;
	break;

	case GET_COL_VALUE:
	str = value;
	break;

	case GET_COL_SOURCE:
	switch (sourcetype) {
	case ZPROP_SRC_NONE:
	str = "-";
	break;

	case ZPROP_SRC_DEFAULT:
	str = "default";
	break;

	case ZPROP_SRC_LOCAL:
	str = "local";
	break;

	case ZPROP_SRC_TEMPORARY:
	str = "temporary";
	break;

	case ZPROP_SRC_INHERITED:
	(void) snprintf(buf, sizeof (buf),
	"inherited from %s", source);
	str = buf;
	break;
	case ZPROP_SRC_RECEIVED:
	str = "received";
	break;

	default:
	str = NULL;
	assert(!"unhandled zprop_source_t");
	}
	break;

	case GET_COL_RECVD:
	str = (recvd_value == NULL ? "-" : recvd_value);
	break;

	default:
	continue;
	}

	if (i == (ZFS_GET_NCOLS - 1) \|\|
	cbp->cb_columns[i + 1] == GET_COL_NONE)
	(void) printf("%s", str);
	else if (cbp->cb_scripted)
	(void) printf("%s\t", str);
	else
	(void) printf("%-*s ",
	cbp->cb_colwidths[cbp->cb_columns[i]],
	str);
	}

	(void) printf("\n");
	}

	/*
	* Given a numeric suffix, convert the value into a number of bits that the
	* resulting value must be shifted.
	*/
	static int
	str2shift(libzfs_handle_t hdl, const char buf)
	{
	const char *ends = "BKMGTPEZ";
	int i;

	if (buf[0] == '\0')
	return (0);
	for (i = 0; i < strlen(ends); i++) {
	if (toupper(buf[0]) == ends[i])
	break;
	}
	if (i == strlen(ends)) {
	if (hdl)
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"invalid numeric suffix '%s'"), buf);
	return (-1);
	}

	/*
	* Allow 'G' = 'GB' = 'GiB', case-insensitively.
	* However, 'BB' and 'BiB' are disallowed.
	*/
	if (buf[1] == '\0' \|\|
	(toupper(buf[0]) != 'B' &&
	((toupper(buf[1]) == 'B' && buf[2] == '\0') \|\|
	(toupper(buf[1]) == 'I' && toupper(buf[2]) == 'B' &&
	buf[3] == '\0'))))
	return (10 * i);

	if (hdl)
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"invalid numeric suffix '%s'"), buf);
	return (-1);
	}

	/*
	* Convert a string of the form '100G' into a real number. Used when setting
	* properties or creating a volume. 'buf' is used to place an extended error
	* message for the caller to use.
	*/
	int
	zfs_nicestrtonum(libzfs_handle_t hdl, const char value, uint64_t *num)
	{
	char *end;
	int shift;

	*num = 0;

	/* Check to see if this looks like a number. */
	if ((value[0] < '0' \|\| value[0] > '9') && value[0] != '.') {
	if (hdl)
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"bad numeric value '%s'"), value);
	return (-1);
	}

	/* Rely on strtoull() to process the numeric portion. */
	errno = 0;
	*num = strtoull(value, &end, 10);

	/*
	* Check for ERANGE, which indicates that the value is too large to fit
	* in a 64-bit value.
	*/
	if (errno == ERANGE) {
	if (hdl)
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"numeric value is too large"));
	return (-1);
	}

	/*
	* If we have a decimal value, then do the computation with floating
	* point arithmetic. Otherwise, use standard arithmetic.
	*/
	if (*end == '.') {
	double fval = strtod(value, &end);

	if ((shift = str2shift(hdl, end)) == -1)
	return (-1);

	fval *= pow(2, shift);

	/*
	* UINT64_MAX is not exactly representable as a double.
	* The closest representation is UINT64_MAX + 1, so we
	* use a >= comparison instead of > for the bounds check.
	*/
	if (fval >= (double)UINT64_MAX) {
	if (hdl)
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"numeric value is too large"));
	return (-1);
	}

	*num = (uint64_t)fval;
	} else {
	if ((shift = str2shift(hdl, end)) == -1)
	return (-1);

	/* Check for overflow */
	if (shift >= 64 \|\| (num << shift) >> shift != num) {
	if (hdl)
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"numeric value is too large"));
	return (-1);
	}

	*num <<= shift;
	}

	return (0);
	}

	/*
	* Given a propname=value nvpair to set, parse any numeric properties
	* (index, boolean, etc) if they are specified as strings and add the
	* resulting nvpair to the returned nvlist.
	*
	* At the DSL layer, all properties are either 64-bit numbers or strings.
	* We want the user to be able to ignore this fact and specify properties
	* as native values (numbers, for example) or as strings (to simplify
	* command line utilities). This also handles converting index types
	* (compression, checksum, etc) from strings to their on-disk index.
	*/
	int
	zprop_parse_value(libzfs_handle_t hdl, nvpair_t elem, int prop,
	zfs_type_t type, nvlist_t ret, const char svalp, uint64_t ivalp,
	const char *errbuf)
	{
	data_type_t datatype = nvpair_type(elem);
	zprop_type_t proptype;
	const char *propname;
	const char *value;
	boolean_t isnone = B_FALSE;
	boolean_t isauto = B_FALSE;
	int err = 0;

	if (type == ZFS_TYPE_POOL) {
	proptype = zpool_prop_get_type(prop);
	propname = zpool_prop_to_name(prop);
	} else if (type == ZFS_TYPE_VDEV) {
	proptype = vdev_prop_get_type(prop);
	propname = vdev_prop_to_name(prop);
	} else {
	proptype = zfs_prop_get_type(prop);
	propname = zfs_prop_to_name(prop);
	}

	/*
	* Convert any properties to the internal DSL value types.
	*/
	*svalp = NULL;
	*ivalp = 0;

	switch (proptype) {
	case PROP_TYPE_STRING:
	if (datatype != DATA_TYPE_STRING) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"'%s' must be a string"), nvpair_name(elem));
	goto error;
	}
	err = nvpair_value_string(elem, svalp);
	if (err != 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"'%s' is invalid"), nvpair_name(elem));
	goto error;
	}
	if (strlen(*svalp) >= ZFS_MAXPROPLEN) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"'%s' is too long"), nvpair_name(elem));
	goto error;
	}
	break;

	case PROP_TYPE_NUMBER:
	if (datatype == DATA_TYPE_STRING) {
	(void) nvpair_value_string(elem, &value);
	if (strcmp(value, "none") == 0) {
	isnone = B_TRUE;
	} else if (strcmp(value, "auto") == 0) {
	isauto = B_TRUE;
	} else if (zfs_nicestrtonum(hdl, value, ivalp) != 0) {
	goto error;
	}
	} else if (datatype == DATA_TYPE_UINT64) {
	(void) nvpair_value_uint64(elem, ivalp);
	} else {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"'%s' must be a number"), nvpair_name(elem));
	goto error;
	}

	/*
	* Quota special: force 'none' and don't allow 0.
	*/
	if ((type & ZFS_TYPE_DATASET) && *ivalp == 0 && !isnone &&
	(prop == ZFS_PROP_QUOTA \|\| prop == ZFS_PROP_REFQUOTA)) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"use 'none' to disable quota/refquota"));
	goto error;
	}

	/*
	* Special handling for "*_limit=none". In this case it's not
	* 0 but UINT64_MAX.
	*/
	if ((type & ZFS_TYPE_DATASET) && isnone &&
	(prop == ZFS_PROP_FILESYSTEM_LIMIT \|\|
	prop == ZFS_PROP_SNAPSHOT_LIMIT)) {
	*ivalp = UINT64_MAX;
	}

	/*
	* Special handling for "checksum_*=none". In this case it's not
	* 0 but UINT64_MAX.
	*/
	if ((type & ZFS_TYPE_VDEV) && isnone &&
	(prop == VDEV_PROP_CHECKSUM_N \|\|
	prop == VDEV_PROP_CHECKSUM_T \|\|
	prop == VDEV_PROP_IO_N \|\|
	- prop == VDEV_PROP_IO_T)) {
	+ prop == VDEV_PROP_IO_T \|\|
	+ prop == VDEV_PROP_SLOW_IO_N \|\|
	+ prop == VDEV_PROP_SLOW_IO_T)) {
	*ivalp = UINT64_MAX;
	}

	/*
	* Special handling for setting 'refreservation' to 'auto'. Use
	* UINT64_MAX to tell the caller to use zfs_fix_auto_resv().
	* 'auto' is only allowed on volumes.
	*/
	if (isauto) {
	switch (prop) {
	case ZFS_PROP_REFRESERVATION:
	if ((type & ZFS_TYPE_VOLUME) == 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"'%s=auto' only allowed on "
	"volumes"), nvpair_name(elem));
	goto error;
	}
	*ivalp = UINT64_MAX;
	break;
	default:
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"'auto' is invalid value for '%s'"),
	nvpair_name(elem));
	goto error;
	}
	}

	break;

	case PROP_TYPE_INDEX:
	if (datatype != DATA_TYPE_STRING) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"'%s' must be a string"), nvpair_name(elem));
	goto error;
	}

	(void) nvpair_value_string(elem, &value);

	if (zprop_string_to_index(prop, value, ivalp, type) != 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"'%s' must be one of '%s'"), propname,
	zprop_values(prop, type));
	goto error;
	}
	break;

	default:
	abort();
	}

	/*
	* Add the result to our return set of properties.
	*/
	if (*svalp != NULL) {
	if (nvlist_add_string(ret, propname, *svalp) != 0) {
	(void) no_memory(hdl);
	return (-1);
	}
	} else {
	if (nvlist_add_uint64(ret, propname, *ivalp) != 0) {
	(void) no_memory(hdl);
	return (-1);
	}
	}

	return (0);
	error:
	(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
	return (-1);
	}

	static int
	addlist(libzfs_handle_t hdl, const char propname, zprop_list_t **listp,
	zfs_type_t type)
	{
	int prop = zprop_name_to_prop(propname, type);
	if (prop != ZPROP_INVAL && !zprop_valid_for_type(prop, type, B_FALSE))
	prop = ZPROP_INVAL;

	/*
	* Return failure if no property table entry was found and this isn't
	* a user-defined property.
	*/
	if (prop == ZPROP_USERPROP && ((type == ZFS_TYPE_POOL &&
	!zfs_prop_user(propname) &&
	!zpool_prop_feature(propname) &&
	!zpool_prop_unsupported(propname)) \|\|
	((type == ZFS_TYPE_DATASET) && !zfs_prop_user(propname) &&
	!zfs_prop_userquota(propname) && !zfs_prop_written(propname)) \|\|
	((type == ZFS_TYPE_VDEV) && !vdev_prop_user(propname)))) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"invalid property '%s'"), propname);
	return (zfs_error(hdl, EZFS_BADPROP,
	dgettext(TEXT_DOMAIN, "bad property list")));
	}

	zprop_list_t entry = zfs_alloc(hdl, sizeof (entry));

	entry->pl_prop = prop;
	if (prop == ZPROP_USERPROP) {
	entry->pl_user_prop = zfs_strdup(hdl, propname);
	entry->pl_width = strlen(propname);
	} else {
	entry->pl_width = zprop_width(prop, &entry->pl_fixed,
	type);
	}

	*listp = entry;

	return (0);
	}

	/*
	* Given a comma-separated list of properties, construct a property list
	* containing both user-defined and native properties. This function will
	* return a NULL list if 'all' is specified, which can later be expanded
	* by zprop_expand_list().
	*/
	int
	zprop_get_list(libzfs_handle_t hdl, char props, zprop_list_t **listp,
	zfs_type_t type)
	{
	*listp = NULL;

	/*
	* If 'all' is specified, return a NULL list.
	*/
	if (strcmp(props, "all") == 0)
	return (0);

	/*
	* If no props were specified, return an error.
	*/
	if (props[0] == '\0') {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
	"no properties specified"));
	return (zfs_error(hdl, EZFS_BADPROP, dgettext(TEXT_DOMAIN,
	"bad property list")));
	}

	for (char *p; (p = strsep(&props, ",")); )
	if (strcmp(p, "space") == 0) {
	static const char *const spaceprops[] = {
	"name", "avail", "used", "usedbysnapshots",
	"usedbydataset", "usedbyrefreservation",
	"usedbychildren"
	};

	for (int i = 0; i < ARRAY_SIZE(spaceprops); i++) {
	if (addlist(hdl, spaceprops[i], listp, type))
	return (-1);
	listp = &(*listp)->pl_next;
	}
	} else {
	if (addlist(hdl, p, listp, type))
	return (-1);
	listp = &(*listp)->pl_next;
	}

	return (0);
	}

	void
	zprop_free_list(zprop_list_t *pl)
	{
	zprop_list_t *next;

	while (pl != NULL) {
	next = pl->pl_next;
	free(pl->pl_user_prop);
	free(pl);
	pl = next;
	}
	}

	typedef struct expand_data {
	zprop_list_t **last;
	libzfs_handle_t *hdl;
	zfs_type_t type;
	} expand_data_t;

	static int
	zprop_expand_list_cb(int prop, void *cb)
	{
	zprop_list_t *entry;
	expand_data_t *edp = cb;

	entry = zfs_alloc(edp->hdl, sizeof (zprop_list_t));

	entry->pl_prop = prop;
	entry->pl_width = zprop_width(prop, &entry->pl_fixed, edp->type);
	entry->pl_all = B_TRUE;

	*(edp->last) = entry;
	edp->last = &entry->pl_next;

	return (ZPROP_CONT);
	}

	int
	zprop_expand_list(libzfs_handle_t hdl, zprop_list_t *plp, zfs_type_t type)
	{
	zprop_list_t *entry;
	zprop_list_t **last;
	expand_data_t exp;

	if (*plp == NULL) {
	/*
	* If this is the very first time we've been called for an 'all'
	* specification, expand the list to include all native
	* properties.
	*/
	last = plp;

	exp.last = last;
	exp.hdl = hdl;
	exp.type = type;

	if (zprop_iter_common(zprop_expand_list_cb, &exp, B_FALSE,
	B_FALSE, type) == ZPROP_INVAL)
	return (-1);

	/*
	* Add 'name' to the beginning of the list, which is handled
	* specially.
	*/
	entry = zfs_alloc(hdl, sizeof (zprop_list_t));
	entry->pl_prop = ((type == ZFS_TYPE_POOL) ? ZPOOL_PROP_NAME :
	((type == ZFS_TYPE_VDEV) ? VDEV_PROP_NAME : ZFS_PROP_NAME));
	entry->pl_width = zprop_width(entry->pl_prop,
	&entry->pl_fixed, type);
	entry->pl_all = B_TRUE;
	entry->pl_next = *plp;
	*plp = entry;
	}
	return (0);
	}

	int
	zprop_iter(zprop_func func, void *cb, boolean_t show_all, boolean_t ordered,
	zfs_type_t type)
	{
	return (zprop_iter_common(func, cb, show_all, ordered, type));
	}

	const char *
	zfs_version_userland(void)
	{
	return (ZFS_META_ALIAS);
	}

	/*
	* Prints both zfs userland and kernel versions
	* Returns 0 on success, and -1 on error
	*/
	int
	zfs_version_print(void)
	{
	(void) puts(ZFS_META_ALIAS);

	char *kver = zfs_version_kernel();
	if (kver == NULL) {
	fprintf(stderr, "zfs_version_kernel() failed: %s\n",
	strerror(errno));
	return (-1);
	}

	(void) printf("zfs-kmod-%s\n", kver);
	free(kver);
	return (0);
	}

	/*
	* Return 1 if the user requested ANSI color output, and our terminal supports
	* it. Return 0 for no color.
	*/
	int
	use_color(void)
	{
	static int use_color = -1;
	char *term;

	/*
	* Optimization:
	*
	* For each zpool invocation, we do a single check to see if we should
	* be using color or not, and cache that value for the lifetime of the
	* the zpool command. That makes it cheap to call use_color() when
	* we're printing with color. We assume that the settings are not going
	* to change during the invocation of a zpool command (the user isn't
	* going to change the ZFS_COLOR value while zpool is running, for
	* example).
	*/
	if (use_color != -1) {
	/*
	* We've already figured out if we should be using color or
	* not. Return the cached value.
	*/
	return (use_color);
	}

	term = getenv("TERM");
	/*
	* The user sets the ZFS_COLOR env var set to enable zpool ANSI color
	* output. However if NO_COLOR is set (https://no-color.org/) then
	* don't use it. Also, don't use color if terminal doesn't support
	* it.
	*/
	if (libzfs_envvar_is_set("ZFS_COLOR") &&
	!libzfs_envvar_is_set("NO_COLOR") &&
	isatty(STDOUT_FILENO) && term && strcmp("dumb", term) != 0 &&
	strcmp("unknown", term) != 0) {
	/* Color supported */
	use_color = 1;
	} else {
	use_color = 0;
	}

	return (use_color);
	}

	/*
	* The functions color_start() and color_end() are used for when you want
	* to colorize a block of text.
	*
	* For example:
	* color_start(ANSI_RED)
	* printf("hello");
	* printf("world");
	* color_end();
	*/
	void
	color_start(const char *color)
	{
	if (color && use_color()) {
	fputs(color, stdout);
	fflush(stdout);
	}
	}

	void
	color_end(void)
	{
	if (use_color()) {
	fputs(ANSI_RESET, stdout);
	fflush(stdout);
	}

	}

	/*
	* printf() with a color. If color is NULL, then do a normal printf.
	*/
	int
	printf_color(const char color, const char format, ...)
	{
	va_list aptr;
	int rc;

	if (color)
	color_start(color);

	va_start(aptr, format);
	rc = vprintf(format, aptr);
	va_end(aptr);

	if (color)
	color_end();

	return (rc);
	}

	/* PATH + 5 env vars + a NULL entry = 7 */
	#define ZPOOL_VDEV_SCRIPT_ENV_COUNT 7

	/*
	* There's a few places where ZFS will call external scripts (like the script
	* in zpool.d/ and `zfs_prepare_disk`). These scripts are called with a
	* reduced $PATH, and some vdev specific environment vars set. This function
	* will allocate an populate the environment variable array that is passed to
	* these scripts. The user must free the arrays with zpool_vdev_free_env() when
	* they are done.
	*
	* The following env vars will be set (but value could be blank):
	*
	* POOL_NAME
	* VDEV_PATH
	* VDEV_UPATH
	* VDEV_ENC_SYSFS_PATH
	*
	* In addition, you can set an optional environment variable named 'opt_key'
	* to 'opt_val' if you want.
	*
	* Returns allocated env[] array on success, NULL otherwise.
	*/
	char **
	zpool_vdev_script_alloc_env(const char *pool_name,
	const char vdev_path, const char vdev_upath,
	const char vdev_enc_sysfs_path, const char opt_key, const char *opt_val)
	{
	char **env = NULL;
	int rc;

	env = calloc(ZPOOL_VDEV_SCRIPT_ENV_COUNT, sizeof (*env));
	if (!env)
	return (NULL);

	env[0] = strdup("PATH=/bin:/sbin:/usr/bin:/usr/sbin");
	if (!env[0])
	goto error;

	/* Setup our custom environment variables */
	rc = asprintf(&env[1], "POOL_NAME=%s", pool_name ? pool_name : "");
	if (rc == -1) {
	env[1] = NULL;
	goto error;
	}

	rc = asprintf(&env[2], "VDEV_PATH=%s", vdev_path ? vdev_path : "");
	if (rc == -1) {
	env[2] = NULL;
	goto error;
	}

	rc = asprintf(&env[3], "VDEV_UPATH=%s", vdev_upath ? vdev_upath : "");
	if (rc == -1) {
	env[3] = NULL;
	goto error;
	}

	rc = asprintf(&env[4], "VDEV_ENC_SYSFS_PATH=%s",
	vdev_enc_sysfs_path ? vdev_enc_sysfs_path : "");
	if (rc == -1) {
	env[4] = NULL;
	goto error;
	}

	if (opt_key != NULL) {
	rc = asprintf(&env[5], "%s=%s", opt_key,
	opt_val ? opt_val : "");
	if (rc == -1) {
	env[5] = NULL;
	goto error;
	}
	}

	return (env);

	error:
	for (int i = 0; i < ZPOOL_VDEV_SCRIPT_ENV_COUNT; i++)
	free(env[i]);

	free(env);

	return (NULL);
	}

	/*
	* Free the env[] array that was allocated by zpool_vdev_script_alloc_env().
	*/
	void
	zpool_vdev_script_free_env(char **env)
	{
	for (int i = 0; i < ZPOOL_VDEV_SCRIPT_ENV_COUNT; i++)
	free(env[i]);

	free(env);
	}

	/*
	* Prepare a disk by (optionally) running a program before labeling the disk.
	* This can be useful for installing disk firmware or doing some pre-flight
	* checks on the disk before it becomes part of the pool. The program run is
	* located at ZFSEXECDIR/zfs_prepare_disk
	* (E.x: /usr/local/libexec/zfs/zfs_prepare_disk).
	*
	* Return 0 on success, non-zero on failure.
	*/
	int
	zpool_prepare_disk(zpool_handle_t zhp, nvlist_t vdev_nv,
	const char prepare_str, char lines[], int lines_cnt)
	{
	const char *script_path = ZFSEXECDIR "/zfs_prepare_disk";
	const char *pool_name;
	int rc = 0;

	/* Path to script and a NULL entry */
	char argv[2] = {(char )script_path};
	char **env = NULL;
	const char path = NULL, enc_sysfs_path = NULL;
	char *upath;
	*lines_cnt = 0;

	if (access(script_path, X_OK) != 0) {
	/* No script, nothing to do */
	return (0);
	}

	(void) nvlist_lookup_string(vdev_nv, ZPOOL_CONFIG_PATH, &path);
	(void) nvlist_lookup_string(vdev_nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH,
	&enc_sysfs_path);

	upath = zfs_get_underlying_path(path);
	pool_name = zhp ? zpool_get_name(zhp) : NULL;

	env = zpool_vdev_script_alloc_env(pool_name, path, upath,
	enc_sysfs_path, "VDEV_PREPARE", prepare_str);

	free(upath);

	if (env == NULL) {
	return (ENOMEM);
	}

	rc = libzfs_run_process_get_stdout(script_path, argv, env, lines,
	lines_cnt);

	zpool_vdev_script_free_env(env);

	return (rc);
	}

	/*
	* Optionally run a script and then label a disk. The script can be used to
	* prepare a disk for inclusion into the pool. For example, it might update
	* the disk's firmware or check its health.
	*
	* The 'name' provided is the short name, stripped of any leading
	* /dev path, and is passed to zpool_label_disk. vdev_nv is the nvlist for
	* the vdev. prepare_str is a string that gets passed as the VDEV_PREPARE
	* env variable to the script.
	*
	* The following env vars are passed to the script:
	*
	* POOL_NAME: The pool name (blank during zpool create)
	* VDEV_PREPARE: Reason why the disk is being prepared for inclusion:
	* "create", "add", "replace", or "autoreplace"
	* VDEV_PATH: Path to the disk
	* VDEV_UPATH: One of the 'underlying paths' to the disk. This is
	* useful for DM devices.
	* VDEV_ENC_SYSFS_PATH: Path to the disk's enclosure sysfs path, if available.
	*
	* Note, some of these values can be blank.
	*
	* Return 0 on success, non-zero otherwise.
	*/
	int
	zpool_prepare_and_label_disk(libzfs_handle_t hdl, zpool_handle_t zhp,
	const char name, nvlist_t vdev_nv, const char *prepare_str,
	char *lines[], int lines_cnt)
	{
	int rc;
	char vdev_path[MAXPATHLEN];
	(void) snprintf(vdev_path, sizeof (vdev_path), "%s/%s", DISK_ROOT,
	name);

	/* zhp will be NULL when creating a pool */
	rc = zpool_prepare_disk(zhp, vdev_nv, prepare_str, lines, lines_cnt);
	if (rc != 0)
	return (rc);

	rc = zpool_label_disk(hdl, zhp, name);
	return (rc);
	}
	diff --git a/sys/contrib/openzfs/lib/libzfs/os/linux/libzfs_pool_os.c b/sys/contrib/openzfs/lib/libzfs/os/linux/libzfs_pool_os.c
	index 401151b1afb5..86eef3255bc2 100644
	--- a/sys/contrib/openzfs/lib/libzfs/os/linux/libzfs_pool_os.c
	+++ b/sys/contrib/openzfs/lib/libzfs/os/linux/libzfs_pool_os.c
	@@ -1,340 +1,350 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright 2015 Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2011, 2018 by Delphix. All rights reserved.
	* Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>
	* Copyright (c) 2018 Datto Inc.
	* Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
	* Copyright (c) 2017, Intel Corporation.
	* Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>
	*/

	#include <errno.h>
	#include <libintl.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <unistd.h>
	#include <libgen.h>
	#include <zone.h>
	#include <sys/stat.h>
	#include <sys/efi_partition.h>
	#include <sys/systeminfo.h>
	#include <sys/zfs_ioctl.h>
	#include <sys/vdev_disk.h>
	#include <dlfcn.h>
	#include <libzutil.h>

	#include "zfs_namecheck.h"
	#include "zfs_prop.h"
	#include "../../libzfs_impl.h"
	#include "zfs_comutil.h"
	#include "zfeature_common.h"

	/*
	* If the device has being dynamically expanded then we need to relabel
	* the disk to use the new unallocated space.
	*/
	int
	zpool_relabel_disk(libzfs_handle_t hdl, const char path, const char *msg)
	{
	int fd, error;

	if ((fd = open(path, O_RDWR\|O_DIRECT\|O_CLOEXEC)) < 0) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "cannot "
	"relabel '%s': unable to open device: %d"), path, errno);
	return (zfs_error(hdl, EZFS_OPENFAILED, msg));
	}

	/*
	* It's possible that we might encounter an error if the device
	* does not have any unallocated space left. If so, we simply
	* ignore that error and continue on.
	*/
	error = efi_use_whole_disk(fd);

	/* Flush the buffers to disk and invalidate the page cache. */
	(void) fsync(fd);
	(void) ioctl(fd, BLKFLSBUF);

	(void) close(fd);
	if (error && error != VT_ENOSPC) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "cannot "
	"relabel '%s': unable to read disk capacity"), path);
	return (zfs_error(hdl, EZFS_NOCAP, msg));
	}
	return (0);
	}

	/*
	* Read the EFI label from the config, if a label does not exist then
	* pass back the error to the caller. If the caller has passed a non-NULL
	* diskaddr argument then we set it to the starting address of the EFI
	* partition.
	*/
	static int
	read_efi_label(nvlist_t config, diskaddr_t sb)
	{
	const char *path;
	int fd;
	char diskname[MAXPATHLEN];
	int err = -1;

	if (nvlist_lookup_string(config, ZPOOL_CONFIG_PATH, &path) != 0)
	return (err);

	(void) snprintf(diskname, sizeof (diskname), "%s%s", DISK_ROOT,
	strrchr(path, '/'));
	if ((fd = open(diskname, O_RDONLY\|O_DIRECT\|O_CLOEXEC)) >= 0) {
	struct dk_gpt *vtoc;

	if ((err = efi_alloc_and_read(fd, &vtoc)) >= 0) {
	if (sb != NULL)
	*sb = vtoc->efi_parts[0].p_start;
	efi_free(vtoc);
	}
	(void) close(fd);
	}
	return (err);
	}

	/*
	* determine where a partition starts on a disk in the current
	* configuration
	*/
	static diskaddr_t
	find_start_block(nvlist_t *config)
	{
	nvlist_t **child;
	uint_t c, children;
	diskaddr_t sb = MAXOFFSET_T;
	uint64_t wholedisk;

	if (nvlist_lookup_nvlist_array(config,
	ZPOOL_CONFIG_CHILDREN, &child, &children) != 0) {
	if (nvlist_lookup_uint64(config,
	ZPOOL_CONFIG_WHOLE_DISK,
	&wholedisk) != 0 \|\| !wholedisk) {
	return (MAXOFFSET_T);
	}
	if (read_efi_label(config, &sb) < 0)
	sb = MAXOFFSET_T;
	return (sb);
	}

	for (c = 0; c < children; c++) {
	sb = find_start_block(child[c]);
	if (sb != MAXOFFSET_T) {
	return (sb);
	}
	}
	return (MAXOFFSET_T);
	}

	static int
	zpool_label_disk_check(char *path)
	{
	struct dk_gpt *vtoc;
	int fd, err;

	if ((fd = open(path, O_RDONLY\|O_DIRECT\|O_CLOEXEC)) < 0)
	return (errno);

	if ((err = efi_alloc_and_read(fd, &vtoc)) != 0) {
	(void) close(fd);
	return (err);
	}

	if (vtoc->efi_flags & EFI_GPT_PRIMARY_CORRUPT) {
	efi_free(vtoc);
	(void) close(fd);
	return (EIDRM);
	}

	efi_free(vtoc);
	(void) close(fd);
	return (0);
	}

	/*
	* Generate a unique partition name for the ZFS member. Partitions must
	* have unique names to ensure udev will be able to create symlinks under
	* /dev/disk/by-partlabel/ for all pool members. The partition names are
	* of the form <pool>-<unique-id>.
	*/
	static void
	zpool_label_name(char *label_name, int label_size)
	{
	uint64_t id = 0;
	int fd;

	fd = open("/dev/urandom", O_RDONLY\|O_CLOEXEC);
	if (fd >= 0) {
	if (read(fd, &id, sizeof (id)) != sizeof (id))
	id = 0;

	close(fd);
	}

	if (id == 0)
	id = (((uint64_t)rand()) << 32) \| (uint64_t)rand();

	snprintf(label_name, label_size, "zfs-%016llx", (u_longlong_t)id);
	}

	/*
	* Label an individual disk. The name provided is the short name,
	* stripped of any leading /dev path.
	*/
	int
	zpool_label_disk(libzfs_handle_t hdl, zpool_handle_t zhp, const char *name)
	{
	char path[MAXPATHLEN];
	struct dk_gpt *vtoc;
	int rval, fd;
	size_t resv = EFI_MIN_RESV_SIZE;
	uint64_t slice_size;
	diskaddr_t start_block;
	char errbuf[ERRBUFLEN];

	/* prepare an error message just in case */
	(void) snprintf(errbuf, sizeof (errbuf),
	dgettext(TEXT_DOMAIN, "cannot label '%s'"), name);

	if (zhp) {
	nvlist_t *nvroot = fnvlist_lookup_nvlist(zhp->zpool_config,
	ZPOOL_CONFIG_VDEV_TREE);

	if (zhp->zpool_start_block == 0)
	start_block = find_start_block(nvroot);
	else
	start_block = zhp->zpool_start_block;
	zhp->zpool_start_block = start_block;
	} else {
	/* new pool */
	start_block = NEW_START_BLOCK;
	}

	(void) snprintf(path, sizeof (path), "%s/%s", DISK_ROOT, name);

	if ((fd = open(path, O_RDWR\|O_DIRECT\|O_EXCL\|O_CLOEXEC)) < 0) {
	/*
	* This shouldn't happen. We've long since verified that this
	* is a valid device.
	*/
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "cannot "
	"label '%s': unable to open device: %d"), path, errno);
	return (zfs_error(hdl, EZFS_OPENFAILED, errbuf));
	}

	if (efi_alloc_and_init(fd, EFI_NUMPAR, &vtoc) != 0) {
	/*
	* The only way this can fail is if we run out of memory, or we
	* were unable to read the disk's capacity
	*/
	if (errno == ENOMEM)
	(void) no_memory(hdl);

	(void) close(fd);
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "cannot "
	"label '%s': unable to read disk capacity"), path);

	return (zfs_error(hdl, EZFS_NOCAP, errbuf));
	}

	slice_size = vtoc->efi_last_u_lba + 1;
	slice_size -= EFI_MIN_RESV_SIZE;
	if (start_block == MAXOFFSET_T)
	start_block = NEW_START_BLOCK;
	slice_size -= start_block;
	slice_size = P2ALIGN(slice_size, PARTITION_END_ALIGNMENT);

	vtoc->efi_parts[0].p_start = start_block;
	vtoc->efi_parts[0].p_size = slice_size;

	+ if (vtoc->efi_parts[0].p_size * vtoc->efi_lbasize < SPA_MINDEVSIZE) {
	+ (void) close(fd);
	+ efi_free(vtoc);
	+
	+ zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "cannot "
	+ "label '%s': partition would be less than the minimum "
	+ "device size (64M)"), path);
	+ return (zfs_error(hdl, EZFS_LABELFAILED, errbuf));
	+ }
	+
	/*
	* Why we use V_USR: V_BACKUP confuses users, and is considered
	* disposable by some EFI utilities (since EFI doesn't have a backup
	* slice). V_UNASSIGNED is supposed to be used only for zero size
	* partitions, and efi_write() will fail if we use it.
	* Other available types were all pretty specific.
	* V_USR is as close to reality as we
	* can get, in the absence of V_OTHER.
	*/
	vtoc->efi_parts[0].p_tag = V_USR;
	zpool_label_name(vtoc->efi_parts[0].p_name, EFI_PART_NAME_LEN);

	vtoc->efi_parts[8].p_start = slice_size + start_block;
	vtoc->efi_parts[8].p_size = resv;
	vtoc->efi_parts[8].p_tag = V_RESERVED;

	rval = efi_write(fd, vtoc);

	/* Flush the buffers to disk and invalidate the page cache. */
	(void) fsync(fd);
	(void) ioctl(fd, BLKFLSBUF);

	if (rval == 0)
	rval = efi_rescan(fd);

	/*
	* Some block drivers (like pcata) may not support EFI GPT labels.
	* Print out a helpful error message directing the user to manually
	* label the disk and give a specific slice.
	*/
	if (rval != 0) {
	(void) close(fd);
	efi_free(vtoc);

	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "try using "
	"parted(8) and then provide a specific slice: %d"), rval);
	return (zfs_error(hdl, EZFS_LABELFAILED, errbuf));
	}

	(void) close(fd);
	efi_free(vtoc);

	(void) snprintf(path, sizeof (path), "%s/%s", DISK_ROOT, name);
	(void) zfs_append_partition(path, MAXPATHLEN);

	/* Wait to udev to signal use the device has settled. */
	rval = zpool_label_disk_wait(path, DISK_LABEL_WAIT);
	if (rval) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "failed to "
	"detect device partitions on '%s': %d"), path, rval);
	return (zfs_error(hdl, EZFS_LABELFAILED, errbuf));
	}

	/* We can't be to paranoid. Read the label back and verify it. */
	(void) snprintf(path, sizeof (path), "%s/%s", DISK_ROOT, name);
	rval = zpool_label_disk_check(path);
	if (rval) {
	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "freshly written "
	"EFI label on '%s' is damaged. Ensure\nthis device "
	"is not in use, and is functioning properly: %d"),
	path, rval);
	return (zfs_error(hdl, EZFS_LABELFAILED, errbuf));
	}
	return (0);
	}
	diff --git a/sys/contrib/openzfs/man/Makefile.am b/sys/contrib/openzfs/man/Makefile.am
	index 45156571eec3..43bb014ddd32 100644
	--- a/sys/contrib/openzfs/man/Makefile.am
	+++ b/sys/contrib/openzfs/man/Makefile.am
	@@ -1,136 +1,136 @@
	dist_noinst_man_MANS = \
	%D%/man1/cstyle.1

	dist_man_MANS = \
	%D%/man1/arcstat.1 \
	%D%/man1/raidz_test.1 \
	%D%/man1/test-runner.1 \
	%D%/man1/zhack.1 \
	%D%/man1/ztest.1 \
	%D%/man1/zvol_wait.1 \
	\
	%D%/man5/vdev_id.conf.5 \
	\
	%D%/man4/spl.4 \
	%D%/man4/zfs.4 \
	\
	%D%/man7/dracut.zfs.7 \
	%D%/man7/vdevprops.7 \
	%D%/man7/zfsconcepts.7 \
	%D%/man7/zfsprops.7 \
	%D%/man7/zpool-features.7 \
	%D%/man7/zpoolconcepts.7 \
	%D%/man7/zpoolprops.7 \
	\
	%D%/man8/fsck.zfs.8 \
	%D%/man8/mount.zfs.8 \
	%D%/man8/vdev_id.8 \
	%D%/man8/zdb.8 \
	%D%/man8/zfs.8 \
	%D%/man8/zfs-allow.8 \
	%D%/man8/zfs-bookmark.8 \
	%D%/man8/zfs-change-key.8 \
	%D%/man8/zfs-clone.8 \
	%D%/man8/zfs-create.8 \
	%D%/man8/zfs-destroy.8 \
	%D%/man8/zfs-diff.8 \
	%D%/man8/zfs-get.8 \
	%D%/man8/zfs-groupspace.8 \
	%D%/man8/zfs-hold.8 \
	%D%/man8/zfs-inherit.8 \
	%D%/man8/zfs-list.8 \
	%D%/man8/zfs-load-key.8 \
	%D%/man8/zfs-mount.8 \
	%D%/man8/zfs-program.8 \
	%D%/man8/zfs-project.8 \
	%D%/man8/zfs-projectspace.8 \
	%D%/man8/zfs-promote.8 \
	%D%/man8/zfs-receive.8 \
	%D%/man8/zfs-recv.8 \
	%D%/man8/zfs-redact.8 \
	%D%/man8/zfs-release.8 \
	%D%/man8/zfs-rename.8 \
	%D%/man8/zfs-rollback.8 \
	%D%/man8/zfs-send.8 \
	%D%/man8/zfs-set.8 \
	%D%/man8/zfs-share.8 \
	%D%/man8/zfs-snapshot.8 \
	%D%/man8/zfs-unallow.8 \
	%D%/man8/zfs-unload-key.8 \
	%D%/man8/zfs-unmount.8 \
	%D%/man8/zfs-upgrade.8 \
	%D%/man8/zfs-userspace.8 \
	%D%/man8/zfs-wait.8 \
	%D%/man8/zfs_ids_to_path.8 \
	- %D%/man8/zfs_prepare_disk.8 \
	%D%/man8/zgenhostid.8 \
	%D%/man8/zinject.8 \
	%D%/man8/zpool.8 \
	%D%/man8/zpool-add.8 \
	%D%/man8/zpool-attach.8 \
	%D%/man8/zpool-checkpoint.8 \
	%D%/man8/zpool-clear.8 \
	%D%/man8/zpool-create.8 \
	%D%/man8/zpool-destroy.8 \
	%D%/man8/zpool-detach.8 \
	%D%/man8/zpool-events.8 \
	%D%/man8/zpool-export.8 \
	%D%/man8/zpool-get.8 \
	%D%/man8/zpool-history.8 \
	%D%/man8/zpool-import.8 \
	%D%/man8/zpool-initialize.8 \
	%D%/man8/zpool-iostat.8 \
	%D%/man8/zpool-labelclear.8 \
	%D%/man8/zpool-list.8 \
	%D%/man8/zpool-offline.8 \
	%D%/man8/zpool-online.8 \
	%D%/man8/zpool-reguid.8 \
	%D%/man8/zpool-remove.8 \
	%D%/man8/zpool-reopen.8 \
	%D%/man8/zpool-replace.8 \
	%D%/man8/zpool-resilver.8 \
	%D%/man8/zpool-scrub.8 \
	%D%/man8/zpool-set.8 \
	%D%/man8/zpool-split.8 \
	%D%/man8/zpool-status.8 \
	%D%/man8/zpool-sync.8 \
	%D%/man8/zpool-trim.8 \
	%D%/man8/zpool-upgrade.8 \
	%D%/man8/zpool-wait.8 \
	%D%/man8/zstream.8 \
	%D%/man8/zstreamdump.8 \
	%D%/man8/zpool_influxdb.8

	if BUILD_FREEBSD
	dist_man_MANS += \
	%D%/man8/zfs-jail.8 \
	%D%/man8/zfs-unjail.8
	endif

	if BUILD_LINUX
	dist_man_MANS += \
	%D%/man8/zfs-unzone.8 \
	%D%/man8/zfs-zone.8
	endif

	nodist_man_MANS = \
	%D%/man8/zed.8 \
	- %D%/man8/zfs-mount-generator.8
	+ %D%/man8/zfs-mount-generator.8 \
	+ %D%/man8/zfs_prepare_disk.8

	dist_noinst_DATA += $(dist_noinst_man_MANS) $(dist_man_MANS)

	SUBSTFILES += $(nodist_man_MANS)

	CHECKS += mancheck
	mancheck:
	$(top_srcdir)/scripts/mancheck.sh $(srcdir)/%D%


	if BUILD_LINUX
	# The manual pager in most Linux distros defaults to "BSD" when .Os is blank,
	# but leaving it blank makes things a lot easier on
	# FreeBSD when OpenZFS is vendored in the base system.
	INSTALL_DATA_HOOKS += man-install-data-hook
	man-install-data-hook:
	cd $(DESTDIR)$(mandir) && $(SED) $(ac_inplace) 's/^\.Os$$/.Os OpenZFS/' $(subst %D%/,,$(dist_man_MANS) $(nodist_man_MANS))
	endif
	diff --git a/sys/contrib/openzfs/man/man4/spl.4 b/sys/contrib/openzfs/man/man4/spl.4
	index 414a92394858..5cc12764e18c 100644
	--- a/sys/contrib/openzfs/man/man4/spl.4
	+++ b/sys/contrib/openzfs/man/man4/spl.4
	@@ -1,203 +1,193 @@
	.\"
	.\" The contents of this file are subject to the terms of the Common Development
	.\" and Distribution License (the "License"). You may not use this file except
	.\" in compliance with the License. You can obtain a copy of the license at
	.\" usr/src/OPENSOLARIS.LICENSE or https://opensource.org/licenses/CDDL-1.0.
	.\"
	.\" See the License for the specific language governing permissions and
	.\" limitations under the License. When distributing Covered Code, include this
	.\" CDDL HEADER in each file and include the License file at
	.\" usr/src/OPENSOLARIS.LICENSE. If applicable, add the following below this
	.\" CDDL HEADER, with the fields enclosed by brackets "[]" replaced with your
	.\" own identifying information:
	.\" Portions Copyright [yyyy] [name of copyright owner]
	.\"
	.\" Copyright 2013 Turbo Fredriksson <turbo@bayour.com>. All rights reserved.
	.\"
	.Dd August 24, 2020
	.Dt SPL 4
	.Os
	.
	.Sh NAME
	.Nm spl
	.Nd parameters of the SPL kernel module
	.
	.Sh DESCRIPTION
	.Bl -tag -width Ds
	.It Sy spl_kmem_cache_kmem_threads Ns = Ns Sy 4 Pq uint
	The number of threads created for the spl_kmem_cache task queue.
	This task queue is responsible for allocating new slabs
	for use by the kmem caches.
	For the majority of systems and workloads only a small number of threads are
	required.
	.
	.It Sy spl_kmem_cache_obj_per_slab Ns = Ns Sy 8 Pq uint
	The preferred number of objects per slab in the cache.
	In general, a larger value will increase the caches memory footprint
	while decreasing the time required to perform an allocation.
	Conversely, a smaller value will minimize the footprint
	and improve cache reclaim time but individual allocations may take longer.
	.
	.It Sy spl_kmem_cache_max_size Ns = Ns Sy 32 Po 64-bit Pc or Sy 4 Po 32-bit Pc Pq uint
	The maximum size of a kmem cache slab in MiB.
	This effectively limits the maximum cache object size to
	.Sy spl_kmem_cache_max_size Ns / Ns Sy spl_kmem_cache_obj_per_slab .
	.Pp
	Caches may not be created with
	object sized larger than this limit.
	.
	.It Sy spl_kmem_cache_slab_limit Ns = Ns Sy 16384 Pq uint
	For small objects the Linux slab allocator should be used to make the most
	efficient use of the memory.
	However, large objects are not supported by
	the Linux slab and therefore the SPL implementation is preferred.
	This value is used to determine the cutoff between a small and large object.
	.Pp
	Objects of size
	.Sy spl_kmem_cache_slab_limit
	or smaller will be allocated using the Linux slab allocator,
	large objects use the SPL allocator.
	A cutoff of 16K was determined to be optimal for architectures using 4K pages.
	.
	.It Sy spl_kmem_alloc_warn Ns = Ns Sy 32768 Pq uint
	As a general rule
	.Fn kmem_alloc
	allocations should be small,
	preferably just a few pages, since they must by physically contiguous.
	Therefore, a rate limited warning will be printed to the console for any
	.Fn kmem_alloc
	which exceeds a reasonable threshold.
	.Pp
	The default warning threshold is set to eight pages but capped at 32K to
	accommodate systems using large pages.
	This value was selected to be small enough to ensure
	the largest allocations are quickly noticed and fixed.
	But large enough to avoid logging any warnings when a allocation size is
	larger than optimal but not a serious concern.
	Since this value is tunable, developers are encouraged to set it lower
	when testing so any new largish allocations are quickly caught.
	These warnings may be disabled by setting the threshold to zero.
	.
	.It Sy spl_kmem_alloc_max Ns = Ns Sy KMALLOC_MAX_SIZE Ns / Ns Sy 4 Pq uint
	Large
	.Fn kmem_alloc
	allocations will fail if they exceed
	.Sy KMALLOC_MAX_SIZE .
	Allocations which are marginally smaller than this limit may succeed but
	should still be avoided due to the expense of locating a contiguous range
	of free pages.
	Therefore, a maximum kmem size with reasonable safely margin of 4x is set.
	.Fn kmem_alloc
	allocations larger than this maximum will quickly fail.
	.Fn vmem_alloc
	allocations less than or equal to this value will use
	.Fn kmalloc ,
	but shift to
	.Fn vmalloc
	when exceeding this value.
	.
	.It Sy spl_kmem_cache_magazine_size Ns = Ns Sy 0 Pq uint
	Cache magazines are an optimization designed to minimize the cost of
	allocating memory.
	They do this by keeping a per-cpu cache of recently
	freed objects, which can then be reallocated without taking a lock.
	This can improve performance on highly contended caches.
	However, because objects in magazines will prevent otherwise empty slabs
	from being immediately released this may not be ideal for low memory machines.
	.Pp
	For this reason,
	.Sy spl_kmem_cache_magazine_size
	can be used to set a maximum magazine size.
	When this value is set to 0 the magazine size will
	be automatically determined based on the object size.
	Otherwise magazines will be limited to 2-256 objects per magazine (i.e per cpu).
	Magazines may never be entirely disabled in this implementation.
	.
	.It Sy spl_hostid Ns = Ns Sy 0 Pq ulong
	The system hostid, when set this can be used to uniquely identify a system.
	By default this value is set to zero which indicates the hostid is disabled.
	It can be explicitly enabled by placing a unique non-zero value in
	.Pa /etc/hostid .
	.
	.It Sy spl_hostid_path Ns = Ns Pa /etc/hostid Pq charp
	The expected path to locate the system hostid when specified.
	This value may be overridden for non-standard configurations.
	.
	.It Sy spl_panic_halt Ns = Ns Sy 0 Pq uint
	Cause a kernel panic on assertion failures.
	When not enabled, the thread is halted to facilitate further debugging.
	.Pp
	Set to a non-zero value to enable.
	.
	.It Sy spl_taskq_kick Ns = Ns Sy 0 Pq uint
	Kick stuck taskq to spawn threads.
	When writing a non-zero value to it, it will scan all the taskqs.
	If any of them have a pending task more than 5 seconds old,
	it will kick it to spawn more threads.
	This can be used if you find a rare
	deadlock occurs because one or more taskqs didn't spawn a thread when it should.
	.
	.It Sy spl_taskq_thread_bind Ns = Ns Sy 0 Pq int
	Bind taskq threads to specific CPUs.
	When enabled all taskq threads will be distributed evenly
	across the available CPUs.
	By default, this behavior is disabled to allow the Linux scheduler
	the maximum flexibility to determine where a thread should run.
	.
	.It Sy spl_taskq_thread_dynamic Ns = Ns Sy 1 Pq int
	Allow dynamic taskqs.
	When enabled taskqs which set the
	.Sy TASKQ_DYNAMIC
	flag will by default create only a single thread.
	New threads will be created on demand up to a maximum allowed number
	to facilitate the completion of outstanding tasks.
	Threads which are no longer needed will be promptly destroyed.
	By default this behavior is enabled but it can be disabled to
	aid performance analysis or troubleshooting.
	.
	.It Sy spl_taskq_thread_priority Ns = Ns Sy 1 Pq int
	Allow newly created taskq threads to set a non-default scheduler priority.
	When enabled, the priority specified when a taskq is created will be applied
	to all threads created by that taskq.
	When disabled all threads will use the default Linux kernel thread priority.
	By default, this behavior is enabled.
	.
	.It Sy spl_taskq_thread_sequential Ns = Ns Sy 4 Pq int
	The number of items a taskq worker thread must handle without interruption
	before requesting a new worker thread be spawned.
	This is used to control
	how quickly taskqs ramp up the number of threads processing the queue.
	Because Linux thread creation and destruction are relatively inexpensive a
	small default value has been selected.
	This means that normally threads will be created aggressively which is
	desirable.
	Increasing this value will
	result in a slower thread creation rate which may be preferable for some
	configurations.
	.
	.It Sy spl_max_show_tasks Ns = Ns Sy 512 Pq uint
	The maximum number of tasks per pending list in each taskq shown in
	.Pa /proc/spl/taskq{,-all} .
	Write
	.Sy 0
	to turn off the limit.
	The proc file will walk the lists with lock held,
	reading it could cause a lock-up if the list grow too large
	without limiting the output.
	"(truncated)" will be shown if the list is larger than the limit.
	.
	-.It Sy spl_taskq_thread_timeout_ms Ns = Ns Sy 10000 Pq uint
	-(Linux-only)
	-How long a taskq has to have had no work before we tear it down.
	-Previously, we would tear down a dynamic taskq worker as soon
	-as we noticed it had no work, but it was observed that this led
	-to a lot of churn in tearing down things we then immediately
	-spawned anew.
	-In practice, it seems any nonzero value will remove the vast
	-majority of this churn, while the nontrivially larger value
	-was chosen to help filter out the little remaining churn on
	-a mostly idle system.
	-Setting this value to
	-.Sy 0
	-will revert to the previous behavior.
	+.It Sy spl_taskq_thread_timeout_ms Ns = Ns Sy 5000 Pq uint
	+Minimum idle threads exit interval for dynamic taskqs.
	+Smaller values allow idle threads exit more often and potentially be
	+respawned again on demand, causing more churn.
	.El
	diff --git a/sys/contrib/openzfs/man/man4/zfs.4 b/sys/contrib/openzfs/man/man4/zfs.4
	index 352990e02daf..1191cc962492 100644
	--- a/sys/contrib/openzfs/man/man4/zfs.4
	+++ b/sys/contrib/openzfs/man/man4/zfs.4
	@@ -1,2622 +1,2683 @@
	.\"
	.\" Copyright (c) 2013 by Turbo Fredriksson <turbo@bayour.com>. All rights reserved.
	.\" Copyright (c) 2019, 2021 by Delphix. All rights reserved.
	.\" Copyright (c) 2019 Datto Inc.
	+.\" Copyright (c) 2023, 2024 Klara, Inc.
	.\" The contents of this file are subject to the terms of the Common Development
	.\" and Distribution License (the "License"). You may not use this file except
	.\" in compliance with the License. You can obtain a copy of the license at
	.\" usr/src/OPENSOLARIS.LICENSE or https://opensource.org/licenses/CDDL-1.0.
	.\"
	.\" See the License for the specific language governing permissions and
	.\" limitations under the License. When distributing Covered Code, include this
	.\" CDDL HEADER in each file and include the License file at
	.\" usr/src/OPENSOLARIS.LICENSE. If applicable, add the following below this
	.\" CDDL HEADER, with the fields enclosed by brackets "[]" replaced with your
	.\" own identifying information:
	.\" Portions Copyright [yyyy] [name of copyright owner]
	.\"
	-.Dd July 21, 2023
	+.Dd January 9, 2024
	.Dt ZFS 4
	.Os
	.
	.Sh NAME
	.Nm zfs
	.Nd tuning of the ZFS kernel module
	.
	.Sh DESCRIPTION
	The ZFS module supports these parameters:
	.Bl -tag -width Ds
	.It Sy dbuf_cache_max_bytes Ns = Ns Sy UINT64_MAX Ns B Pq u64
	Maximum size in bytes of the dbuf cache.
	The target size is determined by the MIN versus
	.No 1/2^ Ns Sy dbuf_cache_shift Pq 1/32nd
	of the target ARC size.
	The behavior of the dbuf cache and its associated settings
	can be observed via the
	.Pa /proc/spl/kstat/zfs/dbufstats
	kstat.
	.
	.It Sy dbuf_metadata_cache_max_bytes Ns = Ns Sy UINT64_MAX Ns B Pq u64
	Maximum size in bytes of the metadata dbuf cache.
	The target size is determined by the MIN versus
	.No 1/2^ Ns Sy dbuf_metadata_cache_shift Pq 1/64th
	of the target ARC size.
	The behavior of the metadata dbuf cache and its associated settings
	can be observed via the
	.Pa /proc/spl/kstat/zfs/dbufstats
	kstat.
	.
	.It Sy dbuf_cache_hiwater_pct Ns = Ns Sy 10 Ns % Pq uint
	The percentage over
	.Sy dbuf_cache_max_bytes
	when dbufs must be evicted directly.
	.
	.It Sy dbuf_cache_lowater_pct Ns = Ns Sy 10 Ns % Pq uint
	The percentage below
	.Sy dbuf_cache_max_bytes
	when the evict thread stops evicting dbufs.
	.
	.It Sy dbuf_cache_shift Ns = Ns Sy 5 Pq uint
	Set the size of the dbuf cache
	.Pq Sy dbuf_cache_max_bytes
	to a log2 fraction of the target ARC size.
	.
	.It Sy dbuf_metadata_cache_shift Ns = Ns Sy 6 Pq uint
	Set the size of the dbuf metadata cache
	.Pq Sy dbuf_metadata_cache_max_bytes
	to a log2 fraction of the target ARC size.
	.
	.It Sy dbuf_mutex_cache_shift Ns = Ns Sy 0 Pq uint
	Set the size of the mutex array for the dbuf cache.
	When set to
	.Sy 0
	the array is dynamically sized based on total system memory.
	.
	.It Sy dmu_object_alloc_chunk_shift Ns = Ns Sy 7 Po 128 Pc Pq uint
	dnode slots allocated in a single operation as a power of 2.
	The default value minimizes lock contention for the bulk operation performed.
	.
	.It Sy dmu_prefetch_max Ns = Ns Sy 134217728 Ns B Po 128 MiB Pc Pq uint
	Limit the amount we can prefetch with one call to this amount in bytes.
	This helps to limit the amount of memory that can be used by prefetching.
	.
	.It Sy ignore_hole_birth Pq int
	Alias for
	.Sy send_holes_without_birth_time .
	.
	.It Sy l2arc_feed_again Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Turbo L2ARC warm-up.
	When the L2ARC is cold the fill interval will be set as fast as possible.
	.
	.It Sy l2arc_feed_min_ms Ns = Ns Sy 200 Pq u64
	Min feed interval in milliseconds.
	Requires
	.Sy l2arc_feed_again Ns = Ns Ar 1
	and only applicable in related situations.
	.
	.It Sy l2arc_feed_secs Ns = Ns Sy 1 Pq u64
	Seconds between L2ARC writing.
	.
	.It Sy l2arc_headroom Ns = Ns Sy 2 Pq u64
	How far through the ARC lists to search for L2ARC cacheable content,
	expressed as a multiplier of
	.Sy l2arc_write_max .
	ARC persistence across reboots can be achieved with persistent L2ARC
	by setting this parameter to
	.Sy 0 ,
	allowing the full length of ARC lists to be searched for cacheable content.
	.
	.It Sy l2arc_headroom_boost Ns = Ns Sy 200 Ns % Pq u64
	Scales
	.Sy l2arc_headroom
	by this percentage when L2ARC contents are being successfully compressed
	before writing.
	A value of
	.Sy 100
	disables this feature.
	.
	.It Sy l2arc_exclude_special Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Controls whether buffers present on special vdevs are eligible for caching
	into L2ARC.
	If set to 1, exclude dbufs on special vdevs from being cached to L2ARC.
	.
	.It Sy l2arc_mfuonly Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Controls whether only MFU metadata and data are cached from ARC into L2ARC.
	This may be desired to avoid wasting space on L2ARC when reading/writing large
	amounts of data that are not expected to be accessed more than once.
	.Pp
	The default is off,
	meaning both MRU and MFU data and metadata are cached.
	When turning off this feature, some MRU buffers will still be present
	in ARC and eventually cached on L2ARC.
	.No If Sy l2arc_noprefetch Ns = Ns Sy 0 ,
	some prefetched buffers will be cached to L2ARC, and those might later
	transition to MRU, in which case the
	.Sy l2arc_mru_asize No arcstat will not be Sy 0 .
	.Pp
	Regardless of
	.Sy l2arc_noprefetch ,
	some MFU buffers might be evicted from ARC,
	accessed later on as prefetches and transition to MRU as prefetches.
	If accessed again they are counted as MRU and the
	.Sy l2arc_mru_asize No arcstat will not be Sy 0 .
	.Pp
	The ARC status of L2ARC buffers when they were first cached in
	L2ARC can be seen in the
	.Sy l2arc_mru_asize , Sy l2arc_mfu_asize , No and Sy l2arc_prefetch_asize
	arcstats when importing the pool or onlining a cache
	device if persistent L2ARC is enabled.
	.Pp
	The
	.Sy evict_l2_eligible_mru
	arcstat does not take into account if this option is enabled as the information
	provided by the
	.Sy evict_l2_eligible_m[rf]u
	arcstats can be used to decide if toggling this option is appropriate
	for the current workload.
	.
	.It Sy l2arc_meta_percent Ns = Ns Sy 33 Ns % Pq uint
	Percent of ARC size allowed for L2ARC-only headers.
	Since L2ARC buffers are not evicted on memory pressure,
	too many headers on a system with an irrationally large L2ARC
	can render it slow or unusable.
	This parameter limits L2ARC writes and rebuilds to achieve the target.
	.
	.It Sy l2arc_trim_ahead Ns = Ns Sy 0 Ns % Pq u64
	Trims ahead of the current write size
	.Pq Sy l2arc_write_max
	on L2ARC devices by this percentage of write size if we have filled the device.
	If set to
	.Sy 100
	we TRIM twice the space required to accommodate upcoming writes.
	A minimum of
	.Sy 64 MiB
	will be trimmed.
	It also enables TRIM of the whole L2ARC device upon creation
	or addition to an existing pool or if the header of the device is
	invalid upon importing a pool or onlining a cache device.
	A value of
	.Sy 0
	disables TRIM on L2ARC altogether and is the default as it can put significant
	stress on the underlying storage devices.
	This will vary depending of how well the specific device handles these commands.
	.
	.It Sy l2arc_noprefetch Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Do not write buffers to L2ARC if they were prefetched but not used by
	applications.
	In case there are prefetched buffers in L2ARC and this option
	is later set, we do not read the prefetched buffers from L2ARC.
	Unsetting this option is useful for caching sequential reads from the
	disks to L2ARC and serve those reads from L2ARC later on.
	This may be beneficial in case the L2ARC device is significantly faster
	in sequential reads than the disks of the pool.
	.Pp
	Use
	.Sy 1
	to disable and
	.Sy 0
	to enable caching/reading prefetches to/from L2ARC.
	.
	.It Sy l2arc_norw Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	No reads during writes.
	.
	.It Sy l2arc_write_boost Ns = Ns Sy 8388608 Ns B Po 8 MiB Pc Pq u64
	Cold L2ARC devices will have
	.Sy l2arc_write_max
	increased by this amount while they remain cold.
	.
	.It Sy l2arc_write_max Ns = Ns Sy 8388608 Ns B Po 8 MiB Pc Pq u64
	Max write bytes per interval.
	.
	.It Sy l2arc_rebuild_enabled Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Rebuild the L2ARC when importing a pool (persistent L2ARC).
	This can be disabled if there are problems importing a pool
	or attaching an L2ARC device (e.g. the L2ARC device is slow
	in reading stored log metadata, or the metadata
	has become somehow fragmented/unusable).
	.
	.It Sy l2arc_rebuild_blocks_min_l2size Ns = Ns Sy 1073741824 Ns B Po 1 GiB Pc Pq u64
	Mininum size of an L2ARC device required in order to write log blocks in it.
	The log blocks are used upon importing the pool to rebuild the persistent L2ARC.
	.Pp
	For L2ARC devices less than 1 GiB, the amount of data
	.Fn l2arc_evict
	evicts is significant compared to the amount of restored L2ARC data.
	In this case, do not write log blocks in L2ARC in order not to waste space.
	.
	.It Sy metaslab_aliquot Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq u64
	Metaslab granularity, in bytes.
	This is roughly similar to what would be referred to as the "stripe size"
	in traditional RAID arrays.
	In normal operation, ZFS will try to write this amount of data to each disk
	before moving on to the next top-level vdev.
	.
	.It Sy metaslab_bias_enabled Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Enable metaslab group biasing based on their vdevs' over- or under-utilization
	relative to the pool.
	.
	.It Sy metaslab_force_ganging Ns = Ns Sy 16777217 Ns B Po 16 MiB + 1 B Pc Pq u64
	Make some blocks above a certain size be gang blocks.
	This option is used by the test suite to facilitate testing.
	.
	.It Sy metaslab_force_ganging_pct Ns = Ns Sy 3 Ns % Pq uint
	For blocks that could be forced to be a gang block (due to
	.Sy metaslab_force_ganging ) ,
	force this many of them to be gang blocks.
	.
	-.It Sy zfs_ddt_zap_default_bs Ns = Ns Sy 15 Po 32 KiB Pc Pq int
	+.It Sy brt_zap_prefetch Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	+Controls prefetching BRT records for blocks which are going to be cloned.
	+.
	+.It Sy brt_zap_default_bs Ns = Ns Sy 12 Po 4 KiB Pc Pq int
	+Default BRT ZAP data block size as a power of 2. Note that changing this after
	+creating a BRT on the pool will not affect existing BRTs, only newly created
	+ones.
	+.
	+.It Sy brt_zap_default_ibs Ns = Ns Sy 12 Po 4 KiB Pc Pq int
	+Default BRT ZAP indirect block size as a power of 2. Note that changing this
	+after creating a BRT on the pool will not affect existing BRTs, only newly
	+created ones.
	+.
	+.It Sy ddt_zap_default_bs Ns = Ns Sy 15 Po 32 KiB Pc Pq int
	Default DDT ZAP data block size as a power of 2. Note that changing this after
	creating a DDT on the pool will not affect existing DDTs, only newly created
	ones.
	.
	-.It Sy zfs_ddt_zap_default_ibs Ns = Ns Sy 15 Po 32 KiB Pc Pq int
	+.It Sy ddt_zap_default_ibs Ns = Ns Sy 15 Po 32 KiB Pc Pq int
	Default DDT ZAP indirect block size as a power of 2. Note that changing this
	after creating a DDT on the pool will not affect existing DDTs, only newly
	created ones.
	.
	.It Sy zfs_default_bs Ns = Ns Sy 9 Po 512 B Pc Pq int
	Default dnode block size as a power of 2.
	.
	.It Sy zfs_default_ibs Ns = Ns Sy 17 Po 128 KiB Pc Pq int
	Default dnode indirect block size as a power of 2.
	.
	.It Sy zfs_history_output_max Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq u64
	When attempting to log an output nvlist of an ioctl in the on-disk history,
	the output will not be stored if it is larger than this size (in bytes).
	This must be less than
	.Sy DMU_MAX_ACCESS Pq 64 MiB .
	This applies primarily to
	.Fn zfs_ioc_channel_program Pq cf. Xr zfs-program 8 .
	.
	.It Sy zfs_keep_log_spacemaps_at_export Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Prevent log spacemaps from being destroyed during pool exports and destroys.
	.
	.It Sy zfs_metaslab_segment_weight_enabled Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Enable/disable segment-based metaslab selection.
	.
	.It Sy zfs_metaslab_switch_threshold Ns = Ns Sy 2 Pq int
	When using segment-based metaslab selection, continue allocating
	from the active metaslab until this option's
	worth of buckets have been exhausted.
	.
	.It Sy metaslab_debug_load Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Load all metaslabs during pool import.
	.
	.It Sy metaslab_debug_unload Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Prevent metaslabs from being unloaded.
	.
	.It Sy metaslab_fragmentation_factor_enabled Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Enable use of the fragmentation metric in computing metaslab weights.
	.
	.It Sy metaslab_df_max_search Ns = Ns Sy 16777216 Ns B Po 16 MiB Pc Pq uint
	Maximum distance to search forward from the last offset.
	Without this limit, fragmented pools can see
	.Em >100`000
	iterations and
	.Fn metaslab_block_picker
	becomes the performance limiting factor on high-performance storage.
	.Pp
	With the default setting of
	.Sy 16 MiB ,
	we typically see less than
	.Em 500
	iterations, even with very fragmented
	.Sy ashift Ns = Ns Sy 9
	pools.
	The maximum number of iterations possible is
	.Sy metaslab_df_max_search / 2^(ashift+1) .
	With the default setting of
	.Sy 16 MiB
	this is
	.Em 16*1024 Pq with Sy ashift Ns = Ns Sy 9
	or
	.Em 2*1024 Pq with Sy ashift Ns = Ns Sy 12 .
	.
	.It Sy metaslab_df_use_largest_segment Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	If not searching forward (due to
	.Sy metaslab_df_max_search , metaslab_df_free_pct ,
	.No or Sy metaslab_df_alloc_threshold ) ,
	this tunable controls which segment is used.
	If set, we will use the largest free segment.
	If unset, we will use a segment of at least the requested size.
	.
	.It Sy zfs_metaslab_max_size_cache_sec Ns = Ns Sy 3600 Ns s Po 1 hour Pc Pq u64
	When we unload a metaslab, we cache the size of the largest free chunk.
	We use that cached size to determine whether or not to load a metaslab
	for a given allocation.
	As more frees accumulate in that metaslab while it's unloaded,
	the cached max size becomes less and less accurate.
	After a number of seconds controlled by this tunable,
	we stop considering the cached max size and start
	considering only the histogram instead.
	.
	.It Sy zfs_metaslab_mem_limit Ns = Ns Sy 25 Ns % Pq uint
	When we are loading a new metaslab, we check the amount of memory being used
	to store metaslab range trees.
	If it is over a threshold, we attempt to unload the least recently used metaslab
	to prevent the system from clogging all of its memory with range trees.
	This tunable sets the percentage of total system memory that is the threshold.
	.
	.It Sy zfs_metaslab_try_hard_before_gang Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	.Bl -item -compact
	.It
	If unset, we will first try normal allocation.
	.It
	If that fails then we will do a gang allocation.
	.It
	If that fails then we will do a "try hard" gang allocation.
	.It
	If that fails then we will have a multi-layer gang block.
	.El
	.Pp
	.Bl -item -compact
	.It
	If set, we will first try normal allocation.
	.It
	If that fails then we will do a "try hard" allocation.
	.It
	If that fails we will do a gang allocation.
	.It
	If that fails we will do a "try hard" gang allocation.
	.It
	If that fails then we will have a multi-layer gang block.
	.El
	.
	.It Sy zfs_metaslab_find_max_tries Ns = Ns Sy 100 Pq uint
	When not trying hard, we only consider this number of the best metaslabs.
	This improves performance, especially when there are many metaslabs per vdev
	and the allocation can't actually be satisfied
	(so we would otherwise iterate all metaslabs).
	.
	.It Sy zfs_vdev_default_ms_count Ns = Ns Sy 200 Pq uint
	When a vdev is added, target this number of metaslabs per top-level vdev.
	.
	.It Sy zfs_vdev_default_ms_shift Ns = Ns Sy 29 Po 512 MiB Pc Pq uint
	Default lower limit for metaslab size.
	.
	.It Sy zfs_vdev_max_ms_shift Ns = Ns Sy 34 Po 16 GiB Pc Pq uint
	Default upper limit for metaslab size.
	.
	.It Sy zfs_vdev_max_auto_ashift Ns = Ns Sy 14 Pq uint
	Maximum ashift used when optimizing for logical \[->] physical sector size on
	new
	top-level vdevs.
	May be increased up to
	.Sy ASHIFT_MAX Po 16 Pc ,
	but this may negatively impact pool space efficiency.
	.
	.It Sy zfs_vdev_min_auto_ashift Ns = Ns Sy ASHIFT_MIN Po 9 Pc Pq uint
	Minimum ashift used when creating new top-level vdevs.
	.
	.It Sy zfs_vdev_min_ms_count Ns = Ns Sy 16 Pq uint
	Minimum number of metaslabs to create in a top-level vdev.
	.
	.It Sy vdev_validate_skip Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Skip label validation steps during pool import.
	Changing is not recommended unless you know what you're doing
	and are recovering a damaged label.
	.
	.It Sy zfs_vdev_ms_count_limit Ns = Ns Sy 131072 Po 128k Pc Pq uint
	Practical upper limit of total metaslabs per top-level vdev.
	.
	.It Sy metaslab_preload_enabled Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Enable metaslab group preloading.
	.
	.It Sy metaslab_preload_limit Ns = Ns Sy 10 Pq uint
	Maximum number of metaslabs per group to preload
	.
	.It Sy metaslab_preload_pct Ns = Ns Sy 50 Pq uint
	Percentage of CPUs to run a metaslab preload taskq
	.
	.It Sy metaslab_lba_weighting_enabled Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Give more weight to metaslabs with lower LBAs,
	assuming they have greater bandwidth,
	as is typically the case on a modern constant angular velocity disk drive.
	.
	.It Sy metaslab_unload_delay Ns = Ns Sy 32 Pq uint
	After a metaslab is used, we keep it loaded for this many TXGs, to attempt to
	reduce unnecessary reloading.
	Note that both this many TXGs and
	.Sy metaslab_unload_delay_ms
	milliseconds must pass before unloading will occur.
	.
	.It Sy metaslab_unload_delay_ms Ns = Ns Sy 600000 Ns ms Po 10 min Pc Pq uint
	After a metaslab is used, we keep it loaded for this many milliseconds,
	to attempt to reduce unnecessary reloading.
	Note, that both this many milliseconds and
	.Sy metaslab_unload_delay
	TXGs must pass before unloading will occur.
	.
	.It Sy reference_history Ns = Ns Sy 3 Pq uint
	Maximum reference holders being tracked when reference_tracking_enable is
	active.
	.
	.It Sy reference_tracking_enable Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Track reference holders to
	.Sy refcount_t
	objects (debug builds only).
	.
	.It Sy send_holes_without_birth_time Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	When set, the
	.Sy hole_birth
	optimization will not be used, and all holes will always be sent during a
	.Nm zfs Cm send .
	This is useful if you suspect your datasets are affected by a bug in
	.Sy hole_birth .
	.
	.It Sy spa_config_path Ns = Ns Pa /etc/zfs/zpool.cache Pq charp
	SPA config file.
	.
	.It Sy spa_asize_inflation Ns = Ns Sy 24 Pq uint
	Multiplication factor used to estimate actual disk consumption from the
	size of data being written.
	The default value is a worst case estimate,
	but lower values may be valid for a given pool depending on its configuration.
	Pool administrators who understand the factors involved
	may wish to specify a more realistic inflation factor,
	particularly if they operate close to quota or capacity limits.
	.
	.It Sy spa_load_print_vdev_tree Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Whether to print the vdev tree in the debugging message buffer during pool
	import.
	.
	.It Sy spa_load_verify_data Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Whether to traverse data blocks during an "extreme rewind"
	.Pq Fl X
	import.
	.Pp
	An extreme rewind import normally performs a full traversal of all
	blocks in the pool for verification.
	If this parameter is unset, the traversal skips non-metadata blocks.
	It can be toggled once the
	import has started to stop or start the traversal of non-metadata blocks.
	.
	.It Sy spa_load_verify_metadata Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Whether to traverse blocks during an "extreme rewind"
	.Pq Fl X
	pool import.
	.Pp
	An extreme rewind import normally performs a full traversal of all
	blocks in the pool for verification.
	If this parameter is unset, the traversal is not performed.
	It can be toggled once the import has started to stop or start the traversal.
	.
	.It Sy spa_load_verify_shift Ns = Ns Sy 4 Po 1/16th Pc Pq uint
	Sets the maximum number of bytes to consume during pool import to the log2
	fraction of the target ARC size.
	.
	.It Sy spa_slop_shift Ns = Ns Sy 5 Po 1/32nd Pc Pq int
	Normally, we don't allow the last
	.Sy 3.2% Pq Sy 1/2^spa_slop_shift
	of space in the pool to be consumed.
	This ensures that we don't run the pool completely out of space,
	due to unaccounted changes (e.g. to the MOS).
	It also limits the worst-case time to allocate space.
	If we have less than this amount of free space,
	most ZPL operations (e.g. write, create) will return
	.Sy ENOSPC .
	.
	.It Sy spa_upgrade_errlog_limit Ns = Ns Sy 0 Pq uint
	Limits the number of on-disk error log entries that will be converted to the
	new format when enabling the
	.Sy head_errlog
	feature.
	The default is to convert all log entries.
	.
	.It Sy vdev_removal_max_span Ns = Ns Sy 32768 Ns B Po 32 KiB Pc Pq uint
	During top-level vdev removal, chunks of data are copied from the vdev
	which may include free space in order to trade bandwidth for IOPS.
	This parameter determines the maximum span of free space, in bytes,
	which will be included as "unnecessary" data in a chunk of copied data.
	.Pp
	The default value here was chosen to align with
	.Sy zfs_vdev_read_gap_limit ,
	which is a similar concept when doing
	regular reads (but there's no reason it has to be the same).
	.
	.It Sy vdev_file_logical_ashift Ns = Ns Sy 9 Po 512 B Pc Pq u64
	Logical ashift for file-based devices.
	.
	.It Sy vdev_file_physical_ashift Ns = Ns Sy 9 Po 512 B Pc Pq u64
	Physical ashift for file-based devices.
	.
	.It Sy zap_iterate_prefetch Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	If set, when we start iterating over a ZAP object,
	prefetch the entire object (all leaf blocks).
	However, this is limited by
	.Sy dmu_prefetch_max .
	.
	.It Sy zap_micro_max_size Ns = Ns Sy 131072 Ns B Po 128 KiB Pc Pq int
	Maximum micro ZAP size.
	A micro ZAP is upgraded to a fat ZAP, once it grows beyond the specified size.
	.
	+.It Sy zfetch_hole_shift Ns = Ns Sy 2 Pq uint
	+Log2 fraction of holes in speculative prefetch stream allowed for it to
	+proceed.
	+.
	.It Sy zfetch_min_distance Ns = Ns Sy 4194304 Ns B Po 4 MiB Pc Pq uint
	Min bytes to prefetch per stream.
	Prefetch distance starts from the demand access size and quickly grows to
	this value, doubling on each hit.
	After that it may grow further by 1/8 per hit, but only if some prefetch
	since last time haven't completed in time to satisfy demand request, i.e.
	prefetch depth didn't cover the read latency or the pool got saturated.
	.
	.It Sy zfetch_max_distance Ns = Ns Sy 67108864 Ns B Po 64 MiB Pc Pq uint
	Max bytes to prefetch per stream.
	.
	.It Sy zfetch_max_idistance Ns = Ns Sy 67108864 Ns B Po 64 MiB Pc Pq uint
	Max bytes to prefetch indirects for per stream.
	.
	+.It Sy zfetch_max_reorder Ns = Ns Sy 16777216 Ns B Po 16 MiB Pc Pq uint
	+Requests within this byte distance from the current prefetch stream position
	+are considered parts of the stream, reordered due to parallel processing.
	+Such requests do not advance the stream position immediately unless
	+.Sy zfetch_hole_shift
	+fill threshold is reached, but saved to fill holes in the stream later.
	+.
	.It Sy zfetch_max_streams Ns = Ns Sy 8 Pq uint
	Max number of streams per zfetch (prefetch streams per file).
	.
	.It Sy zfetch_min_sec_reap Ns = Ns Sy 1 Pq uint
	Min time before inactive prefetch stream can be reclaimed
	.
	.It Sy zfetch_max_sec_reap Ns = Ns Sy 2 Pq uint
	Max time before inactive prefetch stream can be deleted
	.
	.It Sy zfs_abd_scatter_enabled Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Enables ARC from using scatter/gather lists and forces all allocations to be
	linear in kernel memory.
	Disabling can improve performance in some code paths
	at the expense of fragmented kernel memory.
	.
	.It Sy zfs_abd_scatter_max_order Ns = Ns Sy MAX_ORDER\-1 Pq uint
	Maximum number of consecutive memory pages allocated in a single block for
	scatter/gather lists.
	.Pp
	The value of
	.Sy MAX_ORDER
	depends on kernel configuration.
	.
	.It Sy zfs_abd_scatter_min_size Ns = Ns Sy 1536 Ns B Po 1.5 KiB Pc Pq uint
	This is the minimum allocation size that will use scatter (page-based) ABDs.
	Smaller allocations will use linear ABDs.
	.
	.It Sy zfs_arc_dnode_limit Ns = Ns Sy 0 Ns B Pq u64
	When the number of bytes consumed by dnodes in the ARC exceeds this number of
	bytes, try to unpin some of it in response to demand for non-metadata.
	This value acts as a ceiling to the amount of dnode metadata, and defaults to
	.Sy 0 ,
	which indicates that a percent which is based on
	.Sy zfs_arc_dnode_limit_percent
	of the ARC meta buffers that may be used for dnodes.
	.It Sy zfs_arc_dnode_limit_percent Ns = Ns Sy 10 Ns % Pq u64
	Percentage that can be consumed by dnodes of ARC meta buffers.
	.Pp
	See also
	.Sy zfs_arc_dnode_limit ,
	which serves a similar purpose but has a higher priority if nonzero.
	.
	.It Sy zfs_arc_dnode_reduce_percent Ns = Ns Sy 10 Ns % Pq u64
	Percentage of ARC dnodes to try to scan in response to demand for non-metadata
	when the number of bytes consumed by dnodes exceeds
	.Sy zfs_arc_dnode_limit .
	.
	.It Sy zfs_arc_average_blocksize Ns = Ns Sy 8192 Ns B Po 8 KiB Pc Pq uint
	The ARC's buffer hash table is sized based on the assumption of an average
	block size of this value.
	This works out to roughly 1 MiB of hash table per 1 GiB of physical memory
	with 8-byte pointers.
	For configurations with a known larger average block size,
	this value can be increased to reduce the memory footprint.
	.
	.It Sy zfs_arc_eviction_pct Ns = Ns Sy 200 Ns % Pq uint
	When
	.Fn arc_is_overflowing ,
	.Fn arc_get_data_impl
	waits for this percent of the requested amount of data to be evicted.
	For example, by default, for every
	.Em 2 KiB
	that's evicted,
	.Em 1 KiB
	of it may be "reused" by a new allocation.
	Since this is above
	.Sy 100 Ns % ,
	it ensures that progress is made towards getting
	.Sy arc_size No under Sy arc_c .
	Since this is finite, it ensures that allocations can still happen,
	even during the potentially long time that
	.Sy arc_size No is more than Sy arc_c .
	.
	.It Sy zfs_arc_evict_batch_limit Ns = Ns Sy 10 Pq uint
	Number ARC headers to evict per sub-list before proceeding to another sub-list.
	This batch-style operation prevents entire sub-lists from being evicted at once
	but comes at a cost of additional unlocking and locking.
	.
	.It Sy zfs_arc_grow_retry Ns = Ns Sy 0 Ns s Pq uint
	If set to a non zero value, it will replace the
	.Sy arc_grow_retry
	value with this value.
	The
	.Sy arc_grow_retry
	.No value Pq default Sy 5 Ns s
	is the number of seconds the ARC will wait before
	trying to resume growth after a memory pressure event.
	.
	.It Sy zfs_arc_lotsfree_percent Ns = Ns Sy 10 Ns % Pq int
	Throttle I/O when free system memory drops below this percentage of total
	system memory.
	Setting this value to
	.Sy 0
	will disable the throttle.
	.
	.It Sy zfs_arc_max Ns = Ns Sy 0 Ns B Pq u64
	Max size of ARC in bytes.
	If
	.Sy 0 ,
	then the max size of ARC is determined by the amount of system memory installed.
	Under Linux, half of system memory will be used as the limit.
	Under
	.Fx ,
	the larger of
	.Sy all_system_memory No \- Sy 1 GiB
	and
	.Sy 5/8 No \(mu Sy all_system_memory
	will be used as the limit.
	This value must be at least
	.Sy 67108864 Ns B Pq 64 MiB .
	.Pp
	This value can be changed dynamically, with some caveats.
	It cannot be set back to
	.Sy 0
	while running, and reducing it below the current ARC size will not cause
	the ARC to shrink without memory pressure to induce shrinking.
	.
	.It Sy zfs_arc_meta_balance Ns = Ns Sy 500 Pq uint
	Balance between metadata and data on ghost hits.
	Values above 100 increase metadata caching by proportionally reducing effect
	of ghost data hits on target data/metadata rate.
	.
	.It Sy zfs_arc_min Ns = Ns Sy 0 Ns B Pq u64
	Min size of ARC in bytes.
	.No If set to Sy 0 , arc_c_min
	will default to consuming the larger of
	.Sy 32 MiB
	and
	.Sy all_system_memory No / Sy 32 .
	.
	.It Sy zfs_arc_min_prefetch_ms Ns = Ns Sy 0 Ns ms Ns Po Ns ≡ Ns 1s Pc Pq uint
	Minimum time prefetched blocks are locked in the ARC.
	.
	.It Sy zfs_arc_min_prescient_prefetch_ms Ns = Ns Sy 0 Ns ms Ns Po Ns ≡ Ns 6s Pc Pq uint
	Minimum time "prescient prefetched" blocks are locked in the ARC.
	These blocks are meant to be prefetched fairly aggressively ahead of
	the code that may use them.
	.
	.It Sy zfs_arc_prune_task_threads Ns = Ns Sy 1 Pq int
	Number of arc_prune threads.
	.Fx
	does not need more than one.
	Linux may theoretically use one per mount point up to number of CPUs,
	but that was not proven to be useful.
	.
	.It Sy zfs_max_missing_tvds Ns = Ns Sy 0 Pq int
	Number of missing top-level vdevs which will be allowed during
	pool import (only in read-only mode).
	.
	.It Sy zfs_max_nvlist_src_size Ns = Sy 0 Pq u64
	Maximum size in bytes allowed to be passed as
	.Sy zc_nvlist_src_size
	for ioctls on
	.Pa /dev/zfs .
	This prevents a user from causing the kernel to allocate
	an excessive amount of memory.
	When the limit is exceeded, the ioctl fails with
	.Sy EINVAL
	and a description of the error is sent to the
	.Pa zfs-dbgmsg
	log.
	This parameter should not need to be touched under normal circumstances.
	If
	.Sy 0 ,
	equivalent to a quarter of the user-wired memory limit under
	.Fx
	and to
	.Sy 134217728 Ns B Pq 128 MiB
	under Linux.
	.
	.It Sy zfs_multilist_num_sublists Ns = Ns Sy 0 Pq uint
	To allow more fine-grained locking, each ARC state contains a series
	of lists for both data and metadata objects.
	Locking is performed at the level of these "sub-lists".
	This parameters controls the number of sub-lists per ARC state,
	and also applies to other uses of the multilist data structure.
	.Pp
	If
	.Sy 0 ,
	equivalent to the greater of the number of online CPUs and
	.Sy 4 .
	.
	.It Sy zfs_arc_overflow_shift Ns = Ns Sy 8 Pq int
	The ARC size is considered to be overflowing if it exceeds the current
	ARC target size
	.Pq Sy arc_c
	by thresholds determined by this parameter.
	Exceeding by
	.Sy ( arc_c No >> Sy zfs_arc_overflow_shift ) No / Sy 2
	starts ARC reclamation process.
	If that appears insufficient, exceeding by
	.Sy ( arc_c No >> Sy zfs_arc_overflow_shift ) No \(mu Sy 1.5
	blocks new buffer allocation until the reclaim thread catches up.
	Started reclamation process continues till ARC size returns below the
	target size.
	.Pp
	The default value of
	.Sy 8
	causes the ARC to start reclamation if it exceeds the target size by
	.Em 0.2%
	of the target size, and block allocations by
	.Em 0.6% .
	.
	.It Sy zfs_arc_shrink_shift Ns = Ns Sy 0 Pq uint
	If nonzero, this will update
	.Sy arc_shrink_shift Pq default Sy 7
	with the new value.
	.
	.It Sy zfs_arc_pc_percent Ns = Ns Sy 0 Ns % Po off Pc Pq uint
	Percent of pagecache to reclaim ARC to.
	.Pp
	This tunable allows the ZFS ARC to play more nicely
	with the kernel's LRU pagecache.
	It can guarantee that the ARC size won't collapse under scanning
	pressure on the pagecache, yet still allows the ARC to be reclaimed down to
	.Sy zfs_arc_min
	if necessary.
	This value is specified as percent of pagecache size (as measured by
	.Sy NR_FILE_PAGES ) ,
	where that percent may exceed
	.Sy 100 .
	This
	only operates during memory pressure/reclaim.
	.
	.It Sy zfs_arc_shrinker_limit Ns = Ns Sy 10000 Pq int
	This is a limit on how many pages the ARC shrinker makes available for
	eviction in response to one page allocation attempt.
	Note that in practice, the kernel's shrinker can ask us to evict
	up to about four times this for one allocation attempt.
	.Pp
	The default limit of
	.Sy 10000 Pq in practice, Em 160 MiB No per allocation attempt with 4 KiB pages
	limits the amount of time spent attempting to reclaim ARC memory to
	less than 100 ms per allocation attempt,
	even with a small average compressed block size of ~8 KiB.
	.Pp
	The parameter can be set to 0 (zero) to disable the limit,
	and only applies on Linux.
	.
	.It Sy zfs_arc_sys_free Ns = Ns Sy 0 Ns B Pq u64
	The target number of bytes the ARC should leave as free memory on the system.
	If zero, equivalent to the bigger of
	.Sy 512 KiB No and Sy all_system_memory/64 .
	.
	.It Sy zfs_autoimport_disable Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Disable pool import at module load by ignoring the cache file
	.Pq Sy spa_config_path .
	.
	.It Sy zfs_checksum_events_per_second Ns = Ns Sy 20 Ns /s Pq uint
	Rate limit checksum events to this many per second.
	Note that this should not be set below the ZED thresholds
	(currently 10 checksums over 10 seconds)
	or else the daemon may not trigger any action.
	.
	-.It Sy zfs_commit_timeout_pct Ns = Ns Sy 5 Ns % Pq uint
	+.It Sy zfs_commit_timeout_pct Ns = Ns Sy 10 Ns % Pq uint
	This controls the amount of time that a ZIL block (lwb) will remain "open"
	when it isn't "full", and it has a thread waiting for it to be committed to
	stable storage.
	The timeout is scaled based on a percentage of the last lwb
	latency to avoid significantly impacting the latency of each individual
	transaction record (itx).
	.
	.It Sy zfs_condense_indirect_commit_entry_delay_ms Ns = Ns Sy 0 Ns ms Pq int
	Vdev indirection layer (used for device removal) sleeps for this many
	milliseconds during mapping generation.
	Intended for use with the test suite to throttle vdev removal speed.
	.
	.It Sy zfs_condense_indirect_obsolete_pct Ns = Ns Sy 25 Ns % Pq uint
	Minimum percent of obsolete bytes in vdev mapping required to attempt to
	condense
	.Pq see Sy zfs_condense_indirect_vdevs_enable .
	Intended for use with the test suite
	to facilitate triggering condensing as needed.
	.
	.It Sy zfs_condense_indirect_vdevs_enable Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Enable condensing indirect vdev mappings.
	When set, attempt to condense indirect vdev mappings
	if the mapping uses more than
	.Sy zfs_condense_min_mapping_bytes
	bytes of memory and if the obsolete space map object uses more than
	.Sy zfs_condense_max_obsolete_bytes
	bytes on-disk.
	The condensing process is an attempt to save memory by removing obsolete
	mappings.
	.
	.It Sy zfs_condense_max_obsolete_bytes Ns = Ns Sy 1073741824 Ns B Po 1 GiB Pc Pq u64
	Only attempt to condense indirect vdev mappings if the on-disk size
	of the obsolete space map object is greater than this number of bytes
	.Pq see Sy zfs_condense_indirect_vdevs_enable .
	.
	.It Sy zfs_condense_min_mapping_bytes Ns = Ns Sy 131072 Ns B Po 128 KiB Pc Pq u64
	Minimum size vdev mapping to attempt to condense
	.Pq see Sy zfs_condense_indirect_vdevs_enable .
	.
	.It Sy zfs_dbgmsg_enable Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Internally ZFS keeps a small log to facilitate debugging.
	The log is enabled by default, and can be disabled by unsetting this option.
	The contents of the log can be accessed by reading
	.Pa /proc/spl/kstat/zfs/dbgmsg .
	Writing
	.Sy 0
	to the file clears the log.
	.Pp
	This setting does not influence debug prints due to
	.Sy zfs_flags .
	.
	.It Sy zfs_dbgmsg_maxsize Ns = Ns Sy 4194304 Ns B Po 4 MiB Pc Pq uint
	Maximum size of the internal ZFS debug log.
	.
	.It Sy zfs_dbuf_state_index Ns = Ns Sy 0 Pq int
	Historically used for controlling what reporting was available under
	.Pa /proc/spl/kstat/zfs .
	No effect.
	.
	.It Sy zfs_deadman_enabled Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	When a pool sync operation takes longer than
	.Sy zfs_deadman_synctime_ms ,
	or when an individual I/O operation takes longer than
	.Sy zfs_deadman_ziotime_ms ,
	then the operation is considered to be "hung".
	If
	.Sy zfs_deadman_enabled
	is set, then the deadman behavior is invoked as described by
	.Sy zfs_deadman_failmode .
	By default, the deadman is enabled and set to
	.Sy wait
	which results in "hung" I/O operations only being logged.
	The deadman is automatically disabled when a pool gets suspended.
	.
	.It Sy zfs_deadman_failmode Ns = Ns Sy wait Pq charp
	Controls the failure behavior when the deadman detects a "hung" I/O operation.
	Valid values are:
	.Bl -tag -compact -offset 4n -width "continue"
	.It Sy wait
	Wait for a "hung" operation to complete.
	For each "hung" operation a "deadman" event will be posted
	describing that operation.
	.It Sy continue
	Attempt to recover from a "hung" operation by re-dispatching it
	to the I/O pipeline if possible.
	.It Sy panic
	Panic the system.
	This can be used to facilitate automatic fail-over
	to a properly configured fail-over partner.
	.El
	.
	.It Sy zfs_deadman_checktime_ms Ns = Ns Sy 60000 Ns ms Po 1 min Pc Pq u64
	Check time in milliseconds.
	This defines the frequency at which we check for hung I/O requests
	and potentially invoke the
	.Sy zfs_deadman_failmode
	behavior.
	.
	.It Sy zfs_deadman_synctime_ms Ns = Ns Sy 600000 Ns ms Po 10 min Pc Pq u64
	Interval in milliseconds after which the deadman is triggered and also
	the interval after which a pool sync operation is considered to be "hung".
	Once this limit is exceeded the deadman will be invoked every
	.Sy zfs_deadman_checktime_ms
	milliseconds until the pool sync completes.
	.
	.It Sy zfs_deadman_ziotime_ms Ns = Ns Sy 300000 Ns ms Po 5 min Pc Pq u64
	Interval in milliseconds after which the deadman is triggered and an
	individual I/O operation is considered to be "hung".
	As long as the operation remains "hung",
	the deadman will be invoked every
	.Sy zfs_deadman_checktime_ms
	milliseconds until the operation completes.
	.
	.It Sy zfs_dedup_prefetch Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Enable prefetching dedup-ed blocks which are going to be freed.
	.
	.It Sy zfs_delay_min_dirty_percent Ns = Ns Sy 60 Ns % Pq uint
	Start to delay each transaction once there is this amount of dirty data,
	expressed as a percentage of
	.Sy zfs_dirty_data_max .
	This value should be at least
	.Sy zfs_vdev_async_write_active_max_dirty_percent .
	.No See Sx ZFS TRANSACTION DELAY .
	.
	.It Sy zfs_delay_scale Ns = Ns Sy 500000 Pq int
	This controls how quickly the transaction delay approaches infinity.
	Larger values cause longer delays for a given amount of dirty data.
	.Pp
	For the smoothest delay, this value should be about 1 billion divided
	by the maximum number of operations per second.
	This will smoothly handle between ten times and a tenth of this number.
	.No See Sx ZFS TRANSACTION DELAY .
	.Pp
	.Sy zfs_delay_scale No \(mu Sy zfs_dirty_data_max Em must No be smaller than Sy 2^64 .
	.
	.It Sy zfs_disable_ivset_guid_check Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Disables requirement for IVset GUIDs to be present and match when doing a raw
	receive of encrypted datasets.
	Intended for users whose pools were created with
	OpenZFS pre-release versions and now have compatibility issues.
	.
	.It Sy zfs_key_max_salt_uses Ns = Ns Sy 400000000 Po 4*10^8 Pc Pq ulong
	Maximum number of uses of a single salt value before generating a new one for
	encrypted datasets.
	The default value is also the maximum.
	.
	.It Sy zfs_object_mutex_size Ns = Ns Sy 64 Pq uint
	Size of the znode hashtable used for holds.
	.Pp
	Due to the need to hold locks on objects that may not exist yet, kernel mutexes
	are not created per-object and instead a hashtable is used where collisions
	will result in objects waiting when there is not actually contention on the
	same object.
	.
	.It Sy zfs_slow_io_events_per_second Ns = Ns Sy 20 Ns /s Pq int
	Rate limit delay and deadman zevents (which report slow I/O operations) to this
	many per
	second.
	.
	.It Sy zfs_unflushed_max_mem_amt Ns = Ns Sy 1073741824 Ns B Po 1 GiB Pc Pq u64
	Upper-bound limit for unflushed metadata changes to be held by the
	log spacemap in memory, in bytes.
	.
	.It Sy zfs_unflushed_max_mem_ppm Ns = Ns Sy 1000 Ns ppm Po 0.1% Pc Pq u64
	Part of overall system memory that ZFS allows to be used
	for unflushed metadata changes by the log spacemap, in millionths.
	.
	.It Sy zfs_unflushed_log_block_max Ns = Ns Sy 131072 Po 128k Pc Pq u64
	Describes the maximum number of log spacemap blocks allowed for each pool.
	The default value means that the space in all the log spacemaps
	can add up to no more than
	.Sy 131072
	blocks (which means
	.Em 16 GiB
	of logical space before compression and ditto blocks,
	assuming that blocksize is
	.Em 128 KiB ) .
	.Pp
	This tunable is important because it involves a trade-off between import
	time after an unclean export and the frequency of flushing metaslabs.
	The higher this number is, the more log blocks we allow when the pool is
	active which means that we flush metaslabs less often and thus decrease
	the number of I/O operations for spacemap updates per TXG.
	At the same time though, that means that in the event of an unclean export,
	there will be more log spacemap blocks for us to read, inducing overhead
	in the import time of the pool.
	The lower the number, the amount of flushing increases, destroying log
	blocks quicker as they become obsolete faster, which leaves less blocks
	to be read during import time after a crash.
	.Pp
	Each log spacemap block existing during pool import leads to approximately
	one extra logical I/O issued.
	This is the reason why this tunable is exposed in terms of blocks rather
	than space used.
	.
	.It Sy zfs_unflushed_log_block_min Ns = Ns Sy 1000 Pq u64
	If the number of metaslabs is small and our incoming rate is high,
	we could get into a situation that we are flushing all our metaslabs every TXG.
	Thus we always allow at least this many log blocks.
	.
	.It Sy zfs_unflushed_log_block_pct Ns = Ns Sy 400 Ns % Pq u64
	Tunable used to determine the number of blocks that can be used for
	the spacemap log, expressed as a percentage of the total number of
	unflushed metaslabs in the pool.
	.
	.It Sy zfs_unflushed_log_txg_max Ns = Ns Sy 1000 Pq u64
	Tunable limiting maximum time in TXGs any metaslab may remain unflushed.
	It effectively limits maximum number of unflushed per-TXG spacemap logs
	that need to be read after unclean pool export.
	.
	.It Sy zfs_unlink_suspend_progress Ns = Ns Sy 0 Ns \| Ns 1 Pq uint
	When enabled, files will not be asynchronously removed from the list of pending
	unlinks and the space they consume will be leaked.
	Once this option has been disabled and the dataset is remounted,
	the pending unlinks will be processed and the freed space returned to the pool.
	This option is used by the test suite.
	.
	.It Sy zfs_delete_blocks Ns = Ns Sy 20480 Pq ulong
	This is the used to define a large file for the purposes of deletion.
	Files containing more than
	.Sy zfs_delete_blocks
	will be deleted asynchronously, while smaller files are deleted synchronously.
	Decreasing this value will reduce the time spent in an
	.Xr unlink 2
	system call, at the expense of a longer delay before the freed space is
	available.
	This only applies on Linux.
	.
	.It Sy zfs_dirty_data_max Ns = Pq int
	Determines the dirty space limit in bytes.
	Once this limit is exceeded, new writes are halted until space frees up.
	This parameter takes precedence over
	.Sy zfs_dirty_data_max_percent .
	.No See Sx ZFS TRANSACTION DELAY .
	.Pp
	Defaults to
	.Sy physical_ram/10 ,
	capped at
	.Sy zfs_dirty_data_max_max .
	.
	.It Sy zfs_dirty_data_max_max Ns = Pq int
	Maximum allowable value of
	.Sy zfs_dirty_data_max ,
	expressed in bytes.
	This limit is only enforced at module load time, and will be ignored if
	.Sy zfs_dirty_data_max
	is later changed.
	This parameter takes precedence over
	.Sy zfs_dirty_data_max_max_percent .
	.No See Sx ZFS TRANSACTION DELAY .
	.Pp
	Defaults to
	.Sy min(physical_ram/4, 4GiB) ,
	or
	.Sy min(physical_ram/4, 1GiB)
	for 32-bit systems.
	.
	.It Sy zfs_dirty_data_max_max_percent Ns = Ns Sy 25 Ns % Pq uint
	Maximum allowable value of
	.Sy zfs_dirty_data_max ,
	expressed as a percentage of physical RAM.
	This limit is only enforced at module load time, and will be ignored if
	.Sy zfs_dirty_data_max
	is later changed.
	The parameter
	.Sy zfs_dirty_data_max_max
	takes precedence over this one.
	.No See Sx ZFS TRANSACTION DELAY .
	.
	.It Sy zfs_dirty_data_max_percent Ns = Ns Sy 10 Ns % Pq uint
	Determines the dirty space limit, expressed as a percentage of all memory.
	Once this limit is exceeded, new writes are halted until space frees up.
	The parameter
	.Sy zfs_dirty_data_max
	takes precedence over this one.
	.No See Sx ZFS TRANSACTION DELAY .
	.Pp
	Subject to
	.Sy zfs_dirty_data_max_max .
	.
	.It Sy zfs_dirty_data_sync_percent Ns = Ns Sy 20 Ns % Pq uint
	Start syncing out a transaction group if there's at least this much dirty data
	.Pq as a percentage of Sy zfs_dirty_data_max .
	This should be less than
	.Sy zfs_vdev_async_write_active_min_dirty_percent .
	.
	.It Sy zfs_wrlog_data_max Ns = Pq int
	The upper limit of write-transaction zil log data size in bytes.
	Write operations are throttled when approaching the limit until log data is
	cleared out after transaction group sync.
	Because of some overhead, it should be set at least 2 times the size of
	.Sy zfs_dirty_data_max
	.No to prevent harming normal write throughput .
	It also should be smaller than the size of the slog device if slog is present.
	.Pp
	Defaults to
	.Sy zfs_dirty_data_max*2
	.
	.It Sy zfs_fallocate_reserve_percent Ns = Ns Sy 110 Ns % Pq uint
	Since ZFS is a copy-on-write filesystem with snapshots, blocks cannot be
	preallocated for a file in order to guarantee that later writes will not
	run out of space.
	Instead,
	.Xr fallocate 2
	space preallocation only checks that sufficient space is currently available
	in the pool or the user's project quota allocation,
	and then creates a sparse file of the requested size.
	The requested space is multiplied by
	.Sy zfs_fallocate_reserve_percent
	to allow additional space for indirect blocks and other internal metadata.
	Setting this to
	.Sy 0
	disables support for
	.Xr fallocate 2
	and causes it to return
	.Sy EOPNOTSUPP .
	.
	.It Sy zfs_fletcher_4_impl Ns = Ns Sy fastest Pq string
	Select a fletcher 4 implementation.
	.Pp
	Supported selectors are:
	.Sy fastest , scalar , sse2 , ssse3 , avx2 , avx512f , avx512bw ,
	.No and Sy aarch64_neon .
	All except
	.Sy fastest No and Sy scalar
	require instruction set extensions to be available,
	and will only appear if ZFS detects that they are present at runtime.
	If multiple implementations of fletcher 4 are available, the
	.Sy fastest
	will be chosen using a micro benchmark.
	Selecting
	.Sy scalar
	results in the original CPU-based calculation being used.
	Selecting any option other than
	.Sy fastest No or Sy scalar
	results in vector instructions
	from the respective CPU instruction set being used.
	.
	.It Sy zfs_bclone_enabled Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Enable the experimental block cloning feature.
	If this setting is 0, then even if feature@block_cloning is enabled,
	attempts to clone blocks will act as though the feature is disabled.
	.
	.It Sy zfs_bclone_wait_dirty Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	When set to 1 the FICLONE and FICLONERANGE ioctls wait for dirty data to be
	written to disk.
	This allows the clone operation to reliably succeed when a file is
	modified and then immediately cloned.
	For small files this may be slower than making a copy of the file.
	Therefore, this setting defaults to 0 which causes a clone operation to
	immediately fail when encountering a dirty block.
	.
	.It Sy zfs_blake3_impl Ns = Ns Sy fastest Pq string
	Select a BLAKE3 implementation.
	.Pp
	Supported selectors are:
	.Sy cycle , fastest , generic , sse2 , sse41 , avx2 , avx512 .
	All except
	.Sy cycle , fastest No and Sy generic
	require instruction set extensions to be available,
	and will only appear if ZFS detects that they are present at runtime.
	If multiple implementations of BLAKE3 are available, the
	.Sy fastest will be chosen using a micro benchmark. You can see the
	benchmark results by reading this kstat file:
	.Pa /proc/spl/kstat/zfs/chksum_bench .
	.
	.It Sy zfs_free_bpobj_enabled Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Enable/disable the processing of the free_bpobj object.
	.
	.It Sy zfs_async_block_max_blocks Ns = Ns Sy UINT64_MAX Po unlimited Pc Pq u64
	Maximum number of blocks freed in a single TXG.
	.
	.It Sy zfs_max_async_dedup_frees Ns = Ns Sy 100000 Po 10^5 Pc Pq u64
	Maximum number of dedup blocks freed in a single TXG.
	.
	.It Sy zfs_vdev_async_read_max_active Ns = Ns Sy 3 Pq uint
	Maximum asynchronous read I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_async_read_min_active Ns = Ns Sy 1 Pq uint
	Minimum asynchronous read I/O operation active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_async_write_active_max_dirty_percent Ns = Ns Sy 60 Ns % Pq uint
	When the pool has more than this much dirty data, use
	.Sy zfs_vdev_async_write_max_active
	to limit active async writes.
	If the dirty data is between the minimum and maximum,
	the active I/O limit is linearly interpolated.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_async_write_active_min_dirty_percent Ns = Ns Sy 30 Ns % Pq uint
	When the pool has less than this much dirty data, use
	.Sy zfs_vdev_async_write_min_active
	to limit active async writes.
	If the dirty data is between the minimum and maximum,
	the active I/O limit is linearly
	interpolated.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_async_write_max_active Ns = Ns Sy 10 Pq uint
	Maximum asynchronous write I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_async_write_min_active Ns = Ns Sy 2 Pq uint
	Minimum asynchronous write I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.Pp
	Lower values are associated with better latency on rotational media but poorer
	resilver performance.
	The default value of
	.Sy 2
	was chosen as a compromise.
	A value of
	.Sy 3
	has been shown to improve resilver performance further at a cost of
	further increasing latency.
	.
	.It Sy zfs_vdev_initializing_max_active Ns = Ns Sy 1 Pq uint
	Maximum initializing I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_initializing_min_active Ns = Ns Sy 1 Pq uint
	Minimum initializing I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_max_active Ns = Ns Sy 1000 Pq uint
	The maximum number of I/O operations active to each device.
	Ideally, this will be at least the sum of each queue's
	.Sy max_active .
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_open_timeout_ms Ns = Ns Sy 1000 Pq uint
	Timeout value to wait before determining a device is missing
	during import.
	This is helpful for transient missing paths due
	to links being briefly removed and recreated in response to
	udev events.
	.
	.It Sy zfs_vdev_rebuild_max_active Ns = Ns Sy 3 Pq uint
	Maximum sequential resilver I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_rebuild_min_active Ns = Ns Sy 1 Pq uint
	Minimum sequential resilver I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_removal_max_active Ns = Ns Sy 2 Pq uint
	Maximum removal I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_removal_min_active Ns = Ns Sy 1 Pq uint
	Minimum removal I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_scrub_max_active Ns = Ns Sy 2 Pq uint
	Maximum scrub I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_scrub_min_active Ns = Ns Sy 1 Pq uint
	Minimum scrub I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_sync_read_max_active Ns = Ns Sy 10 Pq uint
	Maximum synchronous read I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_sync_read_min_active Ns = Ns Sy 10 Pq uint
	Minimum synchronous read I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_sync_write_max_active Ns = Ns Sy 10 Pq uint
	Maximum synchronous write I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_sync_write_min_active Ns = Ns Sy 10 Pq uint
	Minimum synchronous write I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_trim_max_active Ns = Ns Sy 2 Pq uint
	Maximum trim/discard I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_trim_min_active Ns = Ns Sy 1 Pq uint
	Minimum trim/discard I/O operations active to each device.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_nia_delay Ns = Ns Sy 5 Pq uint
	For non-interactive I/O (scrub, resilver, removal, initialize and rebuild),
	the number of concurrently-active I/O operations is limited to
	.Sy zfs_*_min_active ,
	unless the vdev is "idle".
	When there are no interactive I/O operations active (synchronous or otherwise),
	and
	.Sy zfs_vdev_nia_delay
	operations have completed since the last interactive operation,
	then the vdev is considered to be "idle",
	and the number of concurrently-active non-interactive operations is increased to
	.Sy zfs_*_max_active .
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_nia_credit Ns = Ns Sy 5 Pq uint
	Some HDDs tend to prioritize sequential I/O so strongly, that concurrent
	random I/O latency reaches several seconds.
	On some HDDs this happens even if sequential I/O operations
	are submitted one at a time, and so setting
	.Sy zfs_*_max_active Ns = Sy 1
	does not help.
	To prevent non-interactive I/O, like scrub,
	from monopolizing the device, no more than
	.Sy zfs_vdev_nia_credit operations can be sent
	while there are outstanding incomplete interactive operations.
	This enforced wait ensures the HDD services the interactive I/O
	within a reasonable amount of time.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_queue_depth_pct Ns = Ns Sy 1000 Ns % Pq uint
	Maximum number of queued allocations per top-level vdev expressed as
	a percentage of
	.Sy zfs_vdev_async_write_max_active ,
	which allows the system to detect devices that are more capable
	of handling allocations and to allocate more blocks to those devices.
	This allows for dynamic allocation distribution when devices are imbalanced,
	as fuller devices will tend to be slower than empty devices.
	.Pp
	Also see
	.Sy zio_dva_throttle_enabled .
	.
	.It Sy zfs_vdev_def_queue_depth Ns = Ns Sy 32 Pq uint
	Default queue depth for each vdev IO allocator.
	Higher values allow for better coalescing of sequential writes before sending
	them to the disk, but can increase transaction commit times.
	.
	.It Sy zfs_vdev_failfast_mask Ns = Ns Sy 1 Pq uint
	Defines if the driver should retire on a given error type.
	The following options may be bitwise-ored together:
	.TS
	box;
	lbz r l l .
	Value Name Description
	_
	1 Device No driver retries on device errors
	2 Transport No driver retries on transport errors.
	4 Driver No driver retries on driver errors.
	.TE
	.
	+.It Sy zfs_vdev_disk_max_segs Ns = Ns Sy 0 Pq uint
	+Maximum number of segments to add to a BIO (min 4).
	+If this is higher than the maximum allowed by the device queue or the kernel
	+itself, it will be clamped.
	+Setting it to zero will cause the kernel's ideal size to be used.
	+This parameter only applies on Linux.
	+This parameter is ignored if
	+.Sy zfs_vdev_disk_classic Ns = Ns Sy 1 .
	+.
	+.It Sy zfs_vdev_disk_classic Ns = Ns 0 Ns \| Ns Sy 1 Pq uint
	+Controls the method used to submit IO to the Linux block layer
	+(default
	+.Sy 1 "classic" Ns
	+)
	+.Pp
	+If set to 1, the "classic" method is used.
	+This is the method that has been in use since the earliest versions of
	+ZFS-on-Linux.
	+It has known issues with highly fragmented IO requests and is less efficient on
	+many workloads, but it well known and well understood.
	+.Pp
	+If set to 0, the "new" method is used.
	+This method is available since 2.2.4 and should resolve all known issues and be
	+far more efficient, but has not had as much testing.
	+In the 2.2.x series, this parameter defaults to 1, to use the "classic" method.
	+.Pp
	+It is not recommended that you change it except on advice from the OpenZFS
	+developers.
	+If you do change it, please also open a bug report describing why you did so,
	+including the workload involved and any error messages.
	+.Pp
	+This parameter and the "classic" submission method will be removed in a future
	+release of OpenZFS once we have total confidence in the new method.
	+.Pp
	+This parameter only applies on Linux, and can only be set at module load time.
	+.
	.It Sy zfs_expire_snapshot Ns = Ns Sy 300 Ns s Pq int
	Time before expiring
	.Pa .zfs/snapshot .
	.
	.It Sy zfs_admin_snapshot Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Allow the creation, removal, or renaming of entries in the
	.Sy .zfs/snapshot
	directory to cause the creation, destruction, or renaming of snapshots.
	When enabled, this functionality works both locally and over NFS exports
	which have the
	.Em no_root_squash
	option set.
	.
	.It Sy zfs_flags Ns = Ns Sy 0 Pq int
	Set additional debugging flags.
	The following flags may be bitwise-ored together:
	.TS
	box;
	lbz r l l .
	Value Name Description
	_
	1 ZFS_DEBUG_DPRINTF Enable dprintf entries in the debug log.
	* 2 ZFS_DEBUG_DBUF_VERIFY Enable extra dbuf verifications.
	* 4 ZFS_DEBUG_DNODE_VERIFY Enable extra dnode verifications.
	8 ZFS_DEBUG_SNAPNAMES Enable snapshot name verification.
	* 16 ZFS_DEBUG_MODIFY Check for illegally modified ARC buffers.
	64 ZFS_DEBUG_ZIO_FREE Enable verification of block frees.
	128 ZFS_DEBUG_HISTOGRAM_VERIFY Enable extra spacemap histogram verifications.
	256 ZFS_DEBUG_METASLAB_VERIFY Verify space accounting on disk matches in-memory \fBrange_trees\fP.
	512 ZFS_DEBUG_SET_ERROR Enable \fBSET_ERROR\fP and dprintf entries in the debug log.
	1024 ZFS_DEBUG_INDIRECT_REMAP Verify split blocks created by device removal.
	2048 ZFS_DEBUG_TRIM Verify TRIM ranges are always within the allocatable range tree.
	4096 ZFS_DEBUG_LOG_SPACEMAP Verify that the log summary is consistent with the spacemap log
	and enable \fBzfs_dbgmsgs\fP for metaslab loading and flushing.
	.TE
	.Sy \& * No Requires debug build .
	.
	.It Sy zfs_btree_verify_intensity Ns = Ns Sy 0 Pq uint
	Enables btree verification.
	The following settings are culminative:
	.TS
	box;
	lbz r l l .
	Value Description

	1 Verify height.
	2 Verify pointers from children to parent.
	3 Verify element counts.
	4 Verify element order. (expensive)
	* 5 Verify unused memory is poisoned. (expensive)
	.TE
	.Sy \& * No Requires debug build .
	.
	.It Sy zfs_free_leak_on_eio Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	If destroy encounters an
	.Sy EIO
	while reading metadata (e.g. indirect blocks),
	space referenced by the missing metadata can not be freed.
	Normally this causes the background destroy to become "stalled",
	as it is unable to make forward progress.
	While in this stalled state, all remaining space to free
	from the error-encountering filesystem is "temporarily leaked".
	Set this flag to cause it to ignore the
	.Sy EIO ,
	permanently leak the space from indirect blocks that can not be read,
	and continue to free everything else that it can.
	.Pp
	The default "stalling" behavior is useful if the storage partially
	fails (i.e. some but not all I/O operations fail), and then later recovers.
	In this case, we will be able to continue pool operations while it is
	partially failed, and when it recovers, we can continue to free the
	space, with no leaks.
	Note, however, that this case is actually fairly rare.
	.Pp
	Typically pools either
	.Bl -enum -compact -offset 4n -width "1."
	.It
	fail completely (but perhaps temporarily,
	e.g. due to a top-level vdev going offline), or
	.It
	have localized, permanent errors (e.g. disk returns the wrong data
	due to bit flip or firmware bug).
	.El
	In the former case, this setting does not matter because the
	pool will be suspended and the sync thread will not be able to make
	forward progress regardless.
	In the latter, because the error is permanent, the best we can do
	is leak the minimum amount of space,
	which is what setting this flag will do.
	It is therefore reasonable for this flag to normally be set,
	but we chose the more conservative approach of not setting it,
	so that there is no possibility of
	leaking space in the "partial temporary" failure case.
	.
	.It Sy zfs_free_min_time_ms Ns = Ns Sy 1000 Ns ms Po 1s Pc Pq uint
	During a
	.Nm zfs Cm destroy
	operation using the
	.Sy async_destroy
	feature,
	a minimum of this much time will be spent working on freeing blocks per TXG.
	.
	.It Sy zfs_obsolete_min_time_ms Ns = Ns Sy 500 Ns ms Pq uint
	Similar to
	.Sy zfs_free_min_time_ms ,
	but for cleanup of old indirection records for removed vdevs.
	.
	.It Sy zfs_immediate_write_sz Ns = Ns Sy 32768 Ns B Po 32 KiB Pc Pq s64
	Largest data block to write to the ZIL.
	Larger blocks will be treated as if the dataset being written to had the
	.Sy logbias Ns = Ns Sy throughput
	property set.
	.
	.It Sy zfs_initialize_value Ns = Ns Sy 16045690984833335022 Po 0xDEADBEEFDEADBEEE Pc Pq u64
	Pattern written to vdev free space by
	.Xr zpool-initialize 8 .
	.
	.It Sy zfs_initialize_chunk_size Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq u64
	Size of writes used by
	.Xr zpool-initialize 8 .
	This option is used by the test suite.
	.
	.It Sy zfs_livelist_max_entries Ns = Ns Sy 500000 Po 5*10^5 Pc Pq u64
	The threshold size (in block pointers) at which we create a new sub-livelist.
	Larger sublists are more costly from a memory perspective but the fewer
	sublists there are, the lower the cost of insertion.
	.
	.It Sy zfs_livelist_min_percent_shared Ns = Ns Sy 75 Ns % Pq int
	If the amount of shared space between a snapshot and its clone drops below
	this threshold, the clone turns off the livelist and reverts to the old
	deletion method.
	This is in place because livelists no long give us a benefit
	once a clone has been overwritten enough.
	.
	.It Sy zfs_livelist_condense_new_alloc Ns = Ns Sy 0 Pq int
	Incremented each time an extra ALLOC blkptr is added to a livelist entry while
	it is being condensed.
	This option is used by the test suite to track race conditions.
	.
	.It Sy zfs_livelist_condense_sync_cancel Ns = Ns Sy 0 Pq int
	Incremented each time livelist condensing is canceled while in
	.Fn spa_livelist_condense_sync .
	This option is used by the test suite to track race conditions.
	.
	.It Sy zfs_livelist_condense_sync_pause Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	When set, the livelist condense process pauses indefinitely before
	executing the synctask \(em
	.Fn spa_livelist_condense_sync .
	This option is used by the test suite to trigger race conditions.
	.
	.It Sy zfs_livelist_condense_zthr_cancel Ns = Ns Sy 0 Pq int
	Incremented each time livelist condensing is canceled while in
	.Fn spa_livelist_condense_cb .
	This option is used by the test suite to track race conditions.
	.
	.It Sy zfs_livelist_condense_zthr_pause Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	When set, the livelist condense process pauses indefinitely before
	executing the open context condensing work in
	.Fn spa_livelist_condense_cb .
	This option is used by the test suite to trigger race conditions.
	.
	.It Sy zfs_lua_max_instrlimit Ns = Ns Sy 100000000 Po 10^8 Pc Pq u64
	The maximum execution time limit that can be set for a ZFS channel program,
	specified as a number of Lua instructions.
	.
	.It Sy zfs_lua_max_memlimit Ns = Ns Sy 104857600 Po 100 MiB Pc Pq u64
	The maximum memory limit that can be set for a ZFS channel program, specified
	in bytes.
	.
	.It Sy zfs_max_dataset_nesting Ns = Ns Sy 50 Pq int
	The maximum depth of nested datasets.
	This value can be tuned temporarily to
	fix existing datasets that exceed the predefined limit.
	.
	.It Sy zfs_max_log_walking Ns = Ns Sy 5 Pq u64
	The number of past TXGs that the flushing algorithm of the log spacemap
	feature uses to estimate incoming log blocks.
	.
	.It Sy zfs_max_logsm_summary_length Ns = Ns Sy 10 Pq u64
	Maximum number of rows allowed in the summary of the spacemap log.
	.
	.It Sy zfs_max_recordsize Ns = Ns Sy 16777216 Po 16 MiB Pc Pq uint
	We currently support block sizes from
	.Em 512 Po 512 B Pc No to Em 16777216 Po 16 MiB Pc .
	The benefits of larger blocks, and thus larger I/O,
	need to be weighed against the cost of COWing a giant block to modify one byte.
	Additionally, very large blocks can have an impact on I/O latency,
	and also potentially on the memory allocator.
	Therefore, we formerly forbade creating blocks larger than 1M.
	Larger blocks could be created by changing it,
	and pools with larger blocks can always be imported and used,
	regardless of this setting.
	.
	.It Sy zfs_allow_redacted_dataset_mount Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Allow datasets received with redacted send/receive to be mounted.
	Normally disabled because these datasets may be missing key data.
	.
	.It Sy zfs_min_metaslabs_to_flush Ns = Ns Sy 1 Pq u64
	Minimum number of metaslabs to flush per dirty TXG.
	.
	.It Sy zfs_metaslab_fragmentation_threshold Ns = Ns Sy 70 Ns % Pq uint
	Allow metaslabs to keep their active state as long as their fragmentation
	percentage is no more than this value.
	An active metaslab that exceeds this threshold
	will no longer keep its active status allowing better metaslabs to be selected.
	.
	.It Sy zfs_mg_fragmentation_threshold Ns = Ns Sy 95 Ns % Pq uint
	Metaslab groups are considered eligible for allocations if their
	fragmentation metric (measured as a percentage) is less than or equal to
	this value.
	If a metaslab group exceeds this threshold then it will be
	skipped unless all metaslab groups within the metaslab class have also
	crossed this threshold.
	.
	.It Sy zfs_mg_noalloc_threshold Ns = Ns Sy 0 Ns % Pq uint
	Defines a threshold at which metaslab groups should be eligible for allocations.
	The value is expressed as a percentage of free space
	beyond which a metaslab group is always eligible for allocations.
	If a metaslab group's free space is less than or equal to the
	threshold, the allocator will avoid allocating to that group
	unless all groups in the pool have reached the threshold.
	Once all groups have reached the threshold, all groups are allowed to accept
	allocations.
	The default value of
	.Sy 0
	disables the feature and causes all metaslab groups to be eligible for
	allocations.
	.Pp
	This parameter allows one to deal with pools having heavily imbalanced
	vdevs such as would be the case when a new vdev has been added.
	Setting the threshold to a non-zero percentage will stop allocations
	from being made to vdevs that aren't filled to the specified percentage
	and allow lesser filled vdevs to acquire more allocations than they
	otherwise would under the old
	.Sy zfs_mg_alloc_failures
	facility.
	.
	.It Sy zfs_ddt_data_is_special Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	If enabled, ZFS will place DDT data into the special allocation class.
	.
	.It Sy zfs_user_indirect_is_special Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	If enabled, ZFS will place user data indirect blocks
	into the special allocation class.
	.
	.It Sy zfs_multihost_history Ns = Ns Sy 0 Pq uint
	Historical statistics for this many latest multihost updates will be available
	in
	.Pa /proc/spl/kstat/zfs/ Ns Ao Ar pool Ac Ns Pa /multihost .
	.
	.It Sy zfs_multihost_interval Ns = Ns Sy 1000 Ns ms Po 1 s Pc Pq u64
	Used to control the frequency of multihost writes which are performed when the
	.Sy multihost
	pool property is on.
	This is one of the factors used to determine the
	length of the activity check during import.
	.Pp
	The multihost write period is
	.Sy zfs_multihost_interval No / Sy leaf-vdevs .
	On average a multihost write will be issued for each leaf vdev
	every
	.Sy zfs_multihost_interval
	milliseconds.
	In practice, the observed period can vary with the I/O load
	and this observed value is the delay which is stored in the uberblock.
	.
	.It Sy zfs_multihost_import_intervals Ns = Ns Sy 20 Pq uint
	Used to control the duration of the activity test on import.
	Smaller values of
	.Sy zfs_multihost_import_intervals
	will reduce the import time but increase
	the risk of failing to detect an active pool.
	The total activity check time is never allowed to drop below one second.
	.Pp
	On import the activity check waits a minimum amount of time determined by
	.Sy zfs_multihost_interval No \(mu Sy zfs_multihost_import_intervals ,
	or the same product computed on the host which last had the pool imported,
	whichever is greater.
	The activity check time may be further extended if the value of MMP
	delay found in the best uberblock indicates actual multihost updates happened
	at longer intervals than
	.Sy zfs_multihost_interval .
	A minimum of
	.Em 100 ms
	is enforced.
	.Pp
	.Sy 0 No is equivalent to Sy 1 .
	.
	.It Sy zfs_multihost_fail_intervals Ns = Ns Sy 10 Pq uint
	Controls the behavior of the pool when multihost write failures or delays are
	detected.
	.Pp
	When
	.Sy 0 ,
	multihost write failures or delays are ignored.
	The failures will still be reported to the ZED which depending on
	its configuration may take action such as suspending the pool or offlining a
	device.
	.Pp
	Otherwise, the pool will be suspended if
	.Sy zfs_multihost_fail_intervals No \(mu Sy zfs_multihost_interval
	milliseconds pass without a successful MMP write.
	This guarantees the activity test will see MMP writes if the pool is imported.
	.Sy 1 No is equivalent to Sy 2 ;
	this is necessary to prevent the pool from being suspended
	due to normal, small I/O latency variations.
	.
	.It Sy zfs_no_scrub_io Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Set to disable scrub I/O.
	This results in scrubs not actually scrubbing data and
	simply doing a metadata crawl of the pool instead.
	.
	.It Sy zfs_no_scrub_prefetch Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Set to disable block prefetching for scrubs.
	.
	.It Sy zfs_nocacheflush Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Disable cache flush operations on disks when writing.
	Setting this will cause pool corruption on power loss
	if a volatile out-of-order write cache is enabled.
	.
	.It Sy zfs_nopwrite_enabled Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Allow no-operation writes.
	The occurrence of nopwrites will further depend on other pool properties
	.Pq i.a. the checksumming and compression algorithms .
	.
	.It Sy zfs_dmu_offset_next_sync Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Enable forcing TXG sync to find holes.
	When enabled forces ZFS to sync data when
	.Sy SEEK_HOLE No or Sy SEEK_DATA
	flags are used allowing holes in a file to be accurately reported.
	When disabled holes will not be reported in recently dirtied files.
	.
	.It Sy zfs_pd_bytes_max Ns = Ns Sy 52428800 Ns B Po 50 MiB Pc Pq int
	The number of bytes which should be prefetched during a pool traversal, like
	.Nm zfs Cm send
	or other data crawling operations.
	.
	.It Sy zfs_traverse_indirect_prefetch_limit Ns = Ns Sy 32 Pq uint
	The number of blocks pointed by indirect (non-L0) block which should be
	prefetched during a pool traversal, like
	.Nm zfs Cm send
	or other data crawling operations.
	.
	.It Sy zfs_per_txg_dirty_frees_percent Ns = Ns Sy 30 Ns % Pq u64
	Control percentage of dirtied indirect blocks from frees allowed into one TXG.
	After this threshold is crossed, additional frees will wait until the next TXG.
	.Sy 0 No disables this throttle .
	.
	.It Sy zfs_prefetch_disable Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Disable predictive prefetch.
	Note that it leaves "prescient" prefetch
	.Pq for, e.g., Nm zfs Cm send
	intact.
	Unlike predictive prefetch, prescient prefetch never issues I/O
	that ends up not being needed, so it can't hurt performance.
	.
	.It Sy zfs_qat_checksum_disable Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Disable QAT hardware acceleration for SHA256 checksums.
	May be unset after the ZFS modules have been loaded to initialize the QAT
	hardware as long as support is compiled in and the QAT driver is present.
	.
	.It Sy zfs_qat_compress_disable Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Disable QAT hardware acceleration for gzip compression.
	May be unset after the ZFS modules have been loaded to initialize the QAT
	hardware as long as support is compiled in and the QAT driver is present.
	.
	.It Sy zfs_qat_encrypt_disable Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Disable QAT hardware acceleration for AES-GCM encryption.
	May be unset after the ZFS modules have been loaded to initialize the QAT
	hardware as long as support is compiled in and the QAT driver is present.
	.
	.It Sy zfs_vnops_read_chunk_size Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq u64
	Bytes to read per chunk.
	.
	.It Sy zfs_read_history Ns = Ns Sy 0 Pq uint
	Historical statistics for this many latest reads will be available in
	.Pa /proc/spl/kstat/zfs/ Ns Ao Ar pool Ac Ns Pa /reads .
	.
	.It Sy zfs_read_history_hits Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Include cache hits in read history
	.
	.It Sy zfs_rebuild_max_segment Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq u64
	Maximum read segment size to issue when sequentially resilvering a
	top-level vdev.
	.
	.It Sy zfs_rebuild_scrub_enabled Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Automatically start a pool scrub when the last active sequential resilver
	completes in order to verify the checksums of all blocks which have been
	resilvered.
	This is enabled by default and strongly recommended.
	.
	.It Sy zfs_rebuild_vdev_limit Ns = Ns Sy 67108864 Ns B Po 64 MiB Pc Pq u64
	Maximum amount of I/O that can be concurrently issued for a sequential
	resilver per leaf device, given in bytes.
	.
	.It Sy zfs_reconstruct_indirect_combinations_max Ns = Ns Sy 4096 Pq int
	If an indirect split block contains more than this many possible unique
	combinations when being reconstructed, consider it too computationally
	expensive to check them all.
	Instead, try at most this many randomly selected
	combinations each time the block is accessed.
	This allows all segment copies to participate fairly
	in the reconstruction when all combinations
	cannot be checked and prevents repeated use of one bad copy.
	.
	.It Sy zfs_recover Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Set to attempt to recover from fatal errors.
	This should only be used as a last resort,
	as it typically results in leaked space, or worse.
	.
	.It Sy zfs_removal_ignore_errors Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Ignore hard I/O errors during device removal.
	When set, if a device encounters a hard I/O error during the removal process
	the removal will not be cancelled.
	This can result in a normally recoverable block becoming permanently damaged
	and is hence not recommended.
	This should only be used as a last resort when the
	pool cannot be returned to a healthy state prior to removing the device.
	.
	.It Sy zfs_removal_suspend_progress Ns = Ns Sy 0 Ns \| Ns 1 Pq uint
	This is used by the test suite so that it can ensure that certain actions
	happen while in the middle of a removal.
	.
	.It Sy zfs_remove_max_segment Ns = Ns Sy 16777216 Ns B Po 16 MiB Pc Pq uint
	The largest contiguous segment that we will attempt to allocate when removing
	a device.
	If there is a performance problem with attempting to allocate large blocks,
	consider decreasing this.
	The default value is also the maximum.
	.
	.It Sy zfs_resilver_disable_defer Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Ignore the
	.Sy resilver_defer
	feature, causing an operation that would start a resilver to
	immediately restart the one in progress.
	.
	.It Sy zfs_resilver_min_time_ms Ns = Ns Sy 3000 Ns ms Po 3 s Pc Pq uint
	Resilvers are processed by the sync thread.
	While resilvering, it will spend at least this much time
	working on a resilver between TXG flushes.
	.
	.It Sy zfs_scan_ignore_errors Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	If set, remove the DTL (dirty time list) upon completion of a pool scan (scrub),
	even if there were unrepairable errors.
	Intended to be used during pool repair or recovery to
	stop resilvering when the pool is next imported.
	.
	.It Sy zfs_scrub_min_time_ms Ns = Ns Sy 1000 Ns ms Po 1 s Pc Pq uint
	Scrubs are processed by the sync thread.
	While scrubbing, it will spend at least this much time
	working on a scrub between TXG flushes.
	.
	.It Sy zfs_scrub_error_blocks_per_txg Ns = Ns Sy 4096 Pq uint
	Error blocks to be scrubbed in one txg.
	.
	.It Sy zfs_scan_checkpoint_intval Ns = Ns Sy 7200 Ns s Po 2 hour Pc Pq uint
	To preserve progress across reboots, the sequential scan algorithm periodically
	needs to stop metadata scanning and issue all the verification I/O to disk.
	The frequency of this flushing is determined by this tunable.
	.
	.It Sy zfs_scan_fill_weight Ns = Ns Sy 3 Pq uint
	This tunable affects how scrub and resilver I/O segments are ordered.
	A higher number indicates that we care more about how filled in a segment is,
	while a lower number indicates we care more about the size of the extent without
	considering the gaps within a segment.
	This value is only tunable upon module insertion.
	Changing the value afterwards will have no effect on scrub or resilver
	performance.
	.
	.It Sy zfs_scan_issue_strategy Ns = Ns Sy 0 Pq uint
	Determines the order that data will be verified while scrubbing or resilvering:
	.Bl -tag -compact -offset 4n -width "a"
	.It Sy 1
	Data will be verified as sequentially as possible, given the
	amount of memory reserved for scrubbing
	.Pq see Sy zfs_scan_mem_lim_fact .
	This may improve scrub performance if the pool's data is very fragmented.
	.It Sy 2
	The largest mostly-contiguous chunk of found data will be verified first.
	By deferring scrubbing of small segments, we may later find adjacent data
	to coalesce and increase the segment size.
	.It Sy 0
	.No Use strategy Sy 1 No during normal verification
	.No and strategy Sy 2 No while taking a checkpoint .
	.El
	.
	.It Sy zfs_scan_legacy Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	If unset, indicates that scrubs and resilvers will gather metadata in
	memory before issuing sequential I/O.
	Otherwise indicates that the legacy algorithm will be used,
	where I/O is initiated as soon as it is discovered.
	Unsetting will not affect scrubs or resilvers that are already in progress.
	.
	.It Sy zfs_scan_max_ext_gap Ns = Ns Sy 2097152 Ns B Po 2 MiB Pc Pq int
	Sets the largest gap in bytes between scrub/resilver I/O operations
	that will still be considered sequential for sorting purposes.
	Changing this value will not
	affect scrubs or resilvers that are already in progress.
	.
	.It Sy zfs_scan_mem_lim_fact Ns = Ns Sy 20 Ns ^-1 Pq uint
	Maximum fraction of RAM used for I/O sorting by sequential scan algorithm.
	This tunable determines the hard limit for I/O sorting memory usage.
	When the hard limit is reached we stop scanning metadata and start issuing
	data verification I/O.
	This is done until we get below the soft limit.
	.
	.It Sy zfs_scan_mem_lim_soft_fact Ns = Ns Sy 20 Ns ^-1 Pq uint
	The fraction of the hard limit used to determined the soft limit for I/O sorting
	by the sequential scan algorithm.
	When we cross this limit from below no action is taken.
	When we cross this limit from above it is because we are issuing verification
	I/O.
	In this case (unless the metadata scan is done) we stop issuing verification I/O
	and start scanning metadata again until we get to the hard limit.
	.
	.It Sy zfs_scan_report_txgs Ns = Ns Sy 0 Ns \| Ns 1 Pq uint
	When reporting resilver throughput and estimated completion time use the
	performance observed over roughly the last
	.Sy zfs_scan_report_txgs
	TXGs.
	When set to zero performance is calculated over the time between checkpoints.
	.
	.It Sy zfs_scan_strict_mem_lim Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Enforce tight memory limits on pool scans when a sequential scan is in progress.
	When disabled, the memory limit may be exceeded by fast disks.
	.
	.It Sy zfs_scan_suspend_progress Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Freezes a scrub/resilver in progress without actually pausing it.
	Intended for testing/debugging.
	.
	.It Sy zfs_scan_vdev_limit Ns = Ns Sy 16777216 Ns B Po 16 MiB Pc Pq int
	Maximum amount of data that can be concurrently issued at once for scrubs and
	resilvers per leaf device, given in bytes.
	.
	.It Sy zfs_send_corrupt_data Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Allow sending of corrupt data (ignore read/checksum errors when sending).
	.
	.It Sy zfs_send_unmodified_spill_blocks Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Include unmodified spill blocks in the send stream.
	Under certain circumstances, previous versions of ZFS could incorrectly
	remove the spill block from an existing object.
	Including unmodified copies of the spill blocks creates a backwards-compatible
	stream which will recreate a spill block if it was incorrectly removed.
	.
	.It Sy zfs_send_no_prefetch_queue_ff Ns = Ns Sy 20 Ns ^\-1 Pq uint
	The fill fraction of the
	.Nm zfs Cm send
	internal queues.
	The fill fraction controls the timing with which internal threads are woken up.
	.
	.It Sy zfs_send_no_prefetch_queue_length Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq uint
	The maximum number of bytes allowed in
	.Nm zfs Cm send Ns 's
	internal queues.
	.
	.It Sy zfs_send_queue_ff Ns = Ns Sy 20 Ns ^\-1 Pq uint
	The fill fraction of the
	.Nm zfs Cm send
	prefetch queue.
	The fill fraction controls the timing with which internal threads are woken up.
	.
	.It Sy zfs_send_queue_length Ns = Ns Sy 16777216 Ns B Po 16 MiB Pc Pq uint
	The maximum number of bytes allowed that will be prefetched by
	.Nm zfs Cm send .
	This value must be at least twice the maximum block size in use.
	.
	.It Sy zfs_recv_queue_ff Ns = Ns Sy 20 Ns ^\-1 Pq uint
	The fill fraction of the
	.Nm zfs Cm receive
	queue.
	The fill fraction controls the timing with which internal threads are woken up.
	.
	.It Sy zfs_recv_queue_length Ns = Ns Sy 16777216 Ns B Po 16 MiB Pc Pq uint
	The maximum number of bytes allowed in the
	.Nm zfs Cm receive
	queue.
	This value must be at least twice the maximum block size in use.
	.
	.It Sy zfs_recv_write_batch_size Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq uint
	The maximum amount of data, in bytes, that
	.Nm zfs Cm receive
	will write in one DMU transaction.
	This is the uncompressed size, even when receiving a compressed send stream.
	This setting will not reduce the write size below a single block.
	Capped at a maximum of
	.Sy 32 MiB .
	.
	.It Sy zfs_recv_best_effort_corrective Ns = Ns Sy 0 Pq int
	When this variable is set to non-zero a corrective receive:
	.Bl -enum -compact -offset 4n -width "1."
	.It
	Does not enforce the restriction of source & destination snapshot GUIDs
	matching.
	.It
	If there is an error during healing, the healing receive is not
	terminated instead it moves on to the next record.
	.El
	.
	.It Sy zfs_override_estimate_recordsize Ns = Ns Sy 0 Ns \| Ns 1 Pq uint
	Setting this variable overrides the default logic for estimating block
	sizes when doing a
	.Nm zfs Cm send .
	The default heuristic is that the average block size
	will be the current recordsize.
	Override this value if most data in your dataset is not of that size
	and you require accurate zfs send size estimates.
	.
	.It Sy zfs_sync_pass_deferred_free Ns = Ns Sy 2 Pq uint
	Flushing of data to disk is done in passes.
	Defer frees starting in this pass.
	.
	.It Sy zfs_spa_discard_memory_limit Ns = Ns Sy 16777216 Ns B Po 16 MiB Pc Pq int
	Maximum memory used for prefetching a checkpoint's space map on each
	vdev while discarding the checkpoint.
	.
	.It Sy zfs_special_class_metadata_reserve_pct Ns = Ns Sy 25 Ns % Pq uint
	Only allow small data blocks to be allocated on the special and dedup vdev
	types when the available free space percentage on these vdevs exceeds this
	value.
	This ensures reserved space is available for pool metadata as the
	special vdevs approach capacity.
	.
	.It Sy zfs_sync_pass_dont_compress Ns = Ns Sy 8 Pq uint
	Starting in this sync pass, disable compression (including of metadata).
	With the default setting, in practice, we don't have this many sync passes,
	so this has no effect.
	.Pp
	The original intent was that disabling compression would help the sync passes
	to converge.
	However, in practice, disabling compression increases
	the average number of sync passes; because when we turn compression off,
	many blocks' size will change, and thus we have to re-allocate
	(not overwrite) them.
	It also increases the number of
	.Em 128 KiB
	allocations (e.g. for indirect blocks and spacemaps)
	because these will not be compressed.
	The
	.Em 128 KiB
	allocations are especially detrimental to performance
	on highly fragmented systems, which may have very few free segments of this
	size,
	and may need to load new metaslabs to satisfy these allocations.
	.
	.It Sy zfs_sync_pass_rewrite Ns = Ns Sy 2 Pq uint
	Rewrite new block pointers starting in this pass.
	.
	.It Sy zfs_sync_taskq_batch_pct Ns = Ns Sy 75 Ns % Pq int
	This controls the number of threads used by
	.Sy dp_sync_taskq .
	The default value of
	.Sy 75%
	will create a maximum of one thread per CPU.
	.
	.It Sy zfs_trim_extent_bytes_max Ns = Ns Sy 134217728 Ns B Po 128 MiB Pc Pq uint
	Maximum size of TRIM command.
	Larger ranges will be split into chunks no larger than this value before
	issuing.
	.
	.It Sy zfs_trim_extent_bytes_min Ns = Ns Sy 32768 Ns B Po 32 KiB Pc Pq uint
	Minimum size of TRIM commands.
	TRIM ranges smaller than this will be skipped,
	unless they're part of a larger range which was chunked.
	This is done because it's common for these small TRIMs
	to negatively impact overall performance.
	.
	.It Sy zfs_trim_metaslab_skip Ns = Ns Sy 0 Ns \| Ns 1 Pq uint
	Skip uninitialized metaslabs during the TRIM process.
	This option is useful for pools constructed from large thinly-provisioned
	devices
	where TRIM operations are slow.
	As a pool ages, an increasing fraction of the pool's metaslabs
	will be initialized, progressively degrading the usefulness of this option.
	This setting is stored when starting a manual TRIM and will
	persist for the duration of the requested TRIM.
	.
	.It Sy zfs_trim_queue_limit Ns = Ns Sy 10 Pq uint
	Maximum number of queued TRIMs outstanding per leaf vdev.
	The number of concurrent TRIM commands issued to the device is controlled by
	.Sy zfs_vdev_trim_min_active No and Sy zfs_vdev_trim_max_active .
	.
	.It Sy zfs_trim_txg_batch Ns = Ns Sy 32 Pq uint
	The number of transaction groups' worth of frees which should be aggregated
	before TRIM operations are issued to the device.
	This setting represents a trade-off between issuing larger,
	more efficient TRIM operations and the delay
	before the recently trimmed space is available for use by the device.
	.Pp
	Increasing this value will allow frees to be aggregated for a longer time.
	This will result is larger TRIM operations and potentially increased memory
	usage.
	Decreasing this value will have the opposite effect.
	The default of
	.Sy 32
	was determined to be a reasonable compromise.
	.
	.It Sy zfs_txg_history Ns = Ns Sy 0 Pq uint
	Historical statistics for this many latest TXGs will be available in
	.Pa /proc/spl/kstat/zfs/ Ns Ao Ar pool Ac Ns Pa /TXGs .
	.
	.It Sy zfs_txg_timeout Ns = Ns Sy 5 Ns s Pq uint
	Flush dirty data to disk at least every this many seconds (maximum TXG
	duration).
	.
	.It Sy zfs_vdev_aggregation_limit Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq uint
	Max vdev I/O aggregation size.
	.
	.It Sy zfs_vdev_aggregation_limit_non_rotating Ns = Ns Sy 131072 Ns B Po 128 KiB Pc Pq uint
	Max vdev I/O aggregation size for non-rotating media.
	.
	.It Sy zfs_vdev_mirror_rotating_inc Ns = Ns Sy 0 Pq int
	A number by which the balancing algorithm increments the load calculation for
	the purpose of selecting the least busy mirror member when an I/O operation
	immediately follows its predecessor on rotational vdevs
	for the purpose of making decisions based on load.
	.
	.It Sy zfs_vdev_mirror_rotating_seek_inc Ns = Ns Sy 5 Pq int
	A number by which the balancing algorithm increments the load calculation for
	the purpose of selecting the least busy mirror member when an I/O operation
	lacks locality as defined by
	.Sy zfs_vdev_mirror_rotating_seek_offset .
	Operations within this that are not immediately following the previous operation
	are incremented by half.
	.
	.It Sy zfs_vdev_mirror_rotating_seek_offset Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq int
	The maximum distance for the last queued I/O operation in which
	the balancing algorithm considers an operation to have locality.
	.No See Sx ZFS I/O SCHEDULER .
	.
	.It Sy zfs_vdev_mirror_non_rotating_inc Ns = Ns Sy 0 Pq int
	A number by which the balancing algorithm increments the load calculation for
	the purpose of selecting the least busy mirror member on non-rotational vdevs
	when I/O operations do not immediately follow one another.
	.
	.It Sy zfs_vdev_mirror_non_rotating_seek_inc Ns = Ns Sy 1 Pq int
	A number by which the balancing algorithm increments the load calculation for
	the purpose of selecting the least busy mirror member when an I/O operation
	lacks
	locality as defined by the
	.Sy zfs_vdev_mirror_rotating_seek_offset .
	Operations within this that are not immediately following the previous operation
	are incremented by half.
	.
	.It Sy zfs_vdev_read_gap_limit Ns = Ns Sy 32768 Ns B Po 32 KiB Pc Pq uint
	Aggregate read I/O operations if the on-disk gap between them is within this
	threshold.
	.
	.It Sy zfs_vdev_write_gap_limit Ns = Ns Sy 4096 Ns B Po 4 KiB Pc Pq uint
	Aggregate write I/O operations if the on-disk gap between them is within this
	threshold.
	.
	.It Sy zfs_vdev_raidz_impl Ns = Ns Sy fastest Pq string
	Select the raidz parity implementation to use.
	.Pp
	Variants that don't depend on CPU-specific features
	may be selected on module load, as they are supported on all systems.
	The remaining options may only be set after the module is loaded,
	as they are available only if the implementations are compiled in
	and supported on the running system.
	.Pp
	Once the module is loaded,
	.Pa /sys/module/zfs/parameters/zfs_vdev_raidz_impl
	will show the available options,
	with the currently selected one enclosed in square brackets.
	.Pp
	.TS
	lb l l .
	fastest selected by built-in benchmark
	original original implementation
	scalar scalar implementation
	sse2 SSE2 instruction set 64-bit x86
	ssse3 SSSE3 instruction set 64-bit x86
	avx2 AVX2 instruction set 64-bit x86
	avx512f AVX512F instruction set 64-bit x86
	avx512bw AVX512F & AVX512BW instruction sets 64-bit x86
	aarch64_neon NEON Aarch64/64-bit ARMv8
	aarch64_neonx2 NEON with more unrolling Aarch64/64-bit ARMv8
	powerpc_altivec Altivec PowerPC
	.TE
	.
	.It Sy zfs_vdev_scheduler Pq charp
	.Sy DEPRECATED .
	Prints warning to kernel log for compatibility.
	.
	.It Sy zfs_zevent_len_max Ns = Ns Sy 512 Pq uint
	Max event queue length.
	Events in the queue can be viewed with
	.Xr zpool-events 8 .
	.
	.It Sy zfs_zevent_retain_max Ns = Ns Sy 2000 Pq int
	Maximum recent zevent records to retain for duplicate checking.
	Setting this to
	.Sy 0
	disables duplicate detection.
	.
	.It Sy zfs_zevent_retain_expire_secs Ns = Ns Sy 900 Ns s Po 15 min Pc Pq int
	Lifespan for a recent ereport that was retained for duplicate checking.
	.
	.It Sy zfs_zil_clean_taskq_maxalloc Ns = Ns Sy 1048576 Pq int
	The maximum number of taskq entries that are allowed to be cached.
	When this limit is exceeded transaction records (itxs)
	will be cleaned synchronously.
	.
	.It Sy zfs_zil_clean_taskq_minalloc Ns = Ns Sy 1024 Pq int
	The number of taskq entries that are pre-populated when the taskq is first
	created and are immediately available for use.
	.
	.It Sy zfs_zil_clean_taskq_nthr_pct Ns = Ns Sy 100 Ns % Pq int
	This controls the number of threads used by
	.Sy dp_zil_clean_taskq .
	The default value of
	.Sy 100%
	will create a maximum of one thread per cpu.
	.
	.It Sy zil_maxblocksize Ns = Ns Sy 131072 Ns B Po 128 KiB Pc Pq uint
	This sets the maximum block size used by the ZIL.
	On very fragmented pools, lowering this
	.Pq typically to Sy 36 KiB
	can improve performance.
	.
	.It Sy zil_maxcopied Ns = Ns Sy 7680 Ns B Po 7.5 KiB Pc Pq uint
	This sets the maximum number of write bytes logged via WR_COPIED.
	It tunes a tradeoff between additional memory copy and possibly worse log
	space efficiency vs additional range lock/unlock.
	.
	-.It Sy zil_min_commit_timeout Ns = Ns Sy 5000 Pq u64
	-This sets the minimum delay in nanoseconds ZIL care to delay block commit,
	-waiting for more records.
	-If ZIL writes are too fast, kernel may not be able sleep for so short interval,
	-increasing log latency above allowed by
	-.Sy zfs_commit_timeout_pct .
	-.
	.It Sy zil_nocacheflush Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Disable the cache flush commands that are normally sent to disk by
	the ZIL after an LWB write has completed.
	Setting this will cause ZIL corruption on power loss
	if a volatile out-of-order write cache is enabled.
	.
	.It Sy zil_replay_disable Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Disable intent logging replay.
	Can be disabled for recovery from corrupted ZIL.
	.
	.It Sy zil_slog_bulk Ns = Ns Sy 67108864 Ns B Po 64 MiB Pc Pq u64
	Limit SLOG write size per commit executed with synchronous priority.
	Any writes above that will be executed with lower (asynchronous) priority
	to limit potential SLOG device abuse by single active ZIL writer.
	.
	.It Sy zfs_zil_saxattr Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Setting this tunable to zero disables ZIL logging of new
	.Sy xattr Ns = Ns Sy sa
	records if the
	.Sy org.openzfs:zilsaxattr
	feature is enabled on the pool.
	This would only be necessary to work around bugs in the ZIL logging or replay
	code for this record type.
	The tunable has no effect if the feature is disabled.
	.
	.It Sy zfs_embedded_slog_min_ms Ns = Ns Sy 64 Pq uint
	Usually, one metaslab from each normal-class vdev is dedicated for use by
	the ZIL to log synchronous writes.
	However, if there are fewer than
	.Sy zfs_embedded_slog_min_ms
	metaslabs in the vdev, this functionality is disabled.
	This ensures that we don't set aside an unreasonable amount of space for the
	ZIL.
	.
	.It Sy zstd_earlyabort_pass Ns = Ns Sy 1 Pq uint
	Whether heuristic for detection of incompressible data with zstd levels >= 3
	using LZ4 and zstd-1 passes is enabled.
	.
	.It Sy zstd_abort_size Ns = Ns Sy 131072 Pq uint
	Minimal uncompressed size (inclusive) of a record before the early abort
	heuristic will be attempted.
	.
	.It Sy zio_deadman_log_all Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	If non-zero, the zio deadman will produce debugging messages
	.Pq see Sy zfs_dbgmsg_enable
	for all zios, rather than only for leaf zios possessing a vdev.
	This is meant to be used by developers to gain
	diagnostic information for hang conditions which don't involve a mutex
	or other locking primitive: typically conditions in which a thread in
	the zio pipeline is looping indefinitely.
	.
	.It Sy zio_slow_io_ms Ns = Ns Sy 30000 Ns ms Po 30 s Pc Pq int
	When an I/O operation takes more than this much time to complete,
	it's marked as slow.
	Each slow operation causes a delay zevent.
	Slow I/O counters can be seen with
	.Nm zpool Cm status Fl s .
	.
	.It Sy zio_dva_throttle_enabled Ns = Ns Sy 1 Ns \| Ns 0 Pq int
	Throttle block allocations in the I/O pipeline.
	This allows for dynamic allocation distribution when devices are imbalanced.
	When enabled, the maximum number of pending allocations per top-level vdev
	is limited by
	.Sy zfs_vdev_queue_depth_pct .
	.
	.It Sy zfs_xattr_compat Ns = Ns 0 Ns \| Ns 1 Pq int
	Control the naming scheme used when setting new xattrs in the user namespace.
	If
	.Sy 0
	.Pq the default on Linux ,
	user namespace xattr names are prefixed with the namespace, to be backwards
	compatible with previous versions of ZFS on Linux.
	If
	.Sy 1
	.Pq the default on Fx ,
	user namespace xattr names are not prefixed, to be backwards compatible with
	previous versions of ZFS on illumos and
	.Fx .
	.Pp
	Either naming scheme can be read on this and future versions of ZFS, regardless
	of this tunable, but legacy ZFS on illumos or
	.Fx
	are unable to read user namespace xattrs written in the Linux format, and
	legacy versions of ZFS on Linux are unable to read user namespace xattrs written
	in the legacy ZFS format.
	.Pp
	An existing xattr with the alternate naming scheme is removed when overwriting
	the xattr so as to not accumulate duplicates.
	.
	.It Sy zio_requeue_io_start_cut_in_line Ns = Ns Sy 0 Ns \| Ns 1 Pq int
	Prioritize requeued I/O.
	.
	.It Sy zio_taskq_batch_pct Ns = Ns Sy 80 Ns % Pq uint
	Percentage of online CPUs which will run a worker thread for I/O.
	These workers are responsible for I/O work such as compression and
	checksum calculations.
	Fractional number of CPUs will be rounded down.
	.Pp
	The default value of
	.Sy 80%
	was chosen to avoid using all CPUs which can result in
	latency issues and inconsistent application performance,
	especially when slower compression and/or checksumming is enabled.
	.
	.It Sy zio_taskq_batch_tpq Ns = Ns Sy 0 Pq uint
	Number of worker threads per taskq.
	Lower values improve I/O ordering and CPU utilization,
	while higher reduces lock contention.
	.Pp
	If
	.Sy 0 ,
	generate a system-dependent value close to 6 threads per taskq.
	.
	.It Sy zio_taskq_read Ns = Ns Sy fixed,1,8 null scale null Pq charp
	Set the queue and thread configuration for the IO read queues.
	This is an advanced debugging parameter.
	Don't change this unless you understand what it does.
	.
	.It Sy zio_taskq_write Ns = Ns Sy batch fixed,1,5 scale fixed,1,5 Pq charp
	Set the queue and thread configuration for the IO write queues.
	This is an advanced debugging parameter.
	Don't change this unless you understand what it does.
	.
	.It Sy zvol_inhibit_dev Ns = Ns Sy 0 Ns \| Ns 1 Pq uint
	Do not create zvol device nodes.
	This may slightly improve startup time on
	systems with a very large number of zvols.
	.
	.It Sy zvol_major Ns = Ns Sy 230 Pq uint
	Major number for zvol block devices.
	.
	.It Sy zvol_max_discard_blocks Ns = Ns Sy 16384 Pq long
	Discard (TRIM) operations done on zvols will be done in batches of this
	many blocks, where block size is determined by the
	.Sy volblocksize
	property of a zvol.
	.
	.It Sy zvol_prefetch_bytes Ns = Ns Sy 131072 Ns B Po 128 KiB Pc Pq uint
	When adding a zvol to the system, prefetch this many bytes
	from the start and end of the volume.
	Prefetching these regions of the volume is desirable,
	because they are likely to be accessed immediately by
	.Xr blkid 8
	or the kernel partitioner.
	.
	.It Sy zvol_request_sync Ns = Ns Sy 0 Ns \| Ns 1 Pq uint
	When processing I/O requests for a zvol, submit them synchronously.
	This effectively limits the queue depth to
	.Em 1
	for each I/O submitter.
	When unset, requests are handled asynchronously by a thread pool.
	The number of requests which can be handled concurrently is controlled by
	.Sy zvol_threads .
	.Sy zvol_request_sync
	is ignored when running on a kernel that supports block multiqueue
	.Pq Li blk-mq .
	.
	+.It Sy zvol_num_taskqs Ns = Ns Sy 0 Pq uint
	+Number of zvol taskqs.
	+If
	+.Sy 0
	+(the default) then scaling is done internally to prefer 6 threads per taskq.
	+This only applies on Linux.
	+.
	.It Sy zvol_threads Ns = Ns Sy 0 Pq uint
	The number of system wide threads to use for processing zvol block IOs.
	If
	.Sy 0
	(the default) then internally set
	.Sy zvol_threads
	to the number of CPUs present or 32 (whichever is greater).
	.
	.It Sy zvol_blk_mq_threads Ns = Ns Sy 0 Pq uint
	The number of threads per zvol to use for queuing IO requests.
	This parameter will only appear if your kernel supports
	.Li blk-mq
	and is only read and assigned to a zvol at zvol load time.
	If
	.Sy 0
	(the default) then internally set
	.Sy zvol_blk_mq_threads
	to the number of CPUs present.
	.
	.It Sy zvol_use_blk_mq Ns = Ns Sy 0 Ns \| Ns 1 Pq uint
	Set to
	.Sy 1
	to use the
	.Li blk-mq
	API for zvols.
	Set to
	.Sy 0
	(the default) to use the legacy zvol APIs.
	This setting can give better or worse zvol performance depending on
	the workload.
	This parameter will only appear if your kernel supports
	.Li blk-mq
	and is only read and assigned to a zvol at zvol load time.
	.
	.It Sy zvol_blk_mq_blocks_per_thread Ns = Ns Sy 8 Pq uint
	If
	.Sy zvol_use_blk_mq
	is enabled, then process this number of
	.Sy volblocksize Ns -sized blocks per zvol thread.
	This tunable can be use to favor better performance for zvol reads (lower
	values) or writes (higher values).
	If set to
	.Sy 0 ,
	then the zvol layer will process the maximum number of blocks
	per thread that it can.
	This parameter will only appear if your kernel supports
	.Li blk-mq
	and is only applied at each zvol's load time.
	.
	.It Sy zvol_blk_mq_queue_depth Ns = Ns Sy 0 Pq uint
	The queue_depth value for the zvol
	.Li blk-mq
	interface.
	This parameter will only appear if your kernel supports
	.Li blk-mq
	and is only applied at each zvol's load time.
	If
	.Sy 0
	(the default) then use the kernel's default queue depth.
	Values are clamped to the kernel's
	.Dv BLKDEV_MIN_RQ
	and
	.Dv BLKDEV_MAX_RQ Ns / Ns Dv BLKDEV_DEFAULT_RQ
	limits.
	.
	.It Sy zvol_volmode Ns = Ns Sy 1 Pq uint
	Defines zvol block devices behaviour when
	.Sy volmode Ns = Ns Sy default :
	.Bl -tag -compact -offset 4n -width "a"
	.It Sy 1
	.No equivalent to Sy full
	.It Sy 2
	.No equivalent to Sy dev
	.It Sy 3
	.No equivalent to Sy none
	.El
	.
	.It Sy zvol_enforce_quotas Ns = Ns Sy 0 Ns \| Ns 1 Pq uint
	Enable strict ZVOL quota enforcement.
	The strict quota enforcement may have a performance impact.
	.El
	.
	.Sh ZFS I/O SCHEDULER
	ZFS issues I/O operations to leaf vdevs to satisfy and complete I/O operations.
	The scheduler determines when and in what order those operations are issued.
	The scheduler divides operations into five I/O classes,
	prioritized in the following order: sync read, sync write, async read,
	async write, and scrub/resilver.
	Each queue defines the minimum and maximum number of concurrent operations
	that may be issued to the device.
	In addition, the device has an aggregate maximum,
	.Sy zfs_vdev_max_active .
	Note that the sum of the per-queue minima must not exceed the aggregate maximum.
	If the sum of the per-queue maxima exceeds the aggregate maximum,
	then the number of active operations may reach
	.Sy zfs_vdev_max_active ,
	in which case no further operations will be issued,
	regardless of whether all per-queue minima have been met.
	.Pp
	For many physical devices, throughput increases with the number of
	concurrent operations, but latency typically suffers.
	Furthermore, physical devices typically have a limit
	at which more concurrent operations have no
	effect on throughput or can actually cause it to decrease.
	.Pp
	The scheduler selects the next operation to issue by first looking for an
	I/O class whose minimum has not been satisfied.
	Once all are satisfied and the aggregate maximum has not been hit,
	the scheduler looks for classes whose maximum has not been satisfied.
	Iteration through the I/O classes is done in the order specified above.
	No further operations are issued
	if the aggregate maximum number of concurrent operations has been hit,
	or if there are no operations queued for an I/O class that has not hit its
	maximum.
	Every time an I/O operation is queued or an operation completes,
	the scheduler looks for new operations to issue.
	.Pp
	In general, smaller
	.Sy max_active Ns s
	will lead to lower latency of synchronous operations.
	Larger
	.Sy max_active Ns s
	may lead to higher overall throughput, depending on underlying storage.
	.Pp
	The ratio of the queues'
	.Sy max_active Ns s
	determines the balance of performance between reads, writes, and scrubs.
	For example, increasing
	.Sy zfs_vdev_scrub_max_active
	will cause the scrub or resilver to complete more quickly,
	but reads and writes to have higher latency and lower throughput.
	.Pp
	All I/O classes have a fixed maximum number of outstanding operations,
	except for the async write class.
	Asynchronous writes represent the data that is committed to stable storage
	during the syncing stage for transaction groups.
	Transaction groups enter the syncing state periodically,
	so the number of queued async writes will quickly burst up
	and then bleed down to zero.
	Rather than servicing them as quickly as possible,
	the I/O scheduler changes the maximum number of active async write operations
	according to the amount of dirty data in the pool.
	Since both throughput and latency typically increase with the number of
	concurrent operations issued to physical devices, reducing the
	burstiness in the number of simultaneous operations also stabilizes the
	response time of operations from other queues, in particular synchronous ones.
	In broad strokes, the I/O scheduler will issue more concurrent operations
	from the async write queue as there is more dirty data in the pool.
	.
	.Ss Async Writes
	The number of concurrent operations issued for the async write I/O class
	follows a piece-wise linear function defined by a few adjustable points:
	.Bd -literal
	\| o---------\| <-- \fBzfs_vdev_async_write_max_active\fP
	^ \| /^ \|
	\| \| / \| \|
	active \| / \| \|
	I/O \| / \| \|
	count \| / \| \|
	\| / \| \|
	\|-------o \| \| <-- \fBzfs_vdev_async_write_min_active\fP
	0\|_______^______\|_________\|
	0% \| \| 100% of \fBzfs_dirty_data_max\fP
	\| \|
	\| `-- \fBzfs_vdev_async_write_active_max_dirty_percent\fP
	`--------- \fBzfs_vdev_async_write_active_min_dirty_percent\fP
	.Ed
	.Pp
	Until the amount of dirty data exceeds a minimum percentage of the dirty
	data allowed in the pool, the I/O scheduler will limit the number of
	concurrent operations to the minimum.
	As that threshold is crossed, the number of concurrent operations issued
	increases linearly to the maximum at the specified maximum percentage
	of the dirty data allowed in the pool.
	.Pp
	Ideally, the amount of dirty data on a busy pool will stay in the sloped
	part of the function between
	.Sy zfs_vdev_async_write_active_min_dirty_percent
	and
	.Sy zfs_vdev_async_write_active_max_dirty_percent .
	If it exceeds the maximum percentage,
	this indicates that the rate of incoming data is
	greater than the rate that the backend storage can handle.
	In this case, we must further throttle incoming writes,
	as described in the next section.
	.
	.Sh ZFS TRANSACTION DELAY
	We delay transactions when we've determined that the backend storage
	isn't able to accommodate the rate of incoming writes.
	.Pp
	If there is already a transaction waiting, we delay relative to when
	that transaction will finish waiting.
	This way the calculated delay time
	is independent of the number of threads concurrently executing transactions.
	.Pp
	If we are the only waiter, wait relative to when the transaction started,
	rather than the current time.
	This credits the transaction for "time already served",
	e.g. reading indirect blocks.
	.Pp
	The minimum time for a transaction to take is calculated as
	.D1 min_time = min( Ns Sy zfs_delay_scale No \(mu Po Sy dirty No \- Sy min Pc / Po Sy max No \- Sy dirty Pc , 100ms)
	.Pp
	The delay has two degrees of freedom that can be adjusted via tunables.
	The percentage of dirty data at which we start to delay is defined by
	.Sy zfs_delay_min_dirty_percent .
	This should typically be at or above
	.Sy zfs_vdev_async_write_active_max_dirty_percent ,
	so that we only start to delay after writing at full speed
	has failed to keep up with the incoming write rate.
	The scale of the curve is defined by
	.Sy zfs_delay_scale .
	Roughly speaking, this variable determines the amount of delay at the midpoint
	of the curve.
	.Bd -literal
	delay
	10ms +-------------------------------------------------------------*+
	\| *\|
	9ms + *+
	\| *\|
	8ms + *+
	\| * \|
	7ms + * +
	\| * \|
	6ms + * +
	\| * \|
	5ms + * +
	\| * \|
	4ms + * +
	\| * \|
	3ms + * +
	\| * \|
	2ms + (midpoint) * +
	\| \| ** \|
	1ms + v *** +
	\| \fBzfs_delay_scale\fP ----------> ******** \|
	0 +-------------------------------------*********----------------+
	0% <- \fBzfs_dirty_data_max\fP -> 100%
	.Ed
	.Pp
	Note, that since the delay is added to the outstanding time remaining on the
	most recent transaction it's effectively the inverse of IOPS.
	Here, the midpoint of
	.Em 500 us
	translates to
	.Em 2000 IOPS .
	The shape of the curve
	was chosen such that small changes in the amount of accumulated dirty data
	in the first three quarters of the curve yield relatively small differences
	in the amount of delay.
	.Pp
	The effects can be easier to understand when the amount of delay is
	represented on a logarithmic scale:
	.Bd -literal
	delay
	100ms +-------------------------------------------------------------++
	+ +
	\| \|
	+ *+
	10ms + *+
	+ ** +
	\| (midpoint) ** \|
	+ \| ** +
	1ms + v **** +
	+ \fBzfs_delay_scale\fP ----------> ***** +
	\| **** \|
	+ **** +
	100us + ** +
	+ * +
	\| * \|
	+ * +
	10us + * +
	+ +
	\| \|
	+ +
	+--------------------------------------------------------------+
	0% <- \fBzfs_dirty_data_max\fP -> 100%
	.Ed
	.Pp
	Note here that only as the amount of dirty data approaches its limit does
	the delay start to increase rapidly.
	The goal of a properly tuned system should be to keep the amount of dirty data
	out of that range by first ensuring that the appropriate limits are set
	for the I/O scheduler to reach optimal throughput on the back-end storage,
	and then by changing the value of
	.Sy zfs_delay_scale
	to increase the steepness of the curve.
	diff --git a/sys/contrib/openzfs/man/man7/vdevprops.7 b/sys/contrib/openzfs/man/man7/vdevprops.7
	index 6eebfa0060de..5ec37df179de 100644
	--- a/sys/contrib/openzfs/man/man7/vdevprops.7
	+++ b/sys/contrib/openzfs/man/man7/vdevprops.7
	@@ -1,186 +1,192 @@
	.\"
	.\" CDDL HEADER START
	.\"
	.\" The contents of this file are subject to the terms of the
	.\" Common Development and Distribution License (the "License").
	.\" You may not use this file except in compliance with the License.
	.\"
	.\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	.\" or https://opensource.org/licenses/CDDL-1.0.
	.\" See the License for the specific language governing permissions
	.\" and limitations under the License.
	.\"
	.\" When distributing Covered Code, include this CDDL HEADER in each
	.\" file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	.\" If applicable, add the following below this CDDL HEADER, with the
	.\" fields enclosed by brackets "[]" replaced with your own identifying
	.\" information: Portions Copyright [yyyy] [name of copyright owner]
	.\"
	.\" CDDL HEADER END
	.\"
	.\" Copyright (c) 2021 Klara, Inc.
	.\"
	.Dd October 30, 2022
	.Dt VDEVPROPS 7
	.Os
	.
	.Sh NAME
	.Nm vdevprops
	.Nd native and user-defined properties of ZFS vdevs
	.
	.Sh DESCRIPTION
	Properties are divided into two types, native properties and user-defined
	.Pq or Qq user
	properties.
	Native properties either export internal statistics or control ZFS behavior.
	In addition, native properties are either editable or read-only.
	User properties have no effect on ZFS behavior, but you can use them to annotate
	vdevs in a way that is meaningful in your environment.
	For more information about user properties, see the
	.Sx User Properties
	section, below.
	.
	.Ss Native Properties
	Every vdev has a set of properties that export statistics about the vdev
	as well as control various behaviors.
	Properties are not inherited from top-level vdevs, with the exception of
	-checksum_n, checksum_t, io_n, and io_t.
	+checksum_n, checksum_t, io_n, io_t, slow_io_n, and slow_io_t.
	.Pp
	The values of numeric properties can be specified using human-readable suffixes
	.Po for example,
	.Sy k , KB , M , Gb ,
	and so forth, up to
	.Sy Z
	for zettabyte
	.Pc .
	The following are all valid
	.Pq and equal
	specifications:
	.Li 1536M , 1.5g , 1.50GB .
	.Pp
	The values of non-numeric properties are case sensitive and must be lowercase.
	.Pp
	The following native properties consist of read-only statistics about the
	vdev.
	These properties can not be changed.
	.Bl -tag -width "fragmentation"
	.It Sy capacity
	Percentage of vdev space used
	.It Sy state
	state of this vdev such as online, faulted, or offline
	.It Sy guid
	globally unique id of this vdev
	.It Sy asize
	The allocable size of this vdev
	.It Sy psize
	The physical size of this vdev
	.It Sy ashift
	The physical sector size of this vdev expressed as the power of two
	.It Sy size
	The total size of this vdev
	.It Sy free
	The amount of remaining free space on this vdev
	.It Sy allocated
	The amount of allocated space on this vdev
	.It Sy expandsize
	How much this vdev can expand by
	.It Sy fragmentation
	Percent of fragmentation in this vdev
	.It Sy parity
	The level of parity for this vdev
	.It Sy devid
	The device id for this vdev
	.It Sy physpath
	The physical path to the device
	.It Sy encpath
	The enclosure path to the device
	.It Sy fru
	Field Replacable Unit, usually a model number
	.It Sy parent
	Parent of this vdev
	.It Sy children
	Comma separated list of children of this vdev
	.It Sy numchildren
	The number of children belonging to this vdev
	.It Sy read_errors , write_errors , checksum_errors , initialize_errors
	The number of errors of each type encountered by this vdev
	.It Sy null_ops , read_ops , write_ops , free_ops , claim_ops , trim_ops
	The number of I/O operations of each type performed by this vdev
	.It Xo
	.Sy null_bytes , read_bytes , write_bytes , free_bytes , claim_bytes ,
	.Sy trim_bytes
	.Xc
	The cumulative size of all operations of each type performed by this vdev
	.It Sy removing
	If this device is currently being removed from the pool
	.El
	.Pp
	The following native properties can be used to change the behavior of a vdev.
	.Bl -tag -width "allocating"
	-.It Sy checksum_n , checksum_t , io_n , io_t
	+.It Sy checksum_n , checksum_t , io_n , io_t , slow_io_n , slow_io_t
	Tune the fault management daemon by specifying checksum/io thresholds of <N>
	errors in <T> seconds, respectively.
	These properties can be set on leaf and top-level vdevs.
	When the property is set on the leaf and top-level vdev, the value of the leaf
	vdev will be used.
	If the property is only set on the top-level vdev, this value will be used.
	The value of these properties do not persist across vdev replacement.
	For this reason, it is advisable to set the property on the top-level vdev -
	not on the leaf vdev itself.
	-The default values are 10 errors in 600 seconds.
	+The default values for
	+.Sy OpenZFS on Linux
	+are 10 errors in 600 seconds.
	+For
	+.Sy OpenZFS on FreeBSD
	+defaults see
	+.Xr zfsd 8 .
	.It Sy comment
	A text comment up to 8192 characters long
	.It Sy bootsize
	The amount of space to reserve for the EFI system partition
	.It Sy failfast
	If this device should propage BIO errors back to ZFS, used to disable
	failfast.
	.It Sy path
	The path to the device for this vdev
	.It Sy allocating
	If this device should perform new allocations, used to disable a device
	when it is scheduled for later removal.
	See
	.Xr zpool-remove 8 .
	.El
	.Ss User Properties
	In addition to the standard native properties, ZFS supports arbitrary user
	properties.
	User properties have no effect on ZFS behavior, but applications or
	administrators can use them to annotate vdevs.
	.Pp
	User property names must contain a colon
	.Pq Qq Sy \&:
	character to distinguish them from native properties.
	They may contain lowercase letters, numbers, and the following punctuation
	characters: colon
	.Pq Qq Sy \&: ,
	dash
	.Pq Qq Sy - ,
	period
	.Pq Qq Sy \&. ,
	and underscore
	.Pq Qq Sy _ .
	The expected convention is that the property name is divided into two portions
	such as
	.Ar module : Ns Ar property ,
	but this namespace is not enforced by ZFS.
	User property names can be at most 256 characters, and cannot begin with a dash
	.Pq Qq Sy - .
	.Pp
	When making programmatic use of user properties, it is strongly suggested to use
	a reversed DNS domain name for the
	.Ar module
	component of property names to reduce the chance that two
	independently-developed packages use the same property name for different
	purposes.
	.Pp
	The values of user properties are arbitrary strings and
	are never validated.
	Use the
	.Nm zpool Cm set
	command with a blank value to clear a user property.
	Property values are limited to 8192 bytes.
	.Sh SEE ALSO
	.Xr zpoolprops 7 ,
	.Xr zpool-set 8
	diff --git a/sys/contrib/openzfs/man/man7/zfsprops.7 b/sys/contrib/openzfs/man/man7/zfsprops.7
	index e3674b1f8a8d..59f6404379af 100644
	--- a/sys/contrib/openzfs/man/man7/zfsprops.7
	+++ b/sys/contrib/openzfs/man/man7/zfsprops.7
	@@ -1,2136 +1,2153 @@
	.\"
	.\" CDDL HEADER START
	.\"
	.\" The contents of this file are subject to the terms of the
	.\" Common Development and Distribution License (the "License").
	.\" You may not use this file except in compliance with the License.
	.\"
	.\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	.\" or https://opensource.org/licenses/CDDL-1.0.
	.\" See the License for the specific language governing permissions
	.\" and limitations under the License.
	.\"
	.\" When distributing Covered Code, include this CDDL HEADER in each
	.\" file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	.\" If applicable, add the following below this CDDL HEADER, with the
	.\" fields enclosed by brackets "[]" replaced with your own identifying
	.\" information: Portions Copyright [yyyy] [name of copyright owner]
	.\"
	.\" CDDL HEADER END
	.\"
	.\" Copyright (c) 2009 Sun Microsystems, Inc. All Rights Reserved.
	.\" Copyright 2011 Joshua M. Clulow <josh@sysmgr.org>
	.\" Copyright (c) 2011, 2019 by Delphix. All rights reserved.
	.\" Copyright (c) 2011, Pawel Jakub Dawidek <pjd@FreeBSD.org>
	.\" Copyright (c) 2012, Glen Barber <gjb@FreeBSD.org>
	.\" Copyright (c) 2012, Bryan Drewery <bdrewery@FreeBSD.org>
	.\" Copyright (c) 2013, Steven Hartland <smh@FreeBSD.org>
	.\" Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
	.\" Copyright (c) 2014, Joyent, Inc. All rights reserved.
	.\" Copyright (c) 2014 by Adam Stevko. All rights reserved.
	.\" Copyright (c) 2014 Integros [integros.com]
	.\" Copyright (c) 2016 Nexenta Systems, Inc. All Rights Reserved.
	.\" Copyright (c) 2014, Xin LI <delphij@FreeBSD.org>
	.\" Copyright (c) 2014-2015, The FreeBSD Foundation, All Rights Reserved.
	.\" Copyright 2019 Richard Laager. All rights reserved.
	.\" Copyright 2018 Nexenta Systems, Inc.
	.\" Copyright 2019 Joyent, Inc.
	.\" Copyright (c) 2019, Kjeld Schouten-Lebbing
	.\" Copyright (c) 2022 Hewlett Packard Enterprise Development LP.
	.\"
	.Dd August 8, 2023
	.Dt ZFSPROPS 7
	.Os
	.
	.Sh NAME
	.Nm zfsprops
	.Nd native and user-defined properties of ZFS datasets
	.
	.Sh DESCRIPTION
	Properties are divided into two types, native properties and user-defined
	.Po or
	.Qq user
	.Pc
	properties.
	Native properties either export internal statistics or control ZFS behavior.
	In addition, native properties are either editable or read-only.
	User properties have no effect on ZFS behavior, but you can use them to annotate
	datasets in a way that is meaningful in your environment.
	For more information about user properties, see the
	.Sx User Properties
	section, below.
	.
	.Ss Native Properties
	Every dataset has a set of properties that export statistics about the dataset
	as well as control various behaviors.
	Properties are inherited from the parent unless overridden by the child.
	Some properties apply only to certain types of datasets
	.Pq file systems, volumes, or snapshots .
	.Pp
	The values of numeric properties can be specified using human-readable suffixes
	.Po for example,
	.Sy k ,
	.Sy KB ,
	.Sy M ,
	.Sy Gb ,
	and so forth, up to
	.Sy Z
	for zettabyte
	.Pc .
	The following are all valid
	.Pq and equal
	specifications:
	.Li 1536M ,
	.Li 1.5g ,
	.Li 1.50GB .
	.Pp
	The values of non-numeric properties are case sensitive and must be lowercase,
	except for
	.Sy mountpoint ,
	.Sy sharenfs ,
	and
	.Sy sharesmb .
	.Pp
	The following native properties consist of read-only statistics about the
	dataset.
	These properties can be neither set, nor inherited.
	Native properties apply to all dataset types unless otherwise noted.
	.Bl -tag -width "usedbyrefreservation"
	.It Sy available
	The amount of space available to the dataset and all its children, assuming that
	there is no other activity in the pool.
	Because space is shared within a pool, availability can be limited by any number
	of factors, including physical pool size, quotas, reservations, or other
	datasets within the pool.
	.Pp
	This property can also be referred to by its shortened column name,
	.Sy avail .
	.It Sy compressratio
	For non-snapshots, the compression ratio achieved for the
	.Sy used
	space of this dataset, expressed as a multiplier.
	The
	.Sy used
	property includes descendant datasets, and, for clones, does not include the
	space shared with the origin snapshot.
	For snapshots, the
	.Sy compressratio
	is the same as the
	.Sy refcompressratio
	property.
	Compression can be turned on by running:
	.Nm zfs Cm set Sy compression Ns = Ns Sy on Ar dataset .
	The default value is
	.Sy off .
	.It Sy createtxg
	The transaction group (txg) in which the dataset was created.
	Bookmarks have the same
	.Sy createtxg
	as the snapshot they are initially tied to.
	This property is suitable for ordering a list of snapshots,
	e.g. for incremental send and receive.
	.It Sy creation
	The time this dataset was created.
	.It Sy clones
	For snapshots, this property is a comma-separated list of filesystems or volumes
	which are clones of this snapshot.
	The clones'
	.Sy origin
	property is this snapshot.
	If the
	.Sy clones
	property is not empty, then this snapshot can not be destroyed
	.Po even with the
	.Fl r
	or
	.Fl f
	options
	.Pc .
	The roles of origin and clone can be swapped by promoting the clone with the
	.Nm zfs Cm promote
	command.
	.It Sy defer_destroy
	This property is
	.Sy on
	if the snapshot has been marked for deferred destroy by using the
	.Nm zfs Cm destroy Fl d
	command.
	Otherwise, the property is
	.Sy off .
	.It Sy encryptionroot
	For encrypted datasets, indicates where the dataset is currently inheriting its
	encryption key from.
	Loading or unloading a key for the
	.Sy encryptionroot
	will implicitly load / unload the key for any inheriting datasets (see
	.Nm zfs Cm load-key
	and
	.Nm zfs Cm unload-key
	for details).
	Clones will always share an
	encryption key with their origin.
	See the
	.Sx Encryption
	section of
	.Xr zfs-load-key 8
	for details.
	.It Sy filesystem_count
	The total number of filesystems and volumes that exist under this location in
	the dataset tree.
	This value is only available when a
	.Sy filesystem_limit
	has been set somewhere in the tree under which the dataset resides.
	.It Sy keystatus
	Indicates if an encryption key is currently loaded into ZFS.
	The possible values are
	.Sy none ,
	.Sy available ,
	and
	.Sy unavailable .
	See
	.Nm zfs Cm load-key
	and
	.Nm zfs Cm unload-key .
	.It Sy guid
	The 64 bit GUID of this dataset or bookmark which does not change over its
	entire lifetime.
	When a snapshot is sent to another pool, the received snapshot has the same
	GUID.
	Thus, the
	.Sy guid
	is suitable to identify a snapshot across pools.
	.It Sy logicalreferenced
	The amount of space that is
	.Qq logically
	accessible by this dataset.
	See the
	.Sy referenced
	property.
	The logical space ignores the effect of the
	.Sy compression
	and
	.Sy copies
	properties, giving a quantity closer to the amount of data that applications
	see.
	However, it does include space consumed by metadata.
	.Pp
	This property can also be referred to by its shortened column name,
	.Sy lrefer .
	.It Sy logicalused
	The amount of space that is
	.Qq logically
	consumed by this dataset and all its descendents.
	See the
	.Sy used
	property.
	The logical space ignores the effect of the
	.Sy compression
	and
	.Sy copies
	properties, giving a quantity closer to the amount of data that applications
	see.
	However, it does include space consumed by metadata.
	.Pp
	This property can also be referred to by its shortened column name,
	.Sy lused .
	.It Sy mounted
	For file systems, indicates whether the file system is currently mounted.
	This property can be either
	.Sy yes
	or
	.Sy no .
	.It Sy objsetid
	A unique identifier for this dataset within the pool.
	Unlike the dataset's
	.Sy guid , No the Sy objsetid
	of a dataset is not transferred to other pools when the snapshot is copied
	with a send/receive operation.
	The
	.Sy objsetid
	can be reused (for a new dataset) after the dataset is deleted.
	.It Sy origin
	For cloned file systems or volumes, the snapshot from which the clone was
	created.
	See also the
	.Sy clones
	property.
	.It Sy receive_resume_token
	For filesystems or volumes which have saved partially-completed state from
	.Nm zfs Cm receive Fl s ,
	this opaque token can be provided to
	.Nm zfs Cm send Fl t
	to resume and complete the
	.Nm zfs Cm receive .
	.It Sy redact_snaps
	For bookmarks, this is the list of snapshot guids the bookmark contains a
	redaction
	list for.
	For snapshots, this is the list of snapshot guids the snapshot is redacted with
	respect to.
	.It Sy referenced
	The amount of data that is accessible by this dataset, which may or may not be
	shared with other datasets in the pool.
	When a snapshot or clone is created, it initially references the same amount of
	space as the file system or snapshot it was created from, since its contents are
	identical.
	.Pp
	This property can also be referred to by its shortened column name,
	.Sy refer .
	.It Sy refcompressratio
	The compression ratio achieved for the
	.Sy referenced
	space of this dataset, expressed as a multiplier.
	See also the
	.Sy compressratio
	property.
	.It Sy snapshot_count
	The total number of snapshots that exist under this location in the dataset
	tree.
	This value is only available when a
	.Sy snapshot_limit
	has been set somewhere in the tree under which the dataset resides.
	.It Sy type
	The type of dataset:
	.Sy filesystem ,
	.Sy volume ,
	.Sy snapshot ,
	or
	.Sy bookmark .
	.It Sy used
	The amount of space consumed by this dataset and all its descendents.
	This is the value that is checked against this dataset's quota and reservation.
	The space used does not include this dataset's reservation, but does take into
	account the reservations of any descendent datasets.
	The amount of space that a dataset consumes from its parent, as well as the
	amount of space that is freed if this dataset is recursively destroyed, is the
	greater of its space used and its reservation.
	.Pp
	The used space of a snapshot
	.Po see the
	.Sx Snapshots
	section of
	.Xr zfsconcepts 7
	.Pc
	is space that is referenced exclusively by this snapshot.
	If this snapshot is destroyed, the amount of
	.Sy used
	space will be freed.
	Space that is shared by multiple snapshots isn't accounted for in this metric.
	When a snapshot is destroyed, space that was previously shared with this
	snapshot can become unique to snapshots adjacent to it, thus changing the used
	space of those snapshots.
	The used space of the latest snapshot can also be affected by changes in the
	file system.
	Note that the
	.Sy used
	space of a snapshot is a subset of the
	.Sy written
	space of the snapshot.
	.Pp
	The amount of space used, available, or referenced does not take into account
	pending changes.
	Pending changes are generally accounted for within a few seconds.
	Committing a change to a disk using
	.Xr fsync 2
	or
	.Sy O_SYNC
	does not necessarily guarantee that the space usage information is updated
	immediately.
	.It Sy usedby*
	The
	.Sy usedby*
	properties decompose the
	.Sy used
	properties into the various reasons that space is used.
	Specifically,
	.Sy used No =
	.Sy usedbychildren No +
	.Sy usedbydataset No +
	.Sy usedbyrefreservation No +
	.Sy usedbysnapshots .
	These properties are only available for datasets created on
	.Nm zpool
	.Qo version 13 Qc
	pools.
	.It Sy usedbychildren
	The amount of space used by children of this dataset, which would be freed if
	all the dataset's children were destroyed.
	.It Sy usedbydataset
	The amount of space used by this dataset itself, which would be freed if the
	dataset were destroyed
	.Po after first removing any
	.Sy refreservation
	and destroying any necessary snapshots or descendents
	.Pc .
	.It Sy usedbyrefreservation
	The amount of space used by a
	.Sy refreservation
	set on this dataset, which would be freed if the
	.Sy refreservation
	was removed.
	.It Sy usedbysnapshots
	The amount of space consumed by snapshots of this dataset.
	In particular, it is the amount of space that would be freed if all of this
	dataset's snapshots were destroyed.
	Note that this is not simply the sum of the snapshots'
	.Sy used
	properties because space can be shared by multiple snapshots.
	.It Sy userused Ns @ Ns Ar user
	The amount of space consumed by the specified user in this dataset.
	Space is charged to the owner of each file, as displayed by
	.Nm ls Fl l .
	The amount of space charged is displayed by
	.Nm du No and Nm ls Fl s .
	See the
	.Nm zfs Cm userspace
	command for more information.
	.Pp
	Unprivileged users can access only their own space usage.
	The root user, or a user who has been granted the
	.Sy userused
	privilege with
	.Nm zfs Cm allow ,
	can access everyone's usage.
	.Pp
	The
	.Sy userused Ns @ Ns Ar …
	properties are not displayed by
	.Nm zfs Cm get Sy all .
	The user's name must be appended after the
	.Sy @
	symbol, using one of the following forms:
	.Bl -bullet -compact -offset 4n
	.It
	POSIX name
	.Pq Qq joe
	.It
	POSIX numeric ID
	.Pq Qq 789
	.It
	SID name
	.Pq Qq joe.smith@mydomain
	.It
	SID numeric ID
	.Pq Qq S-1-123-456-789
	.El
	.Pp
	Files created on Linux always have POSIX owners.
	.It Sy userobjused Ns @ Ns Ar user
	The
	.Sy userobjused
	property is similar to
	.Sy userused
	but instead it counts the number of objects consumed by a user.
	This property counts all objects allocated on behalf of the user,
	it may differ from the results of system tools such as
	.Nm df Fl i .
	.Pp
	When the property
	.Sy xattr Ns = Ns Sy on
	is set on a file system additional objects will be created per-file to store
	extended attributes.
	These additional objects are reflected in the
	.Sy userobjused
	value and are counted against the user's
	.Sy userobjquota .
	When a file system is configured to use
	.Sy xattr Ns = Ns Sy sa
	no additional internal objects are normally required.
	.It Sy userrefs
	This property is set to the number of user holds on this snapshot.
	User holds are set by using the
	.Nm zfs Cm hold
	command.
	.It Sy groupused Ns @ Ns Ar group
	The amount of space consumed by the specified group in this dataset.
	Space is charged to the group of each file, as displayed by
	.Nm ls Fl l .
	See the
	.Sy userused Ns @ Ns Ar user
	property for more information.
	.Pp
	Unprivileged users can only access their own groups' space usage.
	The root user, or a user who has been granted the
	.Sy groupused
	privilege with
	.Nm zfs Cm allow ,
	can access all groups' usage.
	.It Sy groupobjused Ns @ Ns Ar group
	The number of objects consumed by the specified group in this dataset.
	Multiple objects may be charged to the group for each file when extended
	attributes are in use.
	See the
	.Sy userobjused Ns @ Ns Ar user
	property for more information.
	.Pp
	Unprivileged users can only access their own groups' space usage.
	The root user, or a user who has been granted the
	.Sy groupobjused
	privilege with
	.Nm zfs Cm allow ,
	can access all groups' usage.
	.It Sy projectused Ns @ Ns Ar project
	The amount of space consumed by the specified project in this dataset.
	Project is identified via the project identifier (ID) that is object-based
	numeral attribute.
	An object can inherit the project ID from its parent object (if the
	parent has the flag of inherit project ID that can be set and changed via
	.Nm chattr Fl /+P
	or
	.Nm zfs project Fl s )
	when being created.
	The privileged user can set and change object's project
	ID via
	.Nm chattr Fl p
	or
	.Nm zfs project Fl s
	anytime.
	Space is charged to the project of each file, as displayed by
	.Nm lsattr Fl p
	or
	.Nm zfs project .
	See the
	.Sy userused Ns @ Ns Ar user
	property for more information.
	.Pp
	The root user, or a user who has been granted the
	.Sy projectused
	privilege with
	.Nm zfs allow ,
	can access all projects' usage.
	.It Sy projectobjused Ns @ Ns Ar project
	The
	.Sy projectobjused
	is similar to
	.Sy projectused
	but instead it counts the number of objects consumed by project.
	When the property
	.Sy xattr Ns = Ns Sy on
	is set on a fileset, ZFS will create additional objects per-file to store
	extended attributes.
	These additional objects are reflected in the
	.Sy projectobjused
	value and are counted against the project's
	.Sy projectobjquota .
	When a filesystem is configured to use
	.Sy xattr Ns = Ns Sy sa
	no additional internal objects are required.
	See the
	.Sy userobjused Ns @ Ns Ar user
	property for more information.
	.Pp
	The root user, or a user who has been granted the
	.Sy projectobjused
	privilege with
	.Nm zfs allow ,
	can access all projects' objects usage.
	.It Sy snapshots_changed
	Provides a mechanism to quickly determine whether snapshot list has
	changed without having to mount a dataset or iterate the snapshot list.
	Specifies the time at which a snapshot for a dataset was last
	created or deleted.
	.Pp
	This allows us to be more efficient how often we query snapshots.
	The property is persistent across mount and unmount operations only if the
	.Sy extensible_dataset
	feature is enabled.
	.It Sy volblocksize
	For volumes, specifies the block size of the volume.
	The
	.Sy blocksize
	cannot be changed once the volume has been written, so it should be set at
	volume creation time.
	The default
	.Sy blocksize
	for volumes is 16 Kbytes.
	Any power of 2 from 512 bytes to 128 Kbytes is valid.
	.Pp
	This property can also be referred to by its shortened column name,
	.Sy volblock .
	.It Sy written
	The amount of space
	.Sy referenced
	by this dataset, that was written since the previous snapshot
	.Pq i.e. that is not referenced by the previous snapshot .
	.It Sy written Ns @ Ns Ar snapshot
	The amount of
	.Sy referenced
	space written to this dataset since the specified snapshot.
	This is the space that is referenced by this dataset but was not referenced by
	the specified snapshot.
	.Pp
	The
	.Ar snapshot
	may be specified as a short snapshot name
	.Pq just the part after the Sy @ ,
	in which case it will be interpreted as a snapshot in the same filesystem as
	this dataset.
	The
	.Ar snapshot
	may be a full snapshot name
	.Pq Ar filesystem Ns @ Ns Ar snapshot ,
	which for clones may be a snapshot in the origin's filesystem
	.Pq or the origin of the origin's filesystem, etc.
	.El
	.Pp
	The following native properties can be used to change the behavior of a ZFS
	dataset.
	.Bl -tag -width ""
	.It Xo
	.Sy aclinherit Ns = Ns Sy discard Ns \| Ns Sy noallow Ns \| Ns
	.Sy restricted Ns \| Ns Sy passthrough Ns \| Ns Sy passthrough-x
	.Xc
	Controls how ACEs are inherited when files and directories are created.
	.Bl -tag -compact -offset 4n -width "passthrough-x"
	.It Sy discard
	does not inherit any ACEs.
	.It Sy noallow
	only inherits inheritable ACEs that specify
	.Qq deny
	permissions.
	.It Sy restricted
	default, removes the
	.Sy write_acl
	and
	.Sy write_owner
	permissions when the ACE is inherited.
	.It Sy passthrough
	inherits all inheritable ACEs without any modifications.
	.It Sy passthrough-x
	same meaning as
	.Sy passthrough ,
	except that the
	.Sy owner@ , group@ , No and Sy everyone@
	ACEs inherit the execute permission only if the file creation mode also requests
	the execute bit.
	.El
	.Pp
	When the property value is set to
	.Sy passthrough ,
	files are created with a mode determined by the inheritable ACEs.
	If no inheritable ACEs exist that affect the mode, then the mode is set in
	accordance to the requested mode from the application.
	.Pp
	The
	.Sy aclinherit
	property does not apply to POSIX ACLs.
	.It Xo
	.Sy aclmode Ns = Ns Sy discard Ns \| Ns Sy groupmask Ns \| Ns
	.Sy passthrough Ns \| Ns Sy restricted Ns
	.Xc
	Controls how an ACL is modified during chmod(2) and how inherited ACEs
	are modified by the file creation mode:
	.Bl -tag -compact -offset 4n -width "passthrough"
	.It Sy discard
	default, deletes all
	.Sy ACEs
	except for those representing
	the mode of the file or directory requested by
	.Xr chmod 2 .
	.It Sy groupmask
	reduces permissions granted in all
	.Sy ALLOW
	entries found in the
	.Sy ACL
	such that they are no greater than the group permissions specified by
	.Xr chmod 2 .
	.It Sy passthrough
	indicates that no changes are made to the ACL other than creating or updating
	the necessary ACL entries to represent the new mode of the file or directory.
	.It Sy restricted
	will cause the
	.Xr chmod 2
	operation to return an error when used on any file or directory which has
	a non-trivial ACL whose entries can not be represented by a mode.
	.Xr chmod 2
	is required to change the set user ID, set group ID, or sticky bits on a file
	or directory, as they do not have equivalent ACL entries.
	In order to use
	.Xr chmod 2
	on a file or directory with a non-trivial ACL when
	.Sy aclmode
	is set to
	.Sy restricted ,
	you must first remove all ACL entries which do not represent the current mode.
	.El
	.It Sy acltype Ns = Ns Sy off Ns \| Ns Sy nfsv4 Ns \| Ns Sy posix
	Controls whether ACLs are enabled and if so what type of ACL to use.
	When this property is set to a type of ACL not supported by the current
	platform, the behavior is the same as if it were set to
	.Sy off .
	.Bl -tag -compact -offset 4n -width "posixacl"
	.It Sy off
	default on Linux, when a file system has the
	.Sy acltype
	property set to off then ACLs are disabled.
	.It Sy noacl
	an alias for
	.Sy off
	.It Sy nfsv4
	default on
	.Fx ,
	indicates that NFSv4-style ZFS ACLs should be used.
	These ACLs can be managed with the
	.Xr getfacl 1
	and
	.Xr setfacl 1 .
	The
	.Sy nfsv4
	ZFS ACL type is not yet supported on Linux.
	.It Sy posix
	indicates POSIX ACLs should be used.
	POSIX ACLs are specific to Linux and are not functional on other platforms.
	POSIX ACLs are stored as an extended
	attribute and therefore will not overwrite any existing NFSv4 ACLs which
	may be set.
	.It Sy posixacl
	an alias for
	.Sy posix
	.El
	.Pp
	To obtain the best performance when setting
	.Sy posix
	users are strongly encouraged to set the
	.Sy xattr Ns = Ns Sy sa
	property.
	This will result in the POSIX ACL being stored more efficiently on disk.
	But as a consequence, all new extended attributes will only be
	accessible from OpenZFS implementations which support the
	.Sy xattr Ns = Ns Sy sa
	property.
	See the
	.Sy xattr
	property for more details.
	.It Sy atime Ns = Ns Sy on Ns \| Ns Sy off
	Controls whether the access time for files is updated when they are read.
	Turning this property off avoids producing write traffic when reading files and
	can result in significant performance gains, though it might confuse mailers
	and other similar utilities.
	The values
	.Sy on
	and
	.Sy off
	are equivalent to the
	.Sy atime
	and
	.Sy noatime
	mount options.
	The default value is
	.Sy on .
	See also
	.Sy relatime
	below.
	.It Sy canmount Ns = Ns Sy on Ns \| Ns Sy off Ns \| Ns Sy noauto
	If this property is set to
	.Sy off ,
	the file system cannot be mounted, and is ignored by
	.Nm zfs Cm mount Fl a .
	Setting this property to
	.Sy off
	is similar to setting the
	.Sy mountpoint
	property to
	.Sy none ,
	except that the dataset still has a normal
	.Sy mountpoint
	property, which can be inherited.
	Setting this property to
	.Sy off
	allows datasets to be used solely as a mechanism to inherit properties.
	One example of setting
	.Sy canmount Ns = Ns Sy off
	is to have two datasets with the same
	.Sy mountpoint ,
	so that the children of both datasets appear in the same directory, but might
	have different inherited characteristics.
	.Pp
	When set to
	.Sy noauto ,
	a dataset can only be mounted and unmounted explicitly.
	The dataset is not mounted automatically when the dataset is created or
	imported, nor is it mounted by the
	.Nm zfs Cm mount Fl a
	command or unmounted by the
	.Nm zfs Cm unmount Fl a
	command.
	.Pp
	This property is not inherited.
	.It Xo
	.Sy checksum Ns = Ns Sy on Ns \| Ns Sy off Ns \| Ns Sy fletcher2 Ns \| Ns
	.Sy fletcher4 Ns \| Ns Sy sha256 Ns \| Ns Sy noparity Ns \| Ns
	.Sy sha512 Ns \| Ns Sy skein Ns \| Ns Sy edonr Ns \| Ns Sy blake3
	.Xc
	Controls the checksum used to verify data integrity.
	The default value is
	.Sy on ,
	which automatically selects an appropriate algorithm
	.Po currently,
	.Sy fletcher4 ,
	but this may change in future releases
	.Pc .
	The value
	.Sy off
	disables integrity checking on user data.
	The value
	.Sy noparity
	not only disables integrity but also disables maintaining parity for user data.
	This setting is used internally by a dump device residing on a RAID-Z pool and
	should not be used by any other dataset.
	Disabling checksums is
	.Em NOT
	a recommended practice.
	.Pp
	The
	.Sy sha512 ,
	.Sy skein ,
	.Sy edonr ,
	and
	.Sy blake3
	checksum algorithms require enabling the appropriate features on the pool.
	.Pp
	Please see
	.Xr zpool-features 7
	for more information on these algorithms.
	.Pp
	Changing this property affects only newly-written data.
	.It Xo
	.Sy compression Ns = Ns Sy on Ns \| Ns Sy off Ns \| Ns Sy gzip Ns \| Ns
	.Sy gzip- Ns Ar N Ns \| Ns Sy lz4 Ns \| Ns Sy lzjb Ns \| Ns Sy zle Ns \| Ns Sy zstd Ns \| Ns
	.Sy zstd- Ns Ar N Ns \| Ns Sy zstd-fast Ns \| Ns Sy zstd-fast- Ns Ar N
	.Xc
	Controls the compression algorithm used for this dataset.
	.Pp
	When set to
	.Sy on
	(the default), indicates that the current default compression algorithm should
	be used.
	The default balances compression and decompression speed, with compression ratio
	and is expected to work well on a wide variety of workloads.
	Unlike all other settings for this property,
	.Sy on
	does not select a fixed compression type.
	As new compression algorithms are added to ZFS and enabled on a pool, the
	default compression algorithm may change.
	The current default compression algorithm is either
	.Sy lzjb
	or, if the
	.Sy lz4_compress
	feature is enabled,
	.Sy lz4 .
	.Pp
	The
	.Sy lz4
	compression algorithm is a high-performance replacement for the
	.Sy lzjb
	algorithm.
	It features significantly faster compression and decompression, as well as a
	moderately higher compression ratio than
	.Sy lzjb ,
	but can only be used on pools with the
	.Sy lz4_compress
	feature set to
	.Sy enabled .
	See
	.Xr zpool-features 7
	for details on ZFS feature flags and the
	.Sy lz4_compress
	feature.
	.Pp
	The
	.Sy lzjb
	compression algorithm is optimized for performance while providing decent data
	compression.
	.Pp
	The
	.Sy gzip
	compression algorithm uses the same compression as the
	.Xr gzip 1
	command.
	You can specify the
	.Sy gzip
	level by using the value
	.Sy gzip- Ns Ar N ,
	where
	.Ar N
	is an integer from 1
	.Pq fastest
	to 9
	.Pq best compression ratio .
	Currently,
	.Sy gzip
	is equivalent to
	.Sy gzip-6
	.Po which is also the default for
	.Xr gzip 1
	.Pc .
	.Pp
	The
	.Sy zstd
	compression algorithm provides both high compression ratios and good
	performance.
	You can specify the
	.Sy zstd
	level by using the value
	.Sy zstd- Ns Ar N ,
	where
	.Ar N
	is an integer from 1
	.Pq fastest
	to 19
	.Pq best compression ratio .
	.Sy zstd
	is equivalent to
	.Sy zstd-3 .
	.Pp
	Faster speeds at the cost of the compression ratio can be requested by
	setting a negative
	.Sy zstd
	level.
	This is done using
	.Sy zstd-fast- Ns Ar N ,
	where
	.Ar N
	is an integer in
	.Bq Sy 1 Ns - Ns Sy 10 , 20 , 30 , No … , Sy 100 , 500 , 1000
	which maps to a negative
	.Sy zstd
	level.
	The lower the level the faster the compression \(em
	.Sy 1000
	provides the fastest compression and lowest compression ratio.
	.Sy zstd-fast
	is equivalent to
	.Sy zstd-fast- Ns Ar 1 .
	.Pp
	The
	.Sy zle
	compression algorithm compresses runs of zeros.
	.Pp
	This property can also be referred to by its shortened column name
	.Sy compress .
	Changing this property affects only newly-written data.
	.Pp
	When any setting except
	.Sy off
	is selected, compression will explicitly check for blocks consisting of only
	zeroes (the NUL byte).
	When a zero-filled block is detected, it is stored as
	a hole and not compressed using the indicated compression algorithm.
	.Pp
	Any block being compressed must be no larger than 7/8 of its original size
	after compression, otherwise the compression will not be considered worthwhile
	and the block saved uncompressed.
	Note that when the logical block is less than
	8 times the disk sector size this effectively reduces the necessary compression
	ratio; for example, 8 KiB blocks on disks with 4 KiB disk sectors must compress
	to 1/2
	or less of their original size.
	.It Xo
	.Sy context Ns = Ns Sy none Ns \| Ns
	.Ar SELinux-User : Ns Ar SELinux-Role : Ns Ar SELinux-Type : Ns Ar Sensitivity-Level
	.Xc
	This flag sets the SELinux context for all files in the file system under
	a mount point for that file system.
	See
	.Xr selinux 8
	for more information.
	.It Xo
	.Sy fscontext Ns = Ns Sy none Ns \| Ns
	.Ar SELinux-User : Ns Ar SELinux-Role : Ns Ar SELinux-Type : Ns Ar Sensitivity-Level
	.Xc
	This flag sets the SELinux context for the file system file system being
	mounted.
	See
	.Xr selinux 8
	for more information.
	.It Xo
	.Sy defcontext Ns = Ns Sy none Ns \| Ns
	.Ar SELinux-User : Ns Ar SELinux-Role : Ns Ar SELinux-Type : Ns Ar Sensitivity-Level
	.Xc
	This flag sets the SELinux default context for unlabeled files.
	See
	.Xr selinux 8
	for more information.
	.It Xo
	.Sy rootcontext Ns = Ns Sy none Ns \| Ns
	.Ar SELinux-User : Ns Ar SELinux-Role : Ns Ar SELinux-Type : Ns Ar Sensitivity-Level
	.Xc
	This flag sets the SELinux context for the root inode of the file system.
	See
	.Xr selinux 8
	for more information.
	.It Sy copies Ns = Ns Sy 1 Ns \| Ns Sy 2 Ns \| Ns Sy 3
	Controls the number of copies of data stored for this dataset.
	These copies are in addition to any redundancy provided by the pool, for
	example, mirroring or RAID-Z.
	The copies are stored on different disks, if possible.
	The space used by multiple copies is charged to the associated file and dataset,
	changing the
	.Sy used
	property and counting against quotas and reservations.
	.Pp
	Changing this property only affects newly-written data.
	Therefore, set this property at file system creation time by using the
	.Fl o Sy copies Ns = Ns Ar N
	option.
	.Pp
	Remember that ZFS will not import a pool with a missing top-level vdev.
	Do
	.Em NOT
	create, for example a two-disk striped pool and set
	.Sy copies Ns = Ns Ar 2
	on some datasets thinking you have setup redundancy for them.
	When a disk fails you will not be able to import the pool
	and will have lost all of your data.
	.Pp
	Encrypted datasets may not have
	.Sy copies Ns = Ns Ar 3
	since the implementation stores some encryption metadata where the third copy
	would normally be.
	.It Sy devices Ns = Ns Sy on Ns \| Ns Sy off
	Controls whether device nodes can be opened on this file system.
	The default value is
	.Sy on .
	The values
	.Sy on
	and
	.Sy off
	are equivalent to the
	.Sy dev
	and
	.Sy nodev
	mount options.
	.It Xo
	.Sy dedup Ns = Ns Sy off Ns \| Ns Sy on Ns \| Ns Sy verify Ns \| Ns
	.Sy sha256 Ns Oo , Ns Sy verify Oc Ns \| Ns Sy sha512 Ns Oo , Ns Sy verify Oc Ns \| Ns Sy skein Ns Oo , Ns Sy verify Oc Ns \| Ns
	.Sy edonr , Ns Sy verify Ns \| Ns Sy blake3 Ns Oo , Ns Sy verify Oc Ns
	.Xc
	Configures deduplication for a dataset.
	The default value is
	.Sy off .
	The default deduplication checksum is
	.Sy sha256
	(this may change in the future).
	When
	.Sy dedup
	is enabled, the checksum defined here overrides the
	.Sy checksum
	property.
	Setting the value to
	.Sy verify
	has the same effect as the setting
	.Sy sha256 , Ns Sy verify .
	.Pp
	If set to
	.Sy verify ,
	ZFS will do a byte-to-byte comparison in case of two blocks having the same
	signature to make sure the block contents are identical.
	Specifying
	.Sy verify
	is mandatory for the
	.Sy edonr
	algorithm.
	.Pp
	Unless necessary, deduplication should
	.Em not
	be enabled on a system.
	See the
	.Sx Deduplication
	section of
	.Xr zfsconcepts 7 .
	.It Xo
	.Sy dnodesize Ns = Ns Sy legacy Ns \| Ns Sy auto Ns \| Ns Sy 1k Ns \| Ns
	.Sy 2k Ns \| Ns Sy 4k Ns \| Ns Sy 8k Ns \| Ns Sy 16k
	.Xc
	Specifies a compatibility mode or literal value for the size of dnodes in the
	file system.
	The default value is
	.Sy legacy .
	Setting this property to a value other than
	.Sy legacy No requires the Sy large_dnode No pool feature to be enabled .
	.Pp
	Consider setting
	.Sy dnodesize
	to
	.Sy auto
	if the dataset uses the
	.Sy xattr Ns = Ns Sy sa
	property setting and the workload makes heavy use of extended attributes.
	This
	may be applicable to SELinux-enabled systems, Lustre servers, and Samba
	servers, for example.
	Literal values are supported for cases where the optimal
	size is known in advance and for performance testing.
	.Pp
	Leave
	.Sy dnodesize
	set to
	.Sy legacy
	if you need to receive a send stream of this dataset on a pool that doesn't
	enable the
	.Sy large_dnode
	feature, or if you need to import this pool on a system that doesn't support the
	.Sy large_dnode No feature .
	.Pp
	This property can also be referred to by its shortened column name,
	.Sy dnsize .
	.It Xo
	.Sy encryption Ns = Ns Sy off Ns \| Ns Sy on Ns \| Ns Sy aes-128-ccm Ns \| Ns
	.Sy aes-192-ccm Ns \| Ns Sy aes-256-ccm Ns \| Ns Sy aes-128-gcm Ns \| Ns
	.Sy aes-192-gcm Ns \| Ns Sy aes-256-gcm
	.Xc
	Controls the encryption cipher suite (block cipher, key length, and mode) used
	for this dataset.
	Requires the
	.Sy encryption
	feature to be enabled on the pool.
	Requires a
	.Sy keyformat
	to be set at dataset creation time.
	.Pp
	Selecting
	.Sy encryption Ns = Ns Sy on
	when creating a dataset indicates that the default encryption suite will be
	selected, which is currently
	.Sy aes-256-gcm .
	In order to provide consistent data protection, encryption must be specified at
	dataset creation time and it cannot be changed afterwards.
	.Pp
	For more details and caveats about encryption see the
	.Sx Encryption
	section of
	.Xr zfs-load-key 8 .
	.It Sy keyformat Ns = Ns Sy raw Ns \| Ns Sy hex Ns \| Ns Sy passphrase
	Controls what format the user's encryption key will be provided as.
	This property is only set when the dataset is encrypted.
	.Pp
	Raw keys and hex keys must be 32 bytes long (regardless of the chosen
	encryption suite) and must be randomly generated.
	A raw key can be generated with the following command:
	.Dl # Nm dd Sy if=/dev/urandom bs=32 count=1 Sy of= Ns Pa /path/to/output/key
	.Pp
	Passphrases must be between 8 and 512 bytes long and will be processed through
	PBKDF2 before being used (see the
	.Sy pbkdf2iters
	property).
	Even though the encryption suite cannot be changed after dataset creation,
	the keyformat can be with
	.Nm zfs Cm change-key .
	.It Xo
	.Sy keylocation Ns = Ns Sy prompt Ns \| Ns Sy file:// Ns Ar /absolute/file/path Ns \| Ns Sy https:// Ns Ar address Ns \| Ns Sy http:// Ns Ar address
	.Xc
	Controls where the user's encryption key will be loaded from by default for
	commands such as
	.Nm zfs Cm load-key
	and
	.Nm zfs Cm mount Fl l .
	This property is only set for encrypted datasets which are encryption roots.
	If unspecified, the default is
	.Sy prompt .
	.Pp
	Even though the encryption suite cannot be changed after dataset creation, the
	keylocation can be with either
	.Nm zfs Cm set
	or
	.Nm zfs Cm change-key .
	If
	.Sy prompt
	is selected ZFS will ask for the key at the command prompt when it is required
	to access the encrypted data (see
	.Nm zfs Cm load-key
	for details).
	This setting will also allow the key to be passed in via the standard input
	stream,
	but users should be careful not to place keys which should be kept secret on
	the command line.
	If a file URI is selected, the key will be loaded from the
	specified absolute file path.
	If an HTTPS or HTTP URL is selected, it will be GETted using
	.Xr fetch 3 ,
	libcurl, or nothing, depending on compile-time configuration and run-time
	availability.
	The
	.Sy SSL_CA_CERT_FILE
	environment variable can be set to set the location
	of the concatenated certificate store.
	The
	.Sy SSL_CA_CERT_PATH
	environment variable can be set to override the location
	of the directory containing the certificate authority bundle.
	The
	.Sy SSL_CLIENT_CERT_FILE
	and
	.Sy SSL_CLIENT_KEY_FILE
	environment variables can be set to configure the path
	to the client certificate and its key.
	.It Sy pbkdf2iters Ns = Ns Ar iterations
	Controls the number of PBKDF2 iterations that a
	.Sy passphrase
	encryption key should be run through when processing it into an encryption key.
	This property is only defined when encryption is enabled and a keyformat of
	.Sy passphrase
	is selected.
	The goal of PBKDF2 is to significantly increase the
	computational difficulty needed to brute force a user's passphrase.
	This is accomplished by forcing the attacker to run each passphrase through a
	computationally expensive hashing function many times before they arrive at the
	resulting key.
	A user who actually knows the passphrase will only have to pay this cost once.
	As CPUs become better at processing, this number should be
	raised to ensure that a brute force attack is still not possible.
	The current default is
	.Sy 350000
	and the minimum is
	.Sy 100000 .
	This property may be changed with
	.Nm zfs Cm change-key .
	.It Sy exec Ns = Ns Sy on Ns \| Ns Sy off
	Controls whether processes can be executed from within this file system.
	The default value is
	.Sy on .
	The values
	.Sy on
	and
	.Sy off
	are equivalent to the
	.Sy exec
	and
	.Sy noexec
	mount options.
	.It Sy filesystem_limit Ns = Ns Ar count Ns \| Ns Sy none
	Limits the number of filesystems and volumes that can exist under this point in
	the dataset tree.
	The limit is not enforced if the user is allowed to change the limit.
	Setting a
	.Sy filesystem_limit
	to
	.Sy on
	a descendent of a filesystem that already has a
	.Sy filesystem_limit
	does not override the ancestor's
	.Sy filesystem_limit ,
	but rather imposes an additional limit.
	This feature must be enabled to be used
	.Po see
	.Xr zpool-features 7
	.Pc .
	.It Sy special_small_blocks Ns = Ns Ar size
	This value represents the threshold block size for including small file
	blocks into the special allocation class.
	Blocks smaller than or equal to this
	value will be assigned to the special allocation class while greater blocks
	will be assigned to the regular class.
	Valid values are zero or a power of two from 512 up to 1048576 (1 MiB).
	The default size is 0 which means no small file blocks
	will be allocated in the special class.
	.Pp
	Before setting this property, a special class vdev must be added to the
	pool.
	See
	.Xr zpoolconcepts 7
	for more details on the special allocation class.
	.It Sy mountpoint Ns = Ns Pa path Ns \| Ns Sy none Ns \| Ns Sy legacy
	Controls the mount point used for this file system.
	See the
	.Sx Mount Points
	section of
	.Xr zfsconcepts 7
	for more information on how this property is used.
	.Pp
	When the
	.Sy mountpoint
	property is changed for a file system, the file system and any children that
	inherit the mount point are unmounted.
	If the new value is
	.Sy legacy ,
	then they remain unmounted.
	Otherwise, they are automatically remounted in the new location if the property
	was previously
	.Sy legacy
	or
	.Sy none .
	In addition, any shared file systems are unshared and shared in the new
	location.
	.Pp
	When the
	.Sy mountpoint
	property is set with
	.Nm zfs Cm set Fl u
	, the
	.Sy mountpoint
	property is updated but dataset is not mounted or unmounted and remains
	as it was before.
	.It Sy nbmand Ns = Ns Sy on Ns \| Ns Sy off
	Controls whether the file system should be mounted with
	.Sy nbmand
	.Pq Non-blocking mandatory locks .
	Changes to this property only take effect when the file system is umounted and
	remounted.
	This was only supported by Linux prior to 5.15, and was buggy there,
	and is not supported by
	.Fx .
	On Solaris it's used for SMB clients.
	.It Sy overlay Ns = Ns Sy on Ns \| Ns Sy off
	Allow mounting on a busy directory or a directory which already contains
	files or directories.
	This is the default mount behavior for Linux and
	.Fx
	file systems.
	On these platforms the property is
	.Sy on
	by default.
	Set to
	.Sy off
	to disable overlay mounts for consistency with OpenZFS on other platforms.
	.It Sy primarycache Ns = Ns Sy all Ns \| Ns Sy none Ns \| Ns Sy metadata
	Controls what is cached in the primary cache
	.Pq ARC .
	If this property is set to
	.Sy all ,
	then both user data and metadata is cached.
	If this property is set to
	.Sy none ,
	then neither user data nor metadata is cached.
	If this property is set to
	.Sy metadata ,
	then only metadata is cached.
	The default value is
	.Sy all .
	.It Sy quota Ns = Ns Ar size Ns \| Ns Sy none
	Limits the amount of space a dataset and its descendents can consume.
	This property enforces a hard limit on the amount of space used.
	This includes all space consumed by descendents, including file systems and
	snapshots.
	Setting a quota on a descendent of a dataset that already has a quota does not
	override the ancestor's quota, but rather imposes an additional limit.
	.Pp
	Quotas cannot be set on volumes, as the
	.Sy volsize
	property acts as an implicit quota.
	.It Sy snapshot_limit Ns = Ns Ar count Ns \| Ns Sy none
	Limits the number of snapshots that can be created on a dataset and its
	descendents.
	Setting a
	.Sy snapshot_limit
	on a descendent of a dataset that already has a
	.Sy snapshot_limit
	does not override the ancestor's
	.Sy snapshot_limit ,
	but rather imposes an additional limit.
	The limit is not enforced if the user is allowed to change the limit.
	For example, this means that recursive snapshots taken from the global zone are
	counted against each delegated dataset within a zone.
	This feature must be enabled to be used
	.Po see
	.Xr zpool-features 7
	.Pc .
	.It Sy userquota@ Ns Ar user Ns = Ns Ar size Ns \| Ns Sy none
	Limits the amount of space consumed by the specified user.
	User space consumption is identified by the
	.Sy userspace@ Ns Ar user
	property.
	.Pp
	Enforcement of user quotas may be delayed by several seconds.
	This delay means that a user might exceed their quota before the system notices
	that they are over quota and begins to refuse additional writes with the
	.Er EDQUOT
	error message.
	See the
	.Nm zfs Cm userspace
	command for more information.
	.Pp
	Unprivileged users can only access their own groups' space usage.
	The root user, or a user who has been granted the
	.Sy userquota
	privilege with
	.Nm zfs Cm allow ,
	can get and set everyone's quota.
	.Pp
	This property is not available on volumes, on file systems before version 4, or
	on pools before version 15.
	The
	.Sy userquota@ Ns Ar …
	properties are not displayed by
	.Nm zfs Cm get Sy all .
	The user's name must be appended after the
	.Sy @
	symbol, using one of the following forms:
	.Bl -bullet -compact -offset 4n
	.It
	POSIX name
	.Pq Qq joe
	.It
	POSIX numeric ID
	.Pq Qq 789
	.It
	SID name
	.Pq Qq joe.smith@mydomain
	.It
	SID numeric ID
	.Pq Qq S-1-123-456-789
	.El
	.Pp
	Files created on Linux always have POSIX owners.
	.It Sy userobjquota@ Ns Ar user Ns = Ns Ar size Ns \| Ns Sy none
	The
	.Sy userobjquota
	is similar to
	.Sy userquota
	but it limits the number of objects a user can create.
	Please refer to
	.Sy userobjused
	for more information about how objects are counted.
	.It Sy groupquota@ Ns Ar group Ns = Ns Ar size Ns \| Ns Sy none
	Limits the amount of space consumed by the specified group.
	Group space consumption is identified by the
	.Sy groupused@ Ns Ar group
	property.
	.Pp
	Unprivileged users can access only their own groups' space usage.
	The root user, or a user who has been granted the
	.Sy groupquota
	privilege with
	.Nm zfs Cm allow ,
	can get and set all groups' quotas.
	.It Sy groupobjquota@ Ns Ar group Ns = Ns Ar size Ns \| Ns Sy none
	The
	.Sy groupobjquota
	is similar to
	.Sy groupquota
	but it limits number of objects a group can consume.
	Please refer to
	.Sy userobjused
	for more information about how objects are counted.
	.It Sy projectquota@ Ns Ar project Ns = Ns Ar size Ns \| Ns Sy none
	Limits the amount of space consumed by the specified project.
	Project space consumption is identified by the
	.Sy projectused@ Ns Ar project
	property.
	Please refer to
	.Sy projectused
	for more information about how project is identified and set/changed.
	.Pp
	The root user, or a user who has been granted the
	.Sy projectquota
	privilege with
	.Nm zfs allow ,
	can access all projects' quota.
	.It Sy projectobjquota@ Ns Ar project Ns = Ns Ar size Ns \| Ns Sy none
	The
	.Sy projectobjquota
	is similar to
	.Sy projectquota
	but it limits number of objects a project can consume.
	Please refer to
	.Sy userobjused
	for more information about how objects are counted.
	.It Sy readonly Ns = Ns Sy on Ns \| Ns Sy off
	Controls whether this dataset can be modified.
	The default value is
	.Sy off .
	The values
	.Sy on
	and
	.Sy off
	are equivalent to the
	.Sy ro
	and
	.Sy rw
	mount options.
	.Pp
	This property can also be referred to by its shortened column name,
	.Sy rdonly .
	.It Sy recordsize Ns = Ns Ar size
	Specifies a suggested block size for files in the file system.
	This property is designed solely for use with database workloads that access
	files in fixed-size records.
	ZFS automatically tunes block sizes according to internal algorithms optimized
	for typical access patterns.
	.Pp
	For databases that create very large files but access them in small random
	chunks, these algorithms may be suboptimal.
	Specifying a
	.Sy recordsize
	greater than or equal to the record size of the database can result in
	significant performance gains.
	Use of this property for general purpose file systems is strongly discouraged,
	and may adversely affect performance.
	.Pp
	The size specified must be a power of two greater than or equal to
	.Ar 512 B
	and less than or equal to
	.Ar 128 KiB .
	If the
	.Sy large_blocks
	feature is enabled on the pool, the size may be up to
	.Ar 1 MiB .
	See
	.Xr zpool-features 7
	for details on ZFS feature flags.
	.Pp
	Changing the file system's
	.Sy recordsize
	affects only files created afterward; existing files are unaffected.
	.Pp
	This property can also be referred to by its shortened column name,
	.Sy recsize .
	.It Sy redundant_metadata Ns = Ns Sy all Ns \| Ns Sy most Ns \| Ns Sy some Ns \| Ns Sy none
	Controls what types of metadata are stored redundantly.
	ZFS stores an extra copy of metadata, so that if a single block is corrupted,
	the amount of user data lost is limited.
	This extra copy is in addition to any redundancy provided at the pool level
	.Pq e.g. by mirroring or RAID-Z ,
	and is in addition to an extra copy specified by the
	.Sy copies
	property
	.Pq up to a total of 3 copies .
	For example if the pool is mirrored,
	.Sy copies Ns = Ns 2 ,
	and
	.Sy redundant_metadata Ns = Ns Sy most ,
	then ZFS stores 6 copies of most metadata, and 4 copies of data and some
	metadata.
	.Pp
	When set to
	.Sy all ,
	ZFS stores an extra copy of all metadata.
	If a single on-disk block is corrupt, at worst a single block of user data
	.Po which is
	.Sy recordsize
	bytes long
	.Pc
	can be lost.
	.Pp
	When set to
	.Sy most ,
	ZFS stores an extra copy of most types of metadata.
	This can improve performance of random writes, because less metadata must be
	written.
	In practice, at worst about 1000 blocks
	.Po of
	.Sy recordsize
	bytes each
	.Pc
	of user data can be lost if a single on-disk block is corrupt.
	The exact behavior of which metadata blocks are stored redundantly may change in
	future releases.
	.Pp
	When set to
	.Sy some ,
	ZFS stores an extra copy of only critical metadata.
	This can improve file create performance since less metadata
	needs to be written.
	If a single on-disk block is corrupt, at worst a single user file can be lost.
	.Pp
	When set to
	.Sy none ,
	ZFS does not store any copies of metadata redundantly.
	If a single on-disk block is corrupt, an entire dataset can be lost.
	.Pp
	The default value is
	.Sy all .
	.It Sy refquota Ns = Ns Ar size Ns \| Ns Sy none
	Limits the amount of space a dataset can consume.
	This property enforces a hard limit on the amount of space used.
	This hard limit does not include space used by descendents, including file
	systems and snapshots.
	.It Sy refreservation Ns = Ns Ar size Ns \| Ns Sy none Ns \| Ns Sy auto
	The minimum amount of space guaranteed to a dataset, not including its
	descendents.
	When the amount of space used is below this value, the dataset is treated as if
	it were taking up the amount of space specified by
	.Sy refreservation .
	The
	.Sy refreservation
	reservation is accounted for in the parent datasets' space used, and counts
	against the parent datasets' quotas and reservations.
	.Pp
	If
	.Sy refreservation
	is set, a snapshot is only allowed if there is enough free pool space outside of
	this reservation to accommodate the current number of
	.Qq referenced
	bytes in the dataset.
	.Pp
	If
	.Sy refreservation
	is set to
	.Sy auto ,
	a volume is thick provisioned
	.Po or
	.Qq not sparse
	.Pc .
	.Sy refreservation Ns = Ns Sy auto
	is only supported on volumes.
	See
	.Sy volsize
	in the
	.Sx Native Properties
	section for more information about sparse volumes.
	.Pp
	This property can also be referred to by its shortened column name,
	.Sy refreserv .
	.It Sy relatime Ns = Ns Sy on Ns \| Ns Sy off
	Controls the manner in which the access time is updated when
	.Sy atime Ns = Ns Sy on
	is set.
	Turning this property on causes the access time to be updated relative
	to the modify or change time.
	Access time is only updated if the previous
	access time was earlier than the current modify or change time or if the
	existing access time hasn't been updated within the past 24 hours.
	The default value is
	.Sy on .
	The values
	.Sy on
	and
	.Sy off
	are equivalent to the
	.Sy relatime
	and
	.Sy norelatime
	mount options.
	.It Sy reservation Ns = Ns Ar size Ns \| Ns Sy none
	The minimum amount of space guaranteed to a dataset and its descendants.
	When the amount of space used is below this value, the dataset is treated as if
	it were taking up the amount of space specified by its reservation.
	Reservations are accounted for in the parent datasets' space used, and count
	against the parent datasets' quotas and reservations.
	.Pp
	This property can also be referred to by its shortened column name,
	.Sy reserv .
	.It Sy secondarycache Ns = Ns Sy all Ns \| Ns Sy none Ns \| Ns Sy metadata
	Controls what is cached in the secondary cache
	.Pq L2ARC .
	If this property is set to
	.Sy all ,
	then both user data and metadata is cached.
	If this property is set to
	.Sy none ,
	then neither user data nor metadata is cached.
	If this property is set to
	.Sy metadata ,
	then only metadata is cached.
	The default value is
	.Sy all .
	+.It Sy prefetch Ns = Ns Sy all Ns \| Ns Sy none Ns \| Ns Sy metadata
	+Controls what speculative prefetch does.
	+If this property is set to
	+.Sy all ,
	+then both user data and metadata are prefetched.
	+If this property is set to
	+.Sy none ,
	+then neither user data nor metadata are prefetched.
	+If this property is set to
	+.Sy metadata ,
	+then only metadata are prefetched.
	+The default value is
	+.Sy all .
	+.Pp
	+Please note that the module parameter zfs_disable_prefetch=1 can
	+be used to totally disable speculative prefetch, bypassing anything
	+this property does.
	.It Sy setuid Ns = Ns Sy on Ns \| Ns Sy off
	Controls whether the setuid bit is respected for the file system.
	The default value is
	.Sy on .
	The values
	.Sy on
	and
	.Sy off
	are equivalent to the
	.Sy suid
	and
	.Sy nosuid
	mount options.
	.It Sy sharesmb Ns = Ns Sy on Ns \| Ns Sy off Ns \| Ns Ar opts
	Controls whether the file system is shared by using
	.Sy Samba USERSHARES
	and what options are to be used.
	Otherwise, the file system is automatically shared and unshared with the
	.Nm zfs Cm share
	and
	.Nm zfs Cm unshare
	commands.
	If the property is set to on, the
	.Xr net 8
	command is invoked to create a
	.Sy USERSHARE .
	.Pp
	Because SMB shares requires a resource name, a unique resource name is
	constructed from the dataset name.
	The constructed name is a copy of the
	dataset name except that the characters in the dataset name, which would be
	invalid in the resource name, are replaced with underscore (_) characters.
	Linux does not currently support additional options which might be available
	on Solaris.
	.Pp
	If the
	.Sy sharesmb
	property is set to
	.Sy off ,
	the file systems are unshared.
	.Pp
	The share is created with the ACL (Access Control List) "Everyone:F" ("F"
	stands for "full permissions", i.e. read and write permissions) and no guest
	access (which means Samba must be able to authenticate a real user \(em
	.Xr passwd 5 Ns / Ns Xr shadow 5 Ns - ,
	LDAP- or
	.Xr smbpasswd 5 Ns -based )
	by default.
	This means that any additional access control
	(disallow specific user specific access etc) must be done on the underlying file
	system.
	.Pp
	When the
	.Sy sharesmb
	property is updated with
	.Nm zfs Cm set Fl u
	, the property is set to desired value, but the operation to share, reshare
	or unshare the the dataset is not performed.
	.It Sy sharenfs Ns = Ns Sy on Ns \| Ns Sy off Ns \| Ns Ar opts
	Controls whether the file system is shared via NFS, and what options are to be
	used.
	A file system with a
	.Sy sharenfs
	property of
	.Sy off
	is managed with the
	.Xr exportfs 8
	command and entries in the
	.Pa /etc/exports
	file.
	Otherwise, the file system is automatically shared and unshared with the
	.Nm zfs Cm share
	and
	.Nm zfs Cm unshare
	commands.
	If the property is set to
	.Sy on ,
	the dataset is shared using the default options:
	.Dl sec=sys,rw,crossmnt,no_subtree_check
	.Pp
	Please note that the options are comma-separated, unlike those found in
	.Xr exports 5 .
	This is done to negate the need for quoting, as well as to make parsing
	with scripts easier.
	.Pp
	See
	.Xr exports 5
	for the meaning of the default options.
	Otherwise, the
	.Xr exportfs 8
	command is invoked with options equivalent to the contents of this property.
	.Pp
	When the
	.Sy sharenfs
	property is changed for a dataset, the dataset and any children inheriting the
	property are re-shared with the new options, only if the property was previously
	.Sy off ,
	or if they were shared before the property was changed.
	If the new property is
	.Sy off ,
	the file systems are unshared.
	.Pp
	When the
	.Sy sharenfs
	property is updated with
	.Nm zfs Cm set Fl u
	, the property is set to desired value, but the operation to share, reshare
	or unshare the the dataset is not performed.
	.It Sy logbias Ns = Ns Sy latency Ns \| Ns Sy throughput
	Provide a hint to ZFS about handling of synchronous requests in this dataset.
	If
	.Sy logbias
	is set to
	.Sy latency
	.Pq the default ,
	ZFS will use pool log devices
	.Pq if configured
	to handle the requests at low latency.
	If
	.Sy logbias
	is set to
	.Sy throughput ,
	ZFS will not use configured pool log devices.
	ZFS will instead optimize synchronous operations for global pool throughput and
	efficient use of resources.
	.It Sy snapdev Ns = Ns Sy hidden Ns \| Ns Sy visible
	Controls whether the volume snapshot devices under
	.Pa /dev/zvol/ Ns Aq Ar pool
	are hidden or visible.
	The default value is
	.Sy hidden .
	.It Sy snapdir Ns = Ns Sy hidden Ns \| Ns Sy visible
	Controls whether the
	.Pa .zfs
	directory is hidden or visible in the root of the file system as discussed in
	the
	.Sx Snapshots
	section of
	.Xr zfsconcepts 7 .
	The default value is
	.Sy hidden .
	.It Sy sync Ns = Ns Sy standard Ns \| Ns Sy always Ns \| Ns Sy disabled
	Controls the behavior of synchronous requests
	.Pq e.g. fsync, O_DSYNC .
	.Sy standard
	is the POSIX-specified behavior of ensuring all synchronous requests
	are written to stable storage and all devices are flushed to ensure
	data is not cached by device controllers
	.Pq this is the default .
	.Sy always
	causes every file system transaction to be written and flushed before its
	system call returns.
	This has a large performance penalty.
	.Sy disabled
	disables synchronous requests.
	File system transactions are only committed to stable storage periodically.
	This option will give the highest performance.
	However, it is very dangerous as ZFS would be ignoring the synchronous
	transaction demands of applications such as databases or NFS.
	Administrators should only use this option when the risks are understood.
	.It Sy version Ns = Ns Ar N Ns \| Ns Sy current
	The on-disk version of this file system, which is independent of the pool
	version.
	This property can only be set to later supported versions.
	See the
	.Nm zfs Cm upgrade
	command.
	.It Sy volsize Ns = Ns Ar size
	For volumes, specifies the logical size of the volume.
	By default, creating a volume establishes a reservation of equal size.
	For storage pools with a version number of 9 or higher, a
	.Sy refreservation
	is set instead.
	Any changes to
	.Sy volsize
	are reflected in an equivalent change to the reservation
	.Pq or Sy refreservation .
	The
	.Sy volsize
	can only be set to a multiple of
	.Sy volblocksize ,
	and cannot be zero.
	.Pp
	The reservation is kept equal to the volume's logical size to prevent unexpected
	behavior for consumers.
	Without the reservation, the volume could run out of space, resulting in
	undefined behavior or data corruption, depending on how the volume is used.
	These effects can also occur when the volume size is changed while it is in use
	.Pq particularly when shrinking the size .
	Extreme care should be used when adjusting the volume size.
	.Pp
	Though not recommended, a
	.Qq sparse volume
	.Po also known as
	.Qq thin provisioned
	.Pc
	can be created by specifying the
	.Fl s
	option to the
	.Nm zfs Cm create Fl V
	command, or by changing the value of the
	.Sy refreservation
	property
	.Po or
	.Sy reservation
	property on pool version 8 or earlier
	.Pc
	after the volume has been created.
	A
	.Qq sparse volume
	is a volume where the value of
	.Sy refreservation
	is less than the size of the volume plus the space required to store its
	metadata.
	Consequently, writes to a sparse volume can fail with
	.Er ENOSPC
	when the pool is low on space.
	For a sparse volume, changes to
	.Sy volsize
	are not reflected in the
	.Sy refreservation .
	A volume that is not sparse is said to be
	.Qq thick provisioned .
	A sparse volume can become thick provisioned by setting
	.Sy refreservation
	to
	.Sy auto .
	.It Sy volmode Ns = Ns Sy default Ns \| Ns Sy full Ns \| Ns Sy geom Ns \| Ns Sy dev Ns \| Ns Sy none
	This property specifies how volumes should be exposed to the OS.
	Setting it to
	.Sy full
	exposes volumes as fully fledged block devices, providing maximal
	functionality.
	The value
	.Sy geom
	is just an alias for
	.Sy full
	and is kept for compatibility.
	Setting it to
	.Sy dev
	hides its partitions.
	Volumes with property set to
	.Sy none
	are not exposed outside ZFS, but can be snapshotted, cloned, replicated, etc,
	that can be suitable for backup purposes.
	Value
	.Sy default
	means that volumes exposition is controlled by system-wide tunable
	.Sy zvol_volmode ,
	where
	.Sy full ,
	.Sy dev
	and
	.Sy none
	are encoded as 1, 2 and 3 respectively.
	The default value is
	.Sy full .
	.It Sy vscan Ns = Ns Sy on Ns \| Ns Sy off
	Controls whether regular files should be scanned for viruses when a file is
	opened and closed.
	In addition to enabling this property, the virus scan service must also be
	enabled for virus scanning to occur.
	The default value is
	.Sy off .
	This property is not used by OpenZFS.
	.It Sy xattr Ns = Ns Sy on Ns \| Ns Sy off Ns \| Ns Sy sa
	Controls whether extended attributes are enabled for this file system.
	Two styles of extended attributes are supported: either directory-based
	or system-attribute-based.
	.Pp
	The default value of
	.Sy on
	enables directory-based extended attributes.
	This style of extended attribute imposes no practical limit
	on either the size or number of attributes which can be set on a file.
	Although under Linux the
	.Xr getxattr 2
	and
	.Xr setxattr 2
	system calls limit the maximum size to
	.Sy 64K .
	This is the most compatible
	style of extended attribute and is supported by all ZFS implementations.
	.Pp
	System-attribute-based xattrs can be enabled by setting the value to
	.Sy sa .
	The key advantage of this type of xattr is improved performance.
	Storing extended attributes as system attributes
	significantly decreases the amount of disk I/O required.
	Up to
	.Sy 64K
	of data may be stored per-file in the space reserved for system attributes.
	If there is not enough space available for an extended attribute
	then it will be automatically written as a directory-based xattr.
	System-attribute-based extended attributes are not accessible
	on platforms which do not support the
	.Sy xattr Ns = Ns Sy sa
	feature.
	OpenZFS supports
	.Sy xattr Ns = Ns Sy sa
	on both
	.Fx
	and Linux.
	.Pp
	The use of system-attribute-based xattrs is strongly encouraged for users of
	SELinux or POSIX ACLs.
	Both of these features heavily rely on extended
	attributes and benefit significantly from the reduced access time.
	.Pp
	The values
	.Sy on
	and
	.Sy off
	are equivalent to the
	.Sy xattr
	and
	.Sy noxattr
	mount options.
	.It Sy jailed Ns = Ns Sy off Ns \| Ns Sy on
	Controls whether the dataset is managed from a jail.
	See
	.Xr zfs-jail 8
	for more information.
	Jails are a
	.Fx
	feature and this property is not available on other platforms.
	.It Sy zoned Ns = Ns Sy off Ns \| Ns Sy on
	Controls whether the dataset is managed from a non-global zone or namespace.
	See
	.Xr zfs-zone 8
	for more information.
	Zoning is a
	Linux
	feature and this property is not available on other platforms.
	.El
	.Pp
	The following three properties cannot be changed after the file system is
	created, and therefore, should be set when the file system is created.
	If the properties are not set with the
	.Nm zfs Cm create
	or
	.Nm zpool Cm create
	commands, these properties are inherited from the parent dataset.
	If the parent dataset lacks these properties due to having been created prior to
	these features being supported, the new file system will have the default values
	for these properties.
	.Bl -tag -width ""
	.It Xo
	.Sy casesensitivity Ns = Ns Sy sensitive Ns \| Ns
	.Sy insensitive Ns \| Ns Sy mixed
	.Xc
	Indicates whether the file name matching algorithm used by the file system
	should be case-sensitive, case-insensitive, or allow a combination of both
	styles of matching.
	The default value for the
	.Sy casesensitivity
	property is
	.Sy sensitive .
	Traditionally,
	.Ux
	and POSIX file systems have case-sensitive file names.
	.Pp
	The
	.Sy mixed
	value for the
	.Sy casesensitivity
	property indicates that the file system can support requests for both
	case-sensitive and case-insensitive matching behavior.
	Currently, case-insensitive matching behavior on a file system that supports
	mixed behavior is limited to the SMB server product.
	For more information about the
	.Sy mixed
	value behavior, see the "ZFS Administration Guide".
	.It Xo
	.Sy normalization Ns = Ns Sy none Ns \| Ns Sy formC Ns \| Ns
	.Sy formD Ns \| Ns Sy formKC Ns \| Ns Sy formKD
	.Xc
	Indicates whether the file system should perform a
	.Sy unicode
	normalization of file names whenever two file names are compared, and which
	normalization algorithm should be used.
	File names are always stored unmodified, names are normalized as part of any
	comparison process.
	If this property is set to a legal value other than
	.Sy none ,
	and the
	.Sy utf8only
	property was left unspecified, the
	.Sy utf8only
	property is automatically set to
	.Sy on .
	The default value of the
	.Sy normalization
	property is
	.Sy none .
	This property cannot be changed after the file system is created.
	.It Sy utf8only Ns = Ns Sy on Ns \| Ns Sy off
	Indicates whether the file system should reject file names that include
	characters that are not present in the
	.Sy UTF-8
	character code set.
	If this property is explicitly set to
	.Sy off ,
	the normalization property must either not be explicitly set or be set to
	.Sy none .
	The default value for the
	.Sy utf8only
	property is
	.Sy off .
	This property cannot be changed after the file system is created.
	.El
	.Pp
	The
	.Sy casesensitivity ,
	.Sy normalization ,
	and
	.Sy utf8only
	properties are also new permissions that can be assigned to non-privileged users
	by using the ZFS delegated administration feature.
	.
	.Ss Temporary Mount Point Properties
	When a file system is mounted, either through
	.Xr mount 8
	for legacy mounts or the
	.Nm zfs Cm mount
	command for normal file systems, its mount options are set according to its
	properties.
	The correlation between properties and mount options is as follows:
	.Bl -tag -compact -offset Ds -width "rootcontext="
	.It Sy atime
	atime/noatime
	.It Sy canmount
	auto/noauto
	.It Sy devices
	dev/nodev
	.It Sy exec
	exec/noexec
	.It Sy readonly
	ro/rw
	.It Sy relatime
	relatime/norelatime
	.It Sy setuid
	suid/nosuid
	.It Sy xattr
	xattr/noxattr
	.It Sy nbmand
	mand/nomand
	.It Sy context Ns =
	context=
	.It Sy fscontext Ns =
	fscontext=
	.It Sy defcontext Ns =
	defcontext=
	.It Sy rootcontext Ns =
	rootcontext=
	.El
	.Pp
	In addition, these options can be set on a per-mount basis using the
	.Fl o
	option, without affecting the property that is stored on disk.
	The values specified on the command line override the values stored in the
	dataset.
	The
	.Sy nosuid
	option is an alias for
	.Sy nodevices , Ns Sy nosetuid .
	These properties are reported as
	.Qq temporary
	by the
	.Nm zfs Cm get
	command.
	If the properties are changed while the dataset is mounted, the new setting
	overrides any temporary settings.
	.
	.Ss User Properties
	In addition to the standard native properties, ZFS supports arbitrary user
	properties.
	User properties have no effect on ZFS behavior, but applications or
	administrators can use them to annotate datasets
	.Pq file systems, volumes, and snapshots .
	.Pp
	User property names must contain a colon
	.Pq Qq Sy \&:
	character to distinguish them from native properties.
	They may contain lowercase letters, numbers, and the following punctuation
	characters: colon
	.Pq Qq Sy \&: ,
	dash
	.Pq Qq Sy - ,
	period
	.Pq Qq Sy \&. ,
	and underscore
	.Pq Qq Sy _ .
	The expected convention is that the property name is divided into two portions
	such as
	.Ar module : Ns Ar property ,
	but this namespace is not enforced by ZFS.
	User property names can be at most 256 characters, and cannot begin with a dash
	.Pq Qq Sy - .
	.Pp
	When making programmatic use of user properties, it is strongly suggested to use
	a reversed DNS domain name for the
	.Ar module
	component of property names to reduce the chance that two
	independently-developed packages use the same property name for different
	purposes.
	.Pp
	The values of user properties are arbitrary strings, are always inherited, and
	are never validated.
	All of the commands that operate on properties
	.Po Nm zfs Cm list ,
	.Nm zfs Cm get ,
	.Nm zfs Cm set ,
	and so forth
	.Pc
	can be used to manipulate both native properties and user properties.
	Use the
	.Nm zfs Cm inherit
	command to clear a user property.
	If the property is not defined in any parent dataset, it is removed entirely.
	Property values are limited to 8192 bytes.
	diff --git a/sys/contrib/openzfs/man/man7/zpoolconcepts.7 b/sys/contrib/openzfs/man/man7/zpoolconcepts.7
	index 98f3ee7cd660..18dfca6dc8ac 100644
	--- a/sys/contrib/openzfs/man/man7/zpoolconcepts.7
	+++ b/sys/contrib/openzfs/man/man7/zpoolconcepts.7
	@@ -1,512 +1,512 @@
	.\"
	.\" CDDL HEADER START
	.\"
	.\" The contents of this file are subject to the terms of the
	.\" Common Development and Distribution License (the "License").
	.\" You may not use this file except in compliance with the License.
	.\"
	.\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	.\" or https://opensource.org/licenses/CDDL-1.0.
	.\" See the License for the specific language governing permissions
	.\" and limitations under the License.
	.\"
	.\" When distributing Covered Code, include this CDDL HEADER in each
	.\" file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	.\" If applicable, add the following below this CDDL HEADER, with the
	.\" fields enclosed by brackets "[]" replaced with your own identifying
	.\" information: Portions Copyright [yyyy] [name of copyright owner]
	.\"
	.\" CDDL HEADER END
	.\"
	.\" Copyright (c) 2007, Sun Microsystems, Inc. All Rights Reserved.
	.\" Copyright (c) 2012, 2018 by Delphix. All rights reserved.
	.\" Copyright (c) 2012 Cyril Plisko. All Rights Reserved.
	.\" Copyright (c) 2017 Datto Inc.
	.\" Copyright (c) 2018 George Melikov. All Rights Reserved.
	.\" Copyright 2017 Nexenta Systems, Inc.
	.\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
	.\"
	.Dd April 7, 2023
	.Dt ZPOOLCONCEPTS 7
	.Os
	.
	.Sh NAME
	.Nm zpoolconcepts
	.Nd overview of ZFS storage pools
	.
	.Sh DESCRIPTION
	.Ss Virtual Devices (vdevs)
	A "virtual device" describes a single device or a collection of devices,
	organized according to certain performance and fault characteristics.
	The following virtual devices are supported:
	.Bl -tag -width "special"
	.It Sy disk
	A block device, typically located under
	.Pa /dev .
	ZFS can use individual slices or partitions, though the recommended mode of
	operation is to use whole disks.
	A disk can be specified by a full path, or it can be a shorthand name
	.Po the relative portion of the path under
	.Pa /dev
	.Pc .
	A whole disk can be specified by omitting the slice or partition designation.
	For example,
	.Pa sda
	is equivalent to
	.Pa /dev/sda .
	When given a whole disk, ZFS automatically labels the disk, if necessary.
	.It Sy file
	A regular file.
	The use of files as a backing store is strongly discouraged.
	It is designed primarily for experimental purposes, as the fault tolerance of a
	file is only as good as the file system on which it resides.
	A file must be specified by a full path.
	.It Sy mirror
	A mirror of two or more devices.
	Data is replicated in an identical fashion across all components of a mirror.
	A mirror with
	.Em N No disks of size Em X No can hold Em X No bytes and can withstand Em N-1
	devices failing, without losing data.
	.It Sy raidz , raidz1 , raidz2 , raidz3
	A distributed-parity layout, similar to RAID-5/6, with improved distribution of
	parity, and which does not suffer from the RAID-5/6
	.Qq write hole ,
	.Pq in which data and parity become inconsistent after a power loss .
	Data and parity is striped across all disks within a raidz group, though not
	necessarily in a consistent stripe width.
	.Pp
	A raidz group can have single, double, or triple parity, meaning that the
	raidz group can sustain one, two, or three failures, respectively, without
	losing any data.
	The
	.Sy raidz1
	vdev type specifies a single-parity raidz group; the
	.Sy raidz2
	vdev type specifies a double-parity raidz group; and the
	.Sy raidz3
	vdev type specifies a triple-parity raidz group.
	The
	.Sy raidz
	vdev type is an alias for
	.Sy raidz1 .
	.Pp
	A raidz group with
	.Em N No disks of size Em X No with Em P No parity disks can hold approximately
	.Em (N-P)*X No bytes and can withstand Em P No devices failing without losing data .
	The minimum number of devices in a raidz group is one more than the number of
	parity disks.
	The recommended number is between 3 and 9 to help increase performance.
	.It Sy draid , draid1 , draid2 , draid3
	A variant of raidz that provides integrated distributed hot spares, allowing
	for faster resilvering, while retaining the benefits of raidz.
	A dRAID vdev is constructed from multiple internal raidz groups, each with
	.Em D No data devices and Em P No parity devices .
	These groups are distributed over all of the children in order to fully
	utilize the available disk performance.
	.Pp
	Unlike raidz, dRAID uses a fixed stripe width (padding as necessary with
	zeros) to allow fully sequential resilvering.
	This fixed stripe width significantly affects both usable capacity and IOPS.
	For example, with the default
	.Em D=8 No and Em 4 KiB No disk sectors the minimum allocation size is Em 32 KiB .
	If using compression, this relatively large allocation size can reduce the
	effective compression ratio.
	When using ZFS volumes (zvols) and dRAID, the default of the
	.Sy volblocksize
	property is increased to account for the allocation size.
	If a dRAID pool will hold a significant amount of small blocks, it is
	recommended to also add a mirrored
	.Sy special
	vdev to store those blocks.
	.Pp
	In regards to I/O, performance is similar to raidz since, for any read, all
	.Em D No data disks must be accessed .
	Delivered random IOPS can be reasonably approximated as
	.Sy floor((N-S)/(D+P))*single_drive_IOPS .
	.Pp
	Like raidz, a dRAID can have single-, double-, or triple-parity.
	The
	.Sy draid1 ,
	.Sy draid2 ,
	and
	.Sy draid3
	types can be used to specify the parity level.
	The
	.Sy draid
	vdev type is an alias for
	.Sy draid1 .
	.Pp
	A dRAID with
	.Em N No disks of size Em X , D No data disks per redundancy group , Em P
	.No parity level, and Em S No distributed hot spares can hold approximately
	.Em (N-S)(D/(D+P))X No bytes and can withstand Em P
	devices failing without losing data.
	.It Sy draid Ns Oo Ar parity Oc Ns Oo Sy \&: Ns Ar data Ns Sy d Oc Ns Oo Sy \&: Ns Ar children Ns Sy c Oc Ns Oo Sy \&: Ns Ar spares Ns Sy s Oc
	A non-default dRAID configuration can be specified by appending one or more
	of the following optional arguments to the
	.Sy draid
	keyword:
	.Bl -tag -compact -width "children"
	.It Ar parity
	The parity level (1-3).
	.It Ar data
	The number of data devices per redundancy group.
	In general, a smaller value of
	.Em D No will increase IOPS, improve the compression ratio ,
	and speed up resilvering at the expense of total usable capacity.
	Defaults to
	.Em 8 , No unless Em N-P-S No is less than Em 8 .
	.It Ar children
	The expected number of children.
	Useful as a cross-check when listing a large number of devices.
	An error is returned when the provided number of children differs.
	.It Ar spares
	The number of distributed hot spares.
	Defaults to zero.
	.El
	.It Sy spare
	A pseudo-vdev which keeps track of available hot spares for a pool.
	For more information, see the
	.Sx Hot Spares
	section.
	.It Sy log
	A separate intent log device.
	If more than one log device is specified, then writes are load-balanced between
	devices.
	Log devices can be mirrored.
	However, raidz vdev types are not supported for the intent log.
	For more information, see the
	.Sx Intent Log
	section.
	.It Sy dedup
	A device solely dedicated for deduplication tables.
	The redundancy of this device should match the redundancy of the other normal
	devices in the pool.
	If more than one dedup device is specified, then
	allocations are load-balanced between those devices.
	.It Sy special
	A device dedicated solely for allocating various kinds of internal metadata,
	and optionally small file blocks.
	The redundancy of this device should match the redundancy of the other normal
	devices in the pool.
	If more than one special device is specified, then
	allocations are load-balanced between those devices.
	.Pp
	For more information on special allocations, see the
	.Sx Special Allocation Class
	section.
	.It Sy cache
	A device used to cache storage pool data.
	A cache device cannot be configured as a mirror or raidz group.
	For more information, see the
	.Sx Cache Devices
	section.
	.El
	.Pp
	Virtual devices cannot be nested arbitrarily.
	A mirror, raidz or draid virtual device can only be created with files or disks.
	Mirrors of mirrors or other such combinations are not allowed.
	.Pp
	A pool can have any number of virtual devices at the top of the configuration
	.Po known as
	.Qq root vdevs
	.Pc .
	Data is dynamically distributed across all top-level devices to balance data
	among devices.
	As new virtual devices are added, ZFS automatically places data on the newly
	available devices.
	.Pp
	Virtual devices are specified one at a time on the command line,
	separated by whitespace.
	Keywords like
	.Sy mirror No and Sy raidz
	are used to distinguish where a group ends and another begins.
	For example, the following creates a pool with two root vdevs,
	each a mirror of two disks:
	.Dl # Nm zpool Cm create Ar mypool Sy mirror Ar sda sdb Sy mirror Ar sdc sdd
	.
	.Ss Device Failure and Recovery
	ZFS supports a rich set of mechanisms for handling device failure and data
	corruption.
	All metadata and data is checksummed, and ZFS automatically repairs bad data
	from a good copy, when corruption is detected.
	.Pp
	In order to take advantage of these features, a pool must make use of some form
	of redundancy, using either mirrored or raidz groups.
	While ZFS supports running in a non-redundant configuration, where each root
	vdev is simply a disk or file, this is strongly discouraged.
	A single case of bit corruption can render some or all of your data unavailable.
	.Pp
	A pool's health status is described by one of three states:
	.Sy online , degraded , No or Sy faulted .
	An online pool has all devices operating normally.
	A degraded pool is one in which one or more devices have failed, but the data is
	still available due to a redundant configuration.
	A faulted pool has corrupted metadata, or one or more faulted devices, and
	insufficient replicas to continue functioning.
	.Pp
	The health of the top-level vdev, such as a mirror or raidz device,
	is potentially impacted by the state of its associated vdevs
	or component devices.
	A top-level vdev or component device is in one of the following states:
	.Bl -tag -width "DEGRADED"
	.It Sy DEGRADED
	One or more top-level vdevs is in the degraded state because one or more
	component devices are offline.
	Sufficient replicas exist to continue functioning.
	.Pp
	One or more component devices is in the degraded or faulted state, but
	sufficient replicas exist to continue functioning.
	The underlying conditions are as follows:
	.Bl -bullet -compact
	.It
	-The number of checksum errors exceeds acceptable levels and the device is
	-degraded as an indication that something may be wrong.
	+The number of checksum errors or slow I/Os exceeds acceptable levels and the
	+device is degraded as an indication that something may be wrong.
	ZFS continues to use the device as necessary.
	.It
	The number of I/O errors exceeds acceptable levels.
	The device could not be marked as faulted because there are insufficient
	replicas to continue functioning.
	.El
	.It Sy FAULTED
	One or more top-level vdevs is in the faulted state because one or more
	component devices are offline.
	Insufficient replicas exist to continue functioning.
	.Pp
	One or more component devices is in the faulted state, and insufficient
	replicas exist to continue functioning.
	The underlying conditions are as follows:
	.Bl -bullet -compact
	.It
	The device could be opened, but the contents did not match expected values.
	.It
	The number of I/O errors exceeds acceptable levels and the device is faulted to
	prevent further use of the device.
	.El
	.It Sy OFFLINE
	The device was explicitly taken offline by the
	.Nm zpool Cm offline
	command.
	.It Sy ONLINE
	The device is online and functioning.
	.It Sy REMOVED
	The device was physically removed while the system was running.
	Device removal detection is hardware-dependent and may not be supported on all
	platforms.
	.It Sy UNAVAIL
	The device could not be opened.
	If a pool is imported when a device was unavailable, then the device will be
	identified by a unique identifier instead of its path since the path was never
	correct in the first place.
	.El
	.Pp
	Checksum errors represent events where a disk returned data that was expected
	to be correct, but was not.
	In other words, these are instances of silent data corruption.
	The checksum errors are reported in
	.Nm zpool Cm status
	and
	.Nm zpool Cm events .
	When a block is stored redundantly, a damaged block may be reconstructed
	(e.g. from raidz parity or a mirrored copy).
	In this case, ZFS reports the checksum error against the disks that contained
	damaged data.
	If a block is unable to be reconstructed (e.g. due to 3 disks being damaged
	in a raidz2 group), it is not possible to determine which disks were silently
	corrupted.
	In this case, checksum errors are reported for all disks on which the block
	is stored.
	.Pp
	If a device is removed and later re-attached to the system,
	ZFS attempts to bring the device online automatically.
	Device attachment detection is hardware-dependent
	and might not be supported on all platforms.
	.
	.Ss Hot Spares
	ZFS allows devices to be associated with pools as
	.Qq hot spares .
	These devices are not actively used in the pool.
	But, when an active device
	fails, it is automatically replaced by a hot spare.
	To create a pool with hot spares, specify a
	.Sy spare
	vdev with any number of devices.
	For example,
	.Dl # Nm zpool Cm create Ar pool Sy mirror Ar sda sdb Sy spare Ar sdc sdd
	.Pp
	Spares can be shared across multiple pools, and can be added with the
	.Nm zpool Cm add
	command and removed with the
	.Nm zpool Cm remove
	command.
	Once a spare replacement is initiated, a new
	.Sy spare
	vdev is created within the configuration that will remain there until the
	original device is replaced.
	At this point, the hot spare becomes available again, if another device fails.
	.Pp
	If a pool has a shared spare that is currently being used, the pool cannot be
	exported, since other pools may use this shared spare, which may lead to
	potential data corruption.
	.Pp
	Shared spares add some risk.
	If the pools are imported on different hosts,
	and both pools suffer a device failure at the same time,
	both could attempt to use the spare at the same time.
	This may not be detected, resulting in data corruption.
	.Pp
	An in-progress spare replacement can be cancelled by detaching the hot spare.
	If the original faulted device is detached, then the hot spare assumes its
	place in the configuration, and is removed from the spare list of all active
	pools.
	.Pp
	The
	.Sy draid
	vdev type provides distributed hot spares.
	These hot spares are named after the dRAID vdev they're a part of
	.Po Sy draid1 Ns - Ns Ar 2 Ns - Ns Ar 3 No specifies spare Ar 3 No of vdev Ar 2 ,
	.No which is a single parity dRAID Pc
	and may only be used by that dRAID vdev.
	Otherwise, they behave the same as normal hot spares.
	.Pp
	Spares cannot replace log devices.
	.
	.Ss Intent Log
	The ZFS Intent Log (ZIL) satisfies POSIX requirements for synchronous
	transactions.
	For instance, databases often require their transactions to be on stable storage
	devices when returning from a system call.
	NFS and other applications can also use
	.Xr fsync 2
	to ensure data stability.
	By default, the intent log is allocated from blocks within the main pool.
	However, it might be possible to get better performance using separate intent
	log devices such as NVRAM or a dedicated disk.
	For example:
	.Dl # Nm zpool Cm create Ar pool sda sdb Sy log Ar sdc
	.Pp
	Multiple log devices can also be specified, and they can be mirrored.
	See the
	.Sx EXAMPLES
	section for an example of mirroring multiple log devices.
	.Pp
	Log devices can be added, replaced, attached, detached, and removed.
	In addition, log devices are imported and exported as part of the pool
	that contains them.
	Mirrored devices can be removed by specifying the top-level mirror vdev.
	.
	.Ss Cache Devices
	Devices can be added to a storage pool as
	.Qq cache devices .
	These devices provide an additional layer of caching between main memory and
	disk.
	For read-heavy workloads, where the working set size is much larger than what
	can be cached in main memory, using cache devices allows much more of this
	working set to be served from low latency media.
	Using cache devices provides the greatest performance improvement for random
	read-workloads of mostly static content.
	.Pp
	To create a pool with cache devices, specify a
	.Sy cache
	vdev with any number of devices.
	For example:
	.Dl # Nm zpool Cm create Ar pool sda sdb Sy cache Ar sdc sdd
	.Pp
	Cache devices cannot be mirrored or part of a raidz configuration.
	If a read error is encountered on a cache device, that read I/O is reissued to
	the original storage pool device, which might be part of a mirrored or raidz
	configuration.
	.Pp
	The content of the cache devices is persistent across reboots and restored
	asynchronously when importing the pool in L2ARC (persistent L2ARC).
	This can be disabled by setting
	.Sy l2arc_rebuild_enabled Ns = Ns Sy 0 .
	For cache devices smaller than
	.Em 1 GiB ,
	ZFS does not write the metadata structures
	required for rebuilding the L2ARC, to conserve space.
	This can be changed with
	.Sy l2arc_rebuild_blocks_min_l2size .
	The cache device header
	.Pq Em 512 B
	is updated even if no metadata structures are written.
	Setting
	.Sy l2arc_headroom Ns = Ns Sy 0
	will result in scanning the full-length ARC lists for cacheable content to be
	written in L2ARC (persistent ARC).
	If a cache device is added with
	.Nm zpool Cm add ,
	its label and header will be overwritten and its contents will not be
	restored in L2ARC, even if the device was previously part of the pool.
	If a cache device is onlined with
	.Nm zpool Cm online ,
	its contents will be restored in L2ARC.
	This is useful in case of memory pressure,
	where the contents of the cache device are not fully restored in L2ARC.
	The user can off- and online the cache device when there is less memory
	pressure, to fully restore its contents to L2ARC.
	.
	.Ss Pool checkpoint
	Before starting critical procedures that include destructive actions
	.Pq like Nm zfs Cm destroy ,
	an administrator can checkpoint the pool's state and, in the case of a
	mistake or failure, rewind the entire pool back to the checkpoint.
	Otherwise, the checkpoint can be discarded when the procedure has completed
	successfully.
	.Pp
	A pool checkpoint can be thought of as a pool-wide snapshot and should be used
	with care as it contains every part of the pool's state, from properties to vdev
	configuration.
	Thus, certain operations are not allowed while a pool has a checkpoint.
	Specifically, vdev removal/attach/detach, mirror splitting, and
	changing the pool's GUID.
	Adding a new vdev is supported, but in the case of a rewind it will have to be
	added again.
	Finally, users of this feature should keep in mind that scrubs in a pool that
	has a checkpoint do not repair checkpointed data.
	.Pp
	To create a checkpoint for a pool:
	.Dl # Nm zpool Cm checkpoint Ar pool
	.Pp
	To later rewind to its checkpointed state, you need to first export it and
	then rewind it during import:
	.Dl # Nm zpool Cm export Ar pool
	.Dl # Nm zpool Cm import Fl -rewind-to-checkpoint Ar pool
	.Pp
	To discard the checkpoint from a pool:
	.Dl # Nm zpool Cm checkpoint Fl d Ar pool
	.Pp
	Dataset reservations (controlled by the
	.Sy reservation No and Sy refreservation
	properties) may be unenforceable while a checkpoint exists, because the
	checkpoint is allowed to consume the dataset's reservation.
	Finally, data that is part of the checkpoint but has been freed in the
	current state of the pool won't be scanned during a scrub.
	.
	.Ss Special Allocation Class
	Allocations in the special class are dedicated to specific block types.
	By default, this includes all metadata, the indirect blocks of user data, and
	any deduplication tables.
	The class can also be provisioned to accept small file blocks.
	.Pp
	A pool must always have at least one normal
	.Pq non- Ns Sy dedup Ns /- Ns Sy special
	vdev before
	other devices can be assigned to the special class.
	If the
	.Sy special
	class becomes full, then allocations intended for it
	will spill back into the normal class.
	.Pp
	Deduplication tables can be excluded from the special class by unsetting the
	.Sy zfs_ddt_data_is_special
	ZFS module parameter.
	.Pp
	Inclusion of small file blocks in the special class is opt-in.
	Each dataset can control the size of small file blocks allowed
	in the special class by setting the
	.Sy special_small_blocks
	property to nonzero.
	See
	.Xr zfsprops 7
	for more info on this property.
	diff --git a/sys/contrib/openzfs/man/man8/zfs-mount.8 b/sys/contrib/openzfs/man/man8/zfs-mount.8
	index 35aa187cf063..20dbe4d0e648 100644
	--- a/sys/contrib/openzfs/man/man8/zfs-mount.8
	+++ b/sys/contrib/openzfs/man/man8/zfs-mount.8
	@@ -1,131 +1,133 @@
	.\"
	.\" CDDL HEADER START
	.\"
	.\" The contents of this file are subject to the terms of the
	.\" Common Development and Distribution License (the "License").
	.\" You may not use this file except in compliance with the License.
	.\"
	.\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	.\" or https://opensource.org/licenses/CDDL-1.0.
	.\" See the License for the specific language governing permissions
	.\" and limitations under the License.
	.\"
	.\" When distributing Covered Code, include this CDDL HEADER in each
	.\" file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	.\" If applicable, add the following below this CDDL HEADER, with the
	.\" fields enclosed by brackets "[]" replaced with your own identifying
	.\" information: Portions Copyright [yyyy] [name of copyright owner]
	.\"
	.\" CDDL HEADER END
	.\"
	.\" Copyright (c) 2009 Sun Microsystems, Inc. All Rights Reserved.
	.\" Copyright 2011 Joshua M. Clulow <josh@sysmgr.org>
	.\" Copyright (c) 2011, 2019 by Delphix. All rights reserved.
	.\" Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
	.\" Copyright (c) 2014, Joyent, Inc. All rights reserved.
	.\" Copyright (c) 2014 by Adam Stevko. All rights reserved.
	.\" Copyright (c) 2014 Integros [integros.com]
	.\" Copyright 2019 Richard Laager. All rights reserved.
	.\" Copyright 2018 Nexenta Systems, Inc.
	.\" Copyright 2019 Joyent, Inc.
	.\"
	.Dd February 16, 2019
	.Dt ZFS-MOUNT 8
	.Os
	.
	.Sh NAME
	.Nm zfs-mount
	.Nd manage mount state of ZFS filesystems
	.Sh SYNOPSIS
	.Nm zfs
	.Cm mount
	.Nm zfs
	.Cm mount
	.Op Fl Oflv
	.Op Fl o Ar options
	-.Fl a Ns \| Ns Ar filesystem
	+.Fl a Ns \| Ns Fl R Ar filesystem Ns \| Ns Ar filesystem
	.Nm zfs
	.Cm unmount
	.Op Fl fu
	.Fl a Ns \| Ns Ar filesystem Ns \| Ns Ar mountpoint
	.
	.Sh DESCRIPTION
	.Bl -tag -width ""
	.It Xo
	.Nm zfs
	.Cm mount
	.Xc
	Displays all ZFS file systems currently mounted.
	.It Xo
	.Nm zfs
	.Cm mount
	.Op Fl Oflv
	.Op Fl o Ar options
	-.Fl a Ns \| Ns Ar filesystem
	+.Fl a Ns \| Ns Fl R Ar filesystem Ns \| Ns Ar filesystem
	.Xc
	Mount ZFS filesystem on a path described by its
	.Sy mountpoint
	property, if the path exists and is empty.
	If
	.Sy mountpoint
	is set to
	.Em legacy ,
	the filesystem should be instead mounted using
	.Xr mount 8 .
	.Bl -tag -width "-O"
	.It Fl O
	Perform an overlay mount.
	Allows mounting in non-empty
	.Sy mountpoint .
	See
	.Xr mount 8
	for more information.
	.It Fl a
	Mount all available ZFS file systems.
	Invoked automatically as part of the boot process if configured.
	+.It Fl R
	+Mount the specified filesystems along with all their children.
	.It Ar filesystem
	Mount the specified filesystem.
	.It Fl o Ar options
	An optional, comma-separated list of mount options to use temporarily for the
	duration of the mount.
	See the
	.Em Temporary Mount Point Properties
	section of
	.Xr zfsprops 7
	for details.
	.It Fl l
	Load keys for encrypted filesystems as they are being mounted.
	This is equivalent to executing
	.Nm zfs Cm load-key
	on each encryption root before mounting it.
	Note that if a filesystem has
	.Sy keylocation Ns = Ns Sy prompt ,
	this will cause the terminal to interactively block after asking for the key.
	.It Fl v
	Report mount progress.
	.It Fl f
	Attempt to force mounting of all filesystems, even those that couldn't normally
	be mounted (e.g. redacted datasets).
	.El
	.It Xo
	.Nm zfs
	.Cm unmount
	.Op Fl fu
	.Fl a Ns \| Ns Ar filesystem Ns \| Ns Ar mountpoint
	.Xc
	Unmounts currently mounted ZFS file systems.
	.Bl -tag -width "-a"
	.It Fl a
	Unmount all available ZFS file systems.
	Invoked automatically as part of the shutdown process.
	.It Fl f
	Forcefully unmount the file system, even if it is currently in use.
	This option is not supported on Linux.
	.It Fl u
	Unload keys for any encryption roots unmounted by this command.
	.It Ar filesystem Ns \| Ns Ar mountpoint
	Unmount the specified filesystem.
	The command can also be given a path to a ZFS file system mount point on the
	system.
	.El
	.El
	diff --git a/sys/contrib/openzfs/man/man8/zinject.8 b/sys/contrib/openzfs/man/man8/zinject.8
	index 4f0bbae81212..b692f12130a8 100644
	--- a/sys/contrib/openzfs/man/man8/zinject.8
	+++ b/sys/contrib/openzfs/man/man8/zinject.8
	@@ -1,296 +1,297 @@
	.\"
	.\" CDDL HEADER START
	.\"
	.\" The contents of this file are subject to the terms of the
	.\" Common Development and Distribution License (the "License").
	.\" You may not use this file except in compliance with the License.
	.\"
	.\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	.\" or https://opensource.org/licenses/CDDL-1.0.
	.\" See the License for the specific language governing permissions
	.\" and limitations under the License.
	.\"
	.\" When distributing Covered Code, include this CDDL HEADER in each
	.\" file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	.\" If applicable, add the following below this CDDL HEADER, with the
	.\" fields enclosed by brackets "[]" replaced with your own identifying
	.\" information: Portions Copyright [yyyy] [name of copyright owner]
	.\"
	.\" CDDL HEADER END
	.\"
	.\" Copyright 2013 Darik Horn <dajhorn@vanadac.com>. All rights reserved.
	.\"
	.\" lint-ok: WARNING: sections out of conventional order: Sh SYNOPSIS
	.\"
	.Dd May 26, 2021
	.Dt ZINJECT 8
	.Os
	.
	.Sh NAME
	.Nm zinject
	.Nd ZFS Fault Injector
	.Sh DESCRIPTION
	.Nm
	creates artificial problems in a ZFS pool by simulating data corruption
	or device failures.
	This program is dangerous.
	.
	.Sh SYNOPSIS
	.Bl -tag -width Ds
	.It Xo
	.Nm zinject
	.Xc
	List injection records.
	.
	.It Xo
	.Nm zinject
	.Fl b Ar objset : Ns Ar object : Ns Ar level : Ns Ar start : Ns Ar end
	.Op Fl f Ar frequency
	.Fl amu
	.Op pool
	.Xc
	Force an error into the pool at a bookmark.
	.
	.It Xo
	.Nm zinject
	.Fl c Ar id Ns \| Ns Sy all
	.Xc
	Cancel injection records.
	.
	.It Xo
	.Nm zinject
	.Fl d Ar vdev
	.Fl A Sy degrade Ns \| Ns Sy fault
	.Ar pool
	.Xc
	Force a vdev into the DEGRADED or FAULTED state.
	.
	.It Xo
	.Nm zinject
	.Fl d Ar vdev
	.Fl D Ar latency : Ns Ar lanes
	+.Op Fl T Ar read\|write
	.Ar pool
	.Xc
	Add an artificial delay to I/O requests on a particular
	device, such that the requests take a minimum of
	.Ar latency
	milliseconds to complete.
	Each delay has an associated number of
	.Ar lanes
	which defines the number of concurrent
	I/O requests that can be processed.
	.Pp
	For example, with a single lane delay of 10 ms
	.No (\& Ns Fl D Ar 10 : Ns Ar 1 ) ,
	the device will only be able to service a single I/O request
	at a time with each request taking 10 ms to complete.
	So, if only a single request is submitted every 10 ms, the
	average latency will be 10 ms; but if more than one request
	is submitted every 10 ms, the average latency will be more
	than 10 ms.
	.Pp
	Similarly, if a delay of 10 ms is specified to have two
	lanes
	.No (\& Ns Fl D Ar 10 : Ns Ar 2 ) ,
	then the device will be able to service
	two requests at a time, each with a minimum latency of 10 ms.
	So, if two requests are submitted every 10 ms, then
	the average latency will be 10 ms; but if more than two
	requests are submitted every 10 ms, the average latency
	will be more than 10 ms.
	.Pp
	Also note, these delays are additive.
	So two invocations of
	.Fl D Ar 10 : Ns Ar 1
	are roughly equivalent to a single invocation of
	.Fl D Ar 10 : Ns Ar 2 .
	This also means, that one can specify multiple
	lanes with differing target latencies.
	For example, an invocation of
	.Fl D Ar 10 : Ns Ar 1
	followed by
	.Fl D Ar 25 : Ns Ar 2
	will create 3 lanes on the device: one lane with a latency
	of 10 ms and two lanes with a 25 ms latency.
	.
	.It Xo
	.Nm zinject
	.Fl d Ar vdev
	.Op Fl e Ar device_error
	.Op Fl L Ar label_error
	.Op Fl T Ar failure
	.Op Fl f Ar frequency
	.Op Fl F
	.Ar pool
	.Xc
	Force a vdev error.
	.
	.It Xo
	.Nm zinject
	.Fl I
	.Op Fl s Ar seconds Ns \| Ns Fl g Ar txgs
	.Ar pool
	.Xc
	Simulate a hardware failure that fails to honor a cache flush.
	.
	.It Xo
	.Nm zinject
	.Fl p Ar function
	.Ar pool
	.Xc
	Panic inside the specified function.
	.
	.It Xo
	.Nm zinject
	.Fl t Sy data
	.Fl C Ar dvas
	.Op Fl e Ar device_error
	.Op Fl f Ar frequency
	.Op Fl l Ar level
	.Op Fl r Ar range
	.Op Fl amq
	.Ar path
	.Xc
	Force an error into the contents of a file.
	.
	.It Xo
	.Nm zinject
	.Fl t Sy dnode
	.Fl C Ar dvas
	.Op Fl e Ar device_error
	.Op Fl f Ar frequency
	.Op Fl l Ar level
	.Op Fl amq
	.Ar path
	.Xc
	Force an error into the metadnode for a file or directory.
	.
	.It Xo
	.Nm zinject
	.Fl t Ar mos_type
	.Fl C Ar dvas
	.Op Fl e Ar device_error
	.Op Fl f Ar frequency
	.Op Fl l Ar level
	.Op Fl r Ar range
	.Op Fl amqu
	.Ar pool
	.Xc
	Force an error into the MOS of a pool.
	.El
	.Sh OPTIONS
	.Bl -tag -width "-C dvas"
	.It Fl a
	Flush the ARC before injection.
	.It Fl b Ar objset : Ns Ar object : Ns Ar level : Ns Ar start : Ns Ar end
	Force an error into the pool at this bookmark tuple.
	Each number is in hexadecimal, and only one block can be specified.
	.It Fl C Ar dvas
	Inject the given error only into specific DVAs.
	The mask should be specified as a list of 0-indexed DVAs separated by commas
	.No (e.g . Ar 0,2 Ns No ).
	This option is not applicable to logical data errors such as
	.Sy decompress
	and
	.Sy decrypt .
	.It Fl d Ar vdev
	A vdev specified by path or GUID.
	.It Fl e Ar device_error
	Specify
	.Bl -tag -compact -width "decompress"
	.It Sy checksum
	for an ECKSUM error,
	.It Sy decompress
	for a data decompression error,
	.It Sy decrypt
	for a data decryption error,
	.It Sy corrupt
	to flip a bit in the data after a read,
	.It Sy dtl
	for an ECHILD error,
	.It Sy io
	for an EIO error where reopening the device will succeed, or
	.It Sy nxio
	for an ENXIO error where reopening the device will fail.
	.El
	.Pp
	For EIO and ENXIO, the "failed" reads or writes still occur.
	The probe simply sets the error value reported by the I/O pipeline
	so it appears the read or write failed.
	Decryption errors only currently work with file data.
	.It Fl f Ar frequency
	Only inject errors a fraction of the time.
	Expressed as a real number percentage between
	.Sy 0.0001
	and
	.Sy 100 .
	.It Fl F
	Fail faster.
	Do fewer checks.
	.It Fl f Ar txgs
	Run for this many transaction groups before reporting failure.
	.It Fl h
	Print the usage message.
	.It Fl l Ar level
	Inject an error at a particular block level.
	The default is
	.Sy 0 .
	.It Fl L Ar label_error
	Set the label error region to one of
	.Sy nvlist ,
	.Sy pad1 ,
	.Sy pad2 ,
	or
	.Sy uber .
	.It Fl m
	Automatically remount the underlying filesystem.
	.It Fl q
	Quiet mode.
	Only print the handler number added.
	.It Fl r Ar range
	Inject an error over a particular logical range of an object, which
	will be translated to the appropriate blkid range according to the
	object's properties.
	.It Fl s Ar seconds
	Run for this many seconds before reporting failure.
	.It Fl T Ar failure
	Set the failure type to one of
	.Sy all ,
	.Sy claim ,
	.Sy free ,
	.Sy read ,
	or
	.Sy write .
	.It Fl t Ar mos_type
	Set this to
	.Bl -tag -compact -width "spacemap"
	.It Sy mos
	for any data in the MOS,
	.It Sy mosdir
	for an object directory,
	.It Sy config
	for the pool configuration,
	.It Sy bpobj
	for the block pointer list,
	.It Sy spacemap
	for the space map,
	.It Sy metaslab
	for the metaslab, or
	.It Sy errlog
	for the persistent error log.
	.El
	.It Fl u
	Unload the pool after injection.
	.El
	.
	.Sh ENVIRONMENT VARIABLES
	.Bl -tag -width "ZF"
	.It Ev ZFS_HOSTID
	Run
	.Nm
	in debug mode.
	.El
	.
	.Sh SEE ALSO
	.Xr zfs 8 ,
	.Xr zpool 8
	diff --git a/sys/contrib/openzfs/man/man8/zpool-add.8 b/sys/contrib/openzfs/man/man8/zpool-add.8
	index 8ccdcccc7b06..60b35f1a511a 100644
	--- a/sys/contrib/openzfs/man/man8/zpool-add.8
	+++ b/sys/contrib/openzfs/man/man8/zpool-add.8
	@@ -1,124 +1,138 @@
	.\" CDDL HEADER START
	.\"
	.\" The contents of this file are subject to the terms of the
	.\" Common Development and Distribution License (the "License").
	.\" You may not use this file except in compliance with the License.
	.\"
	.\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	.\" or https://opensource.org/licenses/CDDL-1.0.
	.\" See the License for the specific language governing permissions
	.\" and limitations under the License.
	.\"
	.\" When distributing Covered Code, include this CDDL HEADER in each
	.\" file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	.\" If applicable, add the following below this CDDL HEADER, with the
	.\" fields enclosed by brackets "[]" replaced with your own identifying
	.\" information: Portions Copyright [yyyy] [name of copyright owner]
	.\"
	.\" CDDL HEADER END
	.\"
	.\" Copyright (c) 2007, Sun Microsystems, Inc. All Rights Reserved.
	.\" Copyright (c) 2012, 2018 by Delphix. All rights reserved.
	.\" Copyright (c) 2012 Cyril Plisko. All Rights Reserved.
	.\" Copyright (c) 2017 Datto Inc.
	.\" Copyright (c) 2018 George Melikov. All Rights Reserved.
	.\" Copyright 2017 Nexenta Systems, Inc.
	.\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
	+.\" Copyright (c) 2024 by Delphix. All Rights Reserved.
	.\"
	-.Dd March 16, 2022
	+.Dd March 8, 2024
	.Dt ZPOOL-ADD 8
	.Os
	.
	.Sh NAME
	.Nm zpool-add
	.Nd add vdevs to ZFS storage pool
	.Sh SYNOPSIS
	.Nm zpool
	.Cm add
	.Op Fl fgLnP
	+.Op Fl -allow-in-use -allow-replication-mismatch -allow-ashift-mismatch
	.Oo Fl o Ar property Ns = Ns Ar value Oc
	.Ar pool vdev Ns …
	.
	.Sh DESCRIPTION
	Adds the specified virtual devices to the given pool.
	The
	.Ar vdev
	specification is described in the
	.Em Virtual Devices
	section of
	.Xr zpoolconcepts 7 .
	The behavior of the
	.Fl f
	option, and the device checks performed are described in the
	.Nm zpool Cm create
	subcommand.
	.Bl -tag -width Ds
	.It Fl f
	Forces use of
	.Ar vdev Ns s ,
	-even if they appear in use or specify a conflicting replication level.
	+even if they appear in use, have conflicting ashift values, or specify
	+a conflicting replication level.
	Not all devices can be overridden in this manner.
	.It Fl g
	Display
	.Ar vdev ,
	GUIDs instead of the normal device names.
	These GUIDs can be used in place of
	device names for the zpool detach/offline/remove/replace commands.
	.It Fl L
	Display real paths for
	.Ar vdev Ns s
	resolving all symbolic links.
	This can be used to look up the current block
	device name regardless of the
	.Pa /dev/disk
	path used to open it.
	.It Fl n
	Displays the configuration that would be used without actually adding the
	.Ar vdev Ns s .
	The actual pool creation can still fail due to insufficient privileges or
	device sharing.
	.It Fl P
	Display real paths for
	.Ar vdev Ns s
	instead of only the last component of the path.
	This can be used in conjunction with the
	.Fl L
	flag.
	.It Fl o Ar property Ns = Ns Ar value
	Sets the given pool properties.
	See the
	.Xr zpoolprops 7
	manual page for a list of valid properties that can be set.
	The only property supported at the moment is
	.Sy ashift .
	+.It Fl -allow-ashift-mismatch
	+Disable the ashift validation which allows mismatched ashift values in the
	+pool.
	+Adding top-level
	+.Ar vdev Ns s
	+with different sector sizes will prohibit future device removal operations, see
	+.Xr zpool-remove 8 .
	+.It Fl -allow-in-use
	+Allow vdevs to be added even if they might be in use in another pool.
	+.It Fl -allow-replication-mismatch
	+Allow vdevs with conflicting replication levels to be added to the pool.
	.El
	.
	.Sh EXAMPLES
	.\" These are, respectively, examples 5, 13 from zpool.8
	.\" Make sure to update them bidirectionally
	.Ss Example 1 : No Adding a Mirror to a ZFS Storage Pool
	The following command adds two mirrored disks to the pool
	.Ar tank ,
	assuming the pool is already made up of two-way mirrors.
	The additional space is immediately available to any datasets within the pool.
	.Dl # Nm zpool Cm add Ar tank Sy mirror Pa sda sdb
	.
	.Ss Example 2 : No Adding Cache Devices to a ZFS Pool
	The following command adds two disks for use as cache devices to a ZFS storage
	pool:
	.Dl # Nm zpool Cm add Ar pool Sy cache Pa sdc sdd
	.Pp
	Once added, the cache devices gradually fill with content from main memory.
	Depending on the size of your cache devices, it could take over an hour for
	them to fill.
	Capacity and reads can be monitored using the
	.Cm iostat
	subcommand as follows:
	.Dl # Nm zpool Cm iostat Fl v Ar pool 5
	.
	.Sh SEE ALSO
	.Xr zpool-attach 8 ,
	.Xr zpool-import 8 ,
	.Xr zpool-initialize 8 ,
	.Xr zpool-online 8 ,
	.Xr zpool-remove 8
	diff --git a/sys/contrib/openzfs/man/man8/zpool-clear.8 b/sys/contrib/openzfs/man/man8/zpool-clear.8
	index c61ecae483ac..3e448be87fc2 100644
	--- a/sys/contrib/openzfs/man/man8/zpool-clear.8
	+++ b/sys/contrib/openzfs/man/man8/zpool-clear.8
	@@ -1,70 +1,71 @@
	.\"
	.\" CDDL HEADER START
	.\"
	.\" The contents of this file are subject to the terms of the
	.\" Common Development and Distribution License (the "License").
	.\" You may not use this file except in compliance with the License.
	.\"
	.\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	.\" or https://opensource.org/licenses/CDDL-1.0.
	.\" See the License for the specific language governing permissions
	.\" and limitations under the License.
	.\"
	.\" When distributing Covered Code, include this CDDL HEADER in each
	.\" file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	.\" If applicable, add the following below this CDDL HEADER, with the
	.\" fields enclosed by brackets "[]" replaced with your own identifying
	.\" information: Portions Copyright [yyyy] [name of copyright owner]
	.\"
	.\" CDDL HEADER END
	.\"
	.\" Copyright (c) 2007, Sun Microsystems, Inc. All Rights Reserved.
	.\" Copyright (c) 2012, 2018 by Delphix. All rights reserved.
	.\" Copyright (c) 2012 Cyril Plisko. All Rights Reserved.
	.\" Copyright (c) 2017 Datto Inc.
	.\" Copyright (c) 2018 George Melikov. All Rights Reserved.
	.\" Copyright 2017 Nexenta Systems, Inc.
	.\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
	.\"
	.Dd May 27, 2021
	.Dt ZPOOL-CLEAR 8
	.Os
	.
	.Sh NAME
	.Nm zpool-clear
	.Nd clear device errors in ZFS storage pool
	.Sh SYNOPSIS
	.Nm zpool
	.Cm clear
	.Op Fl -power
	.Ar pool
	.Oo Ar device Oc Ns …
	.
	.Sh DESCRIPTION
	Clears device errors in a pool.
	If no arguments are specified, all device errors within the pool are cleared.
	If one or more devices is specified, only those errors associated with the
	specified device or devices are cleared.
	.Pp
	If the pool was suspended it will be brought back online provided the
	devices can be accessed.
	Pools with
	.Sy multihost
	-enabled which have been suspended cannot be resumed.
	-While the pool was suspended, it may have been imported on
	-another host, and resuming I/O could result in pool damage.
	+enabled which have been suspended cannot be resumed when there is evidence
	+that the pool was imported by another host.
	+The same checks performed during an import will be applied before the clear
	+proceeds.
	.Bl -tag -width Ds
	.It Fl -power
	Power on the devices's slot in the storage enclosure and wait for the device
	to show up before attempting to clear errors.
	This is done on all the devices specified.
	Alternatively, you can set the
	.Sy ZPOOL_AUTO_POWER_ON_SLOT
	environment variable to always enable this behavior.
	Note: This flag currently works on Linux only.
	.El
	.
	.Sh SEE ALSO
	.Xr zdb 8 ,
	.Xr zpool-reopen 8 ,
	.Xr zpool-status 8
	diff --git a/sys/contrib/openzfs/man/man8/zpool-status.8 b/sys/contrib/openzfs/man/man8/zpool-status.8
	index 24ad6e643cae..bbe7a45aa0c6 100644
	--- a/sys/contrib/openzfs/man/man8/zpool-status.8
	+++ b/sys/contrib/openzfs/man/man8/zpool-status.8
	@@ -1,169 +1,169 @@
	.\"
	.\" CDDL HEADER START
	.\"
	.\" The contents of this file are subject to the terms of the
	.\" Common Development and Distribution License (the "License").
	.\" You may not use this file except in compliance with the License.
	.\"
	.\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	.\" or https://opensource.org/licenses/CDDL-1.0.
	.\" See the License for the specific language governing permissions
	.\" and limitations under the License.
	.\"
	.\" When distributing Covered Code, include this CDDL HEADER in each
	.\" file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	.\" If applicable, add the following below this CDDL HEADER, with the
	.\" fields enclosed by brackets "[]" replaced with your own identifying
	.\" information: Portions Copyright [yyyy] [name of copyright owner]
	.\"
	.\" CDDL HEADER END
	.\"
	.\" Copyright (c) 2007, Sun Microsystems, Inc. All Rights Reserved.
	.\" Copyright (c) 2012, 2018 by Delphix. All rights reserved.
	.\" Copyright (c) 2012 Cyril Plisko. All Rights Reserved.
	.\" Copyright (c) 2017 Datto Inc.
	.\" Copyright (c) 2018 George Melikov. All Rights Reserved.
	.\" Copyright 2017 Nexenta Systems, Inc.
	.\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
	.\"
	.Dd March 16, 2022
	.Dt ZPOOL-STATUS 8
	.Os
	.
	.Sh NAME
	.Nm zpool-status
	.Nd show detailed health status for ZFS storage pools
	.Sh SYNOPSIS
	.Nm zpool
	.Cm status
	-.Op Fl DeigLpPstvx
	+.Op Fl DegiLpPstvx
	.Op Fl T Sy u Ns \| Ns Sy d
	.Op Fl c Op Ar SCRIPT1 Ns Oo , Ns Ar SCRIPT2 Oc Ns …
	.Oo Ar pool Oc Ns …
	.Op Ar interval Op Ar count
	.
	.Sh DESCRIPTION
	Displays the detailed health status for the given pools.
	If no
	.Ar pool
	is specified, then the status of each pool in the system is displayed.
	For more information on pool and device health, see the
	.Sx Device Failure and Recovery
	section of
	.Xr zpoolconcepts 7 .
	.Pp
	If a scrub or resilver is in progress, this command reports the percentage done
	and the estimated time to completion.
	Both of these are only approximate, because the amount of data in the pool and
	the other workloads on the system can change.
	.Bl -tag -width Ds
	.It Fl -power
	Display vdev enclosure slot power status (on or off).
	.It Fl c Op Ar SCRIPT1 Ns Oo , Ns Ar SCRIPT2 Oc Ns …
	Run a script (or scripts) on each vdev and include the output as a new column
	in the
	.Nm zpool Cm status
	output.
	See the
	.Fl c
	option of
	.Nm zpool Cm iostat
	for complete details.
	+.It Fl D
	+Display a histogram of deduplication statistics, showing the allocated
	+.Pq physically present on disk
	+and referenced
	+.Pq logically referenced in the pool
	+block counts and sizes by reference count.
	.It Fl e
	Only show unhealthy vdevs (not-ONLINE or with errors).
	-.It Fl i
	-Display vdev initialization status.
	.It Fl g
	Display vdev GUIDs instead of the normal device names
	These GUIDs can be used in place of device names for the zpool
	detach/offline/remove/replace commands.
	+.It Fl i
	+Display vdev initialization status.
	.It Fl L
	Display real paths for vdevs resolving all symbolic links.
	This can be used to look up the current block device name regardless of the
	.Pa /dev/disk/
	path used to open it.
	.It Fl p
	Display numbers in parsable (exact) values.
	.It Fl P
	Display full paths for vdevs instead of only the last component of
	the path.
	This can be used in conjunction with the
	.Fl L
	flag.
	-.It Fl D
	-Display a histogram of deduplication statistics, showing the allocated
	-.Pq physically present on disk
	-and referenced
	-.Pq logically referenced in the pool
	-block counts and sizes by reference count.
	.It Fl s
	Display the number of leaf vdev slow I/O operations.
	This is the number of I/O operations that didn't complete in
	.Sy zio_slow_io_ms
	milliseconds
	.Pq Sy 30000 No by default .
	This does not necessarily mean the I/O operations failed to complete, just took
	an
	unreasonably long amount of time.
	This may indicate a problem with the underlying storage.
	.It Fl t
	Display vdev TRIM status.
	.It Fl T Sy u Ns \| Ns Sy d
	Display a time stamp.
	Specify
	.Sy u
	for a printed representation of the internal representation of time.
	See
	.Xr time 1 .
	Specify
	.Sy d
	for standard date format.
	See
	.Xr date 1 .
	.It Fl v
	Displays verbose data error information, printing out a complete list of all
	data errors since the last complete pool scrub.
	If the head_errlog feature is enabled and files containing errors have been
	removed then the respective filenames will not be reported in subsequent runs
	of this command.
	.It Fl x
	Only display status for pools that are exhibiting errors or are otherwise
	unavailable.
	Warnings about pools not using the latest on-disk format will not be included.
	.El
	.
	.Sh EXAMPLES
	.\" These are, respectively, examples 16 from zpool.8
	.\" Make sure to update them bidirectionally
	.Ss Example 1 : No Adding output columns
	Additional columns can be added to the
	.Nm zpool Cm status No and Nm zpool Cm iostat No output with Fl c .
	.Bd -literal -compact -offset Ds
	.No # Nm zpool Cm status Fl c Pa vendor , Ns Pa model , Ns Pa size
	NAME STATE READ WRITE CKSUM vendor model size
	tank ONLINE 0 0 0
	mirror-0 ONLINE 0 0 0
	U1 ONLINE 0 0 0 SEAGATE ST8000NM0075 7.3T
	U10 ONLINE 0 0 0 SEAGATE ST8000NM0075 7.3T
	U11 ONLINE 0 0 0 SEAGATE ST8000NM0075 7.3T
	U12 ONLINE 0 0 0 SEAGATE ST8000NM0075 7.3T
	U13 ONLINE 0 0 0 SEAGATE ST8000NM0075 7.3T
	U14 ONLINE 0 0 0 SEAGATE ST8000NM0075 7.3T

	.No # Nm zpool Cm iostat Fl vc Pa size
	capacity operations bandwidth
	pool alloc free read write read write size
	---------- ----- ----- ----- ----- ----- ----- ----
	rpool 14.6G 54.9G 4 55 250K 2.69M
	sda1 14.6G 54.9G 4 55 250K 2.69M 70G
	---------- ----- ----- ----- ----- ----- ----- ----
	.Ed
	.
	.Sh SEE ALSO
	.Xr zpool-events 8 ,
	.Xr zpool-history 8 ,
	.Xr zpool-iostat 8 ,
	.Xr zpool-list 8 ,
	.Xr zpool-resilver 8 ,
	.Xr zpool-scrub 8 ,
	.Xr zpool-wait 8
	diff --git a/sys/contrib/openzfs/module/icp/asm-aarch64/blake3/b3_aarch64_sse2.S b/sys/contrib/openzfs/module/icp/asm-aarch64/blake3/b3_aarch64_sse2.S
	index dc2719d142db..e66bb4bc7f26 100644
	--- a/sys/contrib/openzfs/module/icp/asm-aarch64/blake3/b3_aarch64_sse2.S
	+++ b/sys/contrib/openzfs/module/icp/asm-aarch64/blake3/b3_aarch64_sse2.S
	@@ -1,2061 +1,2069 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Based on BLAKE3 v1.3.1, https://github.com/BLAKE3-team/BLAKE3
	* Copyright (c) 2019-2022 Samuel Neves and Matthew Krupcale
	* Copyright (c) 2022-2023 Tino Reichardt <milky-zfs@mcmilk.de>
	*
	* This is converted assembly: SSE2 -> ARMv8-A
	* Used tools: SIMDe https://github.com/simd-everywhere/simde
	*
	* Should work on FreeBSD, Linux and macOS
	* see: https://github.com/mcmilk/BLAKE3-tests/blob/master/contrib/simde.sh
	*/

	#if defined(__aarch64__)
	+
	+/* make gcc <= 9 happy */
	+#if LD_VERSION >= 233010000
	+#define CFI_NEGATE_RA_STATE .cfi_negate_ra_state
	+#else
	+#define CFI_NEGATE_RA_STATE
	+#endif
	+
	.text
	.section .note.gnu.property,"a",@note
	.p2align 3
	.word 4
	.word 16
	.word 5
	.asciz "GNU"
	.word 3221225472
	.word 4
	.word 3
	.word 0
	.Lsec_end0:
	.text
	.globl zfs_blake3_compress_in_place_sse2
	.p2align 2
	.type zfs_blake3_compress_in_place_sse2,@function
	zfs_blake3_compress_in_place_sse2:
	.cfi_startproc
	hint #25
	- .cfi_negate_ra_state
	+ CFI_NEGATE_RA_STATE
	sub sp, sp, #96
	stp x29, x30, [sp, #64]
	add x29, sp, #64
	str x19, [sp, #80]
	.cfi_def_cfa w29, 32
	.cfi_offset w19, -16
	.cfi_offset w30, -24
	.cfi_offset w29, -32
	mov x19, x0
	mov w5, w4
	mov x4, x3
	mov w3, w2
	mov x2, x1
	mov x0, sp
	mov x1, x19
	bl compress_pre
	ldp q0, q1, [sp]
	ldp q2, q3, [sp, #32]
	eor v0.16b, v2.16b, v0.16b
	eor v1.16b, v3.16b, v1.16b
	ldp x29, x30, [sp, #64]
	stp q0, q1, [x19]
	ldr x19, [sp, #80]
	add sp, sp, #96
	hint #29
	ret
	.Lfunc_end0:
	.size zfs_blake3_compress_in_place_sse2, .Lfunc_end0-zfs_blake3_compress_in_place_sse2
	.cfi_endproc

	.section .rodata.cst16,"aM",@progbits,16
	.p2align 4
	.LCPI1_0:
	.xword -4942790177982912921
	.xword -6534734903820487822
	.text
	.p2align 2
	.type compress_pre,@function
	compress_pre:
	.cfi_startproc
	hint #34
	fmov s1, w3
	movi d0, #0x0000ff000000ff
	ldr q2, [x1]
	fmov d3, x4
	adrp x8, .LCPI1_0
	mov v1.s[1], w5
	str q2, [x0]
	ldr q4, [x8, :lo12:.LCPI1_0]
	add x8, x2, #32
	ldr q5, [x1, #16]
	and v0.8b, v1.8b, v0.8b
	stp q5, q4, [x0, #16]
	mov v3.d[1], v0.d[0]
	str q3, [x0, #48]
	ldp q0, q6, [x2]
	uzp1 v1.4s, v0.4s, v6.4s
	uzp2 v0.4s, v0.4s, v6.4s
	add v2.4s, v2.4s, v1.4s
	uzp1 v18.4s, v1.4s, v1.4s
	add v2.4s, v2.4s, v5.4s
	eor v3.16b, v2.16b, v3.16b
	add v2.4s, v2.4s, v0.4s
	rev32 v3.8h, v3.8h
	add v4.4s, v3.4s, v4.4s
	eor v5.16b, v4.16b, v5.16b
	ushr v6.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	orr v5.16b, v5.16b, v6.16b
	add v2.4s, v2.4s, v5.4s
	eor v3.16b, v2.16b, v3.16b
	ushr v6.4s, v3.4s, #8
	shl v3.4s, v3.4s, #24
	orr v3.16b, v3.16b, v6.16b
	ld2 { v6.4s, v7.4s }, [x8]
	add v4.4s, v3.4s, v4.4s
	ext v3.16b, v3.16b, v3.16b, #8
	add v2.4s, v2.4s, v6.4s
	eor v5.16b, v4.16b, v5.16b
	ext v4.16b, v4.16b, v4.16b, #4
	ext v6.16b, v6.16b, v6.16b, #12
	ext v2.16b, v2.16b, v2.16b, #12
	ushr v16.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	orr v5.16b, v5.16b, v16.16b
	ext v16.16b, v7.16b, v7.16b, #12
	add v2.4s, v2.4s, v5.4s
	mov v7.16b, v16.16b
	eor v3.16b, v3.16b, v2.16b
	add v2.4s, v2.4s, v16.4s
	mov v7.s[1], v6.s[2]
	rev32 v3.8h, v3.8h
	add v4.4s, v4.4s, v3.4s
	eor v5.16b, v4.16b, v5.16b
	ushr v17.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	orr v5.16b, v5.16b, v17.16b
	add v2.4s, v2.4s, v5.4s
	eor v3.16b, v2.16b, v3.16b
	ushr v17.4s, v3.4s, #8
	shl v3.4s, v3.4s, #24
	orr v3.16b, v3.16b, v17.16b
	ext v17.16b, v18.16b, v1.16b, #8
	add v4.4s, v3.4s, v4.4s
	uzp2 v17.4s, v17.4s, v0.4s
	ext v3.16b, v3.16b, v3.16b, #8
	eor v5.16b, v4.16b, v5.16b
	add v2.4s, v2.4s, v17.4s
	ext v4.16b, v4.16b, v4.16b, #12
	ushr v18.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	ext v2.16b, v2.16b, v2.16b, #4
	orr v5.16b, v5.16b, v18.16b
	ext v18.16b, v1.16b, v1.16b, #12
	add v2.4s, v2.4s, v5.4s
	ext v1.16b, v1.16b, v18.16b, #12
	zip1 v18.2d, v16.2d, v0.2d
	zip2 v0.4s, v0.4s, v16.4s
	eor v3.16b, v3.16b, v2.16b
	rev64 v1.4s, v1.4s
	mov v18.s[3], v6.s[3]
	zip1 v16.4s, v0.4s, v6.4s
	rev32 v3.8h, v3.8h
	trn2 v1.4s, v1.4s, v7.4s
	zip1 v0.4s, v6.4s, v0.4s
	add v4.4s, v4.4s, v3.4s
	add v2.4s, v2.4s, v1.4s
	ext v6.16b, v0.16b, v16.16b, #8
	eor v5.16b, v4.16b, v5.16b
	ushr v7.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	orr v5.16b, v5.16b, v7.16b
	add v7.4s, v2.4s, v5.4s
	eor v2.16b, v7.16b, v3.16b
	ext v7.16b, v7.16b, v7.16b, #12
	ushr v3.4s, v2.4s, #8
	shl v2.4s, v2.4s, #24
	orr v3.16b, v2.16b, v3.16b
	ext v2.16b, v18.16b, v18.16b, #12
	add v4.4s, v3.4s, v4.4s
	uzp1 v2.4s, v18.4s, v2.4s
	ext v3.16b, v3.16b, v3.16b, #8
	eor v5.16b, v4.16b, v5.16b
	add v7.4s, v7.4s, v2.4s
	ext v4.16b, v4.16b, v4.16b, #4
	ushr v18.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	orr v5.16b, v5.16b, v18.16b
	add v7.4s, v7.4s, v5.4s
	eor v3.16b, v3.16b, v7.16b
	add v7.4s, v7.4s, v6.4s
	rev32 v3.8h, v3.8h
	add v4.4s, v4.4s, v3.4s
	eor v5.16b, v4.16b, v5.16b
	ushr v0.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	orr v0.16b, v5.16b, v0.16b
	add v5.4s, v7.4s, v0.4s
	ext v7.16b, v17.16b, v17.16b, #4
	eor v3.16b, v5.16b, v3.16b
	uzp1 v17.4s, v7.4s, v7.4s
	ushr v16.4s, v3.4s, #8
	shl v3.4s, v3.4s, #24
	orr v3.16b, v3.16b, v16.16b
	ext v16.16b, v17.16b, v7.16b, #8
	add v4.4s, v3.4s, v4.4s
	uzp2 v16.4s, v16.4s, v1.4s
	ext v3.16b, v3.16b, v3.16b, #8
	eor v0.16b, v4.16b, v0.16b
	add v5.4s, v5.4s, v16.4s
	ext v4.16b, v4.16b, v4.16b, #12
	ushr v17.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	ext v5.16b, v5.16b, v5.16b, #4
	orr v0.16b, v0.16b, v17.16b
	ext v17.16b, v7.16b, v7.16b, #12
	add v5.4s, v5.4s, v0.4s
	ext v7.16b, v7.16b, v17.16b, #12
	mov v17.16b, v6.16b
	eor v3.16b, v3.16b, v5.16b
	rev64 v7.4s, v7.4s
	mov v17.s[1], v2.s[2]
	rev32 v3.8h, v3.8h
	add v4.4s, v4.4s, v3.4s
	eor v18.16b, v4.16b, v0.16b
	trn2 v0.4s, v7.4s, v17.4s
	ushr v7.4s, v18.4s, #12
	shl v17.4s, v18.4s, #20
	add v5.4s, v5.4s, v0.4s
	zip1 v18.2d, v6.2d, v1.2d
	zip2 v1.4s, v1.4s, v6.4s
	orr v7.16b, v17.16b, v7.16b
	mov v18.s[3], v2.s[3]
	zip1 v6.4s, v1.4s, v2.4s
	add v5.4s, v5.4s, v7.4s
	zip1 v1.4s, v2.4s, v1.4s
	eor v3.16b, v5.16b, v3.16b
	ext v5.16b, v5.16b, v5.16b, #12
	ext v6.16b, v1.16b, v6.16b, #8
	ushr v17.4s, v3.4s, #8
	shl v3.4s, v3.4s, #24
	orr v17.16b, v3.16b, v17.16b
	ext v3.16b, v18.16b, v18.16b, #12
	add v4.4s, v17.4s, v4.4s
	uzp1 v3.4s, v18.4s, v3.4s
	ext v17.16b, v17.16b, v17.16b, #8
	eor v7.16b, v4.16b, v7.16b
	add v5.4s, v5.4s, v3.4s
	ext v4.16b, v4.16b, v4.16b, #4
	ushr v18.4s, v7.4s, #7
	shl v7.4s, v7.4s, #25
	orr v7.16b, v7.16b, v18.16b
	add v5.4s, v5.4s, v7.4s
	eor v17.16b, v17.16b, v5.16b
	add v5.4s, v5.4s, v6.4s
	rev32 v17.8h, v17.8h
	add v4.4s, v4.4s, v17.4s
	eor v2.16b, v4.16b, v7.16b
	ext v7.16b, v16.16b, v16.16b, #4
	ushr v1.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	orr v1.16b, v2.16b, v1.16b
	add v2.4s, v5.4s, v1.4s
	eor v5.16b, v2.16b, v17.16b
	uzp1 v17.4s, v7.4s, v7.4s
	ushr v16.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	orr v5.16b, v5.16b, v16.16b
	ext v16.16b, v17.16b, v7.16b, #8
	add v4.4s, v5.4s, v4.4s
	uzp2 v16.4s, v16.4s, v0.4s
	ext v5.16b, v5.16b, v5.16b, #8
	eor v1.16b, v4.16b, v1.16b
	add v2.4s, v2.4s, v16.4s
	ext v4.16b, v4.16b, v4.16b, #12
	ushr v17.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	ext v2.16b, v2.16b, v2.16b, #4
	orr v1.16b, v1.16b, v17.16b
	ext v17.16b, v7.16b, v7.16b, #12
	add v2.4s, v2.4s, v1.4s
	ext v7.16b, v7.16b, v17.16b, #12
	mov v17.16b, v6.16b
	eor v5.16b, v5.16b, v2.16b
	rev64 v7.4s, v7.4s
	mov v17.s[1], v3.s[2]
	rev32 v5.8h, v5.8h
	add v4.4s, v4.4s, v5.4s
	eor v18.16b, v4.16b, v1.16b
	trn2 v1.4s, v7.4s, v17.4s
	ushr v7.4s, v18.4s, #12
	shl v17.4s, v18.4s, #20
	add v2.4s, v2.4s, v1.4s
	zip1 v18.2d, v6.2d, v0.2d
	zip2 v0.4s, v0.4s, v6.4s
	orr v7.16b, v17.16b, v7.16b
	mov v18.s[3], v3.s[3]
	add v2.4s, v2.4s, v7.4s
	eor v5.16b, v2.16b, v5.16b
	ext v2.16b, v2.16b, v2.16b, #12
	ushr v17.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	orr v5.16b, v5.16b, v17.16b
	add v17.4s, v5.4s, v4.4s
	ext v4.16b, v18.16b, v18.16b, #12
	ext v5.16b, v5.16b, v5.16b, #8
	eor v7.16b, v17.16b, v7.16b
	uzp1 v4.4s, v18.4s, v4.4s
	ext v17.16b, v17.16b, v17.16b, #4
	ushr v18.4s, v7.4s, #7
	shl v7.4s, v7.4s, #25
	add v2.4s, v2.4s, v4.4s
	orr v7.16b, v7.16b, v18.16b
	add v2.4s, v2.4s, v7.4s
	eor v5.16b, v5.16b, v2.16b
	rev32 v5.8h, v5.8h
	add v6.4s, v17.4s, v5.4s
	zip1 v17.4s, v0.4s, v3.4s
	zip1 v0.4s, v3.4s, v0.4s
	eor v3.16b, v6.16b, v7.16b
	ext v0.16b, v0.16b, v17.16b, #8
	ushr v7.4s, v3.4s, #12
	shl v3.4s, v3.4s, #20
	add v2.4s, v2.4s, v0.4s
	orr v3.16b, v3.16b, v7.16b
	ext v7.16b, v16.16b, v16.16b, #4
	add v2.4s, v2.4s, v3.4s
	uzp1 v17.4s, v7.4s, v7.4s
	eor v5.16b, v2.16b, v5.16b
	ushr v16.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	orr v5.16b, v5.16b, v16.16b
	ext v16.16b, v17.16b, v7.16b, #8
	add v6.4s, v5.4s, v6.4s
	uzp2 v16.4s, v16.4s, v1.4s
	ext v5.16b, v5.16b, v5.16b, #8
	eor v3.16b, v6.16b, v3.16b
	add v2.4s, v2.4s, v16.4s
	ext v6.16b, v6.16b, v6.16b, #12
	ushr v17.4s, v3.4s, #7
	shl v3.4s, v3.4s, #25
	ext v2.16b, v2.16b, v2.16b, #4
	orr v3.16b, v3.16b, v17.16b
	add v17.4s, v2.4s, v3.4s
	eor v2.16b, v5.16b, v17.16b
	ext v5.16b, v7.16b, v7.16b, #12
	rev32 v18.8h, v2.8h
	ext v2.16b, v7.16b, v5.16b, #12
	mov v5.16b, v0.16b
	add v6.4s, v6.4s, v18.4s
	rev64 v2.4s, v2.4s
	mov v5.s[1], v4.s[2]
	eor v3.16b, v6.16b, v3.16b
	trn2 v2.4s, v2.4s, v5.4s
	ushr v5.4s, v3.4s, #12
	shl v3.4s, v3.4s, #20
	add v7.4s, v17.4s, v2.4s
	orr v3.16b, v3.16b, v5.16b
	add v5.4s, v7.4s, v3.4s
	eor v7.16b, v5.16b, v18.16b
	zip1 v18.2d, v0.2d, v1.2d
	ext v5.16b, v5.16b, v5.16b, #12
	zip2 v0.4s, v1.4s, v0.4s
	ushr v17.4s, v7.4s, #8
	shl v7.4s, v7.4s, #24
	mov v18.s[3], v4.s[3]
	orr v7.16b, v7.16b, v17.16b
	ext v17.16b, v18.16b, v18.16b, #12
	add v6.4s, v7.4s, v6.4s
	ext v7.16b, v7.16b, v7.16b, #8
	eor v19.16b, v6.16b, v3.16b
	uzp1 v3.4s, v18.4s, v17.4s
	ext v6.16b, v6.16b, v6.16b, #4
	ushr v17.4s, v19.4s, #7
	shl v18.4s, v19.4s, #25
	add v5.4s, v5.4s, v3.4s
	orr v17.16b, v18.16b, v17.16b
	add v5.4s, v5.4s, v17.4s
	eor v7.16b, v7.16b, v5.16b
	rev32 v7.8h, v7.8h
	add v1.4s, v6.4s, v7.4s
	zip1 v6.4s, v0.4s, v4.4s
	zip1 v0.4s, v4.4s, v0.4s
	eor v4.16b, v1.16b, v17.16b
	ext v6.16b, v0.16b, v6.16b, #8
	ushr v0.4s, v4.4s, #12
	shl v4.4s, v4.4s, #20
	add v5.4s, v5.4s, v6.4s
	zip1 v20.2d, v6.2d, v2.2d
	orr v0.16b, v4.16b, v0.16b
	mov v20.s[3], v3.s[3]
	add v4.4s, v5.4s, v0.4s
	eor v5.16b, v4.16b, v7.16b
	ext v7.16b, v16.16b, v16.16b, #4
	ushr v16.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	uzp1 v17.4s, v7.4s, v7.4s
	orr v5.16b, v5.16b, v16.16b
	ext v16.16b, v17.16b, v7.16b, #8
	add v1.4s, v5.4s, v1.4s
	uzp2 v16.4s, v16.4s, v2.4s
	zip2 v2.4s, v2.4s, v6.4s
	eor v0.16b, v1.16b, v0.16b
	add v4.4s, v4.4s, v16.4s
	ext v1.16b, v1.16b, v1.16b, #12
	ext v16.16b, v16.16b, v16.16b, #4
	ushr v17.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	ext v4.16b, v4.16b, v4.16b, #4
	orr v17.16b, v0.16b, v17.16b
	ext v0.16b, v5.16b, v5.16b, #8
	ext v5.16b, v7.16b, v7.16b, #12
	add v4.4s, v4.4s, v17.4s
	eor v0.16b, v0.16b, v4.16b
	rev32 v18.8h, v0.8h
	ext v0.16b, v7.16b, v5.16b, #12
	mov v5.16b, v6.16b
	add v7.4s, v1.4s, v18.4s
	rev64 v1.4s, v0.4s
	mov v5.s[1], v3.s[2]
	eor v17.16b, v7.16b, v17.16b
	trn2 v1.4s, v1.4s, v5.4s
	ushr v19.4s, v17.4s, #12
	shl v17.4s, v17.4s, #20
	add v4.4s, v4.4s, v1.4s
	orr v17.16b, v17.16b, v19.16b
	add v19.4s, v4.4s, v17.4s
	eor v4.16b, v19.16b, v18.16b
	ext v19.16b, v19.16b, v19.16b, #12
	ushr v18.4s, v4.4s, #8
	shl v4.4s, v4.4s, #24
	orr v18.16b, v4.16b, v18.16b
	ext v4.16b, v20.16b, v20.16b, #12
	add v7.4s, v18.4s, v7.4s
	uzp1 v4.4s, v20.4s, v4.4s
	ext v18.16b, v18.16b, v18.16b, #8
	eor v17.16b, v7.16b, v17.16b
	add v19.4s, v19.4s, v4.4s
	ext v7.16b, v7.16b, v7.16b, #4
	ushr v20.4s, v17.4s, #7
	shl v17.4s, v17.4s, #25
	orr v17.16b, v17.16b, v20.16b
	add v19.4s, v19.4s, v17.4s
	eor v18.16b, v18.16b, v19.16b
	rev32 v18.8h, v18.8h
	add v6.4s, v7.4s, v18.4s
	zip1 v7.4s, v2.4s, v3.4s
	zip1 v2.4s, v3.4s, v2.4s
	eor v3.16b, v6.16b, v17.16b
	ext v2.16b, v2.16b, v7.16b, #8
	ushr v7.4s, v3.4s, #12
	shl v3.4s, v3.4s, #20
	add v17.4s, v19.4s, v2.4s
	zip1 v1.2d, v2.2d, v1.2d
	zip2 v0.4s, v0.4s, v2.4s
	orr v3.16b, v3.16b, v7.16b
	mov v1.s[3], v4.s[3]
	add v7.4s, v17.4s, v3.4s
	eor v17.16b, v7.16b, v18.16b
	ext v7.16b, v7.16b, v7.16b, #4
	ushr v18.4s, v17.4s, #8
	shl v17.4s, v17.4s, #24
	orr v17.16b, v17.16b, v18.16b
	ext v18.16b, v16.16b, v16.16b, #8
	add v6.4s, v17.4s, v6.4s
	uzp2 v5.4s, v18.4s, v5.4s
	eor v3.16b, v6.16b, v3.16b
	ext v5.16b, v5.16b, v18.16b, #4
	ext v6.16b, v6.16b, v6.16b, #12
	ushr v18.4s, v3.4s, #7
	shl v3.4s, v3.4s, #25
	add v5.4s, v7.4s, v5.4s
	ext v7.16b, v17.16b, v17.16b, #8
	ext v17.16b, v16.16b, v16.16b, #12
	orr v3.16b, v3.16b, v18.16b
	ext v16.16b, v16.16b, v17.16b, #12
	add v5.4s, v3.4s, v5.4s
	mov v17.16b, v2.16b
	rev64 v16.4s, v16.4s
	eor v7.16b, v7.16b, v5.16b
	mov v17.s[1], v4.s[2]
	rev32 v7.8h, v7.8h
	trn2 v16.4s, v16.4s, v17.4s
	add v6.4s, v6.4s, v7.4s
	add v5.4s, v5.4s, v16.4s
	eor v3.16b, v6.16b, v3.16b
	ushr v17.4s, v3.4s, #12
	shl v3.4s, v3.4s, #20
	orr v3.16b, v3.16b, v17.16b
	add v5.4s, v5.4s, v3.4s
	eor v7.16b, v5.16b, v7.16b
	ext v5.16b, v5.16b, v5.16b, #12
	ushr v16.4s, v7.4s, #8
	shl v7.4s, v7.4s, #24
	orr v7.16b, v7.16b, v16.16b
	ext v16.16b, v1.16b, v1.16b, #12
	add v6.4s, v7.4s, v6.4s
	uzp1 v1.4s, v1.4s, v16.4s
	eor v3.16b, v6.16b, v3.16b
	add v1.4s, v5.4s, v1.4s
	ext v5.16b, v7.16b, v7.16b, #8
	ext v6.16b, v6.16b, v6.16b, #4
	ushr v16.4s, v3.4s, #7
	shl v3.4s, v3.4s, #25
	orr v3.16b, v3.16b, v16.16b
	add v1.4s, v1.4s, v3.4s
	eor v5.16b, v5.16b, v1.16b
	rev32 v5.8h, v5.8h
	add v2.4s, v6.4s, v5.4s
	zip1 v6.4s, v0.4s, v4.4s
	zip1 v0.4s, v4.4s, v0.4s
	eor v3.16b, v2.16b, v3.16b
	ext v0.16b, v0.16b, v6.16b, #8
	ushr v4.4s, v3.4s, #12
	shl v3.4s, v3.4s, #20
	add v0.4s, v1.4s, v0.4s
	orr v1.16b, v3.16b, v4.16b
	add v0.4s, v0.4s, v1.4s
	eor v3.16b, v0.16b, v5.16b
	ext v0.16b, v0.16b, v0.16b, #4
	ushr v4.4s, v3.4s, #8
	shl v3.4s, v3.4s, #24
	orr v3.16b, v3.16b, v4.16b
	add v2.4s, v3.4s, v2.4s
	ext v3.16b, v3.16b, v3.16b, #8
	eor v1.16b, v2.16b, v1.16b
	ext v2.16b, v2.16b, v2.16b, #12
	ushr v4.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	stp q2, q3, [x0, #32]
	orr v1.16b, v1.16b, v4.16b
	stp q0, q1, [x0]
	ret
	.Lfunc_end1:
	.size compress_pre, .Lfunc_end1-compress_pre
	.cfi_endproc

	.globl zfs_blake3_compress_xof_sse2
	.p2align 2
	.type zfs_blake3_compress_xof_sse2,@function
	zfs_blake3_compress_xof_sse2:
	.cfi_startproc
	hint #25
	- .cfi_negate_ra_state
	+ CFI_NEGATE_RA_STATE
	sub sp, sp, #96
	stp x29, x30, [sp, #64]
	add x29, sp, #64
	stp x20, x19, [sp, #80]
	.cfi_def_cfa w29, 32
	.cfi_offset w19, -8
	.cfi_offset w20, -16
	.cfi_offset w30, -24
	.cfi_offset w29, -32
	mov x20, x0
	mov x19, x5
	mov w5, w4
	mov x4, x3
	mov w3, w2
	mov x2, x1
	mov x0, sp
	mov x1, x20
	bl compress_pre
	ldp q0, q1, [sp]
	ldp q2, q3, [sp, #32]
	eor v0.16b, v2.16b, v0.16b
	eor v1.16b, v3.16b, v1.16b
	ldp x29, x30, [sp, #64]
	stp q0, q1, [x19]
	ldr q0, [x20]
	eor v0.16b, v0.16b, v2.16b
	str q0, [x19, #32]
	ldr q0, [x20, #16]
	eor v0.16b, v0.16b, v3.16b
	str q0, [x19, #48]
	ldp x20, x19, [sp, #80]
	add sp, sp, #96
	hint #29
	ret
	.Lfunc_end2:
	.size zfs_blake3_compress_xof_sse2, .Lfunc_end2-zfs_blake3_compress_xof_sse2
	.cfi_endproc

	.section .rodata.cst16,"aM",@progbits,16
	.p2align 4
	.LCPI3_0:
	.word 0
	.word 1
	.word 2
	.word 3
	.text
	.globl zfs_blake3_hash_many_sse2
	.p2align 2
	.type zfs_blake3_hash_many_sse2,@function
	zfs_blake3_hash_many_sse2:
	.cfi_startproc
	hint #25
	- .cfi_negate_ra_state
	+ CFI_NEGATE_RA_STATE
	stp d15, d14, [sp, #-160]!
	stp d13, d12, [sp, #16]
	stp d11, d10, [sp, #32]
	stp d9, d8, [sp, #48]
	stp x29, x30, [sp, #64]
	add x29, sp, #64
	stp x28, x27, [sp, #80]
	stp x26, x25, [sp, #96]
	stp x24, x23, [sp, #112]
	stp x22, x21, [sp, #128]
	stp x20, x19, [sp, #144]
	sub sp, sp, #464
	.cfi_def_cfa w29, 96
	.cfi_offset w19, -8
	.cfi_offset w20, -16
	.cfi_offset w21, -24
	.cfi_offset w22, -32
	.cfi_offset w23, -40
	.cfi_offset w24, -48
	.cfi_offset w25, -56
	.cfi_offset w26, -64
	.cfi_offset w27, -72
	.cfi_offset w28, -80
	.cfi_offset w30, -88
	.cfi_offset w29, -96
	.cfi_offset b8, -104
	.cfi_offset b9, -112
	.cfi_offset b10, -120
	.cfi_offset b11, -128
	.cfi_offset b12, -136
	.cfi_offset b13, -144
	.cfi_offset b14, -152
	.cfi_offset b15, -160
	mov w19, w6
	mov x20, x4
	mov x24, x1
	ldr x26, [x29, #104]
	ldrb w27, [x29, #96]
	cmp x1, #4
	str x3, [sp, #40]
	b.lo .LBB3_6
	adrp x8, .LCPI3_0
	sbfx w9, w5, #0, #1
	mov w10, #44677
	mov w11, #62322
	movk w10, #47975, lsl #16
	movk w11, #15470, lsl #16
	ldr q0, [x8, :lo12:.LCPI3_0]
	dup v1.4s, w9
	mov w9, #58983
	orr w8, w7, w19
	movk w9, #27145, lsl #16
	and v0.16b, v1.16b, v0.16b
	dup v1.4s, w11
	movi v24.4s, #64
	dup v2.4s, w9
	mov w9, #62778
	movk w9, #42319, lsl #16
	str q0, [sp, #16]
	orr v0.4s, #128, lsl #24
	stp q2, q1, [sp, #48]
	str q0, [sp]
	dup v0.4s, w10
	str q0, [sp, #80]
	b .LBB3_3
	.LBB3_2:
	zip1 v0.4s, v12.4s, v31.4s
	add x10, x20, #4
	zip1 v1.4s, v29.4s, v30.4s
	tst w5, #0x1
	zip1 v2.4s, v28.4s, v23.4s
	csel x20, x10, x20, ne
	zip1 v3.4s, v13.4s, v25.4s
	add x0, x0, #32
	zip2 v6.4s, v12.4s, v31.4s
	sub x24, x24, #4
	zip1 v4.2d, v0.2d, v1.2d
	cmp x24, #3
	zip2 v7.4s, v29.4s, v30.4s
	zip1 v5.2d, v2.2d, v3.2d
	zip2 v0.2d, v0.2d, v1.2d
	zip2 v1.2d, v2.2d, v3.2d
	zip2 v2.4s, v28.4s, v23.4s
	zip2 v3.4s, v13.4s, v25.4s
	stp q4, q5, [x26]
	zip2 v4.2d, v6.2d, v7.2d
	stp q0, q1, [x26, #32]
	zip1 v0.2d, v6.2d, v7.2d
	zip1 v1.2d, v2.2d, v3.2d
	zip2 v2.2d, v2.2d, v3.2d
	stp q0, q1, [x26, #64]
	stp q4, q2, [x26, #96]
	add x26, x26, #128
	b.ls .LBB3_6
	.LBB3_3:
	ldr x14, [sp, #40]
	mov x10, x14
	add x11, x14, #8
	add x12, x14, #12
	add x13, x14, #16
	ld1r { v12.4s }, [x10], #4
	ld1r { v29.4s }, [x11]
	add x11, x14, #20
	ld1r { v30.4s }, [x12]
	add x12, x14, #24
	ld1r { v28.4s }, [x13]
	ld1r { v23.4s }, [x11]
	add x11, x14, #28
	ld1r { v13.4s }, [x12]
	ld1r { v31.4s }, [x10]
	ld1r { v25.4s }, [x11]
	cbz x2, .LBB3_2
	ldr q1, [sp, #16]
	dup v0.4s, w20
	lsr x12, x20, #32
	mov x10, xzr
	ldp x13, x14, [x0, #16]
	add v1.4s, v0.4s, v1.4s
	mov x15, x2
	movi v0.4s, #128, lsl #24
	mov w4, w8
	str q1, [sp, #112]
	eor v0.16b, v1.16b, v0.16b
	ldr q1, [sp]
	cmgt v0.4s, v1.4s, v0.4s
	dup v1.4s, w12
	ldp x11, x12, [x0]
	sub v0.4s, v1.4s, v0.4s
	str q0, [sp, #96]
	.LBB3_5:
	add x17, x11, x10
	add x21, x12, x10
	add x16, x13, x10
	add x6, x14, x10
	subs x15, x15, #1
	add x10, x10, #64
	ldp q0, q1, [x17]
	csel w3, w27, wzr, eq
	orr w3, w3, w4
	mov w4, w19
	and w3, w3, #0xff
	ldp q3, q6, [x21]
	dup v2.4s, w3
	zip1 v21.4s, v0.4s, v3.4s
	zip2 v19.4s, v0.4s, v3.4s
	ldp q5, q7, [x16]
	zip1 v17.4s, v1.4s, v6.4s
	zip2 v22.4s, v1.4s, v6.4s
	ldp q16, q18, [x6]
	zip1 v4.4s, v5.4s, v16.4s
	zip2 v0.4s, v5.4s, v16.4s
	ldp q26, q27, [x17, #32]
	zip1 v1.4s, v7.4s, v18.4s
	zip2 v3.4s, v7.4s, v18.4s
	zip2 v20.2d, v19.2d, v0.2d
	mov v19.d[1], v0.d[0]
	dup v18.4s, w9
	ldp q8, q9, [x21, #32]
	stur q19, [x29, #-208]
	zip2 v7.4s, v26.4s, v8.4s
	zip1 v10.4s, v26.4s, v8.4s
	ldp q11, q5, [x16, #32]
	zip2 v26.2d, v17.2d, v1.2d
	stp q7, q26, [sp, #192]
	mov v17.d[1], v1.d[0]
	add v1.4s, v23.4s, v31.4s
	ldp q16, q6, [x6, #32]
	stur q17, [x29, #-256]
	add v1.4s, v1.4s, v19.4s
	zip1 v8.4s, v11.4s, v16.4s
	zip2 v7.4s, v11.4s, v16.4s
	zip1 v11.4s, v27.4s, v9.4s
	zip2 v9.4s, v27.4s, v9.4s
	zip2 v27.2d, v21.2d, v4.2d
	mov v21.d[1], v4.d[0]
	str q7, [sp, #224]
	add v4.4s, v28.4s, v12.4s
	zip1 v15.4s, v5.4s, v6.4s
	zip2 v14.4s, v5.4s, v6.4s
	stur q27, [x29, #-192]
	zip2 v16.2d, v22.2d, v3.2d
	stp q20, q21, [x29, #-240]
	add v0.4s, v4.4s, v21.4s
	ldp q6, q4, [sp, #96]
	mov v22.d[1], v3.d[0]
	add v5.4s, v25.4s, v30.4s
	add v3.4s, v13.4s, v29.4s
	eor v6.16b, v1.16b, v6.16b
	add v1.4s, v1.4s, v20.4s
	str q22, [sp, #256]
	eor v4.16b, v0.16b, v4.16b
	add v5.4s, v5.4s, v22.4s
	add v3.4s, v3.4s, v17.4s
	ldr q17, [sp, #48]
	rev32 v6.8h, v6.8h
	rev32 v4.8h, v4.8h
	eor v2.16b, v5.16b, v2.16b
	eor v7.16b, v3.16b, v24.16b
	add v0.4s, v0.4s, v27.4s
	add v21.4s, v4.4s, v17.4s
	rev32 v31.8h, v2.8h
	ldr q2, [sp, #80]
	rev32 v7.8h, v7.8h
	mov v27.16b, v16.16b
	eor v17.16b, v21.16b, v28.16b
	add v29.4s, v6.4s, v2.4s
	ldr q2, [sp, #64]
	add v24.4s, v31.4s, v18.4s
	str q27, [sp, #176]
	ushr v19.4s, v17.4s, #12
	shl v17.4s, v17.4s, #20
	add v30.4s, v7.4s, v2.4s
	eor v18.16b, v29.16b, v23.16b
	orr v12.16b, v17.16b, v19.16b
	eor v17.16b, v30.16b, v13.16b
	eor v19.16b, v24.16b, v25.16b
	ushr v23.4s, v18.4s, #12
	shl v18.4s, v18.4s, #20
	ushr v25.4s, v17.4s, #12
	shl v17.4s, v17.4s, #20
	ushr v28.4s, v19.4s, #12
	shl v19.4s, v19.4s, #20
	orr v13.16b, v18.16b, v23.16b
	orr v25.16b, v17.16b, v25.16b
	orr v2.16b, v19.16b, v28.16b
	add v28.4s, v0.4s, v12.4s
	add v0.4s, v3.4s, v26.4s
	add v18.4s, v1.4s, v13.4s
	add v3.4s, v5.4s, v16.4s
	eor v1.16b, v28.16b, v4.16b
	add v17.4s, v0.4s, v25.4s
	eor v0.16b, v18.16b, v6.16b
	add v19.4s, v3.4s, v2.4s
	ushr v16.4s, v1.4s, #8
	shl v3.4s, v1.4s, #24
	eor v4.16b, v17.16b, v7.16b
	ushr v6.4s, v0.4s, #8
	shl v1.4s, v0.4s, #24
	eor v5.16b, v19.16b, v31.16b
	ushr v23.4s, v4.4s, #8
	shl v4.4s, v4.4s, #24
	orr v7.16b, v3.16b, v16.16b
	orr v6.16b, v1.16b, v6.16b
	ushr v31.4s, v5.4s, #8
	shl v0.4s, v5.4s, #24
	orr v5.16b, v4.16b, v23.16b
	add v4.4s, v7.4s, v21.4s
	ldr q21, [sp, #192]
	add v3.4s, v6.4s, v29.4s
	orr v31.16b, v0.16b, v31.16b
	add v23.4s, v5.4s, v30.4s
	eor v0.16b, v4.16b, v12.16b
	eor v1.16b, v3.16b, v13.16b
	add v16.4s, v31.4s, v24.4s
	eor v20.16b, v23.16b, v25.16b
	ushr v24.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	ushr v29.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	ushr v30.4s, v20.4s, #7
	shl v20.4s, v20.4s, #25
	orr v25.16b, v0.16b, v24.16b
	orr v0.16b, v1.16b, v29.16b
	mov v29.16b, v10.16b
	orr v1.16b, v20.16b, v30.16b
	mov v20.16b, v10.16b
	mov v24.16b, v21.16b
	ldr q20, [sp, #224]
	mov v29.d[1], v8.d[0]
	mov v13.16b, v9.16b
	zip2 v30.2d, v10.2d, v8.2d
	zip2 v8.2d, v21.2d, v20.2d
	mov v26.16b, v11.16b
	mov v24.d[1], v20.d[0]
	add v20.4s, v28.4s, v29.4s
	mov v13.d[1], v14.d[0]
	str q8, [sp, #128]
	eor v2.16b, v16.16b, v2.16b
	mov v26.d[1], v15.d[0]
	str q24, [sp, #192]
	add v20.4s, v20.4s, v0.4s
	add v19.4s, v19.4s, v13.4s
	ushr v12.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	zip2 v10.2d, v9.2d, v14.2d
	add v18.4s, v18.4s, v24.4s
	add v17.4s, v17.4s, v26.4s
	mov v14.16b, v26.16b
	eor v26.16b, v20.16b, v31.16b
	stp q10, q30, [sp, #224]
	add v19.4s, v19.4s, v25.4s
	orr v2.16b, v2.16b, v12.16b
	add v18.4s, v18.4s, v1.4s
	rev32 v26.8h, v26.8h
	eor v5.16b, v19.16b, v5.16b
	add v17.4s, v17.4s, v2.4s
	eor v7.16b, v18.16b, v7.16b
	add v23.4s, v23.4s, v26.4s
	rev32 v5.8h, v5.8h
	eor v6.16b, v17.16b, v6.16b
	rev32 v7.8h, v7.8h
	eor v0.16b, v23.16b, v0.16b
	add v3.4s, v3.4s, v5.4s
	rev32 v6.8h, v6.8h
	add v16.4s, v16.4s, v7.4s
	ushr v31.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	eor v25.16b, v3.16b, v25.16b
	add v4.4s, v4.4s, v6.4s
	eor v1.16b, v16.16b, v1.16b
	orr v0.16b, v0.16b, v31.16b
	ushr v31.4s, v25.4s, #12
	shl v25.4s, v25.4s, #20
	add v20.4s, v20.4s, v30.4s
	zip2 v21.2d, v11.2d, v15.2d
	ushr v11.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	eor v2.16b, v4.16b, v2.16b
	orr v25.16b, v25.16b, v31.16b
	add v19.4s, v19.4s, v10.4s
	add v20.4s, v20.4s, v0.4s
	orr v1.16b, v1.16b, v11.16b
	ushr v11.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	add v18.4s, v18.4s, v8.4s
	add v19.4s, v19.4s, v25.4s
	eor v26.16b, v20.16b, v26.16b
	orr v2.16b, v2.16b, v11.16b
	add v17.4s, v17.4s, v21.4s
	add v18.4s, v18.4s, v1.4s
	eor v5.16b, v19.16b, v5.16b
	ushr v31.4s, v26.4s, #8
	shl v26.4s, v26.4s, #24
	add v17.4s, v17.4s, v2.4s
	ushr v11.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	eor v7.16b, v18.16b, v7.16b
	orr v26.16b, v26.16b, v31.16b
	eor v6.16b, v17.16b, v6.16b
	orr v5.16b, v5.16b, v11.16b
	ushr v31.4s, v7.4s, #8
	shl v7.4s, v7.4s, #24
	add v23.4s, v26.4s, v23.4s
	ushr v11.4s, v6.4s, #8
	shl v6.4s, v6.4s, #24
	orr v7.16b, v7.16b, v31.16b
	add v3.4s, v5.4s, v3.4s
	eor v0.16b, v23.16b, v0.16b
	ldp q28, q12, [x29, #-256]
	orr v6.16b, v6.16b, v11.16b
	add v16.4s, v7.4s, v16.4s
	eor v25.16b, v3.16b, v25.16b
	ushr v31.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	add v4.4s, v6.4s, v4.4s
	ushr v11.4s, v25.4s, #7
	shl v25.4s, v25.4s, #25
	eor v1.16b, v16.16b, v1.16b
	orr v0.16b, v0.16b, v31.16b
	add v18.4s, v18.4s, v12.4s
	mov v15.16b, v29.16b
	ldur q29, [x29, #-208]
	eor v2.16b, v4.16b, v2.16b
	orr v25.16b, v25.16b, v11.16b
	ushr v31.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	str q15, [sp, #160]
	add v20.4s, v20.4s, v29.4s
	add v18.4s, v18.4s, v0.4s
	ushr v11.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	orr v1.16b, v1.16b, v31.16b
	add v20.4s, v20.4s, v25.4s
	add v17.4s, v17.4s, v27.4s
	eor v6.16b, v6.16b, v18.16b
	orr v2.16b, v2.16b, v11.16b
	add v19.4s, v19.4s, v28.4s
	eor v7.16b, v7.16b, v20.16b
	add v17.4s, v17.4s, v1.4s
	rev32 v6.8h, v6.8h
	add v19.4s, v19.4s, v2.4s
	rev32 v7.8h, v7.8h
	eor v5.16b, v17.16b, v5.16b
	add v3.4s, v3.4s, v6.4s
	eor v26.16b, v19.16b, v26.16b
	add v4.4s, v4.4s, v7.4s
	rev32 v5.8h, v5.8h
	eor v0.16b, v3.16b, v0.16b
	rev32 v26.8h, v26.8h
	eor v25.16b, v4.16b, v25.16b
	add v23.4s, v23.4s, v5.4s
	ushr v11.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	add v16.4s, v16.4s, v26.4s
	ushr v31.4s, v25.4s, #12
	shl v25.4s, v25.4s, #20
	eor v1.16b, v23.16b, v1.16b
	orr v0.16b, v0.16b, v11.16b
	add v18.4s, v18.4s, v24.4s
	orr v25.16b, v25.16b, v31.16b
	eor v2.16b, v16.16b, v2.16b
	ushr v31.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	add v20.4s, v20.4s, v22.4s
	add v18.4s, v18.4s, v0.4s
	mov v9.16b, v30.16b
	mov v30.16b, v21.16b
	ldur q21, [x29, #-224]
	ushr v11.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	orr v1.16b, v1.16b, v31.16b
	add v20.4s, v20.4s, v25.4s
	str q30, [sp, #144]
	add v17.4s, v17.4s, v21.4s
	ldur q21, [x29, #-192]
	eor v6.16b, v18.16b, v6.16b
	orr v2.16b, v2.16b, v11.16b
	add v19.4s, v19.4s, v30.4s
	eor v7.16b, v20.16b, v7.16b
	add v17.4s, v17.4s, v1.4s
	ushr v11.4s, v6.4s, #8
	shl v6.4s, v6.4s, #24
	add v19.4s, v19.4s, v2.4s
	ushr v31.4s, v7.4s, #8
	shl v7.4s, v7.4s, #24
	eor v5.16b, v17.16b, v5.16b
	orr v6.16b, v6.16b, v11.16b
	eor v26.16b, v19.16b, v26.16b
	orr v7.16b, v7.16b, v31.16b
	ushr v31.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	add v3.4s, v6.4s, v3.4s
	ushr v11.4s, v26.4s, #8
	shl v26.4s, v26.4s, #24
	add v4.4s, v7.4s, v4.4s
	orr v5.16b, v5.16b, v31.16b
	eor v0.16b, v3.16b, v0.16b
	orr v26.16b, v26.16b, v11.16b
	eor v25.16b, v4.16b, v25.16b
	add v23.4s, v5.4s, v23.4s
	ushr v11.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	add v16.4s, v26.4s, v16.4s
	ushr v31.4s, v25.4s, #7
	shl v25.4s, v25.4s, #25
	eor v1.16b, v23.16b, v1.16b
	orr v0.16b, v0.16b, v11.16b
	add v20.4s, v20.4s, v21.4s
	orr v25.16b, v25.16b, v31.16b
	eor v2.16b, v16.16b, v2.16b
	ushr v31.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v20.4s, v20.4s, v0.4s
	add v19.4s, v19.4s, v10.4s
	ushr v11.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	orr v1.16b, v1.16b, v31.16b
	add v18.4s, v18.4s, v14.4s
	eor v26.16b, v20.16b, v26.16b
	add v19.4s, v19.4s, v25.4s
	orr v2.16b, v2.16b, v11.16b
	add v17.4s, v17.4s, v9.4s
	ldr q9, [sp, #208]
	add v18.4s, v18.4s, v1.4s
	rev32 v26.8h, v26.8h
	eor v5.16b, v19.16b, v5.16b
	add v17.4s, v17.4s, v2.4s
	eor v7.16b, v18.16b, v7.16b
	add v23.4s, v23.4s, v26.4s
	rev32 v5.8h, v5.8h
	eor v6.16b, v17.16b, v6.16b
	rev32 v7.8h, v7.8h
	eor v0.16b, v23.16b, v0.16b
	add v3.4s, v3.4s, v5.4s
	rev32 v6.8h, v6.8h
	add v16.4s, v16.4s, v7.4s
	ushr v31.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	eor v25.16b, v3.16b, v25.16b
	add v4.4s, v4.4s, v6.4s
	eor v1.16b, v16.16b, v1.16b
	orr v0.16b, v0.16b, v31.16b
	ushr v31.4s, v25.4s, #12
	shl v25.4s, v25.4s, #20
	add v20.4s, v20.4s, v8.4s
	ushr v11.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	eor v2.16b, v4.16b, v2.16b
	orr v25.16b, v25.16b, v31.16b
	add v19.4s, v19.4s, v15.4s
	add v20.4s, v20.4s, v0.4s
	orr v1.16b, v1.16b, v11.16b
	ushr v11.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	add v18.4s, v18.4s, v9.4s
	add v19.4s, v19.4s, v25.4s
	eor v26.16b, v20.16b, v26.16b
	orr v2.16b, v2.16b, v11.16b
	add v17.4s, v17.4s, v13.4s
	add v18.4s, v18.4s, v1.4s
	eor v5.16b, v19.16b, v5.16b
	ushr v31.4s, v26.4s, #8
	shl v26.4s, v26.4s, #24
	add v17.4s, v17.4s, v2.4s
	ushr v11.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	eor v7.16b, v18.16b, v7.16b
	orr v26.16b, v26.16b, v31.16b
	eor v6.16b, v17.16b, v6.16b
	orr v5.16b, v5.16b, v11.16b
	ushr v31.4s, v7.4s, #8
	shl v7.4s, v7.4s, #24
	add v23.4s, v26.4s, v23.4s
	ushr v11.4s, v6.4s, #8
	shl v6.4s, v6.4s, #24
	orr v7.16b, v7.16b, v31.16b
	add v3.4s, v5.4s, v3.4s
	eor v0.16b, v23.16b, v0.16b
	orr v6.16b, v6.16b, v11.16b
	add v16.4s, v7.4s, v16.4s
	eor v25.16b, v3.16b, v25.16b
	ushr v31.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	add v4.4s, v6.4s, v4.4s
	ushr v11.4s, v25.4s, #7
	shl v25.4s, v25.4s, #25
	eor v1.16b, v16.16b, v1.16b
	orr v0.16b, v0.16b, v31.16b
	add v18.4s, v18.4s, v24.4s
	eor v2.16b, v4.16b, v2.16b
	orr v25.16b, v25.16b, v11.16b
	ushr v31.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v20.4s, v20.4s, v12.4s
	add v18.4s, v18.4s, v0.4s
	ushr v11.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	orr v1.16b, v1.16b, v31.16b
	add v20.4s, v20.4s, v25.4s
	add v17.4s, v17.4s, v30.4s
	eor v6.16b, v6.16b, v18.16b
	orr v2.16b, v2.16b, v11.16b
	add v19.4s, v19.4s, v27.4s
	eor v7.16b, v7.16b, v20.16b
	add v17.4s, v17.4s, v1.4s
	rev32 v6.8h, v6.8h
	add v19.4s, v19.4s, v2.4s
	rev32 v7.8h, v7.8h
	eor v5.16b, v17.16b, v5.16b
	add v3.4s, v3.4s, v6.4s
	eor v26.16b, v19.16b, v26.16b
	add v4.4s, v4.4s, v7.4s
	rev32 v5.8h, v5.8h
	eor v0.16b, v3.16b, v0.16b
	rev32 v26.8h, v26.8h
	eor v25.16b, v4.16b, v25.16b
	add v23.4s, v23.4s, v5.4s
	ushr v11.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	add v16.4s, v16.4s, v26.4s
	ushr v31.4s, v25.4s, #12
	shl v25.4s, v25.4s, #20
	eor v1.16b, v23.16b, v1.16b
	orr v0.16b, v0.16b, v11.16b
	add v18.4s, v18.4s, v14.4s
	orr v25.16b, v25.16b, v31.16b
	eor v2.16b, v16.16b, v2.16b
	ushr v31.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	add v20.4s, v20.4s, v28.4s
	add v18.4s, v18.4s, v0.4s
	mov v10.16b, v13.16b
	ushr v11.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	orr v1.16b, v1.16b, v31.16b
	add v20.4s, v20.4s, v25.4s
	add v17.4s, v17.4s, v29.4s
	eor v6.16b, v18.16b, v6.16b
	orr v2.16b, v2.16b, v11.16b
	add v19.4s, v19.4s, v10.4s
	eor v7.16b, v20.16b, v7.16b
	add v17.4s, v17.4s, v1.4s
	ushr v11.4s, v6.4s, #8
	shl v6.4s, v6.4s, #24
	add v19.4s, v19.4s, v2.4s
	ushr v31.4s, v7.4s, #8
	shl v7.4s, v7.4s, #24
	eor v5.16b, v17.16b, v5.16b
	orr v6.16b, v6.16b, v11.16b
	eor v26.16b, v19.16b, v26.16b
	orr v7.16b, v7.16b, v31.16b
	ushr v31.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	add v3.4s, v6.4s, v3.4s
	ushr v11.4s, v26.4s, #8
	shl v26.4s, v26.4s, #24
	add v4.4s, v7.4s, v4.4s
	orr v5.16b, v5.16b, v31.16b
	eor v0.16b, v3.16b, v0.16b
	mov v22.16b, v8.16b
	ldp q8, q28, [sp, #240]
	orr v26.16b, v26.16b, v11.16b
	eor v25.16b, v4.16b, v25.16b
	add v23.4s, v5.4s, v23.4s
	ushr v11.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	add v16.4s, v26.4s, v16.4s
	ushr v31.4s, v25.4s, #7
	shl v25.4s, v25.4s, #25
	eor v1.16b, v23.16b, v1.16b
	orr v0.16b, v0.16b, v11.16b
	add v20.4s, v20.4s, v28.4s
	orr v25.16b, v25.16b, v31.16b
	eor v2.16b, v16.16b, v2.16b
	ushr v31.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v20.4s, v20.4s, v0.4s
	add v19.4s, v19.4s, v15.4s
	ushr v11.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	orr v1.16b, v1.16b, v31.16b
	add v18.4s, v18.4s, v8.4s
	eor v26.16b, v20.16b, v26.16b
	add v19.4s, v19.4s, v25.4s
	orr v2.16b, v2.16b, v11.16b
	add v17.4s, v17.4s, v22.4s
	ldur q22, [x29, #-256]
	add v18.4s, v18.4s, v1.4s
	rev32 v26.8h, v26.8h
	eor v5.16b, v19.16b, v5.16b
	add v17.4s, v17.4s, v2.4s
	eor v7.16b, v18.16b, v7.16b
	add v23.4s, v23.4s, v26.4s
	rev32 v5.8h, v5.8h
	eor v6.16b, v17.16b, v6.16b
	rev32 v7.8h, v7.8h
	eor v0.16b, v23.16b, v0.16b
	add v3.4s, v3.4s, v5.4s
	rev32 v6.8h, v6.8h
	add v16.4s, v16.4s, v7.4s
	ushr v31.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	eor v25.16b, v3.16b, v25.16b
	add v4.4s, v4.4s, v6.4s
	eor v1.16b, v16.16b, v1.16b
	orr v0.16b, v0.16b, v31.16b
	ushr v31.4s, v25.4s, #12
	shl v25.4s, v25.4s, #20
	add v20.4s, v20.4s, v9.4s
	mov v13.16b, v12.16b
	mov v12.16b, v27.16b
	mov v27.16b, v9.16b
	ldur q9, [x29, #-192]
	mov v21.16b, v15.16b
	ldr q15, [sp, #224]
	ushr v11.4s, v1.4s, #12
	ldur q21, [x29, #-224]
	shl v1.4s, v1.4s, #20
	eor v2.16b, v4.16b, v2.16b
	orr v25.16b, v25.16b, v31.16b
	add v19.4s, v19.4s, v9.4s
	add v20.4s, v20.4s, v0.4s
	orr v1.16b, v1.16b, v11.16b
	ushr v11.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	add v18.4s, v18.4s, v21.4s
	add v19.4s, v19.4s, v25.4s
	eor v26.16b, v20.16b, v26.16b
	orr v2.16b, v2.16b, v11.16b
	add v17.4s, v17.4s, v15.4s
	add v18.4s, v18.4s, v1.4s
	eor v5.16b, v19.16b, v5.16b
	ushr v31.4s, v26.4s, #8
	shl v26.4s, v26.4s, #24
	add v17.4s, v17.4s, v2.4s
	ushr v11.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	eor v7.16b, v18.16b, v7.16b
	orr v26.16b, v26.16b, v31.16b
	eor v6.16b, v17.16b, v6.16b
	orr v5.16b, v5.16b, v11.16b
	ushr v31.4s, v7.4s, #8
	shl v7.4s, v7.4s, #24
	add v23.4s, v26.4s, v23.4s
	ushr v11.4s, v6.4s, #8
	shl v6.4s, v6.4s, #24
	orr v7.16b, v7.16b, v31.16b
	add v3.4s, v5.4s, v3.4s
	eor v0.16b, v23.16b, v0.16b
	orr v6.16b, v6.16b, v11.16b
	add v16.4s, v7.4s, v16.4s
	eor v25.16b, v3.16b, v25.16b
	ushr v31.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	add v4.4s, v6.4s, v4.4s
	ushr v11.4s, v25.4s, #7
	shl v25.4s, v25.4s, #25
	eor v1.16b, v16.16b, v1.16b
	orr v0.16b, v0.16b, v31.16b
	add v18.4s, v18.4s, v14.4s
	eor v2.16b, v4.16b, v2.16b
	orr v25.16b, v25.16b, v11.16b
	ushr v31.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v20.4s, v20.4s, v24.4s
	add v18.4s, v18.4s, v0.4s
	ushr v11.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	orr v1.16b, v1.16b, v31.16b
	add v20.4s, v20.4s, v25.4s
	add v17.4s, v17.4s, v10.4s
	eor v6.16b, v6.16b, v18.16b
	orr v2.16b, v2.16b, v11.16b
	add v19.4s, v19.4s, v30.4s
	eor v7.16b, v7.16b, v20.16b
	add v17.4s, v17.4s, v1.4s
	rev32 v6.8h, v6.8h
	add v19.4s, v19.4s, v2.4s
	rev32 v7.8h, v7.8h
	eor v5.16b, v17.16b, v5.16b
	add v3.4s, v3.4s, v6.4s
	eor v26.16b, v19.16b, v26.16b
	add v4.4s, v4.4s, v7.4s
	rev32 v5.8h, v5.8h
	eor v0.16b, v3.16b, v0.16b
	rev32 v26.8h, v26.8h
	eor v25.16b, v4.16b, v25.16b
	add v23.4s, v23.4s, v5.4s
	ushr v11.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	add v16.4s, v16.4s, v26.4s
	ushr v31.4s, v25.4s, #12
	shl v25.4s, v25.4s, #20
	eor v1.16b, v23.16b, v1.16b
	orr v0.16b, v0.16b, v11.16b
	add v18.4s, v18.4s, v8.4s
	orr v25.16b, v25.16b, v31.16b
	eor v2.16b, v16.16b, v2.16b
	ushr v31.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	add v20.4s, v20.4s, v12.4s
	add v18.4s, v18.4s, v0.4s
	ushr v11.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	orr v1.16b, v1.16b, v31.16b
	add v20.4s, v20.4s, v25.4s
	add v17.4s, v17.4s, v13.4s
	ldr q13, [sp, #160]
	eor v6.16b, v18.16b, v6.16b
	orr v2.16b, v2.16b, v11.16b
	add v19.4s, v19.4s, v15.4s
	eor v7.16b, v20.16b, v7.16b
	add v17.4s, v17.4s, v1.4s
	ushr v11.4s, v6.4s, #8
	shl v6.4s, v6.4s, #24
	add v19.4s, v19.4s, v2.4s
	ushr v31.4s, v7.4s, #8
	shl v7.4s, v7.4s, #24
	eor v5.16b, v17.16b, v5.16b
	orr v6.16b, v6.16b, v11.16b
	eor v26.16b, v19.16b, v26.16b
	orr v7.16b, v7.16b, v31.16b
	ushr v31.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	add v3.4s, v6.4s, v3.4s
	ushr v11.4s, v26.4s, #8
	shl v26.4s, v26.4s, #24
	add v4.4s, v7.4s, v4.4s
	orr v5.16b, v5.16b, v31.16b
	eor v0.16b, v3.16b, v0.16b
	orr v26.16b, v26.16b, v11.16b
	eor v25.16b, v4.16b, v25.16b
	add v23.4s, v5.4s, v23.4s
	ushr v11.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	add v16.4s, v26.4s, v16.4s
	ushr v31.4s, v25.4s, #7
	shl v25.4s, v25.4s, #25
	eor v1.16b, v23.16b, v1.16b
	orr v0.16b, v0.16b, v11.16b
	add v20.4s, v20.4s, v22.4s
	orr v25.16b, v25.16b, v31.16b
	eor v2.16b, v16.16b, v2.16b
	ushr v31.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v20.4s, v20.4s, v0.4s
	add v19.4s, v19.4s, v9.4s
	mov v29.16b, v14.16b
	ldr q14, [sp, #128]
	ushr v11.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	orr v1.16b, v1.16b, v31.16b
	add v18.4s, v18.4s, v14.4s
	eor v26.16b, v20.16b, v26.16b
	add v19.4s, v19.4s, v25.4s
	orr v2.16b, v2.16b, v11.16b
	add v17.4s, v17.4s, v27.4s
	add v18.4s, v18.4s, v1.4s
	rev32 v26.8h, v26.8h
	eor v5.16b, v19.16b, v5.16b
	add v17.4s, v17.4s, v2.4s
	eor v7.16b, v18.16b, v7.16b
	add v23.4s, v23.4s, v26.4s
	rev32 v5.8h, v5.8h
	eor v6.16b, v17.16b, v6.16b
	rev32 v7.8h, v7.8h
	eor v0.16b, v23.16b, v0.16b
	add v3.4s, v3.4s, v5.4s
	rev32 v6.8h, v6.8h
	add v16.4s, v16.4s, v7.4s
	ushr v31.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	eor v25.16b, v3.16b, v25.16b
	add v4.4s, v4.4s, v6.4s
	eor v1.16b, v16.16b, v1.16b
	orr v0.16b, v0.16b, v31.16b
	ushr v31.4s, v25.4s, #12
	shl v25.4s, v25.4s, #20
	add v20.4s, v20.4s, v21.4s
	ushr v11.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	eor v2.16b, v4.16b, v2.16b
	orr v25.16b, v25.16b, v31.16b
	add v19.4s, v19.4s, v28.4s
	add v20.4s, v20.4s, v0.4s
	mov v12.16b, v27.16b
	ldur q27, [x29, #-208]
	orr v1.16b, v1.16b, v11.16b
	ushr v11.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	add v18.4s, v18.4s, v27.4s
	add v19.4s, v19.4s, v25.4s
	eor v26.16b, v20.16b, v26.16b
	orr v2.16b, v2.16b, v11.16b
	add v17.4s, v17.4s, v13.4s
	add v18.4s, v18.4s, v1.4s
	eor v5.16b, v19.16b, v5.16b
	ushr v31.4s, v26.4s, #8
	shl v26.4s, v26.4s, #24
	add v17.4s, v17.4s, v2.4s
	ushr v11.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	eor v7.16b, v18.16b, v7.16b
	orr v26.16b, v26.16b, v31.16b
	eor v6.16b, v17.16b, v6.16b
	orr v5.16b, v5.16b, v11.16b
	ushr v31.4s, v7.4s, #8
	shl v7.4s, v7.4s, #24
	add v23.4s, v26.4s, v23.4s
	ushr v11.4s, v6.4s, #8
	shl v6.4s, v6.4s, #24
	orr v7.16b, v7.16b, v31.16b
	add v3.4s, v5.4s, v3.4s
	eor v0.16b, v23.16b, v0.16b
	orr v6.16b, v6.16b, v11.16b
	add v16.4s, v7.4s, v16.4s
	eor v25.16b, v3.16b, v25.16b
	ushr v31.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	add v4.4s, v6.4s, v4.4s
	ushr v11.4s, v25.4s, #7
	shl v25.4s, v25.4s, #25
	eor v1.16b, v16.16b, v1.16b
	orr v0.16b, v0.16b, v31.16b
	add v18.4s, v18.4s, v8.4s
	eor v2.16b, v4.16b, v2.16b
	orr v25.16b, v25.16b, v11.16b
	ushr v31.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v20.4s, v20.4s, v29.4s
	add v18.4s, v18.4s, v0.4s
	ushr v11.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	orr v1.16b, v1.16b, v31.16b
	add v20.4s, v20.4s, v25.4s
	add v17.4s, v17.4s, v15.4s
	eor v6.16b, v6.16b, v18.16b
	orr v2.16b, v2.16b, v11.16b
	add v19.4s, v19.4s, v10.4s
	eor v7.16b, v7.16b, v20.16b
	add v17.4s, v17.4s, v1.4s
	rev32 v6.8h, v6.8h
	add v19.4s, v19.4s, v2.4s
	rev32 v7.8h, v7.8h
	eor v5.16b, v17.16b, v5.16b
	add v3.4s, v3.4s, v6.4s
	eor v26.16b, v19.16b, v26.16b
	add v4.4s, v4.4s, v7.4s
	rev32 v5.8h, v5.8h
	eor v0.16b, v3.16b, v0.16b
	rev32 v26.8h, v26.8h
	eor v25.16b, v4.16b, v25.16b
	add v23.4s, v23.4s, v5.4s
	ushr v11.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	add v16.4s, v16.4s, v26.4s
	ushr v31.4s, v25.4s, #12
	shl v25.4s, v25.4s, #20
	eor v1.16b, v23.16b, v1.16b
	orr v0.16b, v0.16b, v11.16b
	add v18.4s, v18.4s, v14.4s
	mov v30.16b, v29.16b
	mov v29.16b, v15.16b
	ldr q15, [sp, #144]
	orr v25.16b, v25.16b, v31.16b
	eor v2.16b, v16.16b, v2.16b
	ushr v31.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	add v20.4s, v20.4s, v15.4s
	add v18.4s, v18.4s, v0.4s
	ushr v11.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	orr v1.16b, v1.16b, v31.16b
	add v20.4s, v20.4s, v25.4s
	add v17.4s, v17.4s, v24.4s
	eor v6.16b, v18.16b, v6.16b
	orr v2.16b, v2.16b, v11.16b
	add v19.4s, v19.4s, v13.4s
	eor v7.16b, v20.16b, v7.16b
	add v17.4s, v17.4s, v1.4s
	ushr v11.4s, v6.4s, #8
	shl v6.4s, v6.4s, #24
	add v19.4s, v19.4s, v2.4s
	ushr v31.4s, v7.4s, #8
	shl v7.4s, v7.4s, #24
	eor v5.16b, v17.16b, v5.16b
	orr v6.16b, v6.16b, v11.16b
	eor v26.16b, v19.16b, v26.16b
	orr v7.16b, v7.16b, v31.16b
	ushr v31.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	add v3.4s, v6.4s, v3.4s
	ushr v11.4s, v26.4s, #8
	shl v26.4s, v26.4s, #24
	add v4.4s, v7.4s, v4.4s
	orr v5.16b, v5.16b, v31.16b
	eor v0.16b, v3.16b, v0.16b
	orr v26.16b, v26.16b, v11.16b
	eor v25.16b, v4.16b, v25.16b
	add v23.4s, v5.4s, v23.4s
	ushr v11.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	mov v9.16b, v28.16b
	mov v28.16b, v10.16b
	ldr q10, [sp, #176]
	add v16.4s, v26.4s, v16.4s
	ushr v31.4s, v25.4s, #7
	shl v25.4s, v25.4s, #25
	eor v1.16b, v23.16b, v1.16b
	orr v0.16b, v0.16b, v11.16b
	add v20.4s, v20.4s, v10.4s
	orr v25.16b, v25.16b, v31.16b
	eor v2.16b, v16.16b, v2.16b
	ushr v31.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v20.4s, v20.4s, v0.4s
	add v19.4s, v19.4s, v9.4s
	ushr v11.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	orr v1.16b, v1.16b, v31.16b
	add v18.4s, v18.4s, v12.4s
	eor v26.16b, v20.16b, v26.16b
	add v19.4s, v19.4s, v25.4s
	orr v2.16b, v2.16b, v11.16b
	add v17.4s, v17.4s, v21.4s
	add v18.4s, v18.4s, v1.4s
	rev32 v26.8h, v26.8h
	eor v5.16b, v19.16b, v5.16b
	add v17.4s, v17.4s, v2.4s
	eor v7.16b, v18.16b, v7.16b
	add v23.4s, v23.4s, v26.4s
	rev32 v5.8h, v5.8h
	eor v6.16b, v17.16b, v6.16b
	rev32 v7.8h, v7.8h
	eor v0.16b, v23.16b, v0.16b
	add v3.4s, v3.4s, v5.4s
	rev32 v6.8h, v6.8h
	add v16.4s, v16.4s, v7.4s
	ushr v31.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	eor v25.16b, v3.16b, v25.16b
	add v4.4s, v4.4s, v6.4s
	eor v1.16b, v16.16b, v1.16b
	orr v0.16b, v0.16b, v31.16b
	ushr v31.4s, v25.4s, #12
	shl v25.4s, v25.4s, #20
	ushr v11.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	eor v2.16b, v4.16b, v2.16b
	add v20.4s, v20.4s, v27.4s
	orr v25.16b, v25.16b, v31.16b
	add v19.4s, v19.4s, v22.4s
	mov v9.16b, v22.16b
	ldur q22, [x29, #-240]
	orr v1.16b, v1.16b, v11.16b
	ushr v11.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	add v20.4s, v20.4s, v0.4s
	add v18.4s, v18.4s, v22.4s
	add v19.4s, v19.4s, v25.4s
	mov v24.16b, v21.16b
	ldur q21, [x29, #-192]
	orr v2.16b, v2.16b, v11.16b
	eor v26.16b, v20.16b, v26.16b
	add v17.4s, v17.4s, v21.4s
	add v18.4s, v18.4s, v1.4s
	eor v5.16b, v19.16b, v5.16b
	ushr v31.4s, v26.4s, #8
	add v17.4s, v17.4s, v2.4s
	shl v26.4s, v26.4s, #24
	ushr v11.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	eor v7.16b, v18.16b, v7.16b
	orr v26.16b, v26.16b, v31.16b
	eor v6.16b, v17.16b, v6.16b
	orr v5.16b, v5.16b, v11.16b
	ushr v31.4s, v7.4s, #8
	shl v7.4s, v7.4s, #24
	ushr v11.4s, v6.4s, #8
	shl v6.4s, v6.4s, #24
	add v23.4s, v26.4s, v23.4s
	orr v7.16b, v7.16b, v31.16b
	add v3.4s, v5.4s, v3.4s
	orr v6.16b, v6.16b, v11.16b
	eor v0.16b, v23.16b, v0.16b
	add v16.4s, v7.4s, v16.4s
	eor v25.16b, v3.16b, v25.16b
	add v4.4s, v6.4s, v4.4s
	ushr v31.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	ushr v11.4s, v25.4s, #7
	shl v25.4s, v25.4s, #25
	eor v1.16b, v16.16b, v1.16b
	orr v0.16b, v0.16b, v31.16b
	eor v2.16b, v4.16b, v2.16b
	orr v25.16b, v25.16b, v11.16b
	ushr v31.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v20.4s, v20.4s, v8.4s
	add v18.4s, v18.4s, v14.4s
	ushr v11.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	orr v1.16b, v1.16b, v31.16b
	add v20.4s, v20.4s, v25.4s
	add v17.4s, v17.4s, v13.4s
	add v18.4s, v18.4s, v0.4s
	orr v2.16b, v2.16b, v11.16b
	add v19.4s, v19.4s, v29.4s
	eor v7.16b, v7.16b, v20.16b
	add v17.4s, v17.4s, v1.4s
	eor v6.16b, v6.16b, v18.16b
	add v19.4s, v19.4s, v2.4s
	rev32 v7.8h, v7.8h
	eor v5.16b, v17.16b, v5.16b
	rev32 v6.8h, v6.8h
	eor v26.16b, v19.16b, v26.16b
	add v4.4s, v4.4s, v7.4s
	rev32 v5.8h, v5.8h
	add v3.4s, v3.4s, v6.4s
	rev32 v26.8h, v26.8h
	eor v25.16b, v4.16b, v25.16b
	add v23.4s, v23.4s, v5.4s
	eor v0.16b, v3.16b, v0.16b
	add v16.4s, v16.4s, v26.4s
	ushr v31.4s, v25.4s, #12
	shl v25.4s, v25.4s, #20
	ushr v11.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	eor v1.16b, v23.16b, v1.16b
	orr v25.16b, v25.16b, v31.16b
	eor v2.16b, v16.16b, v2.16b
	orr v0.16b, v0.16b, v11.16b
	ushr v31.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	add v20.4s, v20.4s, v28.4s
	add v18.4s, v18.4s, v12.4s
	ushr v11.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	orr v1.16b, v1.16b, v31.16b
	add v20.4s, v20.4s, v25.4s
	add v17.4s, v17.4s, v30.4s
	add v18.4s, v18.4s, v0.4s
	orr v2.16b, v2.16b, v11.16b
	add v19.4s, v19.4s, v21.4s
	eor v7.16b, v20.16b, v7.16b
	add v17.4s, v17.4s, v1.4s
	eor v6.16b, v18.16b, v6.16b
	add v19.4s, v19.4s, v2.4s
	ushr v31.4s, v7.4s, #8
	shl v7.4s, v7.4s, #24
	ushr v11.4s, v6.4s, #8
	shl v6.4s, v6.4s, #24
	eor v5.16b, v17.16b, v5.16b
	orr v7.16b, v7.16b, v31.16b
	eor v26.16b, v19.16b, v26.16b
	orr v6.16b, v6.16b, v11.16b
	ushr v31.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	ushr v11.4s, v26.4s, #8
	shl v26.4s, v26.4s, #24
	add v4.4s, v7.4s, v4.4s
	orr v5.16b, v5.16b, v31.16b
	add v3.4s, v6.4s, v3.4s
	orr v26.16b, v26.16b, v11.16b
	eor v25.16b, v4.16b, v25.16b
	add v23.4s, v5.4s, v23.4s
	eor v0.16b, v3.16b, v0.16b
	add v16.4s, v26.4s, v16.4s
	ushr v31.4s, v25.4s, #7
	shl v25.4s, v25.4s, #25
	ushr v11.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	eor v1.16b, v23.16b, v1.16b
	orr v25.16b, v25.16b, v31.16b
	eor v2.16b, v16.16b, v2.16b
	orr v0.16b, v0.16b, v11.16b
	ushr v31.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v20.4s, v20.4s, v15.4s
	ushr v11.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	orr v1.16b, v1.16b, v31.16b
	add v18.4s, v18.4s, v24.4s
	add v20.4s, v20.4s, v0.4s
	add v19.4s, v19.4s, v9.4s
	mov v8.16b, v13.16b
	ldur q13, [x29, #-208]
	orr v2.16b, v2.16b, v11.16b
	add v18.4s, v18.4s, v1.4s
	add v17.4s, v17.4s, v13.4s
	eor v26.16b, v20.16b, v26.16b
	add v19.4s, v19.4s, v25.4s
	eor v7.16b, v18.16b, v7.16b
	add v17.4s, v17.4s, v2.4s
	rev32 v26.8h, v26.8h
	eor v5.16b, v19.16b, v5.16b
	rev32 v7.8h, v7.8h
	eor v6.16b, v17.16b, v6.16b
	add v23.4s, v23.4s, v26.4s
	rev32 v5.8h, v5.8h
	add v16.4s, v16.4s, v7.4s
	rev32 v6.8h, v6.8h
	eor v0.16b, v23.16b, v0.16b
	add v3.4s, v3.4s, v5.4s
	eor v1.16b, v16.16b, v1.16b
	add v4.4s, v4.4s, v6.4s
	ushr v31.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	eor v25.16b, v3.16b, v25.16b
	ushr v11.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	orr v0.16b, v0.16b, v31.16b
	eor v2.16b, v4.16b, v2.16b
	ushr v31.4s, v25.4s, #12
	shl v25.4s, v25.4s, #20
	orr v1.16b, v1.16b, v11.16b
	ushr v11.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	add v20.4s, v20.4s, v22.4s
	orr v25.16b, v25.16b, v31.16b
	add v19.4s, v19.4s, v10.4s
	mov v27.16b, v12.16b
	mov v12.16b, v30.16b
	mov v29.16b, v21.16b
	mov v21.16b, v24.16b
	ldr q24, [sp, #192]
	mov v30.16b, v22.16b
	ldr q22, [sp, #256]
	orr v2.16b, v2.16b, v11.16b
	add v20.4s, v20.4s, v0.4s
	add v18.4s, v18.4s, v24.4s
	add v19.4s, v19.4s, v25.4s
	add v17.4s, v17.4s, v22.4s
	eor v26.16b, v20.16b, v26.16b
	add v18.4s, v18.4s, v1.4s
	eor v5.16b, v19.16b, v5.16b
	add v17.4s, v17.4s, v2.4s
	ushr v31.4s, v26.4s, #8
	shl v26.4s, v26.4s, #24
	ushr v11.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	eor v7.16b, v18.16b, v7.16b
	eor v6.16b, v17.16b, v6.16b
	orr v26.16b, v26.16b, v31.16b
	orr v5.16b, v5.16b, v11.16b
	ushr v31.4s, v7.4s, #8
	shl v7.4s, v7.4s, #24
	ushr v11.4s, v6.4s, #8
	shl v6.4s, v6.4s, #24
	add v23.4s, v26.4s, v23.4s
	orr v7.16b, v7.16b, v31.16b
	add v3.4s, v5.4s, v3.4s
	orr v6.16b, v6.16b, v11.16b
	eor v0.16b, v23.16b, v0.16b
	add v16.4s, v7.4s, v16.4s
	eor v25.16b, v3.16b, v25.16b
	add v4.4s, v6.4s, v4.4s
	ushr v31.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	ushr v11.4s, v25.4s, #7
	shl v25.4s, v25.4s, #25
	eor v1.16b, v16.16b, v1.16b
	eor v2.16b, v4.16b, v2.16b
	orr v0.16b, v0.16b, v31.16b
	orr v25.16b, v25.16b, v11.16b
	ushr v31.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	ushr v11.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	add v20.4s, v20.4s, v14.4s
	add v18.4s, v18.4s, v27.4s
	ldr q27, [sp, #224]
	orr v1.16b, v1.16b, v31.16b
	orr v2.16b, v2.16b, v11.16b
	add v20.4s, v20.4s, v25.4s
	add v17.4s, v17.4s, v29.4s
	add v18.4s, v18.4s, v0.4s
	add v19.4s, v19.4s, v8.4s
	eor v7.16b, v7.16b, v20.16b
	add v17.4s, v17.4s, v1.4s
	eor v6.16b, v6.16b, v18.16b
	add v19.4s, v19.4s, v2.4s
	rev32 v7.8h, v7.8h
	eor v5.16b, v17.16b, v5.16b
	rev32 v6.8h, v6.8h
	eor v26.16b, v19.16b, v26.16b
	add v4.4s, v4.4s, v7.4s
	rev32 v5.8h, v5.8h
	add v3.4s, v3.4s, v6.4s
	rev32 v26.8h, v26.8h
	eor v25.16b, v4.16b, v25.16b
	add v23.4s, v23.4s, v5.4s
	eor v0.16b, v3.16b, v0.16b
	add v16.4s, v16.4s, v26.4s
	ushr v29.4s, v25.4s, #12
	shl v25.4s, v25.4s, #20
	ushr v31.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	eor v1.16b, v23.16b, v1.16b
	eor v2.16b, v16.16b, v2.16b
	orr v25.16b, v25.16b, v29.16b
	orr v0.16b, v0.16b, v31.16b
	ushr v29.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	ushr v31.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	add v18.4s, v18.4s, v21.4s
	ldr q21, [sp, #240]
	add v20.4s, v20.4s, v27.4s
	prfm pldl1keep, [x17, #256]
	orr v1.16b, v1.16b, v29.16b
	prfm pldl1keep, [x21, #256]
	orr v2.16b, v2.16b, v31.16b
	prfm pldl1keep, [x16, #256]
	add v18.4s, v18.4s, v0.4s
	prfm pldl1keep, [x6, #256]
	add v17.4s, v17.4s, v21.4s
	add v19.4s, v19.4s, v22.4s
	add v20.4s, v20.4s, v25.4s
	eor v6.16b, v18.16b, v6.16b
	add v17.4s, v17.4s, v1.4s
	add v19.4s, v19.4s, v2.4s
	eor v7.16b, v20.16b, v7.16b
	ushr v22.4s, v6.4s, #8
	shl v6.4s, v6.4s, #24
	eor v5.16b, v17.16b, v5.16b
	eor v26.16b, v19.16b, v26.16b
	ushr v21.4s, v7.4s, #8
	shl v7.4s, v7.4s, #24
	orr v6.16b, v6.16b, v22.16b
	ushr v22.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	ushr v29.4s, v26.4s, #8
	shl v26.4s, v26.4s, #24
	orr v7.16b, v7.16b, v21.16b
	orr v5.16b, v5.16b, v22.16b
	add v3.4s, v6.4s, v3.4s
	orr v21.16b, v26.16b, v29.16b
	add v4.4s, v7.4s, v4.4s
	add v22.4s, v5.4s, v23.4s
	eor v0.16b, v3.16b, v0.16b
	add v16.4s, v21.4s, v16.4s
	eor v23.16b, v4.16b, v25.16b
	eor v1.16b, v22.16b, v1.16b
	ushr v25.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	eor v2.16b, v16.16b, v2.16b
	ushr v26.4s, v23.4s, #7
	shl v23.4s, v23.4s, #25
	orr v0.16b, v0.16b, v25.16b
	ushr v25.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	ushr v29.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	add v20.4s, v20.4s, v28.4s
	orr v23.16b, v23.16b, v26.16b
	orr v1.16b, v1.16b, v25.16b
	orr v2.16b, v2.16b, v29.16b
	add v20.4s, v20.4s, v0.4s
	add v18.4s, v18.4s, v13.4s
	add v17.4s, v17.4s, v30.4s
	add v19.4s, v19.4s, v10.4s
	eor v21.16b, v20.16b, v21.16b
	add v18.4s, v18.4s, v1.4s
	add v17.4s, v17.4s, v2.4s
	add v19.4s, v19.4s, v23.4s
	rev32 v21.8h, v21.8h
	eor v7.16b, v18.16b, v7.16b
	eor v6.16b, v17.16b, v6.16b
	eor v5.16b, v19.16b, v5.16b
	add v22.4s, v22.4s, v21.4s
	rev32 v7.8h, v7.8h
	rev32 v6.8h, v6.8h
	rev32 v5.8h, v5.8h
	eor v0.16b, v22.16b, v0.16b
	add v16.4s, v16.4s, v7.4s
	add v4.4s, v4.4s, v6.4s
	add v3.4s, v3.4s, v5.4s
	ushr v25.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	eor v1.16b, v16.16b, v1.16b
	eor v2.16b, v4.16b, v2.16b
	eor v23.16b, v3.16b, v23.16b
	orr v0.16b, v0.16b, v25.16b
	ushr v25.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	ushr v26.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	ushr v27.4s, v23.4s, #12
	shl v23.4s, v23.4s, #20
	orr v1.16b, v1.16b, v25.16b
	add v20.4s, v20.4s, v24.4s
	orr v2.16b, v2.16b, v26.16b
	orr v23.16b, v23.16b, v27.16b
	add v18.4s, v18.4s, v12.4s
	add v17.4s, v17.4s, v9.4s
	add v19.4s, v19.4s, v15.4s
	add v20.4s, v20.4s, v0.4s
	add v18.4s, v18.4s, v1.4s
	add v17.4s, v17.4s, v2.4s
	add v19.4s, v19.4s, v23.4s
	eor v21.16b, v20.16b, v21.16b
	eor v7.16b, v18.16b, v7.16b
	eor v6.16b, v17.16b, v6.16b
	eor v5.16b, v19.16b, v5.16b
	ushr v24.4s, v21.4s, #8
	shl v21.4s, v21.4s, #24
	ushr v25.4s, v7.4s, #8
	shl v7.4s, v7.4s, #24
	ushr v26.4s, v6.4s, #8
	shl v6.4s, v6.4s, #24
	ushr v27.4s, v5.4s, #8
	shl v5.4s, v5.4s, #24
	orr v21.16b, v21.16b, v24.16b
	orr v7.16b, v7.16b, v25.16b
	orr v6.16b, v6.16b, v26.16b
	orr v5.16b, v5.16b, v27.16b
	add v22.4s, v21.4s, v22.4s
	add v16.4s, v7.4s, v16.4s
	add v4.4s, v6.4s, v4.4s
	add v3.4s, v5.4s, v3.4s
	eor v0.16b, v22.16b, v0.16b
	eor v1.16b, v16.16b, v1.16b
	eor v2.16b, v4.16b, v2.16b
	eor v23.16b, v3.16b, v23.16b
	ushr v24.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	ushr v25.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	ushr v26.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	ushr v27.4s, v23.4s, #7
	shl v23.4s, v23.4s, #25
	orr v0.16b, v0.16b, v24.16b
	orr v1.16b, v1.16b, v25.16b
	orr v2.16b, v2.16b, v26.16b
	orr v23.16b, v23.16b, v27.16b
	movi v24.4s, #64
	eor v12.16b, v4.16b, v20.16b
	eor v31.16b, v18.16b, v3.16b
	eor v29.16b, v17.16b, v22.16b
	eor v30.16b, v16.16b, v19.16b
	eor v28.16b, v7.16b, v23.16b
	eor v23.16b, v6.16b, v0.16b
	eor v13.16b, v1.16b, v5.16b
	eor v25.16b, v2.16b, v21.16b
	cbnz x15, .LBB3_5
	b .LBB3_2
	.LBB3_6:
	cbz x24, .LBB3_14
	orr w8, w7, w19
	and x22, x5, #0x1
	stur w8, [x29, #-192]
	.LBB3_8:
	ldr x8, [sp, #40]
	mov x28, x0
	ldr x25, [x0]
	mov x23, x2
	ldur w5, [x29, #-192]
	ldp q0, q1, [x8]
	mov x8, x2
	b .LBB3_11
	.LBB3_9:
	orr w5, w5, w27
	.LBB3_10:
	sub x0, x29, #144
	sub x1, x29, #176
	mov x2, x25
	mov w3, #64
	mov x4, x20
	bl compress_pre
	ldp q0, q1, [x29, #-144]
	add x25, x25, #64
	mov x8, x21
	mov w5, w19
	ldp q2, q3, [x29, #-112]
	eor v0.16b, v2.16b, v0.16b
	eor v1.16b, v3.16b, v1.16b
	.LBB3_11:
	subs x21, x8, #1
	stp q0, q1, [x29, #-176]
	b.eq .LBB3_9
	cbnz x8, .LBB3_10
	ldp q1, q0, [x29, #-176]
	mov x0, x28
	add x20, x20, x22
	add x0, x28, #8
	subs x24, x24, #1
	mov x2, x23
	stp q1, q0, [x26], #32
	b.ne .LBB3_8
	.LBB3_14:
	add sp, sp, #464
	ldp x20, x19, [sp, #144]
	ldp x22, x21, [sp, #128]
	ldp x24, x23, [sp, #112]
	ldp x26, x25, [sp, #96]
	ldp x28, x27, [sp, #80]
	ldp x29, x30, [sp, #64]
	ldp d9, d8, [sp, #48]
	ldp d11, d10, [sp, #32]
	ldp d13, d12, [sp, #16]
	ldp d15, d14, [sp], #160
	hint #29
	ret
	.Lfunc_end3:
	.size zfs_blake3_hash_many_sse2, .Lfunc_end3-zfs_blake3_hash_many_sse2
	.cfi_endproc
	.section ".note.GNU-stack","",@progbits
	#endif
	\ No newline at end of file
	diff --git a/sys/contrib/openzfs/module/icp/asm-aarch64/blake3/b3_aarch64_sse41.S b/sys/contrib/openzfs/module/icp/asm-aarch64/blake3/b3_aarch64_sse41.S
	index c4c2dfc5bcde..b9fb28dfcf03 100644
	--- a/sys/contrib/openzfs/module/icp/asm-aarch64/blake3/b3_aarch64_sse41.S
	+++ b/sys/contrib/openzfs/module/icp/asm-aarch64/blake3/b3_aarch64_sse41.S
	@@ -1,2398 +1,2406 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Based on BLAKE3 v1.3.1, https://github.com/BLAKE3-team/BLAKE3
	* Copyright (c) 2019-2022 Samuel Neves
	* Copyright (c) 2022-2023 Tino Reichardt <milky-zfs@mcmilk.de>
	*
	* This is converted assembly: SSE4.1 -> ARMv8-A
	* Used tools: SIMDe https://github.com/simd-everywhere/simde
	*
	* Should work on FreeBSD, Linux and macOS
	* see: https://github.com/mcmilk/BLAKE3-tests/blob/master/contrib/simde.sh
	*/

	#if defined(__aarch64__)
	+
	+/* make gcc <= 9 happy */
	+#if LD_VERSION >= 233010000
	+#define CFI_NEGATE_RA_STATE .cfi_negate_ra_state
	+#else
	+#define CFI_NEGATE_RA_STATE
	+#endif
	+
	.text
	.section .note.gnu.property,"a",@note
	.p2align 3
	.word 4
	.word 16
	.word 5
	.asciz "GNU"
	.word 3221225472
	.word 4
	.word 3
	.word 0
	.Lsec_end0:
	.text
	.globl zfs_blake3_compress_in_place_sse41
	.p2align 2
	.type zfs_blake3_compress_in_place_sse41,@function
	zfs_blake3_compress_in_place_sse41:
	.cfi_startproc
	hint #25
	- .cfi_negate_ra_state
	+ CFI_NEGATE_RA_STATE
	sub sp, sp, #96
	stp x29, x30, [sp, #64]
	add x29, sp, #64
	str x19, [sp, #80]
	.cfi_def_cfa w29, 32
	.cfi_offset w19, -16
	.cfi_offset w30, -24
	.cfi_offset w29, -32
	mov x19, x0
	mov w5, w4
	mov x4, x3
	mov w3, w2
	mov x2, x1
	mov x0, sp
	mov x1, x19
	bl compress_pre
	ldp q0, q1, [sp]
	ldp q2, q3, [sp, #32]
	eor v0.16b, v2.16b, v0.16b
	eor v1.16b, v3.16b, v1.16b
	ldp x29, x30, [sp, #64]
	stp q0, q1, [x19]
	ldr x19, [sp, #80]
	add sp, sp, #96
	hint #29
	ret
	.Lfunc_end0:
	.size zfs_blake3_compress_in_place_sse41, .Lfunc_end0-zfs_blake3_compress_in_place_sse41
	.cfi_endproc

	.section .rodata.cst16,"aM",@progbits,16
	.p2align 4
	.LCPI1_0:
	.xword -4942790177982912921
	.xword -6534734903820487822
	.LCPI1_1:
	.byte 2
	.byte 3
	.byte 0
	.byte 1
	.byte 6
	.byte 7
	.byte 4
	.byte 5
	.byte 10
	.byte 11
	.byte 8
	.byte 9
	.byte 14
	.byte 15
	.byte 12
	.byte 13
	.LCPI1_2:
	.byte 1
	.byte 2
	.byte 3
	.byte 0
	.byte 5
	.byte 6
	.byte 7
	.byte 4
	.byte 9
	.byte 10
	.byte 11
	.byte 8
	.byte 13
	.byte 14
	.byte 15
	.byte 12
	.text
	.p2align 2
	.type compress_pre,@function
	compress_pre:
	.cfi_startproc
	hint #34
	fmov s1, w3
	movi d0, #0x0000ff000000ff
	ldr q2, [x1]
	adrp x8, .LCPI1_0
	mov v1.s[1], w5
	str q2, [x0]
	ldr q4, [x8, :lo12:.LCPI1_0]
	ldr q5, [x1, #16]
	adrp x8, .LCPI1_1
	and v0.8b, v1.8b, v0.8b
	fmov d1, x4
	stp q5, q4, [x0, #16]
	mov v1.d[1], v0.d[0]
	str q1, [x0, #48]
	ldp q6, q7, [x2]
	uzp1 v3.4s, v6.4s, v7.4s
	add v0.4s, v2.4s, v3.4s
	uzp2 v2.4s, v6.4s, v7.4s
	add v16.4s, v0.4s, v5.4s
	ldr q0, [x8, :lo12:.LCPI1_1]
	adrp x8, .LCPI1_2
	eor v1.16b, v16.16b, v1.16b
	add v7.4s, v16.4s, v2.4s
	tbl v1.16b, { v1.16b }, v0.16b
	add v4.4s, v1.4s, v4.4s
	eor v5.16b, v4.16b, v5.16b
	ushr v6.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	orr v5.16b, v5.16b, v6.16b
	add v6.4s, v7.4s, v5.4s
	eor v7.16b, v1.16b, v6.16b
	ldr q1, [x8, :lo12:.LCPI1_2]
	add x8, x2, #32
	tbl v7.16b, { v7.16b }, v1.16b
	ld2 { v16.4s, v17.4s }, [x8]
	add v4.4s, v4.4s, v7.4s
	ext v7.16b, v7.16b, v7.16b, #8
	add v6.4s, v6.4s, v16.4s
	eor v5.16b, v4.16b, v5.16b
	ext v4.16b, v4.16b, v4.16b, #4
	ext v16.16b, v16.16b, v16.16b, #12
	ext v6.16b, v6.16b, v6.16b, #12
	ushr v18.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	orr v5.16b, v5.16b, v18.16b
	ext v18.16b, v17.16b, v17.16b, #12
	add v6.4s, v6.4s, v5.4s
	mov v17.16b, v18.16b
	eor v7.16b, v7.16b, v6.16b
	add v6.4s, v6.4s, v18.4s
	mov v17.s[1], v16.s[2]
	tbl v7.16b, { v7.16b }, v0.16b
	add v4.4s, v4.4s, v7.4s
	eor v5.16b, v4.16b, v5.16b
	ushr v19.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	orr v5.16b, v5.16b, v19.16b
	uzp1 v19.4s, v3.4s, v3.4s
	add v6.4s, v6.4s, v5.4s
	ext v19.16b, v19.16b, v3.16b, #8
	eor v7.16b, v7.16b, v6.16b
	uzp2 v19.4s, v19.4s, v2.4s
	tbl v7.16b, { v7.16b }, v1.16b
	add v6.4s, v6.4s, v19.4s
	add v4.4s, v4.4s, v7.4s
	ext v6.16b, v6.16b, v6.16b, #4
	ext v7.16b, v7.16b, v7.16b, #8
	eor v5.16b, v4.16b, v5.16b
	ext v4.16b, v4.16b, v4.16b, #12
	ushr v20.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	orr v5.16b, v5.16b, v20.16b
	ext v20.16b, v3.16b, v3.16b, #12
	add v6.4s, v6.4s, v5.4s
	ext v3.16b, v3.16b, v20.16b, #12
	eor v7.16b, v7.16b, v6.16b
	rev64 v3.4s, v3.4s
	tbl v7.16b, { v7.16b }, v0.16b
	trn2 v3.4s, v3.4s, v17.4s
	add v4.4s, v4.4s, v7.4s
	add v6.4s, v6.4s, v3.4s
	eor v5.16b, v4.16b, v5.16b
	ushr v17.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	orr v5.16b, v5.16b, v17.16b
	zip1 v17.2d, v18.2d, v2.2d
	zip2 v2.4s, v2.4s, v18.4s
	add v6.4s, v6.4s, v5.4s
	mov v17.s[3], v16.s[3]
	zip1 v18.4s, v2.4s, v16.4s
	zip1 v2.4s, v16.4s, v2.4s
	eor v7.16b, v7.16b, v6.16b
	ext v6.16b, v6.16b, v6.16b, #12
	ext v16.16b, v2.16b, v18.16b, #8
	tbl v7.16b, { v7.16b }, v1.16b
	add v20.4s, v4.4s, v7.4s
	ext v4.16b, v17.16b, v17.16b, #12
	ext v7.16b, v7.16b, v7.16b, #8
	eor v5.16b, v20.16b, v5.16b
	uzp1 v4.4s, v17.4s, v4.4s
	ushr v17.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	add v6.4s, v6.4s, v4.4s
	orr v5.16b, v5.16b, v17.16b
	ext v17.16b, v20.16b, v20.16b, #4
	add v6.4s, v6.4s, v5.4s
	eor v7.16b, v7.16b, v6.16b
	add v6.4s, v6.4s, v16.4s
	tbl v7.16b, { v7.16b }, v0.16b
	add v17.4s, v17.4s, v7.4s
	eor v5.16b, v17.16b, v5.16b
	ushr v2.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	orr v2.16b, v5.16b, v2.16b
	add v5.4s, v6.4s, v2.4s
	ext v6.16b, v19.16b, v19.16b, #4
	eor v7.16b, v7.16b, v5.16b
	uzp1 v18.4s, v6.4s, v6.4s
	tbl v7.16b, { v7.16b }, v1.16b
	ext v18.16b, v18.16b, v6.16b, #8
	add v17.4s, v17.4s, v7.4s
	uzp2 v18.4s, v18.4s, v3.4s
	ext v7.16b, v7.16b, v7.16b, #8
	eor v2.16b, v17.16b, v2.16b
	add v5.4s, v5.4s, v18.4s
	ext v17.16b, v17.16b, v17.16b, #12
	ushr v19.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	ext v5.16b, v5.16b, v5.16b, #4
	orr v2.16b, v2.16b, v19.16b
	ext v19.16b, v6.16b, v6.16b, #12
	add v5.4s, v5.4s, v2.4s
	ext v6.16b, v6.16b, v19.16b, #12
	mov v19.16b, v16.16b
	eor v7.16b, v7.16b, v5.16b
	rev64 v6.4s, v6.4s
	mov v19.s[1], v4.s[2]
	tbl v7.16b, { v7.16b }, v0.16b
	add v17.4s, v17.4s, v7.4s
	eor v20.16b, v17.16b, v2.16b
	trn2 v2.4s, v6.4s, v19.4s
	ushr v6.4s, v20.4s, #12
	shl v19.4s, v20.4s, #20
	add v5.4s, v5.4s, v2.4s
	orr v6.16b, v19.16b, v6.16b
	add v19.4s, v5.4s, v6.4s
	eor v5.16b, v7.16b, v19.16b
	zip1 v7.2d, v16.2d, v3.2d
	zip2 v3.4s, v3.4s, v16.4s
	tbl v20.16b, { v5.16b }, v1.16b
	mov v7.s[3], v4.s[3]
	add v17.4s, v17.4s, v20.4s
	ext v5.16b, v7.16b, v7.16b, #12
	eor v6.16b, v17.16b, v6.16b
	uzp1 v5.4s, v7.4s, v5.4s
	ext v7.16b, v19.16b, v19.16b, #12
	ext v17.16b, v17.16b, v17.16b, #4
	ushr v19.4s, v6.4s, #7
	shl v6.4s, v6.4s, #25
	add v7.4s, v7.4s, v5.4s
	orr v6.16b, v6.16b, v19.16b
	ext v19.16b, v20.16b, v20.16b, #8
	add v7.4s, v7.4s, v6.4s
	eor v19.16b, v19.16b, v7.16b
	tbl v19.16b, { v19.16b }, v0.16b
	add v16.4s, v17.4s, v19.4s
	zip1 v17.4s, v3.4s, v4.4s
	zip1 v3.4s, v4.4s, v3.4s
	eor v4.16b, v16.16b, v6.16b
	ext v17.16b, v3.16b, v17.16b, #8
	ushr v3.4s, v4.4s, #12
	shl v4.4s, v4.4s, #20
	add v6.4s, v7.4s, v17.4s
	orr v3.16b, v4.16b, v3.16b
	add v4.4s, v6.4s, v3.4s
	ext v6.16b, v18.16b, v18.16b, #4
	eor v7.16b, v19.16b, v4.16b
	uzp1 v18.4s, v6.4s, v6.4s
	tbl v7.16b, { v7.16b }, v1.16b
	ext v18.16b, v18.16b, v6.16b, #8
	add v16.4s, v16.4s, v7.4s
	uzp2 v18.4s, v18.4s, v2.4s
	ext v7.16b, v7.16b, v7.16b, #8
	eor v3.16b, v16.16b, v3.16b
	add v4.4s, v4.4s, v18.4s
	ext v16.16b, v16.16b, v16.16b, #12
	ushr v19.4s, v3.4s, #7
	shl v3.4s, v3.4s, #25
	ext v4.16b, v4.16b, v4.16b, #4
	orr v3.16b, v3.16b, v19.16b
	ext v19.16b, v6.16b, v6.16b, #12
	add v4.4s, v4.4s, v3.4s
	ext v6.16b, v6.16b, v19.16b, #12
	mov v19.16b, v17.16b
	eor v7.16b, v7.16b, v4.16b
	rev64 v6.4s, v6.4s
	mov v19.s[1], v5.s[2]
	tbl v7.16b, { v7.16b }, v0.16b
	add v16.4s, v16.4s, v7.4s
	eor v20.16b, v16.16b, v3.16b
	trn2 v3.4s, v6.4s, v19.4s
	ushr v6.4s, v20.4s, #12
	shl v19.4s, v20.4s, #20
	add v4.4s, v4.4s, v3.4s
	orr v6.16b, v19.16b, v6.16b
	zip1 v19.2d, v17.2d, v2.2d
	zip2 v2.4s, v2.4s, v17.4s
	add v4.4s, v4.4s, v6.4s
	mov v19.s[3], v5.s[3]
	zip1 v17.4s, v2.4s, v5.4s
	zip1 v2.4s, v5.4s, v2.4s
	eor v7.16b, v7.16b, v4.16b
	ext v20.16b, v19.16b, v19.16b, #12
	ext v4.16b, v4.16b, v4.16b, #12
	ext v2.16b, v2.16b, v17.16b, #8
	tbl v7.16b, { v7.16b }, v1.16b
	add v16.4s, v16.4s, v7.4s
	ext v7.16b, v7.16b, v7.16b, #8
	eor v21.16b, v16.16b, v6.16b
	uzp1 v6.4s, v19.4s, v20.4s
	ext v16.16b, v16.16b, v16.16b, #4
	ushr v19.4s, v21.4s, #7
	shl v20.4s, v21.4s, #25
	add v4.4s, v4.4s, v6.4s
	orr v19.16b, v20.16b, v19.16b
	add v4.4s, v4.4s, v19.4s
	eor v7.16b, v7.16b, v4.16b
	add v4.4s, v4.4s, v2.4s
	tbl v7.16b, { v7.16b }, v0.16b
	add v16.4s, v16.4s, v7.4s
	eor v5.16b, v16.16b, v19.16b
	ushr v17.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	orr v5.16b, v5.16b, v17.16b
	ext v17.16b, v18.16b, v18.16b, #4
	add v4.4s, v4.4s, v5.4s
	uzp1 v18.4s, v17.4s, v17.4s
	eor v7.16b, v7.16b, v4.16b
	ext v18.16b, v18.16b, v17.16b, #8
	tbl v7.16b, { v7.16b }, v1.16b
	uzp2 v18.4s, v18.4s, v3.4s
	add v16.4s, v16.4s, v7.4s
	add v4.4s, v4.4s, v18.4s
	ext v7.16b, v7.16b, v7.16b, #8
	eor v5.16b, v16.16b, v5.16b
	ext v4.16b, v4.16b, v4.16b, #4
	ext v16.16b, v16.16b, v16.16b, #12
	ushr v19.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	orr v5.16b, v5.16b, v19.16b
	add v19.4s, v4.4s, v5.4s
	eor v4.16b, v7.16b, v19.16b
	ext v7.16b, v17.16b, v17.16b, #12
	tbl v20.16b, { v4.16b }, v0.16b
	ext v4.16b, v17.16b, v7.16b, #12
	mov v7.16b, v2.16b
	add v16.4s, v16.4s, v20.4s
	rev64 v4.4s, v4.4s
	mov v7.s[1], v6.s[2]
	eor v5.16b, v16.16b, v5.16b
	trn2 v4.4s, v4.4s, v7.4s
	ushr v7.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	add v17.4s, v19.4s, v4.4s
	zip1 v19.2d, v2.2d, v3.2d
	zip2 v2.4s, v3.4s, v2.4s
	orr v5.16b, v5.16b, v7.16b
	mov v19.s[3], v6.s[3]
	add v7.4s, v17.4s, v5.4s
	eor v17.16b, v20.16b, v7.16b
	ext v20.16b, v19.16b, v19.16b, #12
	ext v7.16b, v7.16b, v7.16b, #12
	tbl v17.16b, { v17.16b }, v1.16b
	add v16.4s, v16.4s, v17.4s
	ext v17.16b, v17.16b, v17.16b, #8
	eor v21.16b, v16.16b, v5.16b
	uzp1 v5.4s, v19.4s, v20.4s
	ext v16.16b, v16.16b, v16.16b, #4
	ushr v19.4s, v21.4s, #7
	shl v20.4s, v21.4s, #25
	add v7.4s, v7.4s, v5.4s
	orr v19.16b, v20.16b, v19.16b
	add v7.4s, v7.4s, v19.4s
	eor v17.16b, v17.16b, v7.16b
	tbl v17.16b, { v17.16b }, v0.16b
	add v3.4s, v16.4s, v17.4s
	zip1 v16.4s, v2.4s, v6.4s
	zip1 v2.4s, v6.4s, v2.4s
	eor v6.16b, v3.16b, v19.16b
	ext v16.16b, v2.16b, v16.16b, #8
	ushr v2.4s, v6.4s, #12
	shl v6.4s, v6.4s, #20
	add v7.4s, v7.4s, v16.4s
	orr v2.16b, v6.16b, v2.16b
	add v6.4s, v7.4s, v2.4s
	ext v7.16b, v18.16b, v18.16b, #4
	eor v17.16b, v17.16b, v6.16b
	uzp1 v18.4s, v7.4s, v7.4s
	tbl v17.16b, { v17.16b }, v1.16b
	ext v18.16b, v18.16b, v7.16b, #8
	add v3.4s, v3.4s, v17.4s
	uzp2 v18.4s, v18.4s, v4.4s
	eor v2.16b, v3.16b, v2.16b
	add v6.4s, v6.4s, v18.4s
	ext v3.16b, v3.16b, v3.16b, #12
	ext v18.16b, v18.16b, v18.16b, #4
	ushr v19.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	ext v6.16b, v6.16b, v6.16b, #4
	orr v19.16b, v2.16b, v19.16b
	ext v2.16b, v17.16b, v17.16b, #8
	ext v17.16b, v7.16b, v7.16b, #12
	add v6.4s, v6.4s, v19.4s
	eor v2.16b, v2.16b, v6.16b
	tbl v20.16b, { v2.16b }, v0.16b
	ext v2.16b, v7.16b, v17.16b, #12
	mov v7.16b, v16.16b
	add v17.4s, v3.4s, v20.4s
	rev64 v3.4s, v2.4s
	mov v7.s[1], v5.s[2]
	eor v19.16b, v17.16b, v19.16b
	trn2 v3.4s, v3.4s, v7.4s
	ushr v21.4s, v19.4s, #12
	shl v19.4s, v19.4s, #20
	add v6.4s, v6.4s, v3.4s
	orr v19.16b, v19.16b, v21.16b
	add v21.4s, v6.4s, v19.4s
	eor v6.16b, v20.16b, v21.16b
	zip1 v20.2d, v16.2d, v4.2d
	zip2 v4.4s, v4.4s, v16.4s
	tbl v22.16b, { v6.16b }, v1.16b
	mov v20.s[3], v5.s[3]
	add v17.4s, v17.4s, v22.4s
	ext v6.16b, v20.16b, v20.16b, #12
	eor v19.16b, v17.16b, v19.16b
	uzp1 v6.4s, v20.4s, v6.4s
	ext v20.16b, v21.16b, v21.16b, #12
	ext v17.16b, v17.16b, v17.16b, #4
	ushr v21.4s, v19.4s, #7
	shl v19.4s, v19.4s, #25
	add v20.4s, v20.4s, v6.4s
	orr v19.16b, v19.16b, v21.16b
	ext v21.16b, v22.16b, v22.16b, #8
	add v20.4s, v20.4s, v19.4s
	eor v21.16b, v21.16b, v20.16b
	tbl v21.16b, { v21.16b }, v0.16b
	add v16.4s, v17.4s, v21.4s
	zip1 v17.4s, v4.4s, v5.4s
	zip1 v4.4s, v5.4s, v4.4s
	eor v5.16b, v16.16b, v19.16b
	ext v4.16b, v4.16b, v17.16b, #8
	ushr v17.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	add v19.4s, v20.4s, v4.4s
	ext v20.16b, v18.16b, v18.16b, #8
	zip1 v3.2d, v4.2d, v3.2d
	orr v5.16b, v5.16b, v17.16b
	zip2 v2.4s, v2.4s, v4.4s
	uzp2 v7.4s, v20.4s, v7.4s
	mov v3.s[3], v6.s[3]
	add v17.4s, v19.4s, v5.4s
	ext v7.16b, v7.16b, v20.16b, #4
	eor v19.16b, v21.16b, v17.16b
	ext v17.16b, v17.16b, v17.16b, #4
	tbl v19.16b, { v19.16b }, v1.16b
	add v7.4s, v17.4s, v7.4s
	add v16.4s, v16.4s, v19.4s
	ext v17.16b, v19.16b, v19.16b, #8
	ext v19.16b, v18.16b, v18.16b, #12
	eor v5.16b, v16.16b, v5.16b
	ext v16.16b, v16.16b, v16.16b, #12
	ext v18.16b, v18.16b, v19.16b, #12
	mov v19.16b, v4.16b
	ushr v20.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	rev64 v18.4s, v18.4s
	mov v19.s[1], v6.s[2]
	orr v5.16b, v5.16b, v20.16b
	trn2 v18.4s, v18.4s, v19.4s
	add v7.4s, v5.4s, v7.4s
	eor v17.16b, v17.16b, v7.16b
	add v7.4s, v7.4s, v18.4s
	ext v18.16b, v3.16b, v3.16b, #12
	tbl v17.16b, { v17.16b }, v0.16b
	uzp1 v3.4s, v3.4s, v18.4s
	add v16.4s, v16.4s, v17.4s
	eor v5.16b, v16.16b, v5.16b
	ushr v19.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	orr v5.16b, v5.16b, v19.16b
	add v7.4s, v7.4s, v5.4s
	eor v17.16b, v17.16b, v7.16b
	ext v7.16b, v7.16b, v7.16b, #12
	tbl v17.16b, { v17.16b }, v1.16b
	add v3.4s, v7.4s, v3.4s
	add v16.4s, v16.4s, v17.4s
	ext v7.16b, v17.16b, v17.16b, #8
	eor v5.16b, v16.16b, v5.16b
	ext v16.16b, v16.16b, v16.16b, #4
	ushr v18.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	orr v5.16b, v5.16b, v18.16b
	add v3.4s, v3.4s, v5.4s
	eor v7.16b, v7.16b, v3.16b
	tbl v0.16b, { v7.16b }, v0.16b
	zip1 v7.4s, v2.4s, v6.4s
	zip1 v2.4s, v6.4s, v2.4s
	add v4.4s, v16.4s, v0.4s
	ext v2.16b, v2.16b, v7.16b, #8
	eor v5.16b, v4.16b, v5.16b
	add v2.4s, v3.4s, v2.4s
	ushr v6.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	orr v3.16b, v5.16b, v6.16b
	add v2.4s, v2.4s, v3.4s
	eor v0.16b, v0.16b, v2.16b
	ext v2.16b, v2.16b, v2.16b, #4
	tbl v0.16b, { v0.16b }, v1.16b
	add v1.4s, v4.4s, v0.4s
	ext v0.16b, v0.16b, v0.16b, #8
	eor v3.16b, v1.16b, v3.16b
	ext v1.16b, v1.16b, v1.16b, #12
	ushr v4.4s, v3.4s, #7
	shl v3.4s, v3.4s, #25
	stp q1, q0, [x0, #32]
	orr v3.16b, v3.16b, v4.16b
	stp q2, q3, [x0]
	ret
	.Lfunc_end1:
	.size compress_pre, .Lfunc_end1-compress_pre
	.cfi_endproc

	.globl zfs_blake3_compress_xof_sse41
	.p2align 2
	.type zfs_blake3_compress_xof_sse41,@function
	zfs_blake3_compress_xof_sse41:
	.cfi_startproc
	hint #25
	- .cfi_negate_ra_state
	+ CFI_NEGATE_RA_STATE
	sub sp, sp, #96
	stp x29, x30, [sp, #64]
	add x29, sp, #64
	stp x20, x19, [sp, #80]
	.cfi_def_cfa w29, 32
	.cfi_offset w19, -8
	.cfi_offset w20, -16
	.cfi_offset w30, -24
	.cfi_offset w29, -32
	mov x20, x0
	mov x19, x5
	mov w5, w4
	mov x4, x3
	mov w3, w2
	mov x2, x1
	mov x0, sp
	mov x1, x20
	bl compress_pre
	ldp q0, q1, [sp]
	ldp q2, q3, [sp, #32]
	eor v0.16b, v2.16b, v0.16b
	eor v1.16b, v3.16b, v1.16b
	ldp x29, x30, [sp, #64]
	stp q0, q1, [x19]
	ldr q0, [x20]
	eor v0.16b, v0.16b, v2.16b
	str q0, [x19, #32]
	ldr q0, [x20, #16]
	eor v0.16b, v0.16b, v3.16b
	str q0, [x19, #48]
	ldp x20, x19, [sp, #80]
	add sp, sp, #96
	hint #29
	ret
	.Lfunc_end2:
	.size zfs_blake3_compress_xof_sse41, .Lfunc_end2-zfs_blake3_compress_xof_sse41
	.cfi_endproc

	.section .rodata.cst16,"aM",@progbits,16
	.p2align 4
	.LCPI3_0:
	.word 0
	.word 1
	.word 2
	.word 3
	.LCPI3_1:
	.byte 2
	.byte 3
	.byte 0
	.byte 1
	.byte 6
	.byte 7
	.byte 4
	.byte 5
	.byte 10
	.byte 11
	.byte 8
	.byte 9
	.byte 14
	.byte 15
	.byte 12
	.byte 13
	.LCPI3_2:
	.byte 1
	.byte 2
	.byte 3
	.byte 0
	.byte 5
	.byte 6
	.byte 7
	.byte 4
	.byte 9
	.byte 10
	.byte 11
	.byte 8
	.byte 13
	.byte 14
	.byte 15
	.byte 12
	.LCPI3_3:
	.word 1779033703
	.word 3144134277
	.word 1013904242
	.word 2773480762
	.text
	.globl zfs_blake3_hash_many_sse41
	.p2align 2
	.type zfs_blake3_hash_many_sse41,@function
	zfs_blake3_hash_many_sse41:
	.cfi_startproc
	hint #34
	stp d15, d14, [sp, #-144]!
	stp d13, d12, [sp, #16]
	stp d11, d10, [sp, #32]
	stp d9, d8, [sp, #48]
	stp x29, x27, [sp, #64]
	stp x26, x25, [sp, #80]
	stp x24, x23, [sp, #96]
	stp x22, x21, [sp, #112]
	stp x20, x19, [sp, #128]
	sub sp, sp, #368
	.cfi_def_cfa_offset 512
	.cfi_offset w19, -8
	.cfi_offset w20, -16
	.cfi_offset w21, -24
	.cfi_offset w22, -32
	.cfi_offset w23, -40
	.cfi_offset w24, -48
	.cfi_offset w25, -56
	.cfi_offset w26, -64
	.cfi_offset w27, -72
	.cfi_offset w29, -80
	.cfi_offset b8, -88
	.cfi_offset b9, -96
	.cfi_offset b10, -104
	.cfi_offset b11, -112
	.cfi_offset b12, -120
	.cfi_offset b13, -128
	.cfi_offset b14, -136
	.cfi_offset b15, -144
	ldr x8, [sp, #520]
	adrp x11, .LCPI3_1
	ldrb w9, [sp, #512]
	adrp x10, .LCPI3_2
	cmp x1, #4
	b.lo .LBB3_6
	adrp x12, .LCPI3_0
	sbfx w13, w5, #0, #1
	mov w15, #58983
	mov w16, #44677
	movk w15, #27145, lsl #16
	movk w16, #47975, lsl #16
	ldr q0, [x12, :lo12:.LCPI3_0]
	dup v1.4s, w13
	movi v13.4s, #64
	mov w13, #62322
	mov w14, #62778
	orr w12, w7, w6
	and v0.16b, v1.16b, v0.16b
	ldr q1, [x11, :lo12:.LCPI3_1]
	movk w13, #15470, lsl #16
	movk w14, #42319, lsl #16
	dup v14.4s, w15
	stp q0, q1, [sp, #16]
	orr v0.4s, #128, lsl #24
	str q0, [sp]
	dup v0.4s, w16
	stp q0, q14, [sp, #48]
	b .LBB3_3
	.LBB3_2:
	zip1 v0.4s, v29.4s, v8.4s
	add x15, x4, #4
	zip1 v1.4s, v30.4s, v31.4s
	tst w5, #0x1
	zip1 v2.4s, v24.4s, v18.4s
	csel x4, x15, x4, ne
	zip1 v3.4s, v25.4s, v26.4s
	add x0, x0, #32
	zip2 v6.4s, v29.4s, v8.4s
	sub x1, x1, #4
	zip1 v4.2d, v0.2d, v1.2d
	cmp x1, #3
	zip2 v7.4s, v30.4s, v31.4s
	zip1 v5.2d, v2.2d, v3.2d
	zip2 v0.2d, v0.2d, v1.2d
	zip2 v1.2d, v2.2d, v3.2d
	zip2 v2.4s, v24.4s, v18.4s
	zip2 v3.4s, v25.4s, v26.4s
	stp q4, q5, [x8]
	zip2 v4.2d, v6.2d, v7.2d
	stp q0, q1, [x8, #32]
	zip1 v0.2d, v6.2d, v7.2d
	zip1 v1.2d, v2.2d, v3.2d
	zip2 v2.2d, v2.2d, v3.2d
	stp q0, q1, [x8, #64]
	stp q4, q2, [x8, #96]
	add x8, x8, #128
	b.ls .LBB3_6
	.LBB3_3:
	mov x15, x3
	add x16, x3, #8
	add x17, x3, #12
	add x19, x3, #16
	add x20, x3, #20
	ld1r { v29.4s }, [x15], #4
	ld1r { v30.4s }, [x16]
	add x16, x3, #24
	ld1r { v31.4s }, [x17]
	add x17, x3, #28
	ld1r { v24.4s }, [x19]
	ld1r { v18.4s }, [x20]
	ld1r { v25.4s }, [x16]
	ld1r { v8.4s }, [x15]
	ld1r { v26.4s }, [x17]
	cbz x2, .LBB3_2
	ldr q1, [sp, #16]
	dup v0.4s, w4
	lsr x17, x4, #32
	mov x15, xzr
	ldp x19, x20, [x0, #16]
	add v1.4s, v0.4s, v1.4s
	mov x21, x2
	movi v0.4s, #128, lsl #24
	mov w26, w12
	str q1, [sp, #96]
	eor v0.16b, v1.16b, v0.16b
	ldr q1, [sp]
	cmgt v0.4s, v1.4s, v0.4s
	dup v1.4s, w17
	ldp x16, x17, [x0]
	sub v0.4s, v1.4s, v0.4s
	str q0, [sp, #80]
	.LBB3_5:
	add x23, x16, x15
	add x24, x17, x15
	add x22, x19, x15
	add x25, x20, x15
	subs x21, x21, #1
	add x15, x15, #64
	ldp q1, q2, [x23]
	csel w27, w9, wzr, eq
	orr w26, w27, w26
	and w26, w26, #0xff
	ldp q4, q5, [x24]
	dup v0.4s, w26
	mov w26, w6
	zip1 v22.4s, v1.4s, v4.4s
	zip2 v20.4s, v1.4s, v4.4s
	ldp q6, q7, [x22]
	zip1 v17.4s, v2.4s, v5.4s
	zip2 v23.4s, v2.4s, v5.4s
	ldp q16, q21, [x25]
	zip1 v19.4s, v6.4s, v16.4s
	zip2 v1.4s, v6.4s, v16.4s
	ldp q27, q28, [x23, #32]
	zip1 v4.4s, v7.4s, v21.4s
	zip2 v5.4s, v7.4s, v21.4s
	zip2 v15.2d, v17.2d, v4.2d
	ldp q9, q10, [x24, #32]
	mov v17.d[1], v4.d[0]
	add v4.4s, v30.4s, v25.4s
	zip2 v11.2d, v23.2d, v5.2d
	zip2 v3.4s, v27.4s, v9.4s
	zip1 v7.4s, v27.4s, v9.4s
	ldp q12, q6, [x22, #32]
	mov v23.d[1], v5.d[0]
	stp q11, q3, [sp, #256]
	add v5.4s, v31.4s, v26.4s
	add v4.4s, v4.4s, v17.4s
	str q23, [sp, #352]
	ldp q16, q2, [x25, #32]
	add v5.4s, v5.4s, v23.4s
	zip1 v3.4s, v12.4s, v16.4s
	eor v0.16b, v5.16b, v0.16b
	zip1 v9.4s, v6.4s, v2.4s
	zip2 v2.4s, v6.4s, v2.4s
	stp q7, q3, [sp, #208]
	zip2 v3.4s, v12.4s, v16.4s
	zip1 v12.4s, v28.4s, v10.4s
	zip2 v10.4s, v28.4s, v10.4s
	stp q17, q2, [sp, #160]
	zip2 v28.2d, v22.2d, v19.2d
	mov v22.d[1], v19.d[0]
	str q3, [sp, #240]
	add v2.4s, v8.4s, v18.4s
	eor v16.16b, v4.16b, v13.16b
	dup v17.4s, w13
	mov v3.16b, v22.16b
	stp q22, q28, [sp, #320]
	zip2 v22.2d, v20.2d, v1.2d
	mov v20.d[1], v1.d[0]
	add v1.4s, v29.4s, v24.4s
	add v4.4s, v4.4s, v15.4s
	add v5.4s, v5.4s, v11.4s
	add v2.4s, v2.4s, v20.4s
	stp q15, q20, [sp, #288]
	add v1.4s, v1.4s, v3.4s
	ldr q3, [sp, #96]
	dup v20.4s, w14
	mov v23.16b, v22.16b
	mov v15.16b, v10.16b
	eor v6.16b, v1.16b, v3.16b
	ldr q3, [sp, #80]
	add v1.4s, v1.4s, v28.4s
	ldr q28, [sp, #272]
	str q23, [sp, #128]
	eor v7.16b, v2.16b, v3.16b
	ldp q27, q3, [sp, #32]
	add v2.4s, v2.4s, v22.4s
	tbl v6.16b, { v6.16b }, v27.16b
	tbl v7.16b, { v7.16b }, v27.16b
	tbl v16.16b, { v16.16b }, v27.16b
	tbl v0.16b, { v0.16b }, v27.16b
	add v19.4s, v6.4s, v14.4s
	add v21.4s, v7.4s, v3.4s
	add v30.4s, v16.4s, v17.4s
	add v31.4s, v0.4s, v20.4s
	eor v24.16b, v19.16b, v24.16b
	eor v17.16b, v21.16b, v18.16b
	ushr v18.4s, v24.4s, #12
	shl v20.4s, v24.4s, #20
	eor v24.16b, v30.16b, v25.16b
	eor v25.16b, v31.16b, v26.16b
	ushr v26.4s, v17.4s, #12
	shl v17.4s, v17.4s, #20
	ushr v29.4s, v24.4s, #12
	shl v24.4s, v24.4s, #20
	ushr v8.4s, v25.4s, #12
	shl v25.4s, v25.4s, #20
	orr v3.16b, v20.16b, v18.16b
	ldr q18, [x10, :lo12:.LCPI3_2]
	orr v13.16b, v17.16b, v26.16b
	orr v24.16b, v24.16b, v29.16b
	orr v14.16b, v25.16b, v8.16b
	add v8.4s, v1.4s, v3.4s
	add v29.4s, v2.4s, v13.4s
	add v17.4s, v4.4s, v24.4s
	add v20.4s, v5.4s, v14.4s
	eor v1.16b, v6.16b, v8.16b
	eor v2.16b, v7.16b, v29.16b
	eor v4.16b, v16.16b, v17.16b
	eor v0.16b, v0.16b, v20.16b
	tbl v25.16b, { v1.16b }, v18.16b
	tbl v16.16b, { v2.16b }, v18.16b
	tbl v6.16b, { v4.16b }, v18.16b
	tbl v4.16b, { v0.16b }, v18.16b
	add v19.4s, v19.4s, v25.4s
	add v21.4s, v21.4s, v16.4s
	add v26.4s, v30.4s, v6.4s
	add v7.4s, v31.4s, v4.4s
	eor v0.16b, v19.16b, v3.16b
	eor v1.16b, v21.16b, v13.16b
	eor v2.16b, v26.16b, v24.16b
	eor v3.16b, v7.16b, v14.16b
	ushr v5.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	ushr v24.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	ushr v30.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	orr v5.16b, v0.16b, v5.16b
	orr v0.16b, v1.16b, v24.16b
	ushr v31.4s, v3.4s, #7
	orr v2.16b, v2.16b, v30.16b
	ldp q24, q30, [sp, #208]
	shl v3.4s, v3.4s, #25
	zip2 v14.2d, v12.2d, v9.2d
	mov v22.16b, v24.16b
	orr v1.16b, v3.16b, v31.16b
	zip2 v3.2d, v24.2d, v30.2d
	mov v24.16b, v28.16b
	mov v22.d[1], v30.d[0]
	ldr q30, [sp, #240]
	mov v31.16b, v12.16b
	stp q22, q14, [sp, #224]
	mov v24.d[1], v30.d[0]
	add v12.4s, v8.4s, v22.4s
	mov v31.d[1], v9.d[0]
	add v22.4s, v29.4s, v24.4s
	ldr q29, [sp, #176]
	zip2 v28.2d, v28.2d, v30.2d
	mov v9.16b, v24.16b
	mov v15.d[1], v29.d[0]
	zip2 v8.2d, v10.2d, v29.2d
	add v10.4s, v12.4s, v0.4s
	add v22.4s, v22.4s, v2.4s
	str q9, [sp, #144]
	add v20.4s, v20.4s, v15.4s
	add v17.4s, v17.4s, v31.4s
	stp q3, q8, [sp, #192]
	eor v4.16b, v4.16b, v10.16b
	eor v25.16b, v25.16b, v22.16b
	add v20.4s, v20.4s, v5.4s
	add v17.4s, v17.4s, v1.4s
	tbl v4.16b, { v4.16b }, v27.16b
	tbl v25.16b, { v25.16b }, v27.16b
	eor v6.16b, v6.16b, v20.16b
	eor v16.16b, v16.16b, v17.16b
	add v26.4s, v26.4s, v4.4s
	add v7.4s, v7.4s, v25.4s
	tbl v6.16b, { v6.16b }, v27.16b
	tbl v16.16b, { v16.16b }, v27.16b
	eor v0.16b, v26.16b, v0.16b
	eor v2.16b, v7.16b, v2.16b
	add v21.4s, v21.4s, v6.4s
	add v19.4s, v19.4s, v16.4s
	ushr v12.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	ushr v13.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	eor v5.16b, v21.16b, v5.16b
	eor v1.16b, v19.16b, v1.16b
	orr v0.16b, v0.16b, v12.16b
	add v10.4s, v10.4s, v3.4s
	orr v2.16b, v2.16b, v13.16b
	ushr v13.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	add v22.4s, v22.4s, v28.4s
	ushr v12.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	add v10.4s, v10.4s, v0.4s
	orr v5.16b, v5.16b, v13.16b
	add v22.4s, v22.4s, v2.4s
	add v20.4s, v20.4s, v8.4s
	orr v1.16b, v1.16b, v12.16b
	add v17.4s, v17.4s, v14.4s
	eor v4.16b, v4.16b, v10.16b
	eor v25.16b, v25.16b, v22.16b
	add v20.4s, v20.4s, v5.4s
	add v17.4s, v17.4s, v1.4s
	tbl v4.16b, { v4.16b }, v18.16b
	tbl v25.16b, { v25.16b }, v18.16b
	eor v6.16b, v6.16b, v20.16b
	eor v16.16b, v16.16b, v17.16b
	add v26.4s, v26.4s, v4.4s
	add v7.4s, v7.4s, v25.4s
	tbl v6.16b, { v6.16b }, v18.16b
	tbl v16.16b, { v16.16b }, v18.16b
	eor v0.16b, v26.16b, v0.16b
	eor v2.16b, v7.16b, v2.16b
	add v21.4s, v21.4s, v6.4s
	add v19.4s, v19.4s, v16.4s
	ushr v12.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	ushr v13.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	eor v5.16b, v21.16b, v5.16b
	eor v1.16b, v19.16b, v1.16b
	orr v0.16b, v0.16b, v12.16b
	add v22.4s, v22.4s, v23.4s
	orr v2.16b, v2.16b, v13.16b
	ushr v13.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	add v17.4s, v17.4s, v11.4s
	mov v30.16b, v28.16b
	mov v28.16b, v23.16b
	ldr q23, [sp, #304]
	ushr v12.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v22.4s, v22.4s, v0.4s
	mov v29.16b, v31.16b
	ldr q31, [sp, #160]
	orr v5.16b, v5.16b, v13.16b
	add v17.4s, v17.4s, v2.4s
	add v10.4s, v10.4s, v23.4s
	orr v1.16b, v1.16b, v12.16b
	str q29, [sp, #272]
	eor v16.16b, v16.16b, v22.16b
	add v20.4s, v20.4s, v31.4s
	eor v6.16b, v6.16b, v17.16b
	add v10.4s, v10.4s, v5.4s
	tbl v16.16b, { v16.16b }, v27.16b
	add v20.4s, v20.4s, v1.4s
	tbl v6.16b, { v6.16b }, v27.16b
	eor v25.16b, v25.16b, v10.16b
	add v21.4s, v21.4s, v16.4s
	eor v4.16b, v4.16b, v20.16b
	add v26.4s, v26.4s, v6.4s
	tbl v25.16b, { v25.16b }, v27.16b
	eor v0.16b, v21.16b, v0.16b
	tbl v4.16b, { v4.16b }, v27.16b
	eor v2.16b, v26.16b, v2.16b
	add v19.4s, v19.4s, v25.4s
	ushr v12.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	add v7.4s, v7.4s, v4.4s
	ushr v13.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	eor v5.16b, v5.16b, v19.16b
	add v22.4s, v22.4s, v24.4s
	ldr q24, [sp, #320]
	orr v0.16b, v0.16b, v12.16b
	eor v1.16b, v7.16b, v1.16b
	orr v2.16b, v2.16b, v13.16b
	ushr v12.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	add v17.4s, v17.4s, v24.4s
	ldr q24, [sp, #352]
	ushr v13.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	add v22.4s, v22.4s, v0.4s
	orr v5.16b, v5.16b, v12.16b
	add v17.4s, v17.4s, v2.4s
	add v10.4s, v10.4s, v24.4s
	ldr q24, [sp, #336]
	orr v1.16b, v1.16b, v13.16b
	eor v16.16b, v16.16b, v22.16b
	add v20.4s, v20.4s, v14.4s
	eor v6.16b, v6.16b, v17.16b
	add v10.4s, v10.4s, v5.4s
	tbl v16.16b, { v16.16b }, v18.16b
	add v20.4s, v20.4s, v1.4s
	tbl v6.16b, { v6.16b }, v18.16b
	eor v25.16b, v25.16b, v10.16b
	add v21.4s, v21.4s, v16.4s
	eor v4.16b, v4.16b, v20.16b
	add v26.4s, v26.4s, v6.4s
	tbl v25.16b, { v25.16b }, v18.16b
	eor v0.16b, v21.16b, v0.16b
	tbl v4.16b, { v4.16b }, v18.16b
	eor v2.16b, v26.16b, v2.16b
	add v19.4s, v19.4s, v25.4s
	ushr v12.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	add v7.4s, v7.4s, v4.4s
	ushr v13.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	eor v5.16b, v19.16b, v5.16b
	orr v0.16b, v0.16b, v12.16b
	eor v1.16b, v7.16b, v1.16b
	add v10.4s, v10.4s, v24.4s
	orr v2.16b, v2.16b, v13.16b
	ushr v12.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	add v22.4s, v22.4s, v29.4s
	ushr v13.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v10.4s, v10.4s, v0.4s
	orr v5.16b, v5.16b, v12.16b
	add v22.4s, v22.4s, v2.4s
	add v20.4s, v20.4s, v8.4s
	ldr q8, [sp, #288]
	orr v1.16b, v1.16b, v13.16b
	add v17.4s, v17.4s, v3.4s
	ldr q3, [sp, #352]
	eor v4.16b, v4.16b, v10.16b
	eor v25.16b, v25.16b, v22.16b
	add v20.4s, v20.4s, v5.4s
	add v17.4s, v17.4s, v1.4s
	tbl v4.16b, { v4.16b }, v27.16b
	tbl v25.16b, { v25.16b }, v27.16b
	eor v6.16b, v6.16b, v20.16b
	eor v16.16b, v16.16b, v17.16b
	add v26.4s, v26.4s, v4.4s
	add v7.4s, v7.4s, v25.4s
	tbl v6.16b, { v6.16b }, v27.16b
	tbl v16.16b, { v16.16b }, v27.16b
	eor v0.16b, v26.16b, v0.16b
	eor v2.16b, v7.16b, v2.16b
	add v21.4s, v21.4s, v6.4s
	add v19.4s, v19.4s, v16.4s
	ushr v12.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	ushr v13.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	eor v5.16b, v21.16b, v5.16b
	eor v1.16b, v19.16b, v1.16b
	orr v0.16b, v0.16b, v12.16b
	add v10.4s, v10.4s, v30.4s
	orr v2.16b, v2.16b, v13.16b
	ushr v13.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	add v22.4s, v22.4s, v8.4s
	mov v24.16b, v30.16b
	mov v30.16b, v15.16b
	add v17.4s, v17.4s, v15.4s
	ldr q15, [sp, #224]
	ushr v12.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	add v10.4s, v10.4s, v0.4s
	str q30, [sp, #176]
	orr v5.16b, v5.16b, v13.16b
	add v22.4s, v22.4s, v2.4s
	add v20.4s, v20.4s, v15.4s
	orr v1.16b, v1.16b, v12.16b
	eor v4.16b, v4.16b, v10.16b
	eor v25.16b, v25.16b, v22.16b
	add v20.4s, v20.4s, v5.4s
	add v17.4s, v17.4s, v1.4s
	tbl v4.16b, { v4.16b }, v18.16b
	tbl v25.16b, { v25.16b }, v18.16b
	eor v6.16b, v6.16b, v20.16b
	eor v16.16b, v16.16b, v17.16b
	add v26.4s, v26.4s, v4.4s
	add v7.4s, v7.4s, v25.4s
	tbl v6.16b, { v6.16b }, v18.16b
	tbl v16.16b, { v16.16b }, v18.16b
	eor v0.16b, v26.16b, v0.16b
	eor v2.16b, v7.16b, v2.16b
	add v21.4s, v21.4s, v6.4s
	add v19.4s, v19.4s, v16.4s
	ushr v12.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	ushr v13.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	eor v5.16b, v21.16b, v5.16b
	eor v1.16b, v19.16b, v1.16b
	orr v0.16b, v0.16b, v12.16b
	add v22.4s, v22.4s, v9.4s
	orr v2.16b, v2.16b, v13.16b
	ushr v13.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	add v17.4s, v17.4s, v14.4s
	ushr v12.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v22.4s, v22.4s, v0.4s
	orr v5.16b, v5.16b, v13.16b
	add v17.4s, v17.4s, v2.4s
	add v10.4s, v10.4s, v28.4s
	orr v1.16b, v1.16b, v12.16b
	eor v16.16b, v16.16b, v22.16b
	add v20.4s, v20.4s, v11.4s
	eor v6.16b, v6.16b, v17.16b
	add v10.4s, v10.4s, v5.4s
	tbl v16.16b, { v16.16b }, v27.16b
	add v20.4s, v20.4s, v1.4s
	tbl v6.16b, { v6.16b }, v27.16b
	eor v25.16b, v25.16b, v10.16b
	add v21.4s, v21.4s, v16.4s
	eor v4.16b, v4.16b, v20.16b
	add v26.4s, v26.4s, v6.4s
	tbl v25.16b, { v25.16b }, v27.16b
	eor v0.16b, v21.16b, v0.16b
	tbl v4.16b, { v4.16b }, v27.16b
	eor v2.16b, v26.16b, v2.16b
	add v19.4s, v19.4s, v25.4s
	ushr v12.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	add v7.4s, v7.4s, v4.4s
	ushr v13.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	eor v5.16b, v5.16b, v19.16b
	orr v0.16b, v0.16b, v12.16b
	eor v1.16b, v7.16b, v1.16b
	add v22.4s, v22.4s, v29.4s
	orr v2.16b, v2.16b, v13.16b
	ushr v12.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	add v17.4s, v17.4s, v23.4s
	ushr v13.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	add v22.4s, v22.4s, v0.4s
	orr v5.16b, v5.16b, v12.16b
	add v17.4s, v17.4s, v2.4s
	add v10.4s, v10.4s, v31.4s
	orr v1.16b, v1.16b, v13.16b
	eor v16.16b, v16.16b, v22.16b
	add v20.4s, v20.4s, v30.4s
	eor v6.16b, v6.16b, v17.16b
	add v10.4s, v10.4s, v5.4s
	tbl v16.16b, { v16.16b }, v18.16b
	add v20.4s, v20.4s, v1.4s
	tbl v6.16b, { v6.16b }, v18.16b
	eor v25.16b, v25.16b, v10.16b
	add v21.4s, v21.4s, v16.4s
	eor v4.16b, v4.16b, v20.16b
	add v26.4s, v26.4s, v6.4s
	tbl v25.16b, { v25.16b }, v18.16b
	eor v0.16b, v21.16b, v0.16b
	tbl v4.16b, { v4.16b }, v18.16b
	eor v2.16b, v26.16b, v2.16b
	add v19.4s, v19.4s, v25.4s
	ushr v12.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	add v7.4s, v7.4s, v4.4s
	ushr v13.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	eor v5.16b, v19.16b, v5.16b
	add v10.4s, v10.4s, v3.4s
	ldr q3, [sp, #192]
	orr v0.16b, v0.16b, v12.16b
	eor v1.16b, v7.16b, v1.16b
	orr v2.16b, v2.16b, v13.16b
	ushr v12.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	add v22.4s, v22.4s, v3.4s
	ushr v13.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v10.4s, v10.4s, v0.4s
	orr v5.16b, v5.16b, v12.16b
	add v22.4s, v22.4s, v2.4s
	add v20.4s, v20.4s, v15.4s
	ldr q15, [sp, #128]
	orr v1.16b, v1.16b, v13.16b
	add v17.4s, v17.4s, v24.4s
	eor v4.16b, v4.16b, v10.16b
	eor v25.16b, v25.16b, v22.16b
	add v20.4s, v20.4s, v5.4s
	add v17.4s, v17.4s, v1.4s
	tbl v4.16b, { v4.16b }, v27.16b
	tbl v25.16b, { v25.16b }, v27.16b
	eor v6.16b, v6.16b, v20.16b
	eor v16.16b, v16.16b, v17.16b
	add v26.4s, v26.4s, v4.4s
	add v7.4s, v7.4s, v25.4s
	tbl v6.16b, { v6.16b }, v27.16b
	tbl v16.16b, { v16.16b }, v27.16b
	eor v0.16b, v26.16b, v0.16b
	eor v2.16b, v7.16b, v2.16b
	add v21.4s, v21.4s, v6.4s
	add v19.4s, v19.4s, v16.4s
	ushr v12.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	ushr v13.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	eor v5.16b, v21.16b, v5.16b
	ldp q23, q11, [sp, #320]
	eor v1.16b, v19.16b, v1.16b
	orr v0.16b, v0.16b, v12.16b
	add v10.4s, v10.4s, v8.4s
	orr v2.16b, v2.16b, v13.16b
	ushr v13.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	add v22.4s, v22.4s, v23.4s
	ushr v12.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	add v10.4s, v10.4s, v0.4s
	mov v28.16b, v31.16b
	mov v31.16b, v8.16b
	ldr q8, [sp, #208]
	orr v5.16b, v5.16b, v13.16b
	add v22.4s, v22.4s, v2.4s
	add v20.4s, v20.4s, v11.4s
	orr v1.16b, v1.16b, v12.16b
	add v17.4s, v17.4s, v8.4s
	eor v4.16b, v4.16b, v10.16b
	eor v25.16b, v25.16b, v22.16b
	add v20.4s, v20.4s, v5.4s
	add v17.4s, v17.4s, v1.4s
	tbl v4.16b, { v4.16b }, v18.16b
	tbl v25.16b, { v25.16b }, v18.16b
	eor v6.16b, v6.16b, v20.16b
	eor v16.16b, v16.16b, v17.16b
	add v26.4s, v26.4s, v4.4s
	add v7.4s, v7.4s, v25.4s
	tbl v6.16b, { v6.16b }, v18.16b
	tbl v16.16b, { v16.16b }, v18.16b
	eor v0.16b, v26.16b, v0.16b
	eor v2.16b, v7.16b, v2.16b
	add v21.4s, v21.4s, v6.4s
	add v19.4s, v19.4s, v16.4s
	ushr v12.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	ushr v13.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	eor v5.16b, v21.16b, v5.16b
	eor v1.16b, v19.16b, v1.16b
	orr v0.16b, v0.16b, v12.16b
	add v22.4s, v22.4s, v29.4s
	orr v2.16b, v2.16b, v13.16b
	ushr v13.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	add v17.4s, v17.4s, v30.4s
	ushr v12.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v22.4s, v22.4s, v0.4s
	orr v5.16b, v5.16b, v13.16b
	add v17.4s, v17.4s, v2.4s
	add v10.4s, v10.4s, v9.4s
	orr v1.16b, v1.16b, v12.16b
	eor v16.16b, v16.16b, v22.16b
	add v20.4s, v20.4s, v14.4s
	ldr q14, [sp, #256]
	eor v6.16b, v6.16b, v17.16b
	add v10.4s, v10.4s, v5.4s
	tbl v16.16b, { v16.16b }, v27.16b
	add v20.4s, v20.4s, v1.4s
	tbl v6.16b, { v6.16b }, v27.16b
	eor v25.16b, v25.16b, v10.16b
	add v21.4s, v21.4s, v16.4s
	eor v4.16b, v4.16b, v20.16b
	add v26.4s, v26.4s, v6.4s
	tbl v25.16b, { v25.16b }, v27.16b
	eor v0.16b, v21.16b, v0.16b
	tbl v4.16b, { v4.16b }, v27.16b
	eor v2.16b, v26.16b, v2.16b
	add v19.4s, v19.4s, v25.4s
	ushr v12.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	add v7.4s, v7.4s, v4.4s
	ushr v13.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	eor v5.16b, v5.16b, v19.16b
	orr v0.16b, v0.16b, v12.16b
	eor v1.16b, v7.16b, v1.16b
	add v22.4s, v22.4s, v3.4s
	orr v2.16b, v2.16b, v13.16b
	ushr v12.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	add v17.4s, v17.4s, v15.4s
	ushr v13.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	add v22.4s, v22.4s, v0.4s
	orr v5.16b, v5.16b, v12.16b
	add v17.4s, v17.4s, v2.4s
	add v10.4s, v10.4s, v14.4s
	orr v1.16b, v1.16b, v13.16b
	eor v16.16b, v16.16b, v22.16b
	add v20.4s, v20.4s, v8.4s
	eor v6.16b, v6.16b, v17.16b
	add v10.4s, v10.4s, v5.4s
	tbl v16.16b, { v16.16b }, v18.16b
	add v20.4s, v20.4s, v1.4s
	tbl v6.16b, { v6.16b }, v18.16b
	eor v25.16b, v25.16b, v10.16b
	add v21.4s, v21.4s, v16.4s
	eor v4.16b, v4.16b, v20.16b
	add v26.4s, v26.4s, v6.4s
	tbl v25.16b, { v25.16b }, v18.16b
	eor v0.16b, v21.16b, v0.16b
	tbl v4.16b, { v4.16b }, v18.16b
	eor v2.16b, v26.16b, v2.16b
	add v19.4s, v19.4s, v25.4s
	ushr v12.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	add v7.4s, v7.4s, v4.4s
	ushr v13.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	eor v5.16b, v19.16b, v5.16b
	orr v0.16b, v0.16b, v12.16b
	eor v1.16b, v7.16b, v1.16b
	add v10.4s, v10.4s, v28.4s
	orr v2.16b, v2.16b, v13.16b
	ushr v12.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	add v22.4s, v22.4s, v24.4s
	ushr v13.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v10.4s, v10.4s, v0.4s
	orr v5.16b, v5.16b, v12.16b
	add v22.4s, v22.4s, v2.4s
	add v20.4s, v20.4s, v11.4s
	ldr q11, [sp, #304]
	orr v1.16b, v1.16b, v13.16b
	add v17.4s, v17.4s, v31.4s
	ldr q31, [sp, #224]
	eor v4.16b, v4.16b, v10.16b
	eor v25.16b, v25.16b, v22.16b
	add v20.4s, v20.4s, v5.4s
	add v17.4s, v17.4s, v1.4s
	tbl v4.16b, { v4.16b }, v27.16b
	tbl v25.16b, { v25.16b }, v27.16b
	eor v6.16b, v6.16b, v20.16b
	eor v16.16b, v16.16b, v17.16b
	add v26.4s, v26.4s, v4.4s
	add v7.4s, v7.4s, v25.4s
	tbl v6.16b, { v6.16b }, v27.16b
	tbl v16.16b, { v16.16b }, v27.16b
	eor v0.16b, v26.16b, v0.16b
	eor v2.16b, v7.16b, v2.16b
	add v21.4s, v21.4s, v6.4s
	add v19.4s, v19.4s, v16.4s
	ushr v12.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	ushr v13.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	eor v5.16b, v21.16b, v5.16b
	eor v1.16b, v19.16b, v1.16b
	orr v0.16b, v0.16b, v12.16b
	add v10.4s, v10.4s, v23.4s
	ldr q23, [sp, #240]
	orr v2.16b, v2.16b, v13.16b
	ushr v13.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	add v22.4s, v22.4s, v11.4s
	mov v30.16b, v8.16b
	mov v8.16b, v24.16b
	ldr q24, [sp, #352]
	ushr v12.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	add v10.4s, v10.4s, v0.4s
	orr v5.16b, v5.16b, v13.16b
	str q8, [sp, #112]
	add v22.4s, v22.4s, v2.4s
	add v20.4s, v20.4s, v24.4s
	orr v1.16b, v1.16b, v12.16b
	add v17.4s, v17.4s, v31.4s
	eor v4.16b, v4.16b, v10.16b
	eor v25.16b, v25.16b, v22.16b
	add v20.4s, v20.4s, v5.4s
	add v17.4s, v17.4s, v1.4s
	tbl v4.16b, { v4.16b }, v18.16b
	tbl v25.16b, { v25.16b }, v18.16b
	eor v6.16b, v6.16b, v20.16b
	eor v16.16b, v16.16b, v17.16b
	add v26.4s, v26.4s, v4.4s
	add v7.4s, v7.4s, v25.4s
	tbl v6.16b, { v6.16b }, v18.16b
	tbl v16.16b, { v16.16b }, v18.16b
	eor v0.16b, v26.16b, v0.16b
	eor v2.16b, v7.16b, v2.16b
	add v21.4s, v21.4s, v6.4s
	mov v29.16b, v3.16b
	add v19.4s, v19.4s, v16.4s
	ushr v12.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	ushr v13.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	eor v5.16b, v21.16b, v5.16b
	eor v1.16b, v19.16b, v1.16b
	orr v0.16b, v0.16b, v12.16b
	add v22.4s, v22.4s, v29.4s
	orr v2.16b, v2.16b, v13.16b
	ushr v13.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	add v17.4s, v17.4s, v30.4s
	ldr q30, [sp, #272]
	ushr v12.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v22.4s, v22.4s, v0.4s
	mov v3.16b, v28.16b
	ldr q28, [sp, #176]
	orr v5.16b, v5.16b, v13.16b
	add v17.4s, v17.4s, v2.4s
	add v10.4s, v10.4s, v30.4s
	orr v1.16b, v1.16b, v12.16b
	eor v16.16b, v16.16b, v22.16b
	add v20.4s, v20.4s, v28.4s
	eor v6.16b, v6.16b, v17.16b
	add v10.4s, v10.4s, v5.4s
	tbl v16.16b, { v16.16b }, v27.16b
	add v20.4s, v20.4s, v1.4s
	tbl v6.16b, { v6.16b }, v27.16b
	eor v25.16b, v25.16b, v10.16b
	add v21.4s, v21.4s, v16.4s
	eor v4.16b, v4.16b, v20.16b
	add v26.4s, v26.4s, v6.4s
	tbl v25.16b, { v25.16b }, v27.16b
	eor v0.16b, v21.16b, v0.16b
	tbl v4.16b, { v4.16b }, v27.16b
	eor v2.16b, v26.16b, v2.16b
	add v19.4s, v19.4s, v25.4s
	ushr v12.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	add v7.4s, v7.4s, v4.4s
	ushr v13.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	eor v5.16b, v5.16b, v19.16b
	orr v0.16b, v0.16b, v12.16b
	eor v1.16b, v7.16b, v1.16b
	add v22.4s, v22.4s, v8.4s
	orr v2.16b, v2.16b, v13.16b
	ushr v12.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	add v17.4s, v17.4s, v9.4s
	ldr q9, [sp, #320]
	ushr v13.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	add v22.4s, v22.4s, v0.4s
	orr v5.16b, v5.16b, v12.16b
	add v17.4s, v17.4s, v2.4s
	add v10.4s, v10.4s, v23.4s
	orr v1.16b, v1.16b, v13.16b
	eor v16.16b, v16.16b, v22.16b
	add v20.4s, v20.4s, v31.4s
	eor v6.16b, v6.16b, v17.16b
	add v10.4s, v10.4s, v5.4s
	tbl v16.16b, { v16.16b }, v18.16b
	add v20.4s, v20.4s, v1.4s
	tbl v6.16b, { v6.16b }, v18.16b
	eor v25.16b, v25.16b, v10.16b
	add v21.4s, v21.4s, v16.4s
	eor v4.16b, v4.16b, v20.16b
	add v26.4s, v26.4s, v6.4s
	tbl v25.16b, { v25.16b }, v18.16b
	eor v0.16b, v21.16b, v0.16b
	tbl v4.16b, { v4.16b }, v18.16b
	eor v2.16b, v26.16b, v2.16b
	add v19.4s, v19.4s, v25.4s
	ushr v12.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	add v7.4s, v7.4s, v4.4s
	ushr v13.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	eor v5.16b, v19.16b, v5.16b
	add v10.4s, v10.4s, v14.4s
	ldr q14, [sp, #288]
	orr v0.16b, v0.16b, v12.16b
	eor v1.16b, v7.16b, v1.16b
	orr v2.16b, v2.16b, v13.16b
	ushr v12.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	add v22.4s, v22.4s, v14.4s
	ushr v13.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	add v10.4s, v10.4s, v0.4s
	orr v5.16b, v5.16b, v12.16b
	add v22.4s, v22.4s, v2.4s
	add v20.4s, v20.4s, v24.4s
	orr v1.16b, v1.16b, v13.16b
	eor v4.16b, v4.16b, v10.16b
	add v17.4s, v17.4s, v9.4s
	eor v25.16b, v25.16b, v22.16b
	add v20.4s, v20.4s, v5.4s
	tbl v4.16b, { v4.16b }, v27.16b
	add v17.4s, v17.4s, v1.4s
	tbl v25.16b, { v25.16b }, v27.16b
	eor v6.16b, v6.16b, v20.16b
	add v26.4s, v26.4s, v4.4s
	eor v16.16b, v16.16b, v17.16b
	add v7.4s, v7.4s, v25.4s
	tbl v6.16b, { v6.16b }, v27.16b
	eor v0.16b, v26.16b, v0.16b
	tbl v16.16b, { v16.16b }, v27.16b
	eor v2.16b, v7.16b, v2.16b
	add v21.4s, v21.4s, v6.4s
	ushr v12.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	add v19.4s, v19.4s, v16.4s
	ushr v13.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	eor v5.16b, v21.16b, v5.16b
	orr v0.16b, v0.16b, v12.16b
	eor v1.16b, v19.16b, v1.16b
	add v10.4s, v10.4s, v11.4s
	orr v2.16b, v2.16b, v13.16b
	ushr v13.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	ushr v12.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	add v10.4s, v10.4s, v0.4s
	add v22.4s, v22.4s, v15.4s
	orr v5.16b, v5.16b, v13.16b
	add v20.4s, v20.4s, v3.4s
	mov v24.16b, v3.16b
	ldr q3, [sp, #336]
	orr v1.16b, v1.16b, v12.16b
	eor v4.16b, v4.16b, v10.16b
	add v22.4s, v22.4s, v2.4s
	add v17.4s, v17.4s, v3.4s
	add v20.4s, v20.4s, v5.4s
	tbl v4.16b, { v4.16b }, v18.16b
	eor v25.16b, v25.16b, v22.16b
	add v17.4s, v17.4s, v1.4s
	eor v6.16b, v6.16b, v20.16b
	add v26.4s, v26.4s, v4.4s
	tbl v25.16b, { v25.16b }, v18.16b
	eor v16.16b, v16.16b, v17.16b
	tbl v6.16b, { v6.16b }, v18.16b
	eor v0.16b, v26.16b, v0.16b
	add v7.4s, v7.4s, v25.4s
	tbl v16.16b, { v16.16b }, v18.16b
	add v21.4s, v21.4s, v6.4s
	ushr v12.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	eor v2.16b, v7.16b, v2.16b
	add v19.4s, v19.4s, v16.4s
	eor v5.16b, v21.16b, v5.16b
	orr v0.16b, v0.16b, v12.16b
	ushr v12.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	eor v1.16b, v19.16b, v1.16b
	ushr v13.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	add v22.4s, v22.4s, v8.4s
	orr v2.16b, v2.16b, v12.16b
	ushr v12.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	orr v5.16b, v5.16b, v13.16b
	add v22.4s, v22.4s, v0.4s
	add v10.4s, v10.4s, v29.4s
	ldr q29, [sp, #208]
	add v17.4s, v17.4s, v31.4s
	orr v1.16b, v1.16b, v12.16b
	add v20.4s, v20.4s, v29.4s
	eor v16.16b, v16.16b, v22.16b
	add v10.4s, v10.4s, v5.4s
	add v17.4s, v17.4s, v2.4s
	add v20.4s, v20.4s, v1.4s
	tbl v16.16b, { v16.16b }, v27.16b
	eor v25.16b, v25.16b, v10.16b
	eor v6.16b, v6.16b, v17.16b
	eor v4.16b, v4.16b, v20.16b
	add v21.4s, v21.4s, v16.4s
	tbl v25.16b, { v25.16b }, v27.16b
	tbl v6.16b, { v6.16b }, v27.16b
	tbl v4.16b, { v4.16b }, v27.16b
	eor v0.16b, v21.16b, v0.16b
	add v19.4s, v19.4s, v25.4s
	add v26.4s, v26.4s, v6.4s
	add v7.4s, v7.4s, v4.4s
	ushr v12.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	eor v5.16b, v5.16b, v19.16b
	eor v2.16b, v26.16b, v2.16b
	eor v1.16b, v7.16b, v1.16b
	orr v0.16b, v0.16b, v12.16b
	ushr v12.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	add v22.4s, v22.4s, v14.4s
	mov v8.16b, v31.16b
	ushr v13.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	mov v31.16b, v14.16b
	ushr v14.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	orr v5.16b, v5.16b, v12.16b
	add v22.4s, v22.4s, v0.4s
	add v10.4s, v10.4s, v28.4s
	ldr q28, [sp, #352]
	orr v2.16b, v2.16b, v13.16b
	orr v1.16b, v1.16b, v14.16b
	add v17.4s, v17.4s, v30.4s
	add v20.4s, v20.4s, v3.4s
	eor v16.16b, v16.16b, v22.16b
	add v10.4s, v10.4s, v5.4s
	add v17.4s, v17.4s, v2.4s
	add v20.4s, v20.4s, v1.4s
	tbl v16.16b, { v16.16b }, v18.16b
	eor v25.16b, v25.16b, v10.16b
	eor v6.16b, v6.16b, v17.16b
	eor v4.16b, v4.16b, v20.16b
	add v21.4s, v21.4s, v16.4s
	tbl v25.16b, { v25.16b }, v18.16b
	tbl v6.16b, { v6.16b }, v18.16b
	tbl v4.16b, { v4.16b }, v18.16b
	eor v0.16b, v21.16b, v0.16b
	add v19.4s, v19.4s, v25.4s
	add v26.4s, v26.4s, v6.4s
	add v7.4s, v7.4s, v4.4s
	ushr v12.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	eor v5.16b, v19.16b, v5.16b
	eor v2.16b, v26.16b, v2.16b
	eor v1.16b, v7.16b, v1.16b
	orr v0.16b, v0.16b, v12.16b
	ushr v12.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	add v10.4s, v10.4s, v23.4s
	ushr v13.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	ushr v14.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	orr v5.16b, v5.16b, v12.16b
	add v10.4s, v10.4s, v0.4s
	add v20.4s, v20.4s, v24.4s
	ldr q24, [sp, #144]
	orr v2.16b, v2.16b, v13.16b
	orr v1.16b, v1.16b, v14.16b
	add v22.4s, v22.4s, v9.4s
	add v17.4s, v17.4s, v11.4s
	eor v4.16b, v4.16b, v10.16b
	add v20.4s, v20.4s, v5.4s
	add v22.4s, v22.4s, v2.4s
	add v17.4s, v17.4s, v1.4s
	tbl v4.16b, { v4.16b }, v27.16b
	eor v6.16b, v6.16b, v20.16b
	eor v25.16b, v25.16b, v22.16b
	eor v16.16b, v16.16b, v17.16b
	add v26.4s, v26.4s, v4.4s
	tbl v6.16b, { v6.16b }, v27.16b
	tbl v25.16b, { v25.16b }, v27.16b
	tbl v16.16b, { v16.16b }, v27.16b
	eor v0.16b, v26.16b, v0.16b
	add v21.4s, v21.4s, v6.4s
	add v7.4s, v7.4s, v25.4s
	add v19.4s, v19.4s, v16.4s
	ushr v12.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	eor v5.16b, v21.16b, v5.16b
	eor v2.16b, v7.16b, v2.16b
	eor v1.16b, v19.16b, v1.16b
	orr v0.16b, v0.16b, v12.16b
	add v10.4s, v10.4s, v15.4s
	ushr v14.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	mov v30.16b, v3.16b
	ldr q3, [sp, #256]
	ushr v12.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	ushr v13.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	add v10.4s, v10.4s, v0.4s
	orr v5.16b, v5.16b, v14.16b
	add v20.4s, v20.4s, v3.4s
	orr v2.16b, v2.16b, v12.16b
	orr v1.16b, v1.16b, v13.16b
	add v22.4s, v22.4s, v24.4s
	add v17.4s, v17.4s, v28.4s
	eor v4.16b, v4.16b, v10.16b
	add v20.4s, v20.4s, v5.4s
	add v22.4s, v22.4s, v2.4s
	add v17.4s, v17.4s, v1.4s
	tbl v4.16b, { v4.16b }, v18.16b
	eor v6.16b, v6.16b, v20.16b
	eor v25.16b, v25.16b, v22.16b
	eor v16.16b, v16.16b, v17.16b
	add v26.4s, v26.4s, v4.4s
	tbl v6.16b, { v6.16b }, v18.16b
	tbl v25.16b, { v25.16b }, v18.16b
	tbl v16.16b, { v16.16b }, v18.16b
	eor v0.16b, v26.16b, v0.16b
	add v21.4s, v21.4s, v6.4s
	add v7.4s, v7.4s, v25.4s
	add v19.4s, v19.4s, v16.4s
	ushr v12.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	eor v5.16b, v21.16b, v5.16b
	eor v2.16b, v7.16b, v2.16b
	eor v1.16b, v19.16b, v1.16b
	orr v0.16b, v0.16b, v12.16b
	ushr v12.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	mov v23.16b, v9.16b
	ldr q9, [sp, #112]
	ushr v13.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	ushr v14.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	orr v5.16b, v5.16b, v12.16b
	add v9.4s, v10.4s, v9.4s
	orr v2.16b, v2.16b, v13.16b
	orr v1.16b, v1.16b, v14.16b
	ldr q14, [sp, #64]
	add v22.4s, v22.4s, v31.4s
	add v17.4s, v17.4s, v30.4s
	add v20.4s, v20.4s, v8.4s
	add v9.4s, v9.4s, v5.4s
	add v22.4s, v22.4s, v0.4s
	add v17.4s, v17.4s, v2.4s
	add v20.4s, v20.4s, v1.4s
	eor v25.16b, v25.16b, v9.16b
	eor v16.16b, v16.16b, v22.16b
	eor v6.16b, v6.16b, v17.16b
	eor v4.16b, v4.16b, v20.16b
	tbl v25.16b, { v25.16b }, v27.16b
	tbl v16.16b, { v16.16b }, v27.16b
	tbl v6.16b, { v6.16b }, v27.16b
	tbl v4.16b, { v4.16b }, v27.16b
	add v19.4s, v19.4s, v25.4s
	add v21.4s, v21.4s, v16.4s
	add v26.4s, v26.4s, v6.4s
	add v7.4s, v7.4s, v4.4s
	eor v5.16b, v5.16b, v19.16b
	eor v0.16b, v21.16b, v0.16b
	eor v2.16b, v26.16b, v2.16b
	eor v1.16b, v7.16b, v1.16b
	ushr v30.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	ushr v10.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	ushr v12.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	ushr v13.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	orr v5.16b, v5.16b, v30.16b
	add v30.4s, v9.4s, v29.4s
	add v22.4s, v22.4s, v23.4s
	ldr q23, [sp, #192]
	orr v0.16b, v0.16b, v10.16b
	orr v2.16b, v2.16b, v12.16b
	orr v1.16b, v1.16b, v13.16b
	add v17.4s, v17.4s, v23.4s
	add v20.4s, v20.4s, v28.4s
	add v23.4s, v30.4s, v5.4s
	add v22.4s, v22.4s, v0.4s
	add v17.4s, v17.4s, v2.4s
	add v20.4s, v20.4s, v1.4s
	eor v25.16b, v25.16b, v23.16b
	eor v16.16b, v16.16b, v22.16b
	eor v6.16b, v6.16b, v17.16b
	eor v4.16b, v4.16b, v20.16b
	tbl v25.16b, { v25.16b }, v18.16b
	tbl v16.16b, { v16.16b }, v18.16b
	tbl v6.16b, { v6.16b }, v18.16b
	tbl v4.16b, { v4.16b }, v18.16b
	add v19.4s, v19.4s, v25.4s
	add v21.4s, v21.4s, v16.4s
	add v26.4s, v26.4s, v6.4s
	add v7.4s, v7.4s, v4.4s
	eor v5.16b, v19.16b, v5.16b
	eor v0.16b, v21.16b, v0.16b
	eor v2.16b, v26.16b, v2.16b
	eor v1.16b, v7.16b, v1.16b
	ushr v28.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	ushr v30.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	ushr v31.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	ushr v8.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	orr v5.16b, v5.16b, v28.16b
	ldr q28, [sp, #176]
	orr v0.16b, v0.16b, v30.16b
	orr v2.16b, v2.16b, v31.16b
	orr v1.16b, v1.16b, v8.16b
	add v23.4s, v23.4s, v28.4s
	add v22.4s, v22.4s, v11.4s
	add v17.4s, v17.4s, v15.4s
	add v20.4s, v20.4s, v3.4s
	ldr q3, [sp, #272]
	add v23.4s, v23.4s, v0.4s
	add v22.4s, v22.4s, v2.4s
	add v17.4s, v17.4s, v1.4s
	add v20.4s, v20.4s, v5.4s
	eor v4.16b, v4.16b, v23.16b
	eor v25.16b, v25.16b, v22.16b
	eor v16.16b, v16.16b, v17.16b
	eor v6.16b, v6.16b, v20.16b
	tbl v4.16b, { v4.16b }, v27.16b
	tbl v25.16b, { v25.16b }, v27.16b
	tbl v16.16b, { v16.16b }, v27.16b
	tbl v6.16b, { v6.16b }, v27.16b
	add v26.4s, v26.4s, v4.4s
	add v7.4s, v7.4s, v25.4s
	add v19.4s, v19.4s, v16.4s
	add v21.4s, v21.4s, v6.4s
	eor v0.16b, v26.16b, v0.16b
	eor v2.16b, v7.16b, v2.16b
	eor v1.16b, v19.16b, v1.16b
	eor v5.16b, v21.16b, v5.16b
	add v3.4s, v22.4s, v3.4s
	ldr q22, [sp, #160]
	ushr v28.4s, v0.4s, #12
	shl v0.4s, v0.4s, #20
	ushr v29.4s, v2.4s, #12
	shl v2.4s, v2.4s, #20
	ushr v30.4s, v1.4s, #12
	shl v1.4s, v1.4s, #20
	ushr v31.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	add v17.4s, v17.4s, v22.4s
	ldr q22, [sp, #240]
	orr v0.16b, v0.16b, v28.16b
	prfm pldl1keep, [x23, #256]
	orr v2.16b, v2.16b, v29.16b
	prfm pldl1keep, [x24, #256]
	orr v1.16b, v1.16b, v30.16b
	prfm pldl1keep, [x22, #256]
	orr v5.16b, v5.16b, v31.16b
	prfm pldl1keep, [x25, #256]
	add v23.4s, v23.4s, v24.4s
	add v20.4s, v20.4s, v22.4s
	add v3.4s, v3.4s, v2.4s
	add v17.4s, v17.4s, v1.4s
	add v22.4s, v23.4s, v0.4s
	add v20.4s, v20.4s, v5.4s
	eor v23.16b, v25.16b, v3.16b
	eor v16.16b, v16.16b, v17.16b
	eor v4.16b, v4.16b, v22.16b
	eor v6.16b, v6.16b, v20.16b
	tbl v23.16b, { v23.16b }, v18.16b
	tbl v16.16b, { v16.16b }, v18.16b
	tbl v4.16b, { v4.16b }, v18.16b
	tbl v6.16b, { v6.16b }, v18.16b
	add v7.4s, v7.4s, v23.4s
	add v19.4s, v19.4s, v16.4s
	add v18.4s, v26.4s, v4.4s
	add v21.4s, v21.4s, v6.4s
	eor v2.16b, v7.16b, v2.16b
	eor v1.16b, v19.16b, v1.16b
	eor v0.16b, v18.16b, v0.16b
	eor v5.16b, v21.16b, v5.16b
	ushr v25.4s, v2.4s, #7
	shl v2.4s, v2.4s, #25
	ushr v24.4s, v0.4s, #7
	shl v0.4s, v0.4s, #25
	ushr v26.4s, v1.4s, #7
	shl v1.4s, v1.4s, #25
	ushr v27.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	orr v0.16b, v0.16b, v24.16b
	orr v2.16b, v2.16b, v25.16b
	orr v1.16b, v1.16b, v26.16b
	orr v5.16b, v5.16b, v27.16b
	movi v13.4s, #64
	eor v29.16b, v19.16b, v22.16b
	eor v8.16b, v21.16b, v3.16b
	eor v30.16b, v17.16b, v18.16b
	eor v31.16b, v20.16b, v7.16b
	eor v24.16b, v5.16b, v23.16b
	eor v18.16b, v0.16b, v16.16b
	eor v25.16b, v2.16b, v6.16b
	eor v26.16b, v1.16b, v4.16b
	cbnz x21, .LBB3_5
	b .LBB3_2
	.LBB3_6:
	cbz x1, .LBB3_14
	adrp x12, .LCPI3_3
	ldr q0, [x11, :lo12:.LCPI3_1]
	orr w11, w7, w6
	ldr q2, [x10, :lo12:.LCPI3_2]
	ldr q1, [x12, :lo12:.LCPI3_3]
	and x12, x5, #0x1
	.LBB3_8:
	movi v3.4s, #64
	lsr x13, x4, #32
	ldp q5, q4, [x3]
	mov x15, x2
	mov w14, w11
	mov v3.s[0], w4
	ldr x10, [x0]
	mov v3.s[1], w13
	b .LBB3_11
	.LBB3_9:
	orr w14, w14, w9
	.LBB3_10:
	ldp q6, q7, [x10]
	mov v16.16b, v3.16b
	and w14, w14, #0xff
	add v5.4s, v5.4s, v4.4s
	mov x15, x13
	mov v16.s[3], w14
	add x14, x10, #32
	uzp1 v17.4s, v6.4s, v7.4s
	add x10, x10, #64
	add v5.4s, v5.4s, v17.4s
	eor v16.16b, v5.16b, v16.16b
	tbl v16.16b, { v16.16b }, v0.16b
	add v18.4s, v16.4s, v1.4s
	eor v19.16b, v18.16b, v4.16b
	uzp2 v4.4s, v6.4s, v7.4s
	ushr v6.4s, v19.4s, #12
	shl v7.4s, v19.4s, #20
	ld2 { v19.4s, v20.4s }, [x14]
	add v5.4s, v5.4s, v4.4s
	mov w14, w6
	orr v6.16b, v7.16b, v6.16b
	add v5.4s, v5.4s, v6.4s
	eor v7.16b, v16.16b, v5.16b
	add v5.4s, v5.4s, v19.4s
	tbl v7.16b, { v7.16b }, v2.16b
	ext v5.16b, v5.16b, v5.16b, #12
	add v16.4s, v18.4s, v7.4s
	ext v7.16b, v7.16b, v7.16b, #8
	eor v6.16b, v6.16b, v16.16b
	ext v16.16b, v16.16b, v16.16b, #4
	ushr v18.4s, v6.4s, #7
	shl v6.4s, v6.4s, #25
	orr v6.16b, v6.16b, v18.16b
	ext v18.16b, v20.16b, v20.16b, #12
	add v5.4s, v5.4s, v6.4s
	eor v7.16b, v5.16b, v7.16b
	add v5.4s, v5.4s, v18.4s
	tbl v7.16b, { v7.16b }, v0.16b
	add v16.4s, v16.4s, v7.4s
	eor v6.16b, v6.16b, v16.16b
	ushr v21.4s, v6.4s, #12
	shl v6.4s, v6.4s, #20
	orr v6.16b, v6.16b, v21.16b
	uzp1 v21.4s, v17.4s, v17.4s
	add v5.4s, v5.4s, v6.4s
	ext v21.16b, v21.16b, v17.16b, #8
	eor v7.16b, v7.16b, v5.16b
	uzp2 v21.4s, v21.4s, v4.4s
	tbl v7.16b, { v7.16b }, v2.16b
	add v5.4s, v5.4s, v21.4s
	add v16.4s, v16.4s, v7.4s
	ext v5.16b, v5.16b, v5.16b, #4
	ext v7.16b, v7.16b, v7.16b, #8
	eor v6.16b, v6.16b, v16.16b
	ushr v22.4s, v6.4s, #7
	shl v6.4s, v6.4s, #25
	orr v6.16b, v6.16b, v22.16b
	add v22.4s, v5.4s, v6.4s
	eor v5.16b, v22.16b, v7.16b
	ext v7.16b, v16.16b, v16.16b, #12
	tbl v16.16b, { v5.16b }, v0.16b
	ext v5.16b, v17.16b, v17.16b, #12
	add v7.4s, v7.4s, v16.4s
	ext v5.16b, v17.16b, v5.16b, #12
	ext v17.16b, v19.16b, v19.16b, #12
	mov v19.16b, v18.16b
	eor v6.16b, v6.16b, v7.16b
	rev64 v5.4s, v5.4s
	mov v19.s[1], v17.s[2]
	ushr v20.4s, v6.4s, #12
	shl v6.4s, v6.4s, #20
	trn2 v5.4s, v5.4s, v19.4s
	orr v6.16b, v6.16b, v20.16b
	zip1 v20.2d, v18.2d, v4.2d
	zip2 v4.4s, v4.4s, v18.4s
	add v19.4s, v6.4s, v5.4s
	mov v20.s[3], v17.s[3]
	add v19.4s, v19.4s, v22.4s
	ext v22.16b, v20.16b, v20.16b, #12
	eor v16.16b, v16.16b, v19.16b
	ext v19.16b, v19.16b, v19.16b, #12
	tbl v16.16b, { v16.16b }, v2.16b
	add v7.4s, v7.4s, v16.4s
	ext v16.16b, v16.16b, v16.16b, #8
	eor v6.16b, v6.16b, v7.16b
	ext v7.16b, v7.16b, v7.16b, #4
	ushr v23.4s, v6.4s, #7
	shl v24.4s, v6.4s, #25
	uzp1 v6.4s, v20.4s, v22.4s
	orr v20.16b, v24.16b, v23.16b
	add v22.4s, v20.4s, v6.4s
	add v19.4s, v22.4s, v19.4s
	eor v16.16b, v19.16b, v16.16b
	tbl v16.16b, { v16.16b }, v0.16b
	add v7.4s, v7.4s, v16.4s
	eor v18.16b, v20.16b, v7.16b
	zip1 v20.4s, v4.4s, v17.4s
	zip1 v4.4s, v17.4s, v4.4s
	ushr v17.4s, v18.4s, #12
	shl v18.4s, v18.4s, #20
	ext v20.16b, v4.16b, v20.16b, #8
	orr v4.16b, v18.16b, v17.16b
	ext v18.16b, v21.16b, v21.16b, #4
	add v17.4s, v4.4s, v20.4s
	add v17.4s, v17.4s, v19.4s
	uzp1 v19.4s, v18.4s, v18.4s
	eor v16.16b, v16.16b, v17.16b
	ext v19.16b, v19.16b, v18.16b, #8
	tbl v16.16b, { v16.16b }, v2.16b
	uzp2 v19.4s, v19.4s, v5.4s
	add v7.4s, v7.4s, v16.4s
	add v17.4s, v17.4s, v19.4s
	ext v16.16b, v16.16b, v16.16b, #8
	eor v4.16b, v4.16b, v7.16b
	ext v17.16b, v17.16b, v17.16b, #4
	ext v7.16b, v7.16b, v7.16b, #12
	ushr v21.4s, v4.4s, #7
	shl v4.4s, v4.4s, #25
	orr v4.16b, v4.16b, v21.16b
	ext v21.16b, v18.16b, v18.16b, #12
	add v17.4s, v17.4s, v4.4s
	ext v18.16b, v18.16b, v21.16b, #12
	mov v21.16b, v20.16b
	eor v16.16b, v17.16b, v16.16b
	rev64 v18.4s, v18.4s
	mov v21.s[1], v6.s[2]
	tbl v16.16b, { v16.16b }, v0.16b
	add v7.4s, v7.4s, v16.4s
	eor v4.16b, v4.16b, v7.16b
	ushr v22.4s, v4.4s, #12
	shl v23.4s, v4.4s, #20
	trn2 v4.4s, v18.4s, v21.4s
	orr v18.16b, v23.16b, v22.16b
	add v21.4s, v18.4s, v4.4s
	add v17.4s, v21.4s, v17.4s
	zip1 v21.2d, v20.2d, v5.2d
	zip2 v5.4s, v5.4s, v20.4s
	eor v16.16b, v16.16b, v17.16b
	mov v21.s[3], v6.s[3]
	ext v17.16b, v17.16b, v17.16b, #12
	zip1 v20.4s, v5.4s, v6.4s
	tbl v16.16b, { v16.16b }, v2.16b
	zip1 v5.4s, v6.4s, v5.4s
	add v22.4s, v7.4s, v16.4s
	ext v16.16b, v16.16b, v16.16b, #8
	ext v20.16b, v5.16b, v20.16b, #8
	eor v7.16b, v18.16b, v22.16b
	ext v18.16b, v21.16b, v21.16b, #12
	ushr v23.4s, v7.4s, #7
	shl v24.4s, v7.4s, #25
	uzp1 v7.4s, v21.4s, v18.4s
	orr v18.16b, v24.16b, v23.16b
	add v21.4s, v18.4s, v7.4s
	add v17.4s, v21.4s, v17.4s
	ext v21.16b, v22.16b, v22.16b, #4
	eor v16.16b, v17.16b, v16.16b
	tbl v16.16b, { v16.16b }, v0.16b
	add v21.4s, v21.4s, v16.4s
	eor v18.16b, v18.16b, v21.16b
	ushr v6.4s, v18.4s, #12
	shl v18.4s, v18.4s, #20
	orr v5.16b, v18.16b, v6.16b
	add v6.4s, v5.4s, v20.4s
	add v6.4s, v6.4s, v17.4s
	ext v17.16b, v19.16b, v19.16b, #4
	eor v16.16b, v16.16b, v6.16b
	uzp1 v18.4s, v17.4s, v17.4s
	tbl v16.16b, { v16.16b }, v2.16b
	ext v18.16b, v18.16b, v17.16b, #8
	add v19.4s, v21.4s, v16.4s
	uzp2 v18.4s, v18.4s, v4.4s
	ext v16.16b, v16.16b, v16.16b, #8
	eor v5.16b, v5.16b, v19.16b
	add v6.4s, v6.4s, v18.4s
	ext v19.16b, v19.16b, v19.16b, #12
	ushr v21.4s, v5.4s, #7
	shl v5.4s, v5.4s, #25
	ext v6.16b, v6.16b, v6.16b, #4
	orr v5.16b, v5.16b, v21.16b
	ext v21.16b, v17.16b, v17.16b, #12
	add v6.4s, v6.4s, v5.4s
	ext v17.16b, v17.16b, v21.16b, #12
	mov v21.16b, v20.16b
	eor v16.16b, v6.16b, v16.16b
	rev64 v17.4s, v17.4s
	mov v21.s[1], v7.s[2]
	tbl v16.16b, { v16.16b }, v0.16b
	add v19.4s, v19.4s, v16.4s
	eor v5.16b, v5.16b, v19.16b
	ushr v22.4s, v5.4s, #12
	shl v23.4s, v5.4s, #20
	trn2 v5.4s, v17.4s, v21.4s
	orr v17.16b, v23.16b, v22.16b
	add v21.4s, v17.4s, v5.4s
	add v6.4s, v21.4s, v6.4s
	eor v16.16b, v16.16b, v6.16b
	ext v6.16b, v6.16b, v6.16b, #12
	tbl v21.16b, { v16.16b }, v2.16b
	zip1 v16.2d, v20.2d, v4.2d
	zip2 v4.4s, v4.4s, v20.4s
	add v19.4s, v19.4s, v21.4s
	mov v16.s[3], v7.s[3]
	ext v21.16b, v21.16b, v21.16b, #8
	zip1 v20.4s, v4.4s, v7.4s
	eor v17.16b, v17.16b, v19.16b
	ext v22.16b, v16.16b, v16.16b, #12
	ext v19.16b, v19.16b, v19.16b, #4
	zip1 v4.4s, v7.4s, v4.4s
	ushr v23.4s, v17.4s, #7
	shl v17.4s, v17.4s, #25
	uzp1 v16.4s, v16.4s, v22.4s
	ext v4.16b, v4.16b, v20.16b, #8
	orr v17.16b, v17.16b, v23.16b
	add v22.4s, v17.4s, v16.4s
	add v6.4s, v22.4s, v6.4s
	eor v21.16b, v6.16b, v21.16b
	tbl v21.16b, { v21.16b }, v0.16b
	add v19.4s, v19.4s, v21.4s
	eor v17.16b, v17.16b, v19.16b
	ushr v7.4s, v17.4s, #12
	shl v17.4s, v17.4s, #20
	orr v7.16b, v17.16b, v7.16b
	add v17.4s, v7.4s, v4.4s
	add v6.4s, v17.4s, v6.4s
	ext v17.16b, v18.16b, v18.16b, #4
	eor v18.16b, v21.16b, v6.16b
	uzp1 v20.4s, v17.4s, v17.4s
	tbl v18.16b, { v18.16b }, v2.16b
	ext v20.16b, v20.16b, v17.16b, #8
	add v19.4s, v19.4s, v18.4s
	uzp2 v20.4s, v20.4s, v5.4s
	ext v18.16b, v18.16b, v18.16b, #8
	eor v7.16b, v7.16b, v19.16b
	add v6.4s, v6.4s, v20.4s
	ushr v21.4s, v7.4s, #7
	shl v7.4s, v7.4s, #25
	ext v6.16b, v6.16b, v6.16b, #4
	orr v7.16b, v7.16b, v21.16b
	add v21.4s, v6.4s, v7.4s
	eor v6.16b, v21.16b, v18.16b
	ext v18.16b, v19.16b, v19.16b, #12
	tbl v19.16b, { v6.16b }, v0.16b
	ext v6.16b, v17.16b, v17.16b, #12
	add v18.4s, v18.4s, v19.4s
	ext v6.16b, v17.16b, v6.16b, #12
	mov v17.16b, v4.16b
	eor v7.16b, v7.16b, v18.16b
	rev64 v6.4s, v6.4s
	mov v17.s[1], v16.s[2]
	ushr v22.4s, v7.4s, #12
	shl v7.4s, v7.4s, #20
	trn2 v6.4s, v6.4s, v17.4s
	orr v7.16b, v7.16b, v22.16b
	add v17.4s, v7.4s, v6.4s
	add v17.4s, v17.4s, v21.4s
	zip1 v21.2d, v4.2d, v5.2d
	zip2 v4.4s, v5.4s, v4.4s
	eor v19.16b, v19.16b, v17.16b
	mov v21.s[3], v16.s[3]
	ext v17.16b, v17.16b, v17.16b, #12
	tbl v19.16b, { v19.16b }, v2.16b
	ext v22.16b, v21.16b, v21.16b, #12
	add v18.4s, v18.4s, v19.4s
	ext v19.16b, v19.16b, v19.16b, #8
	eor v7.16b, v7.16b, v18.16b
	ext v18.16b, v18.16b, v18.16b, #4
	ushr v23.4s, v7.4s, #7
	shl v24.4s, v7.4s, #25
	uzp1 v7.4s, v21.4s, v22.4s
	orr v21.16b, v24.16b, v23.16b
	add v22.4s, v21.4s, v7.4s
	add v17.4s, v22.4s, v17.4s
	eor v19.16b, v17.16b, v19.16b
	tbl v19.16b, { v19.16b }, v0.16b
	add v18.4s, v18.4s, v19.4s
	eor v5.16b, v21.16b, v18.16b
	zip1 v21.4s, v4.4s, v16.4s
	zip1 v4.4s, v16.4s, v4.4s
	ushr v16.4s, v5.4s, #12
	shl v5.4s, v5.4s, #20
	ext v21.16b, v4.16b, v21.16b, #8
	orr v4.16b, v5.16b, v16.16b
	ext v16.16b, v20.16b, v20.16b, #4
	mov v23.16b, v21.16b
	add v5.4s, v4.4s, v21.4s
	mov v23.s[1], v7.s[2]
	add v5.4s, v5.4s, v17.4s
	eor v17.16b, v19.16b, v5.16b
	uzp1 v19.4s, v16.4s, v16.4s
	tbl v17.16b, { v17.16b }, v2.16b
	ext v19.16b, v19.16b, v16.16b, #8
	add v18.4s, v18.4s, v17.4s
	uzp2 v19.4s, v19.4s, v6.4s
	eor v4.16b, v4.16b, v18.16b
	add v5.4s, v5.4s, v19.4s
	ext v19.16b, v19.16b, v19.16b, #4
	ushr v20.4s, v4.4s, #7
	shl v4.4s, v4.4s, #25
	ext v5.16b, v5.16b, v5.16b, #4
	orr v20.16b, v4.16b, v20.16b
	ext v4.16b, v17.16b, v17.16b, #8
	add v17.4s, v5.4s, v20.4s
	ext v5.16b, v18.16b, v18.16b, #12
	eor v4.16b, v17.16b, v4.16b
	tbl v18.16b, { v4.16b }, v0.16b
	ext v4.16b, v16.16b, v16.16b, #12
	add v22.4s, v5.4s, v18.4s
	ext v4.16b, v16.16b, v4.16b, #12
	eor v5.16b, v20.16b, v22.16b
	rev64 v16.4s, v4.4s
	ushr v20.4s, v5.4s, #12
	shl v24.4s, v5.4s, #20
	trn2 v5.4s, v16.4s, v23.4s
	orr v16.16b, v24.16b, v20.16b
	add v20.4s, v16.4s, v5.4s
	add v17.4s, v20.4s, v17.4s
	zip1 v20.2d, v21.2d, v6.2d
	zip2 v6.4s, v6.4s, v21.4s
	eor v18.16b, v18.16b, v17.16b
	mov v20.s[3], v7.s[3]
	ext v17.16b, v17.16b, v17.16b, #12
	zip1 v21.4s, v6.4s, v7.4s
	tbl v18.16b, { v18.16b }, v2.16b
	ext v24.16b, v20.16b, v20.16b, #12
	zip1 v6.4s, v7.4s, v6.4s
	add v22.4s, v22.4s, v18.4s
	ext v18.16b, v18.16b, v18.16b, #8
	ext v6.16b, v6.16b, v21.16b, #8
	eor v16.16b, v16.16b, v22.16b
	ext v22.16b, v22.16b, v22.16b, #4
	zip1 v5.2d, v6.2d, v5.2d
	zip2 v4.4s, v4.4s, v6.4s
	ushr v25.4s, v16.4s, #7
	shl v26.4s, v16.4s, #25
	uzp1 v16.4s, v20.4s, v24.4s
	orr v20.16b, v26.16b, v25.16b
	mov v5.s[3], v16.s[3]
	add v24.4s, v20.4s, v16.4s
	add v17.4s, v24.4s, v17.4s
	eor v18.16b, v17.16b, v18.16b
	tbl v18.16b, { v18.16b }, v0.16b
	add v22.4s, v22.4s, v18.4s
	eor v20.16b, v20.16b, v22.16b
	ushr v7.4s, v20.4s, #12
	shl v20.4s, v20.4s, #20
	orr v7.16b, v20.16b, v7.16b
	add v20.4s, v7.4s, v6.4s
	add v17.4s, v20.4s, v17.4s
	ext v20.16b, v19.16b, v19.16b, #8
	eor v18.16b, v18.16b, v17.16b
	ext v17.16b, v17.16b, v17.16b, #4
	tbl v18.16b, { v18.16b }, v2.16b
	add v21.4s, v22.4s, v18.4s
	uzp2 v22.4s, v20.4s, v23.4s
	ext v18.16b, v18.16b, v18.16b, #8
	eor v7.16b, v7.16b, v21.16b
	ext v20.16b, v22.16b, v20.16b, #4
	ushr v22.4s, v7.4s, #7
	shl v7.4s, v7.4s, #25
	add v17.4s, v17.4s, v20.4s
	ext v20.16b, v21.16b, v21.16b, #12
	ext v21.16b, v19.16b, v19.16b, #12
	orr v7.16b, v7.16b, v22.16b
	ext v19.16b, v19.16b, v21.16b, #12
	add v17.4s, v17.4s, v7.4s
	mov v21.16b, v6.16b
	rev64 v19.4s, v19.4s
	eor v18.16b, v17.16b, v18.16b
	mov v21.s[1], v16.s[2]
	tbl v18.16b, { v18.16b }, v0.16b
	trn2 v19.4s, v19.4s, v21.4s
	add v20.4s, v20.4s, v18.4s
	eor v7.16b, v7.16b, v20.16b
	ushr v22.4s, v7.4s, #12
	shl v7.4s, v7.4s, #20
	orr v7.16b, v7.16b, v22.16b
	add v19.4s, v7.4s, v19.4s
	add v17.4s, v19.4s, v17.4s
	eor v18.16b, v18.16b, v17.16b
	ext v17.16b, v17.16b, v17.16b, #12
	tbl v18.16b, { v18.16b }, v2.16b
	add v19.4s, v20.4s, v18.4s
	ext v20.16b, v5.16b, v5.16b, #12
	ext v18.16b, v18.16b, v18.16b, #8
	eor v7.16b, v7.16b, v19.16b
	uzp1 v5.4s, v5.4s, v20.4s
	ushr v21.4s, v7.4s, #7
	shl v7.4s, v7.4s, #25
	orr v7.16b, v7.16b, v21.16b
	add v5.4s, v7.4s, v5.4s
	add v5.4s, v5.4s, v17.4s
	eor v17.16b, v5.16b, v18.16b
	ext v18.16b, v19.16b, v19.16b, #4
	tbl v17.16b, { v17.16b }, v0.16b
	add v18.4s, v18.4s, v17.4s
	eor v6.16b, v7.16b, v18.16b
	zip1 v7.4s, v4.4s, v16.4s
	zip1 v4.4s, v16.4s, v4.4s
	ushr v16.4s, v6.4s, #12
	shl v6.4s, v6.4s, #20
	ext v4.16b, v4.16b, v7.16b, #8
	orr v6.16b, v6.16b, v16.16b
	add v4.4s, v6.4s, v4.4s
	add v4.4s, v4.4s, v5.4s
	eor v5.16b, v17.16b, v4.16b
	ext v4.16b, v4.16b, v4.16b, #4
	tbl v5.16b, { v5.16b }, v2.16b
	add v7.4s, v18.4s, v5.4s
	eor v6.16b, v6.16b, v7.16b
	ext v7.16b, v7.16b, v7.16b, #12
	ushr v16.4s, v6.4s, #7
	shl v6.4s, v6.4s, #25
	orr v6.16b, v6.16b, v16.16b
	ext v16.16b, v5.16b, v5.16b, #8
	eor v5.16b, v4.16b, v7.16b
	eor v4.16b, v6.16b, v16.16b
	.LBB3_11:
	subs x13, x15, #1
	b.eq .LBB3_9
	cbnz x15, .LBB3_10
	add x4, x4, x12
	add x0, x0, #8
	subs x1, x1, #1
	stp q5, q4, [x8], #32
	b.ne .LBB3_8
	.LBB3_14:
	add sp, sp, #368
	ldp x20, x19, [sp, #128]
	ldp x22, x21, [sp, #112]
	ldp x24, x23, [sp, #96]
	ldp x26, x25, [sp, #80]
	ldp x29, x27, [sp, #64]
	ldp d9, d8, [sp, #48]
	ldp d11, d10, [sp, #32]
	ldp d13, d12, [sp, #16]
	ldp d15, d14, [sp], #144
	ret
	.Lfunc_end3:
	.size zfs_blake3_hash_many_sse41, .Lfunc_end3-zfs_blake3_hash_many_sse41
	.cfi_endproc
	.section ".note.GNU-stack","",@progbits
	#endif
	\ No newline at end of file
	diff --git a/sys/contrib/openzfs/module/icp/asm-aarch64/sha2/sha256-armv8.S b/sys/contrib/openzfs/module/icp/asm-aarch64/sha2/sha256-armv8.S
	index 7ae486e4e229..4dcdd3b65d0b 100644
	--- a/sys/contrib/openzfs/module/icp/asm-aarch64/sha2/sha256-armv8.S
	+++ b/sys/contrib/openzfs/module/icp/asm-aarch64/sha2/sha256-armv8.S
	@@ -1,2002 +1,2012 @@
	/*
	* Copyright 2004-2022 The OpenSSL Project Authors. All Rights Reserved.
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* https://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	/*
	* Portions Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
	* - modified assembly to fit into OpenZFS
	*/

	#if defined(__aarch64__)

	+ .section .note.gnu.property,"a",@note
	+ .p2align 3
	+ .word 4
	+ .word 16
	+ .word 5
	+ .asciz "GNU"
	+ .word 3221225472
	+ .word 4
	+ .word 3
	+ .word 0
	.text

	.align 6
	.type .LK256,%object
	.LK256:
	.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
	.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
	.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
	.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
	.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
	.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
	.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
	.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
	.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
	.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
	.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
	.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
	.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
	.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
	.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
	.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
	.long 0 //terminator
	.size .LK256,.-.LK256

	.globl zfs_sha256_block_armv7
	.type zfs_sha256_block_armv7,%function
	.align 6
	zfs_sha256_block_armv7:
	hint #34 // bti c
	stp x29,x30,[sp,#-128]!
	add x29,sp,#0

	stp x19,x20,[sp,#16]
	stp x21,x22,[sp,#32]
	stp x23,x24,[sp,#48]
	stp x25,x26,[sp,#64]
	stp x27,x28,[sp,#80]
	sub sp,sp,#4*4

	ldp w20,w21,[x0] // load context
	ldp w22,w23,[x0,#2*4]
	ldp w24,w25,[x0,#4*4]
	add x2,x1,x2,lsl#6 // end of input
	ldp w26,w27,[x0,#6*4]
	adr x30,.LK256
	stp x0,x2,[x29,#96]

	.Loop:
	ldp w3,w4,[x1],#2*4
	ldr w19,[x30],#4 // *K++
	eor w28,w21,w22 // magic seed
	str x1,[x29,#112]
	#ifndef __AARCH64EB__
	rev w3,w3 // 0
	#endif
	ror w16,w24,#6
	add w27,w27,w19 // h+=K[i]
	eor w6,w24,w24,ror#14
	and w17,w25,w24
	bic w19,w26,w24
	add w27,w27,w3 // h+=X[i]
	orr w17,w17,w19 // Ch(e,f,g)
	eor w19,w20,w21 // a^b, b^c in next round
	eor w16,w16,w6,ror#11 // Sigma1(e)
	ror w6,w20,#2
	add w27,w27,w17 // h+=Ch(e,f,g)
	eor w17,w20,w20,ror#9
	add w27,w27,w16 // h+=Sigma1(e)
	and w28,w28,w19 // (b^c)&=(a^b)
	add w23,w23,w27 // d+=h
	eor w28,w28,w21 // Maj(a,b,c)
	eor w17,w6,w17,ror#13 // Sigma0(a)
	add w27,w27,w28 // h+=Maj(a,b,c)
	ldr w28,[x30],#4 // *K++, w19 in next round
	//add w27,w27,w17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev w4,w4 // 1
	#endif
	ldp w5,w6,[x1],#2*4
	add w27,w27,w17 // h+=Sigma0(a)
	ror w16,w23,#6
	add w26,w26,w28 // h+=K[i]
	eor w7,w23,w23,ror#14
	and w17,w24,w23
	bic w28,w25,w23
	add w26,w26,w4 // h+=X[i]
	orr w17,w17,w28 // Ch(e,f,g)
	eor w28,w27,w20 // a^b, b^c in next round
	eor w16,w16,w7,ror#11 // Sigma1(e)
	ror w7,w27,#2
	add w26,w26,w17 // h+=Ch(e,f,g)
	eor w17,w27,w27,ror#9
	add w26,w26,w16 // h+=Sigma1(e)
	and w19,w19,w28 // (b^c)&=(a^b)
	add w22,w22,w26 // d+=h
	eor w19,w19,w20 // Maj(a,b,c)
	eor w17,w7,w17,ror#13 // Sigma0(a)
	add w26,w26,w19 // h+=Maj(a,b,c)
	ldr w19,[x30],#4 // *K++, w28 in next round
	//add w26,w26,w17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev w5,w5 // 2
	#endif
	add w26,w26,w17 // h+=Sigma0(a)
	ror w16,w22,#6
	add w25,w25,w19 // h+=K[i]
	eor w8,w22,w22,ror#14
	and w17,w23,w22
	bic w19,w24,w22
	add w25,w25,w5 // h+=X[i]
	orr w17,w17,w19 // Ch(e,f,g)
	eor w19,w26,w27 // a^b, b^c in next round
	eor w16,w16,w8,ror#11 // Sigma1(e)
	ror w8,w26,#2
	add w25,w25,w17 // h+=Ch(e,f,g)
	eor w17,w26,w26,ror#9
	add w25,w25,w16 // h+=Sigma1(e)
	and w28,w28,w19 // (b^c)&=(a^b)
	add w21,w21,w25 // d+=h
	eor w28,w28,w27 // Maj(a,b,c)
	eor w17,w8,w17,ror#13 // Sigma0(a)
	add w25,w25,w28 // h+=Maj(a,b,c)
	ldr w28,[x30],#4 // *K++, w19 in next round
	//add w25,w25,w17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev w6,w6 // 3
	#endif
	ldp w7,w8,[x1],#2*4
	add w25,w25,w17 // h+=Sigma0(a)
	ror w16,w21,#6
	add w24,w24,w28 // h+=K[i]
	eor w9,w21,w21,ror#14
	and w17,w22,w21
	bic w28,w23,w21
	add w24,w24,w6 // h+=X[i]
	orr w17,w17,w28 // Ch(e,f,g)
	eor w28,w25,w26 // a^b, b^c in next round
	eor w16,w16,w9,ror#11 // Sigma1(e)
	ror w9,w25,#2
	add w24,w24,w17 // h+=Ch(e,f,g)
	eor w17,w25,w25,ror#9
	add w24,w24,w16 // h+=Sigma1(e)
	and w19,w19,w28 // (b^c)&=(a^b)
	add w20,w20,w24 // d+=h
	eor w19,w19,w26 // Maj(a,b,c)
	eor w17,w9,w17,ror#13 // Sigma0(a)
	add w24,w24,w19 // h+=Maj(a,b,c)
	ldr w19,[x30],#4 // *K++, w28 in next round
	//add w24,w24,w17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev w7,w7 // 4
	#endif
	add w24,w24,w17 // h+=Sigma0(a)
	ror w16,w20,#6
	add w23,w23,w19 // h+=K[i]
	eor w10,w20,w20,ror#14
	and w17,w21,w20
	bic w19,w22,w20
	add w23,w23,w7 // h+=X[i]
	orr w17,w17,w19 // Ch(e,f,g)
	eor w19,w24,w25 // a^b, b^c in next round
	eor w16,w16,w10,ror#11 // Sigma1(e)
	ror w10,w24,#2
	add w23,w23,w17 // h+=Ch(e,f,g)
	eor w17,w24,w24,ror#9
	add w23,w23,w16 // h+=Sigma1(e)
	and w28,w28,w19 // (b^c)&=(a^b)
	add w27,w27,w23 // d+=h
	eor w28,w28,w25 // Maj(a,b,c)
	eor w17,w10,w17,ror#13 // Sigma0(a)
	add w23,w23,w28 // h+=Maj(a,b,c)
	ldr w28,[x30],#4 // *K++, w19 in next round
	//add w23,w23,w17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev w8,w8 // 5
	#endif
	ldp w9,w10,[x1],#2*4
	add w23,w23,w17 // h+=Sigma0(a)
	ror w16,w27,#6
	add w22,w22,w28 // h+=K[i]
	eor w11,w27,w27,ror#14
	and w17,w20,w27
	bic w28,w21,w27
	add w22,w22,w8 // h+=X[i]
	orr w17,w17,w28 // Ch(e,f,g)
	eor w28,w23,w24 // a^b, b^c in next round
	eor w16,w16,w11,ror#11 // Sigma1(e)
	ror w11,w23,#2
	add w22,w22,w17 // h+=Ch(e,f,g)
	eor w17,w23,w23,ror#9
	add w22,w22,w16 // h+=Sigma1(e)
	and w19,w19,w28 // (b^c)&=(a^b)
	add w26,w26,w22 // d+=h
	eor w19,w19,w24 // Maj(a,b,c)
	eor w17,w11,w17,ror#13 // Sigma0(a)
	add w22,w22,w19 // h+=Maj(a,b,c)
	ldr w19,[x30],#4 // *K++, w28 in next round
	//add w22,w22,w17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev w9,w9 // 6
	#endif
	add w22,w22,w17 // h+=Sigma0(a)
	ror w16,w26,#6
	add w21,w21,w19 // h+=K[i]
	eor w12,w26,w26,ror#14
	and w17,w27,w26
	bic w19,w20,w26
	add w21,w21,w9 // h+=X[i]
	orr w17,w17,w19 // Ch(e,f,g)
	eor w19,w22,w23 // a^b, b^c in next round
	eor w16,w16,w12,ror#11 // Sigma1(e)
	ror w12,w22,#2
	add w21,w21,w17 // h+=Ch(e,f,g)
	eor w17,w22,w22,ror#9
	add w21,w21,w16 // h+=Sigma1(e)
	and w28,w28,w19 // (b^c)&=(a^b)
	add w25,w25,w21 // d+=h
	eor w28,w28,w23 // Maj(a,b,c)
	eor w17,w12,w17,ror#13 // Sigma0(a)
	add w21,w21,w28 // h+=Maj(a,b,c)
	ldr w28,[x30],#4 // *K++, w19 in next round
	//add w21,w21,w17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev w10,w10 // 7
	#endif
	ldp w11,w12,[x1],#2*4
	add w21,w21,w17 // h+=Sigma0(a)
	ror w16,w25,#6
	add w20,w20,w28 // h+=K[i]
	eor w13,w25,w25,ror#14
	and w17,w26,w25
	bic w28,w27,w25
	add w20,w20,w10 // h+=X[i]
	orr w17,w17,w28 // Ch(e,f,g)
	eor w28,w21,w22 // a^b, b^c in next round
	eor w16,w16,w13,ror#11 // Sigma1(e)
	ror w13,w21,#2
	add w20,w20,w17 // h+=Ch(e,f,g)
	eor w17,w21,w21,ror#9
	add w20,w20,w16 // h+=Sigma1(e)
	and w19,w19,w28 // (b^c)&=(a^b)
	add w24,w24,w20 // d+=h
	eor w19,w19,w22 // Maj(a,b,c)
	eor w17,w13,w17,ror#13 // Sigma0(a)
	add w20,w20,w19 // h+=Maj(a,b,c)
	ldr w19,[x30],#4 // *K++, w28 in next round
	//add w20,w20,w17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev w11,w11 // 8
	#endif
	add w20,w20,w17 // h+=Sigma0(a)
	ror w16,w24,#6
	add w27,w27,w19 // h+=K[i]
	eor w14,w24,w24,ror#14
	and w17,w25,w24
	bic w19,w26,w24
	add w27,w27,w11 // h+=X[i]
	orr w17,w17,w19 // Ch(e,f,g)
	eor w19,w20,w21 // a^b, b^c in next round
	eor w16,w16,w14,ror#11 // Sigma1(e)
	ror w14,w20,#2
	add w27,w27,w17 // h+=Ch(e,f,g)
	eor w17,w20,w20,ror#9
	add w27,w27,w16 // h+=Sigma1(e)
	and w28,w28,w19 // (b^c)&=(a^b)
	add w23,w23,w27 // d+=h
	eor w28,w28,w21 // Maj(a,b,c)
	eor w17,w14,w17,ror#13 // Sigma0(a)
	add w27,w27,w28 // h+=Maj(a,b,c)
	ldr w28,[x30],#4 // *K++, w19 in next round
	//add w27,w27,w17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev w12,w12 // 9
	#endif
	ldp w13,w14,[x1],#2*4
	add w27,w27,w17 // h+=Sigma0(a)
	ror w16,w23,#6
	add w26,w26,w28 // h+=K[i]
	eor w15,w23,w23,ror#14
	and w17,w24,w23
	bic w28,w25,w23
	add w26,w26,w12 // h+=X[i]
	orr w17,w17,w28 // Ch(e,f,g)
	eor w28,w27,w20 // a^b, b^c in next round
	eor w16,w16,w15,ror#11 // Sigma1(e)
	ror w15,w27,#2
	add w26,w26,w17 // h+=Ch(e,f,g)
	eor w17,w27,w27,ror#9
	add w26,w26,w16 // h+=Sigma1(e)
	and w19,w19,w28 // (b^c)&=(a^b)
	add w22,w22,w26 // d+=h
	eor w19,w19,w20 // Maj(a,b,c)
	eor w17,w15,w17,ror#13 // Sigma0(a)
	add w26,w26,w19 // h+=Maj(a,b,c)
	ldr w19,[x30],#4 // *K++, w28 in next round
	//add w26,w26,w17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev w13,w13 // 10
	#endif
	add w26,w26,w17 // h+=Sigma0(a)
	ror w16,w22,#6
	add w25,w25,w19 // h+=K[i]
	eor w0,w22,w22,ror#14
	and w17,w23,w22
	bic w19,w24,w22
	add w25,w25,w13 // h+=X[i]
	orr w17,w17,w19 // Ch(e,f,g)
	eor w19,w26,w27 // a^b, b^c in next round
	eor w16,w16,w0,ror#11 // Sigma1(e)
	ror w0,w26,#2
	add w25,w25,w17 // h+=Ch(e,f,g)
	eor w17,w26,w26,ror#9
	add w25,w25,w16 // h+=Sigma1(e)
	and w28,w28,w19 // (b^c)&=(a^b)
	add w21,w21,w25 // d+=h
	eor w28,w28,w27 // Maj(a,b,c)
	eor w17,w0,w17,ror#13 // Sigma0(a)
	add w25,w25,w28 // h+=Maj(a,b,c)
	ldr w28,[x30],#4 // *K++, w19 in next round
	//add w25,w25,w17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev w14,w14 // 11
	#endif
	ldp w15,w0,[x1],#2*4
	add w25,w25,w17 // h+=Sigma0(a)
	str w6,[sp,#12]
	ror w16,w21,#6
	add w24,w24,w28 // h+=K[i]
	eor w6,w21,w21,ror#14
	and w17,w22,w21
	bic w28,w23,w21
	add w24,w24,w14 // h+=X[i]
	orr w17,w17,w28 // Ch(e,f,g)
	eor w28,w25,w26 // a^b, b^c in next round
	eor w16,w16,w6,ror#11 // Sigma1(e)
	ror w6,w25,#2
	add w24,w24,w17 // h+=Ch(e,f,g)
	eor w17,w25,w25,ror#9
	add w24,w24,w16 // h+=Sigma1(e)
	and w19,w19,w28 // (b^c)&=(a^b)
	add w20,w20,w24 // d+=h
	eor w19,w19,w26 // Maj(a,b,c)
	eor w17,w6,w17,ror#13 // Sigma0(a)
	add w24,w24,w19 // h+=Maj(a,b,c)
	ldr w19,[x30],#4 // *K++, w28 in next round
	//add w24,w24,w17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev w15,w15 // 12
	#endif
	add w24,w24,w17 // h+=Sigma0(a)
	str w7,[sp,#0]
	ror w16,w20,#6
	add w23,w23,w19 // h+=K[i]
	eor w7,w20,w20,ror#14
	and w17,w21,w20
	bic w19,w22,w20
	add w23,w23,w15 // h+=X[i]
	orr w17,w17,w19 // Ch(e,f,g)
	eor w19,w24,w25 // a^b, b^c in next round
	eor w16,w16,w7,ror#11 // Sigma1(e)
	ror w7,w24,#2
	add w23,w23,w17 // h+=Ch(e,f,g)
	eor w17,w24,w24,ror#9
	add w23,w23,w16 // h+=Sigma1(e)
	and w28,w28,w19 // (b^c)&=(a^b)
	add w27,w27,w23 // d+=h
	eor w28,w28,w25 // Maj(a,b,c)
	eor w17,w7,w17,ror#13 // Sigma0(a)
	add w23,w23,w28 // h+=Maj(a,b,c)
	ldr w28,[x30],#4 // *K++, w19 in next round
	//add w23,w23,w17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev w0,w0 // 13
	#endif
	ldp w1,w2,[x1]
	add w23,w23,w17 // h+=Sigma0(a)
	str w8,[sp,#4]
	ror w16,w27,#6
	add w22,w22,w28 // h+=K[i]
	eor w8,w27,w27,ror#14
	and w17,w20,w27
	bic w28,w21,w27
	add w22,w22,w0 // h+=X[i]
	orr w17,w17,w28 // Ch(e,f,g)
	eor w28,w23,w24 // a^b, b^c in next round
	eor w16,w16,w8,ror#11 // Sigma1(e)
	ror w8,w23,#2
	add w22,w22,w17 // h+=Ch(e,f,g)
	eor w17,w23,w23,ror#9
	add w22,w22,w16 // h+=Sigma1(e)
	and w19,w19,w28 // (b^c)&=(a^b)
	add w26,w26,w22 // d+=h
	eor w19,w19,w24 // Maj(a,b,c)
	eor w17,w8,w17,ror#13 // Sigma0(a)
	add w22,w22,w19 // h+=Maj(a,b,c)
	ldr w19,[x30],#4 // *K++, w28 in next round
	//add w22,w22,w17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev w1,w1 // 14
	#endif
	ldr w6,[sp,#12]
	add w22,w22,w17 // h+=Sigma0(a)
	str w9,[sp,#8]
	ror w16,w26,#6
	add w21,w21,w19 // h+=K[i]
	eor w9,w26,w26,ror#14
	and w17,w27,w26
	bic w19,w20,w26
	add w21,w21,w1 // h+=X[i]
	orr w17,w17,w19 // Ch(e,f,g)
	eor w19,w22,w23 // a^b, b^c in next round
	eor w16,w16,w9,ror#11 // Sigma1(e)
	ror w9,w22,#2
	add w21,w21,w17 // h+=Ch(e,f,g)
	eor w17,w22,w22,ror#9
	add w21,w21,w16 // h+=Sigma1(e)
	and w28,w28,w19 // (b^c)&=(a^b)
	add w25,w25,w21 // d+=h
	eor w28,w28,w23 // Maj(a,b,c)
	eor w17,w9,w17,ror#13 // Sigma0(a)
	add w21,w21,w28 // h+=Maj(a,b,c)
	ldr w28,[x30],#4 // *K++, w19 in next round
	//add w21,w21,w17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev w2,w2 // 15
	#endif
	ldr w7,[sp,#0]
	add w21,w21,w17 // h+=Sigma0(a)
	str w10,[sp,#12]
	ror w16,w25,#6
	add w20,w20,w28 // h+=K[i]
	ror w9,w4,#7
	and w17,w26,w25
	ror w8,w1,#17
	bic w28,w27,w25
	ror w10,w21,#2
	add w20,w20,w2 // h+=X[i]
	eor w16,w16,w25,ror#11
	eor w9,w9,w4,ror#18
	orr w17,w17,w28 // Ch(e,f,g)
	eor w28,w21,w22 // a^b, b^c in next round
	eor w16,w16,w25,ror#25 // Sigma1(e)
	eor w10,w10,w21,ror#13
	add w20,w20,w17 // h+=Ch(e,f,g)
	and w19,w19,w28 // (b^c)&=(a^b)
	eor w8,w8,w1,ror#19
	eor w9,w9,w4,lsr#3 // sigma0(X[i+1])
	add w20,w20,w16 // h+=Sigma1(e)
	eor w19,w19,w22 // Maj(a,b,c)
	eor w17,w10,w21,ror#22 // Sigma0(a)
	eor w8,w8,w1,lsr#10 // sigma1(X[i+14])
	add w3,w3,w12
	add w24,w24,w20 // d+=h
	add w20,w20,w19 // h+=Maj(a,b,c)
	ldr w19,[x30],#4 // *K++, w28 in next round
	add w3,w3,w9
	add w20,w20,w17 // h+=Sigma0(a)
	add w3,w3,w8
	.Loop_16_xx:
	ldr w8,[sp,#4]
	str w11,[sp,#0]
	ror w16,w24,#6
	add w27,w27,w19 // h+=K[i]
	ror w10,w5,#7
	and w17,w25,w24
	ror w9,w2,#17
	bic w19,w26,w24
	ror w11,w20,#2
	add w27,w27,w3 // h+=X[i]
	eor w16,w16,w24,ror#11
	eor w10,w10,w5,ror#18
	orr w17,w17,w19 // Ch(e,f,g)
	eor w19,w20,w21 // a^b, b^c in next round
	eor w16,w16,w24,ror#25 // Sigma1(e)
	eor w11,w11,w20,ror#13
	add w27,w27,w17 // h+=Ch(e,f,g)
	and w28,w28,w19 // (b^c)&=(a^b)
	eor w9,w9,w2,ror#19
	eor w10,w10,w5,lsr#3 // sigma0(X[i+1])
	add w27,w27,w16 // h+=Sigma1(e)
	eor w28,w28,w21 // Maj(a,b,c)
	eor w17,w11,w20,ror#22 // Sigma0(a)
	eor w9,w9,w2,lsr#10 // sigma1(X[i+14])
	add w4,w4,w13
	add w23,w23,w27 // d+=h
	add w27,w27,w28 // h+=Maj(a,b,c)
	ldr w28,[x30],#4 // *K++, w19 in next round
	add w4,w4,w10
	add w27,w27,w17 // h+=Sigma0(a)
	add w4,w4,w9
	ldr w9,[sp,#8]
	str w12,[sp,#4]
	ror w16,w23,#6
	add w26,w26,w28 // h+=K[i]
	ror w11,w6,#7
	and w17,w24,w23
	ror w10,w3,#17
	bic w28,w25,w23
	ror w12,w27,#2
	add w26,w26,w4 // h+=X[i]
	eor w16,w16,w23,ror#11
	eor w11,w11,w6,ror#18
	orr w17,w17,w28 // Ch(e,f,g)
	eor w28,w27,w20 // a^b, b^c in next round
	eor w16,w16,w23,ror#25 // Sigma1(e)
	eor w12,w12,w27,ror#13
	add w26,w26,w17 // h+=Ch(e,f,g)
	and w19,w19,w28 // (b^c)&=(a^b)
	eor w10,w10,w3,ror#19
	eor w11,w11,w6,lsr#3 // sigma0(X[i+1])
	add w26,w26,w16 // h+=Sigma1(e)
	eor w19,w19,w20 // Maj(a,b,c)
	eor w17,w12,w27,ror#22 // Sigma0(a)
	eor w10,w10,w3,lsr#10 // sigma1(X[i+14])
	add w5,w5,w14
	add w22,w22,w26 // d+=h
	add w26,w26,w19 // h+=Maj(a,b,c)
	ldr w19,[x30],#4 // *K++, w28 in next round
	add w5,w5,w11
	add w26,w26,w17 // h+=Sigma0(a)
	add w5,w5,w10
	ldr w10,[sp,#12]
	str w13,[sp,#8]
	ror w16,w22,#6
	add w25,w25,w19 // h+=K[i]
	ror w12,w7,#7
	and w17,w23,w22
	ror w11,w4,#17
	bic w19,w24,w22
	ror w13,w26,#2
	add w25,w25,w5 // h+=X[i]
	eor w16,w16,w22,ror#11
	eor w12,w12,w7,ror#18
	orr w17,w17,w19 // Ch(e,f,g)
	eor w19,w26,w27 // a^b, b^c in next round
	eor w16,w16,w22,ror#25 // Sigma1(e)
	eor w13,w13,w26,ror#13
	add w25,w25,w17 // h+=Ch(e,f,g)
	and w28,w28,w19 // (b^c)&=(a^b)
	eor w11,w11,w4,ror#19
	eor w12,w12,w7,lsr#3 // sigma0(X[i+1])
	add w25,w25,w16 // h+=Sigma1(e)
	eor w28,w28,w27 // Maj(a,b,c)
	eor w17,w13,w26,ror#22 // Sigma0(a)
	eor w11,w11,w4,lsr#10 // sigma1(X[i+14])
	add w6,w6,w15
	add w21,w21,w25 // d+=h
	add w25,w25,w28 // h+=Maj(a,b,c)
	ldr w28,[x30],#4 // *K++, w19 in next round
	add w6,w6,w12
	add w25,w25,w17 // h+=Sigma0(a)
	add w6,w6,w11
	ldr w11,[sp,#0]
	str w14,[sp,#12]
	ror w16,w21,#6
	add w24,w24,w28 // h+=K[i]
	ror w13,w8,#7
	and w17,w22,w21
	ror w12,w5,#17
	bic w28,w23,w21
	ror w14,w25,#2
	add w24,w24,w6 // h+=X[i]
	eor w16,w16,w21,ror#11
	eor w13,w13,w8,ror#18
	orr w17,w17,w28 // Ch(e,f,g)
	eor w28,w25,w26 // a^b, b^c in next round
	eor w16,w16,w21,ror#25 // Sigma1(e)
	eor w14,w14,w25,ror#13
	add w24,w24,w17 // h+=Ch(e,f,g)
	and w19,w19,w28 // (b^c)&=(a^b)
	eor w12,w12,w5,ror#19
	eor w13,w13,w8,lsr#3 // sigma0(X[i+1])
	add w24,w24,w16 // h+=Sigma1(e)
	eor w19,w19,w26 // Maj(a,b,c)
	eor w17,w14,w25,ror#22 // Sigma0(a)
	eor w12,w12,w5,lsr#10 // sigma1(X[i+14])
	add w7,w7,w0
	add w20,w20,w24 // d+=h
	add w24,w24,w19 // h+=Maj(a,b,c)
	ldr w19,[x30],#4 // *K++, w28 in next round
	add w7,w7,w13
	add w24,w24,w17 // h+=Sigma0(a)
	add w7,w7,w12
	ldr w12,[sp,#4]
	str w15,[sp,#0]
	ror w16,w20,#6
	add w23,w23,w19 // h+=K[i]
	ror w14,w9,#7
	and w17,w21,w20
	ror w13,w6,#17
	bic w19,w22,w20
	ror w15,w24,#2
	add w23,w23,w7 // h+=X[i]
	eor w16,w16,w20,ror#11
	eor w14,w14,w9,ror#18
	orr w17,w17,w19 // Ch(e,f,g)
	eor w19,w24,w25 // a^b, b^c in next round
	eor w16,w16,w20,ror#25 // Sigma1(e)
	eor w15,w15,w24,ror#13
	add w23,w23,w17 // h+=Ch(e,f,g)
	and w28,w28,w19 // (b^c)&=(a^b)
	eor w13,w13,w6,ror#19
	eor w14,w14,w9,lsr#3 // sigma0(X[i+1])
	add w23,w23,w16 // h+=Sigma1(e)
	eor w28,w28,w25 // Maj(a,b,c)
	eor w17,w15,w24,ror#22 // Sigma0(a)
	eor w13,w13,w6,lsr#10 // sigma1(X[i+14])
	add w8,w8,w1
	add w27,w27,w23 // d+=h
	add w23,w23,w28 // h+=Maj(a,b,c)
	ldr w28,[x30],#4 // *K++, w19 in next round
	add w8,w8,w14
	add w23,w23,w17 // h+=Sigma0(a)
	add w8,w8,w13
	ldr w13,[sp,#8]
	str w0,[sp,#4]
	ror w16,w27,#6
	add w22,w22,w28 // h+=K[i]
	ror w15,w10,#7
	and w17,w20,w27
	ror w14,w7,#17
	bic w28,w21,w27
	ror w0,w23,#2
	add w22,w22,w8 // h+=X[i]
	eor w16,w16,w27,ror#11
	eor w15,w15,w10,ror#18
	orr w17,w17,w28 // Ch(e,f,g)
	eor w28,w23,w24 // a^b, b^c in next round
	eor w16,w16,w27,ror#25 // Sigma1(e)
	eor w0,w0,w23,ror#13
	add w22,w22,w17 // h+=Ch(e,f,g)
	and w19,w19,w28 // (b^c)&=(a^b)
	eor w14,w14,w7,ror#19
	eor w15,w15,w10,lsr#3 // sigma0(X[i+1])
	add w22,w22,w16 // h+=Sigma1(e)
	eor w19,w19,w24 // Maj(a,b,c)
	eor w17,w0,w23,ror#22 // Sigma0(a)
	eor w14,w14,w7,lsr#10 // sigma1(X[i+14])
	add w9,w9,w2
	add w26,w26,w22 // d+=h
	add w22,w22,w19 // h+=Maj(a,b,c)
	ldr w19,[x30],#4 // *K++, w28 in next round
	add w9,w9,w15
	add w22,w22,w17 // h+=Sigma0(a)
	add w9,w9,w14
	ldr w14,[sp,#12]
	str w1,[sp,#8]
	ror w16,w26,#6
	add w21,w21,w19 // h+=K[i]
	ror w0,w11,#7
	and w17,w27,w26
	ror w15,w8,#17
	bic w19,w20,w26
	ror w1,w22,#2
	add w21,w21,w9 // h+=X[i]
	eor w16,w16,w26,ror#11
	eor w0,w0,w11,ror#18
	orr w17,w17,w19 // Ch(e,f,g)
	eor w19,w22,w23 // a^b, b^c in next round
	eor w16,w16,w26,ror#25 // Sigma1(e)
	eor w1,w1,w22,ror#13
	add w21,w21,w17 // h+=Ch(e,f,g)
	and w28,w28,w19 // (b^c)&=(a^b)
	eor w15,w15,w8,ror#19
	eor w0,w0,w11,lsr#3 // sigma0(X[i+1])
	add w21,w21,w16 // h+=Sigma1(e)
	eor w28,w28,w23 // Maj(a,b,c)
	eor w17,w1,w22,ror#22 // Sigma0(a)
	eor w15,w15,w8,lsr#10 // sigma1(X[i+14])
	add w10,w10,w3
	add w25,w25,w21 // d+=h
	add w21,w21,w28 // h+=Maj(a,b,c)
	ldr w28,[x30],#4 // *K++, w19 in next round
	add w10,w10,w0
	add w21,w21,w17 // h+=Sigma0(a)
	add w10,w10,w15
	ldr w15,[sp,#0]
	str w2,[sp,#12]
	ror w16,w25,#6
	add w20,w20,w28 // h+=K[i]
	ror w1,w12,#7
	and w17,w26,w25
	ror w0,w9,#17
	bic w28,w27,w25
	ror w2,w21,#2
	add w20,w20,w10 // h+=X[i]
	eor w16,w16,w25,ror#11
	eor w1,w1,w12,ror#18
	orr w17,w17,w28 // Ch(e,f,g)
	eor w28,w21,w22 // a^b, b^c in next round
	eor w16,w16,w25,ror#25 // Sigma1(e)
	eor w2,w2,w21,ror#13
	add w20,w20,w17 // h+=Ch(e,f,g)
	and w19,w19,w28 // (b^c)&=(a^b)
	eor w0,w0,w9,ror#19
	eor w1,w1,w12,lsr#3 // sigma0(X[i+1])
	add w20,w20,w16 // h+=Sigma1(e)
	eor w19,w19,w22 // Maj(a,b,c)
	eor w17,w2,w21,ror#22 // Sigma0(a)
	eor w0,w0,w9,lsr#10 // sigma1(X[i+14])
	add w11,w11,w4
	add w24,w24,w20 // d+=h
	add w20,w20,w19 // h+=Maj(a,b,c)
	ldr w19,[x30],#4 // *K++, w28 in next round
	add w11,w11,w1
	add w20,w20,w17 // h+=Sigma0(a)
	add w11,w11,w0
	ldr w0,[sp,#4]
	str w3,[sp,#0]
	ror w16,w24,#6
	add w27,w27,w19 // h+=K[i]
	ror w2,w13,#7
	and w17,w25,w24
	ror w1,w10,#17
	bic w19,w26,w24
	ror w3,w20,#2
	add w27,w27,w11 // h+=X[i]
	eor w16,w16,w24,ror#11
	eor w2,w2,w13,ror#18
	orr w17,w17,w19 // Ch(e,f,g)
	eor w19,w20,w21 // a^b, b^c in next round
	eor w16,w16,w24,ror#25 // Sigma1(e)
	eor w3,w3,w20,ror#13
	add w27,w27,w17 // h+=Ch(e,f,g)
	and w28,w28,w19 // (b^c)&=(a^b)
	eor w1,w1,w10,ror#19
	eor w2,w2,w13,lsr#3 // sigma0(X[i+1])
	add w27,w27,w16 // h+=Sigma1(e)
	eor w28,w28,w21 // Maj(a,b,c)
	eor w17,w3,w20,ror#22 // Sigma0(a)
	eor w1,w1,w10,lsr#10 // sigma1(X[i+14])
	add w12,w12,w5
	add w23,w23,w27 // d+=h
	add w27,w27,w28 // h+=Maj(a,b,c)
	ldr w28,[x30],#4 // *K++, w19 in next round
	add w12,w12,w2
	add w27,w27,w17 // h+=Sigma0(a)
	add w12,w12,w1
	ldr w1,[sp,#8]
	str w4,[sp,#4]
	ror w16,w23,#6
	add w26,w26,w28 // h+=K[i]
	ror w3,w14,#7
	and w17,w24,w23
	ror w2,w11,#17
	bic w28,w25,w23
	ror w4,w27,#2
	add w26,w26,w12 // h+=X[i]
	eor w16,w16,w23,ror#11
	eor w3,w3,w14,ror#18
	orr w17,w17,w28 // Ch(e,f,g)
	eor w28,w27,w20 // a^b, b^c in next round
	eor w16,w16,w23,ror#25 // Sigma1(e)
	eor w4,w4,w27,ror#13
	add w26,w26,w17 // h+=Ch(e,f,g)
	and w19,w19,w28 // (b^c)&=(a^b)
	eor w2,w2,w11,ror#19
	eor w3,w3,w14,lsr#3 // sigma0(X[i+1])
	add w26,w26,w16 // h+=Sigma1(e)
	eor w19,w19,w20 // Maj(a,b,c)
	eor w17,w4,w27,ror#22 // Sigma0(a)
	eor w2,w2,w11,lsr#10 // sigma1(X[i+14])
	add w13,w13,w6
	add w22,w22,w26 // d+=h
	add w26,w26,w19 // h+=Maj(a,b,c)
	ldr w19,[x30],#4 // *K++, w28 in next round
	add w13,w13,w3
	add w26,w26,w17 // h+=Sigma0(a)
	add w13,w13,w2
	ldr w2,[sp,#12]
	str w5,[sp,#8]
	ror w16,w22,#6
	add w25,w25,w19 // h+=K[i]
	ror w4,w15,#7
	and w17,w23,w22
	ror w3,w12,#17
	bic w19,w24,w22
	ror w5,w26,#2
	add w25,w25,w13 // h+=X[i]
	eor w16,w16,w22,ror#11
	eor w4,w4,w15,ror#18
	orr w17,w17,w19 // Ch(e,f,g)
	eor w19,w26,w27 // a^b, b^c in next round
	eor w16,w16,w22,ror#25 // Sigma1(e)
	eor w5,w5,w26,ror#13
	add w25,w25,w17 // h+=Ch(e,f,g)
	and w28,w28,w19 // (b^c)&=(a^b)
	eor w3,w3,w12,ror#19
	eor w4,w4,w15,lsr#3 // sigma0(X[i+1])
	add w25,w25,w16 // h+=Sigma1(e)
	eor w28,w28,w27 // Maj(a,b,c)
	eor w17,w5,w26,ror#22 // Sigma0(a)
	eor w3,w3,w12,lsr#10 // sigma1(X[i+14])
	add w14,w14,w7
	add w21,w21,w25 // d+=h
	add w25,w25,w28 // h+=Maj(a,b,c)
	ldr w28,[x30],#4 // *K++, w19 in next round
	add w14,w14,w4
	add w25,w25,w17 // h+=Sigma0(a)
	add w14,w14,w3
	ldr w3,[sp,#0]
	str w6,[sp,#12]
	ror w16,w21,#6
	add w24,w24,w28 // h+=K[i]
	ror w5,w0,#7
	and w17,w22,w21
	ror w4,w13,#17
	bic w28,w23,w21
	ror w6,w25,#2
	add w24,w24,w14 // h+=X[i]
	eor w16,w16,w21,ror#11
	eor w5,w5,w0,ror#18
	orr w17,w17,w28 // Ch(e,f,g)
	eor w28,w25,w26 // a^b, b^c in next round
	eor w16,w16,w21,ror#25 // Sigma1(e)
	eor w6,w6,w25,ror#13
	add w24,w24,w17 // h+=Ch(e,f,g)
	and w19,w19,w28 // (b^c)&=(a^b)
	eor w4,w4,w13,ror#19
	eor w5,w5,w0,lsr#3 // sigma0(X[i+1])
	add w24,w24,w16 // h+=Sigma1(e)
	eor w19,w19,w26 // Maj(a,b,c)
	eor w17,w6,w25,ror#22 // Sigma0(a)
	eor w4,w4,w13,lsr#10 // sigma1(X[i+14])
	add w15,w15,w8
	add w20,w20,w24 // d+=h
	add w24,w24,w19 // h+=Maj(a,b,c)
	ldr w19,[x30],#4 // *K++, w28 in next round
	add w15,w15,w5
	add w24,w24,w17 // h+=Sigma0(a)
	add w15,w15,w4
	ldr w4,[sp,#4]
	str w7,[sp,#0]
	ror w16,w20,#6
	add w23,w23,w19 // h+=K[i]
	ror w6,w1,#7
	and w17,w21,w20
	ror w5,w14,#17
	bic w19,w22,w20
	ror w7,w24,#2
	add w23,w23,w15 // h+=X[i]
	eor w16,w16,w20,ror#11
	eor w6,w6,w1,ror#18
	orr w17,w17,w19 // Ch(e,f,g)
	eor w19,w24,w25 // a^b, b^c in next round
	eor w16,w16,w20,ror#25 // Sigma1(e)
	eor w7,w7,w24,ror#13
	add w23,w23,w17 // h+=Ch(e,f,g)
	and w28,w28,w19 // (b^c)&=(a^b)
	eor w5,w5,w14,ror#19
	eor w6,w6,w1,lsr#3 // sigma0(X[i+1])
	add w23,w23,w16 // h+=Sigma1(e)
	eor w28,w28,w25 // Maj(a,b,c)
	eor w17,w7,w24,ror#22 // Sigma0(a)
	eor w5,w5,w14,lsr#10 // sigma1(X[i+14])
	add w0,w0,w9
	add w27,w27,w23 // d+=h
	add w23,w23,w28 // h+=Maj(a,b,c)
	ldr w28,[x30],#4 // *K++, w19 in next round
	add w0,w0,w6
	add w23,w23,w17 // h+=Sigma0(a)
	add w0,w0,w5
	ldr w5,[sp,#8]
	str w8,[sp,#4]
	ror w16,w27,#6
	add w22,w22,w28 // h+=K[i]
	ror w7,w2,#7
	and w17,w20,w27
	ror w6,w15,#17
	bic w28,w21,w27
	ror w8,w23,#2
	add w22,w22,w0 // h+=X[i]
	eor w16,w16,w27,ror#11
	eor w7,w7,w2,ror#18
	orr w17,w17,w28 // Ch(e,f,g)
	eor w28,w23,w24 // a^b, b^c in next round
	eor w16,w16,w27,ror#25 // Sigma1(e)
	eor w8,w8,w23,ror#13
	add w22,w22,w17 // h+=Ch(e,f,g)
	and w19,w19,w28 // (b^c)&=(a^b)
	eor w6,w6,w15,ror#19
	eor w7,w7,w2,lsr#3 // sigma0(X[i+1])
	add w22,w22,w16 // h+=Sigma1(e)
	eor w19,w19,w24 // Maj(a,b,c)
	eor w17,w8,w23,ror#22 // Sigma0(a)
	eor w6,w6,w15,lsr#10 // sigma1(X[i+14])
	add w1,w1,w10
	add w26,w26,w22 // d+=h
	add w22,w22,w19 // h+=Maj(a,b,c)
	ldr w19,[x30],#4 // *K++, w28 in next round
	add w1,w1,w7
	add w22,w22,w17 // h+=Sigma0(a)
	add w1,w1,w6
	ldr w6,[sp,#12]
	str w9,[sp,#8]
	ror w16,w26,#6
	add w21,w21,w19 // h+=K[i]
	ror w8,w3,#7
	and w17,w27,w26
	ror w7,w0,#17
	bic w19,w20,w26
	ror w9,w22,#2
	add w21,w21,w1 // h+=X[i]
	eor w16,w16,w26,ror#11
	eor w8,w8,w3,ror#18
	orr w17,w17,w19 // Ch(e,f,g)
	eor w19,w22,w23 // a^b, b^c in next round
	eor w16,w16,w26,ror#25 // Sigma1(e)
	eor w9,w9,w22,ror#13
	add w21,w21,w17 // h+=Ch(e,f,g)
	and w28,w28,w19 // (b^c)&=(a^b)
	eor w7,w7,w0,ror#19
	eor w8,w8,w3,lsr#3 // sigma0(X[i+1])
	add w21,w21,w16 // h+=Sigma1(e)
	eor w28,w28,w23 // Maj(a,b,c)
	eor w17,w9,w22,ror#22 // Sigma0(a)
	eor w7,w7,w0,lsr#10 // sigma1(X[i+14])
	add w2,w2,w11
	add w25,w25,w21 // d+=h
	add w21,w21,w28 // h+=Maj(a,b,c)
	ldr w28,[x30],#4 // *K++, w19 in next round
	add w2,w2,w8
	add w21,w21,w17 // h+=Sigma0(a)
	add w2,w2,w7
	ldr w7,[sp,#0]
	str w10,[sp,#12]
	ror w16,w25,#6
	add w20,w20,w28 // h+=K[i]
	ror w9,w4,#7
	and w17,w26,w25
	ror w8,w1,#17
	bic w28,w27,w25
	ror w10,w21,#2
	add w20,w20,w2 // h+=X[i]
	eor w16,w16,w25,ror#11
	eor w9,w9,w4,ror#18
	orr w17,w17,w28 // Ch(e,f,g)
	eor w28,w21,w22 // a^b, b^c in next round
	eor w16,w16,w25,ror#25 // Sigma1(e)
	eor w10,w10,w21,ror#13
	add w20,w20,w17 // h+=Ch(e,f,g)
	and w19,w19,w28 // (b^c)&=(a^b)
	eor w8,w8,w1,ror#19
	eor w9,w9,w4,lsr#3 // sigma0(X[i+1])
	add w20,w20,w16 // h+=Sigma1(e)
	eor w19,w19,w22 // Maj(a,b,c)
	eor w17,w10,w21,ror#22 // Sigma0(a)
	eor w8,w8,w1,lsr#10 // sigma1(X[i+14])
	add w3,w3,w12
	add w24,w24,w20 // d+=h
	add w20,w20,w19 // h+=Maj(a,b,c)
	ldr w19,[x30],#4 // *K++, w28 in next round
	add w3,w3,w9
	add w20,w20,w17 // h+=Sigma0(a)
	add w3,w3,w8
	cbnz w19,.Loop_16_xx

	ldp x0,x2,[x29,#96]
	ldr x1,[x29,#112]
	sub x30,x30,#260 // rewind

	ldp w3,w4,[x0]
	ldp w5,w6,[x0,#2*4]
	add x1,x1,#14*4 // advance input pointer
	ldp w7,w8,[x0,#4*4]
	add w20,w20,w3
	ldp w9,w10,[x0,#6*4]
	add w21,w21,w4
	add w22,w22,w5
	add w23,w23,w6
	stp w20,w21,[x0]
	add w24,w24,w7
	add w25,w25,w8
	stp w22,w23,[x0,#2*4]
	add w26,w26,w9
	add w27,w27,w10
	cmp x1,x2
	stp w24,w25,[x0,#4*4]
	stp w26,w27,[x0,#6*4]
	b.ne .Loop

	ldp x19,x20,[x29,#16]
	add sp,sp,#4*4
	ldp x21,x22,[x29,#32]
	ldp x23,x24,[x29,#48]
	ldp x25,x26,[x29,#64]
	ldp x27,x28,[x29,#80]
	ldp x29,x30,[sp],#128
	ret
	.size zfs_sha256_block_armv7,.-zfs_sha256_block_armv7

	.globl zfs_sha256_block_armv8
	.type zfs_sha256_block_armv8,%function
	.align 6
	zfs_sha256_block_armv8:
	hint #34 // bti c
	.Lv8_entry:
	stp x29,x30,[sp,#-16]!
	add x29,sp,#0

	ld1 {v0.4s,v1.4s},[x0]
	adr x3,.LK256

	.Loop_hw:
	ld1 {v4.16b-v7.16b},[x1],#64
	sub x2,x2,#1
	ld1 {v16.4s},[x3],#16
	rev32 v4.16b,v4.16b
	rev32 v5.16b,v5.16b
	rev32 v6.16b,v6.16b
	rev32 v7.16b,v7.16b
	orr v18.16b,v0.16b,v0.16b // offload
	orr v19.16b,v1.16b,v1.16b
	ld1 {v17.4s},[x3],#16
	add v16.4s,v16.4s,v4.4s
	.inst 0x5e2828a4 //sha256su0 v4.16b,v5.16b
	orr v2.16b,v0.16b,v0.16b
	.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s
	.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s
	.inst 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b
	ld1 {v16.4s},[x3],#16
	add v17.4s,v17.4s,v5.4s
	.inst 0x5e2828c5 //sha256su0 v5.16b,v6.16b
	orr v2.16b,v0.16b,v0.16b
	.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s
	.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s
	.inst 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b
	ld1 {v17.4s},[x3],#16
	add v16.4s,v16.4s,v6.4s
	.inst 0x5e2828e6 //sha256su0 v6.16b,v7.16b
	orr v2.16b,v0.16b,v0.16b
	.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s
	.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s
	.inst 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b
	ld1 {v16.4s},[x3],#16
	add v17.4s,v17.4s,v7.4s
	.inst 0x5e282887 //sha256su0 v7.16b,v4.16b
	orr v2.16b,v0.16b,v0.16b
	.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s
	.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s
	.inst 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b
	ld1 {v17.4s},[x3],#16
	add v16.4s,v16.4s,v4.4s
	.inst 0x5e2828a4 //sha256su0 v4.16b,v5.16b
	orr v2.16b,v0.16b,v0.16b
	.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s
	.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s
	.inst 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b
	ld1 {v16.4s},[x3],#16
	add v17.4s,v17.4s,v5.4s
	.inst 0x5e2828c5 //sha256su0 v5.16b,v6.16b
	orr v2.16b,v0.16b,v0.16b
	.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s
	.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s
	.inst 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b
	ld1 {v17.4s},[x3],#16
	add v16.4s,v16.4s,v6.4s
	.inst 0x5e2828e6 //sha256su0 v6.16b,v7.16b
	orr v2.16b,v0.16b,v0.16b
	.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s
	.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s
	.inst 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b
	ld1 {v16.4s},[x3],#16
	add v17.4s,v17.4s,v7.4s
	.inst 0x5e282887 //sha256su0 v7.16b,v4.16b
	orr v2.16b,v0.16b,v0.16b
	.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s
	.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s
	.inst 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b
	ld1 {v17.4s},[x3],#16
	add v16.4s,v16.4s,v4.4s
	.inst 0x5e2828a4 //sha256su0 v4.16b,v5.16b
	orr v2.16b,v0.16b,v0.16b
	.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s
	.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s
	.inst 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b
	ld1 {v16.4s},[x3],#16
	add v17.4s,v17.4s,v5.4s
	.inst 0x5e2828c5 //sha256su0 v5.16b,v6.16b
	orr v2.16b,v0.16b,v0.16b
	.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s
	.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s
	.inst 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b
	ld1 {v17.4s},[x3],#16
	add v16.4s,v16.4s,v6.4s
	.inst 0x5e2828e6 //sha256su0 v6.16b,v7.16b
	orr v2.16b,v0.16b,v0.16b
	.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s
	.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s
	.inst 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b
	ld1 {v16.4s},[x3],#16
	add v17.4s,v17.4s,v7.4s
	.inst 0x5e282887 //sha256su0 v7.16b,v4.16b
	orr v2.16b,v0.16b,v0.16b
	.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s
	.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s
	.inst 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b
	ld1 {v17.4s},[x3],#16
	add v16.4s,v16.4s,v4.4s
	orr v2.16b,v0.16b,v0.16b
	.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s
	.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s

	ld1 {v16.4s},[x3],#16
	add v17.4s,v17.4s,v5.4s
	orr v2.16b,v0.16b,v0.16b
	.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s
	.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s

	ld1 {v17.4s},[x3]
	add v16.4s,v16.4s,v6.4s
	sub x3,x3,#64*4-16 // rewind
	orr v2.16b,v0.16b,v0.16b
	.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s
	.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s

	add v17.4s,v17.4s,v7.4s
	orr v2.16b,v0.16b,v0.16b
	.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s
	.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s

	add v0.4s,v0.4s,v18.4s
	add v1.4s,v1.4s,v19.4s

	cbnz x2,.Loop_hw

	st1 {v0.4s,v1.4s},[x0]

	ldr x29,[sp],#16
	ret
	.size zfs_sha256_block_armv8,.-zfs_sha256_block_armv8

	.globl zfs_sha256_block_neon
	.type zfs_sha256_block_neon,%function
	.align 4
	zfs_sha256_block_neon:
	hint #34 // bti c
	.Lneon_entry:
	stp x29, x30, [sp, #-16]!
	mov x29, sp
	sub sp,sp,#16*4

	adr x16,.LK256
	add x2,x1,x2,lsl#6 // len to point at the end of inp

	ld1 {v0.16b},[x1], #16
	ld1 {v1.16b},[x1], #16
	ld1 {v2.16b},[x1], #16
	ld1 {v3.16b},[x1], #16
	ld1 {v4.4s},[x16], #16
	ld1 {v5.4s},[x16], #16
	ld1 {v6.4s},[x16], #16
	ld1 {v7.4s},[x16], #16
	rev32 v0.16b,v0.16b // yes, even on
	rev32 v1.16b,v1.16b // big-endian
	rev32 v2.16b,v2.16b
	rev32 v3.16b,v3.16b
	mov x17,sp
	add v4.4s,v4.4s,v0.4s
	add v5.4s,v5.4s,v1.4s
	add v6.4s,v6.4s,v2.4s
	st1 {v4.4s-v5.4s},[x17], #32
	add v7.4s,v7.4s,v3.4s
	st1 {v6.4s-v7.4s},[x17]
	sub x17,x17,#32

	ldp w3,w4,[x0]
	ldp w5,w6,[x0,#8]
	ldp w7,w8,[x0,#16]
	ldp w9,w10,[x0,#24]
	ldr w12,[sp,#0]
	mov w13,wzr
	eor w14,w4,w5
	mov w15,wzr
	b .L_00_48

	.align 4
	.L_00_48:
	ext v4.16b,v0.16b,v1.16b,#4
	add w10,w10,w12
	add w3,w3,w15
	and w12,w8,w7
	bic w15,w9,w7
	ext v7.16b,v2.16b,v3.16b,#4
	eor w11,w7,w7,ror#5
	add w3,w3,w13
	mov d19,v3.d[1]
	orr w12,w12,w15
	eor w11,w11,w7,ror#19
	ushr v6.4s,v4.4s,#7
	eor w15,w3,w3,ror#11
	ushr v5.4s,v4.4s,#3
	add w10,w10,w12
	add v0.4s,v0.4s,v7.4s
	ror w11,w11,#6
	sli v6.4s,v4.4s,#25
	eor w13,w3,w4
	eor w15,w15,w3,ror#20
	ushr v7.4s,v4.4s,#18
	add w10,w10,w11
	ldr w12,[sp,#4]
	and w14,w14,w13
	eor v5.16b,v5.16b,v6.16b
	ror w15,w15,#2
	add w6,w6,w10
	sli v7.4s,v4.4s,#14
	eor w14,w14,w4
	ushr v16.4s,v19.4s,#17
	add w9,w9,w12
	add w10,w10,w15
	and w12,w7,w6
	eor v5.16b,v5.16b,v7.16b
	bic w15,w8,w6
	eor w11,w6,w6,ror#5
	sli v16.4s,v19.4s,#15
	add w10,w10,w14
	orr w12,w12,w15
	ushr v17.4s,v19.4s,#10
	eor w11,w11,w6,ror#19
	eor w15,w10,w10,ror#11
	ushr v7.4s,v19.4s,#19
	add w9,w9,w12
	ror w11,w11,#6
	add v0.4s,v0.4s,v5.4s
	eor w14,w10,w3
	eor w15,w15,w10,ror#20
	sli v7.4s,v19.4s,#13
	add w9,w9,w11
	ldr w12,[sp,#8]
	and w13,w13,w14
	eor v17.16b,v17.16b,v16.16b
	ror w15,w15,#2
	add w5,w5,w9
	eor w13,w13,w3
	eor v17.16b,v17.16b,v7.16b
	add w8,w8,w12
	add w9,w9,w15
	and w12,w6,w5
	add v0.4s,v0.4s,v17.4s
	bic w15,w7,w5
	eor w11,w5,w5,ror#5
	add w9,w9,w13
	ushr v18.4s,v0.4s,#17
	orr w12,w12,w15
	ushr v19.4s,v0.4s,#10
	eor w11,w11,w5,ror#19
	eor w15,w9,w9,ror#11
	sli v18.4s,v0.4s,#15
	add w8,w8,w12
	ushr v17.4s,v0.4s,#19
	ror w11,w11,#6
	eor w13,w9,w10
	eor v19.16b,v19.16b,v18.16b
	eor w15,w15,w9,ror#20
	add w8,w8,w11
	sli v17.4s,v0.4s,#13
	ldr w12,[sp,#12]
	and w14,w14,w13
	ror w15,w15,#2
	ld1 {v4.4s},[x16], #16
	add w4,w4,w8
	eor v19.16b,v19.16b,v17.16b
	eor w14,w14,w10
	eor v17.16b,v17.16b,v17.16b
	add w7,w7,w12
	add w8,w8,w15
	and w12,w5,w4
	mov v17.d[1],v19.d[0]
	bic w15,w6,w4
	eor w11,w4,w4,ror#5
	add w8,w8,w14
	add v0.4s,v0.4s,v17.4s
	orr w12,w12,w15
	eor w11,w11,w4,ror#19
	eor w15,w8,w8,ror#11
	add v4.4s,v4.4s,v0.4s
	add w7,w7,w12
	ror w11,w11,#6
	eor w14,w8,w9
	eor w15,w15,w8,ror#20
	add w7,w7,w11
	ldr w12,[sp,#16]
	and w13,w13,w14
	ror w15,w15,#2
	add w3,w3,w7
	eor w13,w13,w9
	st1 {v4.4s},[x17], #16
	ext v4.16b,v1.16b,v2.16b,#4
	add w6,w6,w12
	add w7,w7,w15
	and w12,w4,w3
	bic w15,w5,w3
	ext v7.16b,v3.16b,v0.16b,#4
	eor w11,w3,w3,ror#5
	add w7,w7,w13
	mov d19,v0.d[1]
	orr w12,w12,w15
	eor w11,w11,w3,ror#19
	ushr v6.4s,v4.4s,#7
	eor w15,w7,w7,ror#11
	ushr v5.4s,v4.4s,#3
	add w6,w6,w12
	add v1.4s,v1.4s,v7.4s
	ror w11,w11,#6
	sli v6.4s,v4.4s,#25
	eor w13,w7,w8
	eor w15,w15,w7,ror#20
	ushr v7.4s,v4.4s,#18
	add w6,w6,w11
	ldr w12,[sp,#20]
	and w14,w14,w13
	eor v5.16b,v5.16b,v6.16b
	ror w15,w15,#2
	add w10,w10,w6
	sli v7.4s,v4.4s,#14
	eor w14,w14,w8
	ushr v16.4s,v19.4s,#17
	add w5,w5,w12
	add w6,w6,w15
	and w12,w3,w10
	eor v5.16b,v5.16b,v7.16b
	bic w15,w4,w10
	eor w11,w10,w10,ror#5
	sli v16.4s,v19.4s,#15
	add w6,w6,w14
	orr w12,w12,w15
	ushr v17.4s,v19.4s,#10
	eor w11,w11,w10,ror#19
	eor w15,w6,w6,ror#11
	ushr v7.4s,v19.4s,#19
	add w5,w5,w12
	ror w11,w11,#6
	add v1.4s,v1.4s,v5.4s
	eor w14,w6,w7
	eor w15,w15,w6,ror#20
	sli v7.4s,v19.4s,#13
	add w5,w5,w11
	ldr w12,[sp,#24]
	and w13,w13,w14
	eor v17.16b,v17.16b,v16.16b
	ror w15,w15,#2
	add w9,w9,w5
	eor w13,w13,w7
	eor v17.16b,v17.16b,v7.16b
	add w4,w4,w12
	add w5,w5,w15
	and w12,w10,w9
	add v1.4s,v1.4s,v17.4s
	bic w15,w3,w9
	eor w11,w9,w9,ror#5
	add w5,w5,w13
	ushr v18.4s,v1.4s,#17
	orr w12,w12,w15
	ushr v19.4s,v1.4s,#10
	eor w11,w11,w9,ror#19
	eor w15,w5,w5,ror#11
	sli v18.4s,v1.4s,#15
	add w4,w4,w12
	ushr v17.4s,v1.4s,#19
	ror w11,w11,#6
	eor w13,w5,w6
	eor v19.16b,v19.16b,v18.16b
	eor w15,w15,w5,ror#20
	add w4,w4,w11
	sli v17.4s,v1.4s,#13
	ldr w12,[sp,#28]
	and w14,w14,w13
	ror w15,w15,#2
	ld1 {v4.4s},[x16], #16
	add w8,w8,w4
	eor v19.16b,v19.16b,v17.16b
	eor w14,w14,w6
	eor v17.16b,v17.16b,v17.16b
	add w3,w3,w12
	add w4,w4,w15
	and w12,w9,w8
	mov v17.d[1],v19.d[0]
	bic w15,w10,w8
	eor w11,w8,w8,ror#5
	add w4,w4,w14
	add v1.4s,v1.4s,v17.4s
	orr w12,w12,w15
	eor w11,w11,w8,ror#19
	eor w15,w4,w4,ror#11
	add v4.4s,v4.4s,v1.4s
	add w3,w3,w12
	ror w11,w11,#6
	eor w14,w4,w5
	eor w15,w15,w4,ror#20
	add w3,w3,w11
	ldr w12,[sp,#32]
	and w13,w13,w14
	ror w15,w15,#2
	add w7,w7,w3
	eor w13,w13,w5
	st1 {v4.4s},[x17], #16
	ext v4.16b,v2.16b,v3.16b,#4
	add w10,w10,w12
	add w3,w3,w15
	and w12,w8,w7
	bic w15,w9,w7
	ext v7.16b,v0.16b,v1.16b,#4
	eor w11,w7,w7,ror#5
	add w3,w3,w13
	mov d19,v1.d[1]
	orr w12,w12,w15
	eor w11,w11,w7,ror#19
	ushr v6.4s,v4.4s,#7
	eor w15,w3,w3,ror#11
	ushr v5.4s,v4.4s,#3
	add w10,w10,w12
	add v2.4s,v2.4s,v7.4s
	ror w11,w11,#6
	sli v6.4s,v4.4s,#25
	eor w13,w3,w4
	eor w15,w15,w3,ror#20
	ushr v7.4s,v4.4s,#18
	add w10,w10,w11
	ldr w12,[sp,#36]
	and w14,w14,w13
	eor v5.16b,v5.16b,v6.16b
	ror w15,w15,#2
	add w6,w6,w10
	sli v7.4s,v4.4s,#14
	eor w14,w14,w4
	ushr v16.4s,v19.4s,#17
	add w9,w9,w12
	add w10,w10,w15
	and w12,w7,w6
	eor v5.16b,v5.16b,v7.16b
	bic w15,w8,w6
	eor w11,w6,w6,ror#5
	sli v16.4s,v19.4s,#15
	add w10,w10,w14
	orr w12,w12,w15
	ushr v17.4s,v19.4s,#10
	eor w11,w11,w6,ror#19
	eor w15,w10,w10,ror#11
	ushr v7.4s,v19.4s,#19
	add w9,w9,w12
	ror w11,w11,#6
	add v2.4s,v2.4s,v5.4s
	eor w14,w10,w3
	eor w15,w15,w10,ror#20
	sli v7.4s,v19.4s,#13
	add w9,w9,w11
	ldr w12,[sp,#40]
	and w13,w13,w14
	eor v17.16b,v17.16b,v16.16b
	ror w15,w15,#2
	add w5,w5,w9
	eor w13,w13,w3
	eor v17.16b,v17.16b,v7.16b
	add w8,w8,w12
	add w9,w9,w15
	and w12,w6,w5
	add v2.4s,v2.4s,v17.4s
	bic w15,w7,w5
	eor w11,w5,w5,ror#5
	add w9,w9,w13
	ushr v18.4s,v2.4s,#17
	orr w12,w12,w15
	ushr v19.4s,v2.4s,#10
	eor w11,w11,w5,ror#19
	eor w15,w9,w9,ror#11
	sli v18.4s,v2.4s,#15
	add w8,w8,w12
	ushr v17.4s,v2.4s,#19
	ror w11,w11,#6
	eor w13,w9,w10
	eor v19.16b,v19.16b,v18.16b
	eor w15,w15,w9,ror#20
	add w8,w8,w11
	sli v17.4s,v2.4s,#13
	ldr w12,[sp,#44]
	and w14,w14,w13
	ror w15,w15,#2
	ld1 {v4.4s},[x16], #16
	add w4,w4,w8
	eor v19.16b,v19.16b,v17.16b
	eor w14,w14,w10
	eor v17.16b,v17.16b,v17.16b
	add w7,w7,w12
	add w8,w8,w15
	and w12,w5,w4
	mov v17.d[1],v19.d[0]
	bic w15,w6,w4
	eor w11,w4,w4,ror#5
	add w8,w8,w14
	add v2.4s,v2.4s,v17.4s
	orr w12,w12,w15
	eor w11,w11,w4,ror#19
	eor w15,w8,w8,ror#11
	add v4.4s,v4.4s,v2.4s
	add w7,w7,w12
	ror w11,w11,#6
	eor w14,w8,w9
	eor w15,w15,w8,ror#20
	add w7,w7,w11
	ldr w12,[sp,#48]
	and w13,w13,w14
	ror w15,w15,#2
	add w3,w3,w7
	eor w13,w13,w9
	st1 {v4.4s},[x17], #16
	ext v4.16b,v3.16b,v0.16b,#4
	add w6,w6,w12
	add w7,w7,w15
	and w12,w4,w3
	bic w15,w5,w3
	ext v7.16b,v1.16b,v2.16b,#4
	eor w11,w3,w3,ror#5
	add w7,w7,w13
	mov d19,v2.d[1]
	orr w12,w12,w15
	eor w11,w11,w3,ror#19
	ushr v6.4s,v4.4s,#7
	eor w15,w7,w7,ror#11
	ushr v5.4s,v4.4s,#3
	add w6,w6,w12
	add v3.4s,v3.4s,v7.4s
	ror w11,w11,#6
	sli v6.4s,v4.4s,#25
	eor w13,w7,w8
	eor w15,w15,w7,ror#20
	ushr v7.4s,v4.4s,#18
	add w6,w6,w11
	ldr w12,[sp,#52]
	and w14,w14,w13
	eor v5.16b,v5.16b,v6.16b
	ror w15,w15,#2
	add w10,w10,w6
	sli v7.4s,v4.4s,#14
	eor w14,w14,w8
	ushr v16.4s,v19.4s,#17
	add w5,w5,w12
	add w6,w6,w15
	and w12,w3,w10
	eor v5.16b,v5.16b,v7.16b
	bic w15,w4,w10
	eor w11,w10,w10,ror#5
	sli v16.4s,v19.4s,#15
	add w6,w6,w14
	orr w12,w12,w15
	ushr v17.4s,v19.4s,#10
	eor w11,w11,w10,ror#19
	eor w15,w6,w6,ror#11
	ushr v7.4s,v19.4s,#19
	add w5,w5,w12
	ror w11,w11,#6
	add v3.4s,v3.4s,v5.4s
	eor w14,w6,w7
	eor w15,w15,w6,ror#20
	sli v7.4s,v19.4s,#13
	add w5,w5,w11
	ldr w12,[sp,#56]
	and w13,w13,w14
	eor v17.16b,v17.16b,v16.16b
	ror w15,w15,#2
	add w9,w9,w5
	eor w13,w13,w7
	eor v17.16b,v17.16b,v7.16b
	add w4,w4,w12
	add w5,w5,w15
	and w12,w10,w9
	add v3.4s,v3.4s,v17.4s
	bic w15,w3,w9
	eor w11,w9,w9,ror#5
	add w5,w5,w13
	ushr v18.4s,v3.4s,#17
	orr w12,w12,w15
	ushr v19.4s,v3.4s,#10
	eor w11,w11,w9,ror#19
	eor w15,w5,w5,ror#11
	sli v18.4s,v3.4s,#15
	add w4,w4,w12
	ushr v17.4s,v3.4s,#19
	ror w11,w11,#6
	eor w13,w5,w6
	eor v19.16b,v19.16b,v18.16b
	eor w15,w15,w5,ror#20
	add w4,w4,w11
	sli v17.4s,v3.4s,#13
	ldr w12,[sp,#60]
	and w14,w14,w13
	ror w15,w15,#2
	ld1 {v4.4s},[x16], #16
	add w8,w8,w4
	eor v19.16b,v19.16b,v17.16b
	eor w14,w14,w6
	eor v17.16b,v17.16b,v17.16b
	add w3,w3,w12
	add w4,w4,w15
	and w12,w9,w8
	mov v17.d[1],v19.d[0]
	bic w15,w10,w8
	eor w11,w8,w8,ror#5
	add w4,w4,w14
	add v3.4s,v3.4s,v17.4s
	orr w12,w12,w15
	eor w11,w11,w8,ror#19
	eor w15,w4,w4,ror#11
	add v4.4s,v4.4s,v3.4s
	add w3,w3,w12
	ror w11,w11,#6
	eor w14,w4,w5
	eor w15,w15,w4,ror#20
	add w3,w3,w11
	ldr w12,[x16]
	and w13,w13,w14
	ror w15,w15,#2
	add w7,w7,w3
	eor w13,w13,w5
	st1 {v4.4s},[x17], #16
	cmp w12,#0 // check for K256 terminator
	ldr w12,[sp,#0]
	sub x17,x17,#64
	bne .L_00_48

	sub x16,x16,#256 // rewind x16
	cmp x1,x2
	mov x17, #64
	csel x17, x17, xzr, eq
	sub x1,x1,x17 // avoid SEGV
	mov x17,sp
	add w10,w10,w12
	add w3,w3,w15
	and w12,w8,w7
	ld1 {v0.16b},[x1],#16
	bic w15,w9,w7
	eor w11,w7,w7,ror#5
	ld1 {v4.4s},[x16],#16
	add w3,w3,w13
	orr w12,w12,w15
	eor w11,w11,w7,ror#19
	eor w15,w3,w3,ror#11
	rev32 v0.16b,v0.16b
	add w10,w10,w12
	ror w11,w11,#6
	eor w13,w3,w4
	eor w15,w15,w3,ror#20
	add v4.4s,v4.4s,v0.4s
	add w10,w10,w11
	ldr w12,[sp,#4]
	and w14,w14,w13
	ror w15,w15,#2
	add w6,w6,w10
	eor w14,w14,w4
	add w9,w9,w12
	add w10,w10,w15
	and w12,w7,w6
	bic w15,w8,w6
	eor w11,w6,w6,ror#5
	add w10,w10,w14
	orr w12,w12,w15
	eor w11,w11,w6,ror#19
	eor w15,w10,w10,ror#11
	add w9,w9,w12
	ror w11,w11,#6
	eor w14,w10,w3
	eor w15,w15,w10,ror#20
	add w9,w9,w11
	ldr w12,[sp,#8]
	and w13,w13,w14
	ror w15,w15,#2
	add w5,w5,w9
	eor w13,w13,w3
	add w8,w8,w12
	add w9,w9,w15
	and w12,w6,w5
	bic w15,w7,w5
	eor w11,w5,w5,ror#5
	add w9,w9,w13
	orr w12,w12,w15
	eor w11,w11,w5,ror#19
	eor w15,w9,w9,ror#11
	add w8,w8,w12
	ror w11,w11,#6
	eor w13,w9,w10
	eor w15,w15,w9,ror#20
	add w8,w8,w11
	ldr w12,[sp,#12]
	and w14,w14,w13
	ror w15,w15,#2
	add w4,w4,w8
	eor w14,w14,w10
	add w7,w7,w12
	add w8,w8,w15
	and w12,w5,w4
	bic w15,w6,w4
	eor w11,w4,w4,ror#5
	add w8,w8,w14
	orr w12,w12,w15
	eor w11,w11,w4,ror#19
	eor w15,w8,w8,ror#11
	add w7,w7,w12
	ror w11,w11,#6
	eor w14,w8,w9
	eor w15,w15,w8,ror#20
	add w7,w7,w11
	ldr w12,[sp,#16]
	and w13,w13,w14
	ror w15,w15,#2
	add w3,w3,w7
	eor w13,w13,w9
	st1 {v4.4s},[x17], #16
	add w6,w6,w12
	add w7,w7,w15
	and w12,w4,w3
	ld1 {v1.16b},[x1],#16
	bic w15,w5,w3
	eor w11,w3,w3,ror#5
	ld1 {v4.4s},[x16],#16
	add w7,w7,w13
	orr w12,w12,w15
	eor w11,w11,w3,ror#19
	eor w15,w7,w7,ror#11
	rev32 v1.16b,v1.16b
	add w6,w6,w12
	ror w11,w11,#6
	eor w13,w7,w8
	eor w15,w15,w7,ror#20
	add v4.4s,v4.4s,v1.4s
	add w6,w6,w11
	ldr w12,[sp,#20]
	and w14,w14,w13
	ror w15,w15,#2
	add w10,w10,w6
	eor w14,w14,w8
	add w5,w5,w12
	add w6,w6,w15
	and w12,w3,w10
	bic w15,w4,w10
	eor w11,w10,w10,ror#5
	add w6,w6,w14
	orr w12,w12,w15
	eor w11,w11,w10,ror#19
	eor w15,w6,w6,ror#11
	add w5,w5,w12
	ror w11,w11,#6
	eor w14,w6,w7
	eor w15,w15,w6,ror#20
	add w5,w5,w11
	ldr w12,[sp,#24]
	and w13,w13,w14
	ror w15,w15,#2
	add w9,w9,w5
	eor w13,w13,w7
	add w4,w4,w12
	add w5,w5,w15
	and w12,w10,w9
	bic w15,w3,w9
	eor w11,w9,w9,ror#5
	add w5,w5,w13
	orr w12,w12,w15
	eor w11,w11,w9,ror#19
	eor w15,w5,w5,ror#11
	add w4,w4,w12
	ror w11,w11,#6
	eor w13,w5,w6
	eor w15,w15,w5,ror#20
	add w4,w4,w11
	ldr w12,[sp,#28]
	and w14,w14,w13
	ror w15,w15,#2
	add w8,w8,w4
	eor w14,w14,w6
	add w3,w3,w12
	add w4,w4,w15
	and w12,w9,w8
	bic w15,w10,w8
	eor w11,w8,w8,ror#5
	add w4,w4,w14
	orr w12,w12,w15
	eor w11,w11,w8,ror#19
	eor w15,w4,w4,ror#11
	add w3,w3,w12
	ror w11,w11,#6
	eor w14,w4,w5
	eor w15,w15,w4,ror#20
	add w3,w3,w11
	ldr w12,[sp,#32]
	and w13,w13,w14
	ror w15,w15,#2
	add w7,w7,w3
	eor w13,w13,w5
	st1 {v4.4s},[x17], #16
	add w10,w10,w12
	add w3,w3,w15
	and w12,w8,w7
	ld1 {v2.16b},[x1],#16
	bic w15,w9,w7
	eor w11,w7,w7,ror#5
	ld1 {v4.4s},[x16],#16
	add w3,w3,w13
	orr w12,w12,w15
	eor w11,w11,w7,ror#19
	eor w15,w3,w3,ror#11
	rev32 v2.16b,v2.16b
	add w10,w10,w12
	ror w11,w11,#6
	eor w13,w3,w4
	eor w15,w15,w3,ror#20
	add v4.4s,v4.4s,v2.4s
	add w10,w10,w11
	ldr w12,[sp,#36]
	and w14,w14,w13
	ror w15,w15,#2
	add w6,w6,w10
	eor w14,w14,w4
	add w9,w9,w12
	add w10,w10,w15
	and w12,w7,w6
	bic w15,w8,w6
	eor w11,w6,w6,ror#5
	add w10,w10,w14
	orr w12,w12,w15
	eor w11,w11,w6,ror#19
	eor w15,w10,w10,ror#11
	add w9,w9,w12
	ror w11,w11,#6
	eor w14,w10,w3
	eor w15,w15,w10,ror#20
	add w9,w9,w11
	ldr w12,[sp,#40]
	and w13,w13,w14
	ror w15,w15,#2
	add w5,w5,w9
	eor w13,w13,w3
	add w8,w8,w12
	add w9,w9,w15
	and w12,w6,w5
	bic w15,w7,w5
	eor w11,w5,w5,ror#5
	add w9,w9,w13
	orr w12,w12,w15
	eor w11,w11,w5,ror#19
	eor w15,w9,w9,ror#11
	add w8,w8,w12
	ror w11,w11,#6
	eor w13,w9,w10
	eor w15,w15,w9,ror#20
	add w8,w8,w11
	ldr w12,[sp,#44]
	and w14,w14,w13
	ror w15,w15,#2
	add w4,w4,w8
	eor w14,w14,w10
	add w7,w7,w12
	add w8,w8,w15
	and w12,w5,w4
	bic w15,w6,w4
	eor w11,w4,w4,ror#5
	add w8,w8,w14
	orr w12,w12,w15
	eor w11,w11,w4,ror#19
	eor w15,w8,w8,ror#11
	add w7,w7,w12
	ror w11,w11,#6
	eor w14,w8,w9
	eor w15,w15,w8,ror#20
	add w7,w7,w11
	ldr w12,[sp,#48]
	and w13,w13,w14
	ror w15,w15,#2
	add w3,w3,w7
	eor w13,w13,w9
	st1 {v4.4s},[x17], #16
	add w6,w6,w12
	add w7,w7,w15
	and w12,w4,w3
	ld1 {v3.16b},[x1],#16
	bic w15,w5,w3
	eor w11,w3,w3,ror#5
	ld1 {v4.4s},[x16],#16
	add w7,w7,w13
	orr w12,w12,w15
	eor w11,w11,w3,ror#19
	eor w15,w7,w7,ror#11
	rev32 v3.16b,v3.16b
	add w6,w6,w12
	ror w11,w11,#6
	eor w13,w7,w8
	eor w15,w15,w7,ror#20
	add v4.4s,v4.4s,v3.4s
	add w6,w6,w11
	ldr w12,[sp,#52]
	and w14,w14,w13
	ror w15,w15,#2
	add w10,w10,w6
	eor w14,w14,w8
	add w5,w5,w12
	add w6,w6,w15
	and w12,w3,w10
	bic w15,w4,w10
	eor w11,w10,w10,ror#5
	add w6,w6,w14
	orr w12,w12,w15
	eor w11,w11,w10,ror#19
	eor w15,w6,w6,ror#11
	add w5,w5,w12
	ror w11,w11,#6
	eor w14,w6,w7
	eor w15,w15,w6,ror#20
	add w5,w5,w11
	ldr w12,[sp,#56]
	and w13,w13,w14
	ror w15,w15,#2
	add w9,w9,w5
	eor w13,w13,w7
	add w4,w4,w12
	add w5,w5,w15
	and w12,w10,w9
	bic w15,w3,w9
	eor w11,w9,w9,ror#5
	add w5,w5,w13
	orr w12,w12,w15
	eor w11,w11,w9,ror#19
	eor w15,w5,w5,ror#11
	add w4,w4,w12
	ror w11,w11,#6
	eor w13,w5,w6
	eor w15,w15,w5,ror#20
	add w4,w4,w11
	ldr w12,[sp,#60]
	and w14,w14,w13
	ror w15,w15,#2
	add w8,w8,w4
	eor w14,w14,w6
	add w3,w3,w12
	add w4,w4,w15
	and w12,w9,w8
	bic w15,w10,w8
	eor w11,w8,w8,ror#5
	add w4,w4,w14
	orr w12,w12,w15
	eor w11,w11,w8,ror#19
	eor w15,w4,w4,ror#11
	add w3,w3,w12
	ror w11,w11,#6
	eor w14,w4,w5
	eor w15,w15,w4,ror#20
	add w3,w3,w11
	and w13,w13,w14
	ror w15,w15,#2
	add w7,w7,w3
	eor w13,w13,w5
	st1 {v4.4s},[x17], #16
	add w3,w3,w15 // h+=Sigma0(a) from the past
	ldp w11,w12,[x0,#0]
	add w3,w3,w13 // h+=Maj(a,b,c) from the past
	ldp w13,w14,[x0,#8]
	add w3,w3,w11 // accumulate
	add w4,w4,w12
	ldp w11,w12,[x0,#16]
	add w5,w5,w13
	add w6,w6,w14
	ldp w13,w14,[x0,#24]
	add w7,w7,w11
	add w8,w8,w12
	ldr w12,[sp,#0]
	stp w3,w4,[x0,#0]
	add w9,w9,w13
	mov w13,wzr
	stp w5,w6,[x0,#8]
	add w10,w10,w14
	stp w7,w8,[x0,#16]
	eor w14,w4,w5
	stp w9,w10,[x0,#24]
	mov w15,wzr
	mov x17,sp
	b.ne .L_00_48

	ldr x29,[x29]
	add sp,sp,#16*4+16
	ret
	.size zfs_sha256_block_neon,.-zfs_sha256_block_neon

	#endif
	diff --git a/sys/contrib/openzfs/module/icp/asm-aarch64/sha2/sha512-armv8.S b/sys/contrib/openzfs/module/icp/asm-aarch64/sha2/sha512-armv8.S
	index 9c61eeee4d7b..f6c8f7742912 100644
	--- a/sys/contrib/openzfs/module/icp/asm-aarch64/sha2/sha512-armv8.S
	+++ b/sys/contrib/openzfs/module/icp/asm-aarch64/sha2/sha512-armv8.S
	@@ -1,1560 +1,1570 @@
	/*
	* Copyright 2004-2022 The OpenSSL Project Authors. All Rights Reserved.
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* https://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	/*
	* Portions Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
	* - modified assembly to fit into OpenZFS
	*/

	#if defined(__aarch64__)

	+ .section .note.gnu.property,"a",@note
	+ .p2align 3
	+ .word 4
	+ .word 16
	+ .word 5
	+ .asciz "GNU"
	+ .word 3221225472
	+ .word 4
	+ .word 3
	+ .word 0
	.text

	.align 6
	.type .LK512,%object
	.LK512:
	.quad 0x428a2f98d728ae22,0x7137449123ef65cd
	.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
	.quad 0x3956c25bf348b538,0x59f111f1b605d019
	.quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118
	.quad 0xd807aa98a3030242,0x12835b0145706fbe
	.quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
	.quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1
	.quad 0x9bdc06a725c71235,0xc19bf174cf692694
	.quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3
	.quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
	.quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483
	.quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5
	.quad 0x983e5152ee66dfab,0xa831c66d2db43210
	.quad 0xb00327c898fb213f,0xbf597fc7beef0ee4
	.quad 0xc6e00bf33da88fc2,0xd5a79147930aa725
	.quad 0x06ca6351e003826f,0x142929670a0e6e70
	.quad 0x27b70a8546d22ffc,0x2e1b21385c26c926
	.quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df
	.quad 0x650a73548baf63de,0x766a0abb3c77b2a8
	.quad 0x81c2c92e47edaee6,0x92722c851482353b
	.quad 0xa2bfe8a14cf10364,0xa81a664bbc423001
	.quad 0xc24b8b70d0f89791,0xc76c51a30654be30
	.quad 0xd192e819d6ef5218,0xd69906245565a910
	.quad 0xf40e35855771202a,0x106aa07032bbd1b8
	.quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53
	.quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
	.quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
	.quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
	.quad 0x748f82ee5defb2fc,0x78a5636f43172f60
	.quad 0x84c87814a1f0ab72,0x8cc702081a6439ec
	.quad 0x90befffa23631e28,0xa4506cebde82bde9
	.quad 0xbef9a3f7b2c67915,0xc67178f2e372532b
	.quad 0xca273eceea26619c,0xd186b8c721c0c207
	.quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
	.quad 0x06f067aa72176fba,0x0a637dc5a2c898a6
	.quad 0x113f9804bef90dae,0x1b710b35131c471b
	.quad 0x28db77f523047d84,0x32caab7b40c72493
	.quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
	.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a
	.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817
	.quad 0 // terminator
	.size .LK512,.-.LK512

	.globl zfs_sha512_block_armv7
	.type zfs_sha512_block_armv7,%function
	.align 6
	zfs_sha512_block_armv7:
	hint #34 // bti c
	stp x29,x30,[sp,#-128]!
	add x29,sp,#0

	stp x19,x20,[sp,#16]
	stp x21,x22,[sp,#32]
	stp x23,x24,[sp,#48]
	stp x25,x26,[sp,#64]
	stp x27,x28,[sp,#80]
	sub sp,sp,#4*8

	ldp x20,x21,[x0] // load context
	ldp x22,x23,[x0,#2*8]
	ldp x24,x25,[x0,#4*8]
	add x2,x1,x2,lsl#7 // end of input
	ldp x26,x27,[x0,#6*8]
	adr x30,.LK512
	stp x0,x2,[x29,#96]

	.Loop:
	ldp x3,x4,[x1],#2*8
	ldr x19,[x30],#8 // *K++
	eor x28,x21,x22 // magic seed
	str x1,[x29,#112]
	#ifndef __AARCH64EB__
	rev x3,x3 // 0
	#endif
	ror x16,x24,#14
	add x27,x27,x19 // h+=K[i]
	eor x6,x24,x24,ror#23
	and x17,x25,x24
	bic x19,x26,x24
	add x27,x27,x3 // h+=X[i]
	orr x17,x17,x19 // Ch(e,f,g)
	eor x19,x20,x21 // a^b, b^c in next round
	eor x16,x16,x6,ror#18 // Sigma1(e)
	ror x6,x20,#28
	add x27,x27,x17 // h+=Ch(e,f,g)
	eor x17,x20,x20,ror#5
	add x27,x27,x16 // h+=Sigma1(e)
	and x28,x28,x19 // (b^c)&=(a^b)
	add x23,x23,x27 // d+=h
	eor x28,x28,x21 // Maj(a,b,c)
	eor x17,x6,x17,ror#34 // Sigma0(a)
	add x27,x27,x28 // h+=Maj(a,b,c)
	ldr x28,[x30],#8 // *K++, x19 in next round
	//add x27,x27,x17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev x4,x4 // 1
	#endif
	ldp x5,x6,[x1],#2*8
	add x27,x27,x17 // h+=Sigma0(a)
	ror x16,x23,#14
	add x26,x26,x28 // h+=K[i]
	eor x7,x23,x23,ror#23
	and x17,x24,x23
	bic x28,x25,x23
	add x26,x26,x4 // h+=X[i]
	orr x17,x17,x28 // Ch(e,f,g)
	eor x28,x27,x20 // a^b, b^c in next round
	eor x16,x16,x7,ror#18 // Sigma1(e)
	ror x7,x27,#28
	add x26,x26,x17 // h+=Ch(e,f,g)
	eor x17,x27,x27,ror#5
	add x26,x26,x16 // h+=Sigma1(e)
	and x19,x19,x28 // (b^c)&=(a^b)
	add x22,x22,x26 // d+=h
	eor x19,x19,x20 // Maj(a,b,c)
	eor x17,x7,x17,ror#34 // Sigma0(a)
	add x26,x26,x19 // h+=Maj(a,b,c)
	ldr x19,[x30],#8 // *K++, x28 in next round
	//add x26,x26,x17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev x5,x5 // 2
	#endif
	add x26,x26,x17 // h+=Sigma0(a)
	ror x16,x22,#14
	add x25,x25,x19 // h+=K[i]
	eor x8,x22,x22,ror#23
	and x17,x23,x22
	bic x19,x24,x22
	add x25,x25,x5 // h+=X[i]
	orr x17,x17,x19 // Ch(e,f,g)
	eor x19,x26,x27 // a^b, b^c in next round
	eor x16,x16,x8,ror#18 // Sigma1(e)
	ror x8,x26,#28
	add x25,x25,x17 // h+=Ch(e,f,g)
	eor x17,x26,x26,ror#5
	add x25,x25,x16 // h+=Sigma1(e)
	and x28,x28,x19 // (b^c)&=(a^b)
	add x21,x21,x25 // d+=h
	eor x28,x28,x27 // Maj(a,b,c)
	eor x17,x8,x17,ror#34 // Sigma0(a)
	add x25,x25,x28 // h+=Maj(a,b,c)
	ldr x28,[x30],#8 // *K++, x19 in next round
	//add x25,x25,x17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev x6,x6 // 3
	#endif
	ldp x7,x8,[x1],#2*8
	add x25,x25,x17 // h+=Sigma0(a)
	ror x16,x21,#14
	add x24,x24,x28 // h+=K[i]
	eor x9,x21,x21,ror#23
	and x17,x22,x21
	bic x28,x23,x21
	add x24,x24,x6 // h+=X[i]
	orr x17,x17,x28 // Ch(e,f,g)
	eor x28,x25,x26 // a^b, b^c in next round
	eor x16,x16,x9,ror#18 // Sigma1(e)
	ror x9,x25,#28
	add x24,x24,x17 // h+=Ch(e,f,g)
	eor x17,x25,x25,ror#5
	add x24,x24,x16 // h+=Sigma1(e)
	and x19,x19,x28 // (b^c)&=(a^b)
	add x20,x20,x24 // d+=h
	eor x19,x19,x26 // Maj(a,b,c)
	eor x17,x9,x17,ror#34 // Sigma0(a)
	add x24,x24,x19 // h+=Maj(a,b,c)
	ldr x19,[x30],#8 // *K++, x28 in next round
	//add x24,x24,x17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev x7,x7 // 4
	#endif
	add x24,x24,x17 // h+=Sigma0(a)
	ror x16,x20,#14
	add x23,x23,x19 // h+=K[i]
	eor x10,x20,x20,ror#23
	and x17,x21,x20
	bic x19,x22,x20
	add x23,x23,x7 // h+=X[i]
	orr x17,x17,x19 // Ch(e,f,g)
	eor x19,x24,x25 // a^b, b^c in next round
	eor x16,x16,x10,ror#18 // Sigma1(e)
	ror x10,x24,#28
	add x23,x23,x17 // h+=Ch(e,f,g)
	eor x17,x24,x24,ror#5
	add x23,x23,x16 // h+=Sigma1(e)
	and x28,x28,x19 // (b^c)&=(a^b)
	add x27,x27,x23 // d+=h
	eor x28,x28,x25 // Maj(a,b,c)
	eor x17,x10,x17,ror#34 // Sigma0(a)
	add x23,x23,x28 // h+=Maj(a,b,c)
	ldr x28,[x30],#8 // *K++, x19 in next round
	//add x23,x23,x17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev x8,x8 // 5
	#endif
	ldp x9,x10,[x1],#2*8
	add x23,x23,x17 // h+=Sigma0(a)
	ror x16,x27,#14
	add x22,x22,x28 // h+=K[i]
	eor x11,x27,x27,ror#23
	and x17,x20,x27
	bic x28,x21,x27
	add x22,x22,x8 // h+=X[i]
	orr x17,x17,x28 // Ch(e,f,g)
	eor x28,x23,x24 // a^b, b^c in next round
	eor x16,x16,x11,ror#18 // Sigma1(e)
	ror x11,x23,#28
	add x22,x22,x17 // h+=Ch(e,f,g)
	eor x17,x23,x23,ror#5
	add x22,x22,x16 // h+=Sigma1(e)
	and x19,x19,x28 // (b^c)&=(a^b)
	add x26,x26,x22 // d+=h
	eor x19,x19,x24 // Maj(a,b,c)
	eor x17,x11,x17,ror#34 // Sigma0(a)
	add x22,x22,x19 // h+=Maj(a,b,c)
	ldr x19,[x30],#8 // *K++, x28 in next round
	//add x22,x22,x17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev x9,x9 // 6
	#endif
	add x22,x22,x17 // h+=Sigma0(a)
	ror x16,x26,#14
	add x21,x21,x19 // h+=K[i]
	eor x12,x26,x26,ror#23
	and x17,x27,x26
	bic x19,x20,x26
	add x21,x21,x9 // h+=X[i]
	orr x17,x17,x19 // Ch(e,f,g)
	eor x19,x22,x23 // a^b, b^c in next round
	eor x16,x16,x12,ror#18 // Sigma1(e)
	ror x12,x22,#28
	add x21,x21,x17 // h+=Ch(e,f,g)
	eor x17,x22,x22,ror#5
	add x21,x21,x16 // h+=Sigma1(e)
	and x28,x28,x19 // (b^c)&=(a^b)
	add x25,x25,x21 // d+=h
	eor x28,x28,x23 // Maj(a,b,c)
	eor x17,x12,x17,ror#34 // Sigma0(a)
	add x21,x21,x28 // h+=Maj(a,b,c)
	ldr x28,[x30],#8 // *K++, x19 in next round
	//add x21,x21,x17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev x10,x10 // 7
	#endif
	ldp x11,x12,[x1],#2*8
	add x21,x21,x17 // h+=Sigma0(a)
	ror x16,x25,#14
	add x20,x20,x28 // h+=K[i]
	eor x13,x25,x25,ror#23
	and x17,x26,x25
	bic x28,x27,x25
	add x20,x20,x10 // h+=X[i]
	orr x17,x17,x28 // Ch(e,f,g)
	eor x28,x21,x22 // a^b, b^c in next round
	eor x16,x16,x13,ror#18 // Sigma1(e)
	ror x13,x21,#28
	add x20,x20,x17 // h+=Ch(e,f,g)
	eor x17,x21,x21,ror#5
	add x20,x20,x16 // h+=Sigma1(e)
	and x19,x19,x28 // (b^c)&=(a^b)
	add x24,x24,x20 // d+=h
	eor x19,x19,x22 // Maj(a,b,c)
	eor x17,x13,x17,ror#34 // Sigma0(a)
	add x20,x20,x19 // h+=Maj(a,b,c)
	ldr x19,[x30],#8 // *K++, x28 in next round
	//add x20,x20,x17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev x11,x11 // 8
	#endif
	add x20,x20,x17 // h+=Sigma0(a)
	ror x16,x24,#14
	add x27,x27,x19 // h+=K[i]
	eor x14,x24,x24,ror#23
	and x17,x25,x24
	bic x19,x26,x24
	add x27,x27,x11 // h+=X[i]
	orr x17,x17,x19 // Ch(e,f,g)
	eor x19,x20,x21 // a^b, b^c in next round
	eor x16,x16,x14,ror#18 // Sigma1(e)
	ror x14,x20,#28
	add x27,x27,x17 // h+=Ch(e,f,g)
	eor x17,x20,x20,ror#5
	add x27,x27,x16 // h+=Sigma1(e)
	and x28,x28,x19 // (b^c)&=(a^b)
	add x23,x23,x27 // d+=h
	eor x28,x28,x21 // Maj(a,b,c)
	eor x17,x14,x17,ror#34 // Sigma0(a)
	add x27,x27,x28 // h+=Maj(a,b,c)
	ldr x28,[x30],#8 // *K++, x19 in next round
	//add x27,x27,x17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev x12,x12 // 9
	#endif
	ldp x13,x14,[x1],#2*8
	add x27,x27,x17 // h+=Sigma0(a)
	ror x16,x23,#14
	add x26,x26,x28 // h+=K[i]
	eor x15,x23,x23,ror#23
	and x17,x24,x23
	bic x28,x25,x23
	add x26,x26,x12 // h+=X[i]
	orr x17,x17,x28 // Ch(e,f,g)
	eor x28,x27,x20 // a^b, b^c in next round
	eor x16,x16,x15,ror#18 // Sigma1(e)
	ror x15,x27,#28
	add x26,x26,x17 // h+=Ch(e,f,g)
	eor x17,x27,x27,ror#5
	add x26,x26,x16 // h+=Sigma1(e)
	and x19,x19,x28 // (b^c)&=(a^b)
	add x22,x22,x26 // d+=h
	eor x19,x19,x20 // Maj(a,b,c)
	eor x17,x15,x17,ror#34 // Sigma0(a)
	add x26,x26,x19 // h+=Maj(a,b,c)
	ldr x19,[x30],#8 // *K++, x28 in next round
	//add x26,x26,x17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev x13,x13 // 10
	#endif
	add x26,x26,x17 // h+=Sigma0(a)
	ror x16,x22,#14
	add x25,x25,x19 // h+=K[i]
	eor x0,x22,x22,ror#23
	and x17,x23,x22
	bic x19,x24,x22
	add x25,x25,x13 // h+=X[i]
	orr x17,x17,x19 // Ch(e,f,g)
	eor x19,x26,x27 // a^b, b^c in next round
	eor x16,x16,x0,ror#18 // Sigma1(e)
	ror x0,x26,#28
	add x25,x25,x17 // h+=Ch(e,f,g)
	eor x17,x26,x26,ror#5
	add x25,x25,x16 // h+=Sigma1(e)
	and x28,x28,x19 // (b^c)&=(a^b)
	add x21,x21,x25 // d+=h
	eor x28,x28,x27 // Maj(a,b,c)
	eor x17,x0,x17,ror#34 // Sigma0(a)
	add x25,x25,x28 // h+=Maj(a,b,c)
	ldr x28,[x30],#8 // *K++, x19 in next round
	//add x25,x25,x17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev x14,x14 // 11
	#endif
	ldp x15,x0,[x1],#2*8
	add x25,x25,x17 // h+=Sigma0(a)
	str x6,[sp,#24]
	ror x16,x21,#14
	add x24,x24,x28 // h+=K[i]
	eor x6,x21,x21,ror#23
	and x17,x22,x21
	bic x28,x23,x21
	add x24,x24,x14 // h+=X[i]
	orr x17,x17,x28 // Ch(e,f,g)
	eor x28,x25,x26 // a^b, b^c in next round
	eor x16,x16,x6,ror#18 // Sigma1(e)
	ror x6,x25,#28
	add x24,x24,x17 // h+=Ch(e,f,g)
	eor x17,x25,x25,ror#5
	add x24,x24,x16 // h+=Sigma1(e)
	and x19,x19,x28 // (b^c)&=(a^b)
	add x20,x20,x24 // d+=h
	eor x19,x19,x26 // Maj(a,b,c)
	eor x17,x6,x17,ror#34 // Sigma0(a)
	add x24,x24,x19 // h+=Maj(a,b,c)
	ldr x19,[x30],#8 // *K++, x28 in next round
	//add x24,x24,x17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev x15,x15 // 12
	#endif
	add x24,x24,x17 // h+=Sigma0(a)
	str x7,[sp,#0]
	ror x16,x20,#14
	add x23,x23,x19 // h+=K[i]
	eor x7,x20,x20,ror#23
	and x17,x21,x20
	bic x19,x22,x20
	add x23,x23,x15 // h+=X[i]
	orr x17,x17,x19 // Ch(e,f,g)
	eor x19,x24,x25 // a^b, b^c in next round
	eor x16,x16,x7,ror#18 // Sigma1(e)
	ror x7,x24,#28
	add x23,x23,x17 // h+=Ch(e,f,g)
	eor x17,x24,x24,ror#5
	add x23,x23,x16 // h+=Sigma1(e)
	and x28,x28,x19 // (b^c)&=(a^b)
	add x27,x27,x23 // d+=h
	eor x28,x28,x25 // Maj(a,b,c)
	eor x17,x7,x17,ror#34 // Sigma0(a)
	add x23,x23,x28 // h+=Maj(a,b,c)
	ldr x28,[x30],#8 // *K++, x19 in next round
	//add x23,x23,x17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev x0,x0 // 13
	#endif
	ldp x1,x2,[x1]
	add x23,x23,x17 // h+=Sigma0(a)
	str x8,[sp,#8]
	ror x16,x27,#14
	add x22,x22,x28 // h+=K[i]
	eor x8,x27,x27,ror#23
	and x17,x20,x27
	bic x28,x21,x27
	add x22,x22,x0 // h+=X[i]
	orr x17,x17,x28 // Ch(e,f,g)
	eor x28,x23,x24 // a^b, b^c in next round
	eor x16,x16,x8,ror#18 // Sigma1(e)
	ror x8,x23,#28
	add x22,x22,x17 // h+=Ch(e,f,g)
	eor x17,x23,x23,ror#5
	add x22,x22,x16 // h+=Sigma1(e)
	and x19,x19,x28 // (b^c)&=(a^b)
	add x26,x26,x22 // d+=h
	eor x19,x19,x24 // Maj(a,b,c)
	eor x17,x8,x17,ror#34 // Sigma0(a)
	add x22,x22,x19 // h+=Maj(a,b,c)
	ldr x19,[x30],#8 // *K++, x28 in next round
	//add x22,x22,x17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev x1,x1 // 14
	#endif
	ldr x6,[sp,#24]
	add x22,x22,x17 // h+=Sigma0(a)
	str x9,[sp,#16]
	ror x16,x26,#14
	add x21,x21,x19 // h+=K[i]
	eor x9,x26,x26,ror#23
	and x17,x27,x26
	bic x19,x20,x26
	add x21,x21,x1 // h+=X[i]
	orr x17,x17,x19 // Ch(e,f,g)
	eor x19,x22,x23 // a^b, b^c in next round
	eor x16,x16,x9,ror#18 // Sigma1(e)
	ror x9,x22,#28
	add x21,x21,x17 // h+=Ch(e,f,g)
	eor x17,x22,x22,ror#5
	add x21,x21,x16 // h+=Sigma1(e)
	and x28,x28,x19 // (b^c)&=(a^b)
	add x25,x25,x21 // d+=h
	eor x28,x28,x23 // Maj(a,b,c)
	eor x17,x9,x17,ror#34 // Sigma0(a)
	add x21,x21,x28 // h+=Maj(a,b,c)
	ldr x28,[x30],#8 // *K++, x19 in next round
	//add x21,x21,x17 // h+=Sigma0(a)
	#ifndef __AARCH64EB__
	rev x2,x2 // 15
	#endif
	ldr x7,[sp,#0]
	add x21,x21,x17 // h+=Sigma0(a)
	str x10,[sp,#24]
	ror x16,x25,#14
	add x20,x20,x28 // h+=K[i]
	ror x9,x4,#1
	and x17,x26,x25
	ror x8,x1,#19
	bic x28,x27,x25
	ror x10,x21,#28
	add x20,x20,x2 // h+=X[i]
	eor x16,x16,x25,ror#18
	eor x9,x9,x4,ror#8
	orr x17,x17,x28 // Ch(e,f,g)
	eor x28,x21,x22 // a^b, b^c in next round
	eor x16,x16,x25,ror#41 // Sigma1(e)
	eor x10,x10,x21,ror#34
	add x20,x20,x17 // h+=Ch(e,f,g)
	and x19,x19,x28 // (b^c)&=(a^b)
	eor x8,x8,x1,ror#61
	eor x9,x9,x4,lsr#7 // sigma0(X[i+1])
	add x20,x20,x16 // h+=Sigma1(e)
	eor x19,x19,x22 // Maj(a,b,c)
	eor x17,x10,x21,ror#39 // Sigma0(a)
	eor x8,x8,x1,lsr#6 // sigma1(X[i+14])
	add x3,x3,x12
	add x24,x24,x20 // d+=h
	add x20,x20,x19 // h+=Maj(a,b,c)
	ldr x19,[x30],#8 // *K++, x28 in next round
	add x3,x3,x9
	add x20,x20,x17 // h+=Sigma0(a)
	add x3,x3,x8
	.Loop_16_xx:
	ldr x8,[sp,#8]
	str x11,[sp,#0]
	ror x16,x24,#14
	add x27,x27,x19 // h+=K[i]
	ror x10,x5,#1
	and x17,x25,x24
	ror x9,x2,#19
	bic x19,x26,x24
	ror x11,x20,#28
	add x27,x27,x3 // h+=X[i]
	eor x16,x16,x24,ror#18
	eor x10,x10,x5,ror#8
	orr x17,x17,x19 // Ch(e,f,g)
	eor x19,x20,x21 // a^b, b^c in next round
	eor x16,x16,x24,ror#41 // Sigma1(e)
	eor x11,x11,x20,ror#34
	add x27,x27,x17 // h+=Ch(e,f,g)
	and x28,x28,x19 // (b^c)&=(a^b)
	eor x9,x9,x2,ror#61
	eor x10,x10,x5,lsr#7 // sigma0(X[i+1])
	add x27,x27,x16 // h+=Sigma1(e)
	eor x28,x28,x21 // Maj(a,b,c)
	eor x17,x11,x20,ror#39 // Sigma0(a)
	eor x9,x9,x2,lsr#6 // sigma1(X[i+14])
	add x4,x4,x13
	add x23,x23,x27 // d+=h
	add x27,x27,x28 // h+=Maj(a,b,c)
	ldr x28,[x30],#8 // *K++, x19 in next round
	add x4,x4,x10
	add x27,x27,x17 // h+=Sigma0(a)
	add x4,x4,x9
	ldr x9,[sp,#16]
	str x12,[sp,#8]
	ror x16,x23,#14
	add x26,x26,x28 // h+=K[i]
	ror x11,x6,#1
	and x17,x24,x23
	ror x10,x3,#19
	bic x28,x25,x23
	ror x12,x27,#28
	add x26,x26,x4 // h+=X[i]
	eor x16,x16,x23,ror#18
	eor x11,x11,x6,ror#8
	orr x17,x17,x28 // Ch(e,f,g)
	eor x28,x27,x20 // a^b, b^c in next round
	eor x16,x16,x23,ror#41 // Sigma1(e)
	eor x12,x12,x27,ror#34
	add x26,x26,x17 // h+=Ch(e,f,g)
	and x19,x19,x28 // (b^c)&=(a^b)
	eor x10,x10,x3,ror#61
	eor x11,x11,x6,lsr#7 // sigma0(X[i+1])
	add x26,x26,x16 // h+=Sigma1(e)
	eor x19,x19,x20 // Maj(a,b,c)
	eor x17,x12,x27,ror#39 // Sigma0(a)
	eor x10,x10,x3,lsr#6 // sigma1(X[i+14])
	add x5,x5,x14
	add x22,x22,x26 // d+=h
	add x26,x26,x19 // h+=Maj(a,b,c)
	ldr x19,[x30],#8 // *K++, x28 in next round
	add x5,x5,x11
	add x26,x26,x17 // h+=Sigma0(a)
	add x5,x5,x10
	ldr x10,[sp,#24]
	str x13,[sp,#16]
	ror x16,x22,#14
	add x25,x25,x19 // h+=K[i]
	ror x12,x7,#1
	and x17,x23,x22
	ror x11,x4,#19
	bic x19,x24,x22
	ror x13,x26,#28
	add x25,x25,x5 // h+=X[i]
	eor x16,x16,x22,ror#18
	eor x12,x12,x7,ror#8
	orr x17,x17,x19 // Ch(e,f,g)
	eor x19,x26,x27 // a^b, b^c in next round
	eor x16,x16,x22,ror#41 // Sigma1(e)
	eor x13,x13,x26,ror#34
	add x25,x25,x17 // h+=Ch(e,f,g)
	and x28,x28,x19 // (b^c)&=(a^b)
	eor x11,x11,x4,ror#61
	eor x12,x12,x7,lsr#7 // sigma0(X[i+1])
	add x25,x25,x16 // h+=Sigma1(e)
	eor x28,x28,x27 // Maj(a,b,c)
	eor x17,x13,x26,ror#39 // Sigma0(a)
	eor x11,x11,x4,lsr#6 // sigma1(X[i+14])
	add x6,x6,x15
	add x21,x21,x25 // d+=h
	add x25,x25,x28 // h+=Maj(a,b,c)
	ldr x28,[x30],#8 // *K++, x19 in next round
	add x6,x6,x12
	add x25,x25,x17 // h+=Sigma0(a)
	add x6,x6,x11
	ldr x11,[sp,#0]
	str x14,[sp,#24]
	ror x16,x21,#14
	add x24,x24,x28 // h+=K[i]
	ror x13,x8,#1
	and x17,x22,x21
	ror x12,x5,#19
	bic x28,x23,x21
	ror x14,x25,#28
	add x24,x24,x6 // h+=X[i]
	eor x16,x16,x21,ror#18
	eor x13,x13,x8,ror#8
	orr x17,x17,x28 // Ch(e,f,g)
	eor x28,x25,x26 // a^b, b^c in next round
	eor x16,x16,x21,ror#41 // Sigma1(e)
	eor x14,x14,x25,ror#34
	add x24,x24,x17 // h+=Ch(e,f,g)
	and x19,x19,x28 // (b^c)&=(a^b)
	eor x12,x12,x5,ror#61
	eor x13,x13,x8,lsr#7 // sigma0(X[i+1])
	add x24,x24,x16 // h+=Sigma1(e)
	eor x19,x19,x26 // Maj(a,b,c)
	eor x17,x14,x25,ror#39 // Sigma0(a)
	eor x12,x12,x5,lsr#6 // sigma1(X[i+14])
	add x7,x7,x0
	add x20,x20,x24 // d+=h
	add x24,x24,x19 // h+=Maj(a,b,c)
	ldr x19,[x30],#8 // *K++, x28 in next round
	add x7,x7,x13
	add x24,x24,x17 // h+=Sigma0(a)
	add x7,x7,x12
	ldr x12,[sp,#8]
	str x15,[sp,#0]
	ror x16,x20,#14
	add x23,x23,x19 // h+=K[i]
	ror x14,x9,#1
	and x17,x21,x20
	ror x13,x6,#19
	bic x19,x22,x20
	ror x15,x24,#28
	add x23,x23,x7 // h+=X[i]
	eor x16,x16,x20,ror#18
	eor x14,x14,x9,ror#8
	orr x17,x17,x19 // Ch(e,f,g)
	eor x19,x24,x25 // a^b, b^c in next round
	eor x16,x16,x20,ror#41 // Sigma1(e)
	eor x15,x15,x24,ror#34
	add x23,x23,x17 // h+=Ch(e,f,g)
	and x28,x28,x19 // (b^c)&=(a^b)
	eor x13,x13,x6,ror#61
	eor x14,x14,x9,lsr#7 // sigma0(X[i+1])
	add x23,x23,x16 // h+=Sigma1(e)
	eor x28,x28,x25 // Maj(a,b,c)
	eor x17,x15,x24,ror#39 // Sigma0(a)
	eor x13,x13,x6,lsr#6 // sigma1(X[i+14])
	add x8,x8,x1
	add x27,x27,x23 // d+=h
	add x23,x23,x28 // h+=Maj(a,b,c)
	ldr x28,[x30],#8 // *K++, x19 in next round
	add x8,x8,x14
	add x23,x23,x17 // h+=Sigma0(a)
	add x8,x8,x13
	ldr x13,[sp,#16]
	str x0,[sp,#8]
	ror x16,x27,#14
	add x22,x22,x28 // h+=K[i]
	ror x15,x10,#1
	and x17,x20,x27
	ror x14,x7,#19
	bic x28,x21,x27
	ror x0,x23,#28
	add x22,x22,x8 // h+=X[i]
	eor x16,x16,x27,ror#18
	eor x15,x15,x10,ror#8
	orr x17,x17,x28 // Ch(e,f,g)
	eor x28,x23,x24 // a^b, b^c in next round
	eor x16,x16,x27,ror#41 // Sigma1(e)
	eor x0,x0,x23,ror#34
	add x22,x22,x17 // h+=Ch(e,f,g)
	and x19,x19,x28 // (b^c)&=(a^b)
	eor x14,x14,x7,ror#61
	eor x15,x15,x10,lsr#7 // sigma0(X[i+1])
	add x22,x22,x16 // h+=Sigma1(e)
	eor x19,x19,x24 // Maj(a,b,c)
	eor x17,x0,x23,ror#39 // Sigma0(a)
	eor x14,x14,x7,lsr#6 // sigma1(X[i+14])
	add x9,x9,x2
	add x26,x26,x22 // d+=h
	add x22,x22,x19 // h+=Maj(a,b,c)
	ldr x19,[x30],#8 // *K++, x28 in next round
	add x9,x9,x15
	add x22,x22,x17 // h+=Sigma0(a)
	add x9,x9,x14
	ldr x14,[sp,#24]
	str x1,[sp,#16]
	ror x16,x26,#14
	add x21,x21,x19 // h+=K[i]
	ror x0,x11,#1
	and x17,x27,x26
	ror x15,x8,#19
	bic x19,x20,x26
	ror x1,x22,#28
	add x21,x21,x9 // h+=X[i]
	eor x16,x16,x26,ror#18
	eor x0,x0,x11,ror#8
	orr x17,x17,x19 // Ch(e,f,g)
	eor x19,x22,x23 // a^b, b^c in next round
	eor x16,x16,x26,ror#41 // Sigma1(e)
	eor x1,x1,x22,ror#34
	add x21,x21,x17 // h+=Ch(e,f,g)
	and x28,x28,x19 // (b^c)&=(a^b)
	eor x15,x15,x8,ror#61
	eor x0,x0,x11,lsr#7 // sigma0(X[i+1])
	add x21,x21,x16 // h+=Sigma1(e)
	eor x28,x28,x23 // Maj(a,b,c)
	eor x17,x1,x22,ror#39 // Sigma0(a)
	eor x15,x15,x8,lsr#6 // sigma1(X[i+14])
	add x10,x10,x3
	add x25,x25,x21 // d+=h
	add x21,x21,x28 // h+=Maj(a,b,c)
	ldr x28,[x30],#8 // *K++, x19 in next round
	add x10,x10,x0
	add x21,x21,x17 // h+=Sigma0(a)
	add x10,x10,x15
	ldr x15,[sp,#0]
	str x2,[sp,#24]
	ror x16,x25,#14
	add x20,x20,x28 // h+=K[i]
	ror x1,x12,#1
	and x17,x26,x25
	ror x0,x9,#19
	bic x28,x27,x25
	ror x2,x21,#28
	add x20,x20,x10 // h+=X[i]
	eor x16,x16,x25,ror#18
	eor x1,x1,x12,ror#8
	orr x17,x17,x28 // Ch(e,f,g)
	eor x28,x21,x22 // a^b, b^c in next round
	eor x16,x16,x25,ror#41 // Sigma1(e)
	eor x2,x2,x21,ror#34
	add x20,x20,x17 // h+=Ch(e,f,g)
	and x19,x19,x28 // (b^c)&=(a^b)
	eor x0,x0,x9,ror#61
	eor x1,x1,x12,lsr#7 // sigma0(X[i+1])
	add x20,x20,x16 // h+=Sigma1(e)
	eor x19,x19,x22 // Maj(a,b,c)
	eor x17,x2,x21,ror#39 // Sigma0(a)
	eor x0,x0,x9,lsr#6 // sigma1(X[i+14])
	add x11,x11,x4
	add x24,x24,x20 // d+=h
	add x20,x20,x19 // h+=Maj(a,b,c)
	ldr x19,[x30],#8 // *K++, x28 in next round
	add x11,x11,x1
	add x20,x20,x17 // h+=Sigma0(a)
	add x11,x11,x0
	ldr x0,[sp,#8]
	str x3,[sp,#0]
	ror x16,x24,#14
	add x27,x27,x19 // h+=K[i]
	ror x2,x13,#1
	and x17,x25,x24
	ror x1,x10,#19
	bic x19,x26,x24
	ror x3,x20,#28
	add x27,x27,x11 // h+=X[i]
	eor x16,x16,x24,ror#18
	eor x2,x2,x13,ror#8
	orr x17,x17,x19 // Ch(e,f,g)
	eor x19,x20,x21 // a^b, b^c in next round
	eor x16,x16,x24,ror#41 // Sigma1(e)
	eor x3,x3,x20,ror#34
	add x27,x27,x17 // h+=Ch(e,f,g)
	and x28,x28,x19 // (b^c)&=(a^b)
	eor x1,x1,x10,ror#61
	eor x2,x2,x13,lsr#7 // sigma0(X[i+1])
	add x27,x27,x16 // h+=Sigma1(e)
	eor x28,x28,x21 // Maj(a,b,c)
	eor x17,x3,x20,ror#39 // Sigma0(a)
	eor x1,x1,x10,lsr#6 // sigma1(X[i+14])
	add x12,x12,x5
	add x23,x23,x27 // d+=h
	add x27,x27,x28 // h+=Maj(a,b,c)
	ldr x28,[x30],#8 // *K++, x19 in next round
	add x12,x12,x2
	add x27,x27,x17 // h+=Sigma0(a)
	add x12,x12,x1
	ldr x1,[sp,#16]
	str x4,[sp,#8]
	ror x16,x23,#14
	add x26,x26,x28 // h+=K[i]
	ror x3,x14,#1
	and x17,x24,x23
	ror x2,x11,#19
	bic x28,x25,x23
	ror x4,x27,#28
	add x26,x26,x12 // h+=X[i]
	eor x16,x16,x23,ror#18
	eor x3,x3,x14,ror#8
	orr x17,x17,x28 // Ch(e,f,g)
	eor x28,x27,x20 // a^b, b^c in next round
	eor x16,x16,x23,ror#41 // Sigma1(e)
	eor x4,x4,x27,ror#34
	add x26,x26,x17 // h+=Ch(e,f,g)
	and x19,x19,x28 // (b^c)&=(a^b)
	eor x2,x2,x11,ror#61
	eor x3,x3,x14,lsr#7 // sigma0(X[i+1])
	add x26,x26,x16 // h+=Sigma1(e)
	eor x19,x19,x20 // Maj(a,b,c)
	eor x17,x4,x27,ror#39 // Sigma0(a)
	eor x2,x2,x11,lsr#6 // sigma1(X[i+14])
	add x13,x13,x6
	add x22,x22,x26 // d+=h
	add x26,x26,x19 // h+=Maj(a,b,c)
	ldr x19,[x30],#8 // *K++, x28 in next round
	add x13,x13,x3
	add x26,x26,x17 // h+=Sigma0(a)
	add x13,x13,x2
	ldr x2,[sp,#24]
	str x5,[sp,#16]
	ror x16,x22,#14
	add x25,x25,x19 // h+=K[i]
	ror x4,x15,#1
	and x17,x23,x22
	ror x3,x12,#19
	bic x19,x24,x22
	ror x5,x26,#28
	add x25,x25,x13 // h+=X[i]
	eor x16,x16,x22,ror#18
	eor x4,x4,x15,ror#8
	orr x17,x17,x19 // Ch(e,f,g)
	eor x19,x26,x27 // a^b, b^c in next round
	eor x16,x16,x22,ror#41 // Sigma1(e)
	eor x5,x5,x26,ror#34
	add x25,x25,x17 // h+=Ch(e,f,g)
	and x28,x28,x19 // (b^c)&=(a^b)
	eor x3,x3,x12,ror#61
	eor x4,x4,x15,lsr#7 // sigma0(X[i+1])
	add x25,x25,x16 // h+=Sigma1(e)
	eor x28,x28,x27 // Maj(a,b,c)
	eor x17,x5,x26,ror#39 // Sigma0(a)
	eor x3,x3,x12,lsr#6 // sigma1(X[i+14])
	add x14,x14,x7
	add x21,x21,x25 // d+=h
	add x25,x25,x28 // h+=Maj(a,b,c)
	ldr x28,[x30],#8 // *K++, x19 in next round
	add x14,x14,x4
	add x25,x25,x17 // h+=Sigma0(a)
	add x14,x14,x3
	ldr x3,[sp,#0]
	str x6,[sp,#24]
	ror x16,x21,#14
	add x24,x24,x28 // h+=K[i]
	ror x5,x0,#1
	and x17,x22,x21
	ror x4,x13,#19
	bic x28,x23,x21
	ror x6,x25,#28
	add x24,x24,x14 // h+=X[i]
	eor x16,x16,x21,ror#18
	eor x5,x5,x0,ror#8
	orr x17,x17,x28 // Ch(e,f,g)
	eor x28,x25,x26 // a^b, b^c in next round
	eor x16,x16,x21,ror#41 // Sigma1(e)
	eor x6,x6,x25,ror#34
	add x24,x24,x17 // h+=Ch(e,f,g)
	and x19,x19,x28 // (b^c)&=(a^b)
	eor x4,x4,x13,ror#61
	eor x5,x5,x0,lsr#7 // sigma0(X[i+1])
	add x24,x24,x16 // h+=Sigma1(e)
	eor x19,x19,x26 // Maj(a,b,c)
	eor x17,x6,x25,ror#39 // Sigma0(a)
	eor x4,x4,x13,lsr#6 // sigma1(X[i+14])
	add x15,x15,x8
	add x20,x20,x24 // d+=h
	add x24,x24,x19 // h+=Maj(a,b,c)
	ldr x19,[x30],#8 // *K++, x28 in next round
	add x15,x15,x5
	add x24,x24,x17 // h+=Sigma0(a)
	add x15,x15,x4
	ldr x4,[sp,#8]
	str x7,[sp,#0]
	ror x16,x20,#14
	add x23,x23,x19 // h+=K[i]
	ror x6,x1,#1
	and x17,x21,x20
	ror x5,x14,#19
	bic x19,x22,x20
	ror x7,x24,#28
	add x23,x23,x15 // h+=X[i]
	eor x16,x16,x20,ror#18
	eor x6,x6,x1,ror#8
	orr x17,x17,x19 // Ch(e,f,g)
	eor x19,x24,x25 // a^b, b^c in next round
	eor x16,x16,x20,ror#41 // Sigma1(e)
	eor x7,x7,x24,ror#34
	add x23,x23,x17 // h+=Ch(e,f,g)
	and x28,x28,x19 // (b^c)&=(a^b)
	eor x5,x5,x14,ror#61
	eor x6,x6,x1,lsr#7 // sigma0(X[i+1])
	add x23,x23,x16 // h+=Sigma1(e)
	eor x28,x28,x25 // Maj(a,b,c)
	eor x17,x7,x24,ror#39 // Sigma0(a)
	eor x5,x5,x14,lsr#6 // sigma1(X[i+14])
	add x0,x0,x9
	add x27,x27,x23 // d+=h
	add x23,x23,x28 // h+=Maj(a,b,c)
	ldr x28,[x30],#8 // *K++, x19 in next round
	add x0,x0,x6
	add x23,x23,x17 // h+=Sigma0(a)
	add x0,x0,x5
	ldr x5,[sp,#16]
	str x8,[sp,#8]
	ror x16,x27,#14
	add x22,x22,x28 // h+=K[i]
	ror x7,x2,#1
	and x17,x20,x27
	ror x6,x15,#19
	bic x28,x21,x27
	ror x8,x23,#28
	add x22,x22,x0 // h+=X[i]
	eor x16,x16,x27,ror#18
	eor x7,x7,x2,ror#8
	orr x17,x17,x28 // Ch(e,f,g)
	eor x28,x23,x24 // a^b, b^c in next round
	eor x16,x16,x27,ror#41 // Sigma1(e)
	eor x8,x8,x23,ror#34
	add x22,x22,x17 // h+=Ch(e,f,g)
	and x19,x19,x28 // (b^c)&=(a^b)
	eor x6,x6,x15,ror#61
	eor x7,x7,x2,lsr#7 // sigma0(X[i+1])
	add x22,x22,x16 // h+=Sigma1(e)
	eor x19,x19,x24 // Maj(a,b,c)
	eor x17,x8,x23,ror#39 // Sigma0(a)
	eor x6,x6,x15,lsr#6 // sigma1(X[i+14])
	add x1,x1,x10
	add x26,x26,x22 // d+=h
	add x22,x22,x19 // h+=Maj(a,b,c)
	ldr x19,[x30],#8 // *K++, x28 in next round
	add x1,x1,x7
	add x22,x22,x17 // h+=Sigma0(a)
	add x1,x1,x6
	ldr x6,[sp,#24]
	str x9,[sp,#16]
	ror x16,x26,#14
	add x21,x21,x19 // h+=K[i]
	ror x8,x3,#1
	and x17,x27,x26
	ror x7,x0,#19
	bic x19,x20,x26
	ror x9,x22,#28
	add x21,x21,x1 // h+=X[i]
	eor x16,x16,x26,ror#18
	eor x8,x8,x3,ror#8
	orr x17,x17,x19 // Ch(e,f,g)
	eor x19,x22,x23 // a^b, b^c in next round
	eor x16,x16,x26,ror#41 // Sigma1(e)
	eor x9,x9,x22,ror#34
	add x21,x21,x17 // h+=Ch(e,f,g)
	and x28,x28,x19 // (b^c)&=(a^b)
	eor x7,x7,x0,ror#61
	eor x8,x8,x3,lsr#7 // sigma0(X[i+1])
	add x21,x21,x16 // h+=Sigma1(e)
	eor x28,x28,x23 // Maj(a,b,c)
	eor x17,x9,x22,ror#39 // Sigma0(a)
	eor x7,x7,x0,lsr#6 // sigma1(X[i+14])
	add x2,x2,x11
	add x25,x25,x21 // d+=h
	add x21,x21,x28 // h+=Maj(a,b,c)
	ldr x28,[x30],#8 // *K++, x19 in next round
	add x2,x2,x8
	add x21,x21,x17 // h+=Sigma0(a)
	add x2,x2,x7
	ldr x7,[sp,#0]
	str x10,[sp,#24]
	ror x16,x25,#14
	add x20,x20,x28 // h+=K[i]
	ror x9,x4,#1
	and x17,x26,x25
	ror x8,x1,#19
	bic x28,x27,x25
	ror x10,x21,#28
	add x20,x20,x2 // h+=X[i]
	eor x16,x16,x25,ror#18
	eor x9,x9,x4,ror#8
	orr x17,x17,x28 // Ch(e,f,g)
	eor x28,x21,x22 // a^b, b^c in next round
	eor x16,x16,x25,ror#41 // Sigma1(e)
	eor x10,x10,x21,ror#34
	add x20,x20,x17 // h+=Ch(e,f,g)
	and x19,x19,x28 // (b^c)&=(a^b)
	eor x8,x8,x1,ror#61
	eor x9,x9,x4,lsr#7 // sigma0(X[i+1])
	add x20,x20,x16 // h+=Sigma1(e)
	eor x19,x19,x22 // Maj(a,b,c)
	eor x17,x10,x21,ror#39 // Sigma0(a)
	eor x8,x8,x1,lsr#6 // sigma1(X[i+14])
	add x3,x3,x12
	add x24,x24,x20 // d+=h
	add x20,x20,x19 // h+=Maj(a,b,c)
	ldr x19,[x30],#8 // *K++, x28 in next round
	add x3,x3,x9
	add x20,x20,x17 // h+=Sigma0(a)
	add x3,x3,x8
	cbnz x19,.Loop_16_xx

	ldp x0,x2,[x29,#96]
	ldr x1,[x29,#112]
	sub x30,x30,#648 // rewind

	ldp x3,x4,[x0]
	ldp x5,x6,[x0,#2*8]
	add x1,x1,#14*8 // advance input pointer
	ldp x7,x8,[x0,#4*8]
	add x20,x20,x3
	ldp x9,x10,[x0,#6*8]
	add x21,x21,x4
	add x22,x22,x5
	add x23,x23,x6
	stp x20,x21,[x0]
	add x24,x24,x7
	add x25,x25,x8
	stp x22,x23,[x0,#2*8]
	add x26,x26,x9
	add x27,x27,x10
	cmp x1,x2
	stp x24,x25,[x0,#4*8]
	stp x26,x27,[x0,#6*8]
	b.ne .Loop

	ldp x19,x20,[x29,#16]
	add sp,sp,#4*8
	ldp x21,x22,[x29,#32]
	ldp x23,x24,[x29,#48]
	ldp x25,x26,[x29,#64]
	ldp x27,x28,[x29,#80]
	ldp x29,x30,[sp],#128
	ret
	.size zfs_sha512_block_armv7,.-zfs_sha512_block_armv7


	.globl zfs_sha512_block_armv8
	.type zfs_sha512_block_armv8,%function
	.align 6
	zfs_sha512_block_armv8:
	hint #34 // bti c
	.Lv8_entry:
	// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later
	stp x29,x30,[sp,#-16]!
	add x29,sp,#0

	ld1 {v16.16b-v19.16b},[x1],#64 // load input
	ld1 {v20.16b-v23.16b},[x1],#64

	ld1 {v0.2d-v3.2d},[x0] // load context
	adr x3,.LK512

	rev64 v16.16b,v16.16b
	rev64 v17.16b,v17.16b
	rev64 v18.16b,v18.16b
	rev64 v19.16b,v19.16b
	rev64 v20.16b,v20.16b
	rev64 v21.16b,v21.16b
	rev64 v22.16b,v22.16b
	rev64 v23.16b,v23.16b
	b .Loop_hw

	.align 4
	.Loop_hw:
	ld1 {v24.2d},[x3],#16
	subs x2,x2,#1
	sub x4,x1,#128
	orr v26.16b,v0.16b,v0.16b // offload
	orr v27.16b,v1.16b,v1.16b
	orr v28.16b,v2.16b,v2.16b
	orr v29.16b,v3.16b,v3.16b
	csel x1,x1,x4,ne // conditional rewind
	add v24.2d,v24.2d,v16.2d
	ld1 {v25.2d},[x3],#16
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v2.16b,v3.16b,#8
	ext v6.16b,v1.16b,v2.16b,#8
	add v3.2d,v3.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xcec08230 //sha512su0 v16.16b,v17.16b
	ext v7.16b,v20.16b,v21.16b,#8
	.inst 0xce6680a3 //sha512h v3.16b,v5.16b,v6.16b
	.inst 0xce678af0 //sha512su1 v16.16b,v23.16b,v7.16b
	add v4.2d,v1.2d,v3.2d // "D + T1"
	.inst 0xce608423 //sha512h2 v3.16b,v1.16b,v0.16b
	add v25.2d,v25.2d,v17.2d
	ld1 {v24.2d},[x3],#16
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v4.16b,v2.16b,#8
	ext v6.16b,v0.16b,v4.16b,#8
	add v2.2d,v2.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xcec08251 //sha512su0 v17.16b,v18.16b
	ext v7.16b,v21.16b,v22.16b,#8
	.inst 0xce6680a2 //sha512h v2.16b,v5.16b,v6.16b
	.inst 0xce678a11 //sha512su1 v17.16b,v16.16b,v7.16b
	add v1.2d,v0.2d,v2.2d // "D + T1"
	.inst 0xce638402 //sha512h2 v2.16b,v0.16b,v3.16b
	add v24.2d,v24.2d,v18.2d
	ld1 {v25.2d},[x3],#16
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v1.16b,v4.16b,#8
	ext v6.16b,v3.16b,v1.16b,#8
	add v4.2d,v4.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xcec08272 //sha512su0 v18.16b,v19.16b
	ext v7.16b,v22.16b,v23.16b,#8
	.inst 0xce6680a4 //sha512h v4.16b,v5.16b,v6.16b
	.inst 0xce678a32 //sha512su1 v18.16b,v17.16b,v7.16b
	add v0.2d,v3.2d,v4.2d // "D + T1"
	.inst 0xce628464 //sha512h2 v4.16b,v3.16b,v2.16b
	add v25.2d,v25.2d,v19.2d
	ld1 {v24.2d},[x3],#16
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v0.16b,v1.16b,#8
	ext v6.16b,v2.16b,v0.16b,#8
	add v1.2d,v1.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xcec08293 //sha512su0 v19.16b,v20.16b
	ext v7.16b,v23.16b,v16.16b,#8
	.inst 0xce6680a1 //sha512h v1.16b,v5.16b,v6.16b
	.inst 0xce678a53 //sha512su1 v19.16b,v18.16b,v7.16b
	add v3.2d,v2.2d,v1.2d // "D + T1"
	.inst 0xce648441 //sha512h2 v1.16b,v2.16b,v4.16b
	add v24.2d,v24.2d,v20.2d
	ld1 {v25.2d},[x3],#16
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v3.16b,v0.16b,#8
	ext v6.16b,v4.16b,v3.16b,#8
	add v0.2d,v0.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xcec082b4 //sha512su0 v20.16b,v21.16b
	ext v7.16b,v16.16b,v17.16b,#8
	.inst 0xce6680a0 //sha512h v0.16b,v5.16b,v6.16b
	.inst 0xce678a74 //sha512su1 v20.16b,v19.16b,v7.16b
	add v2.2d,v4.2d,v0.2d // "D + T1"
	.inst 0xce618480 //sha512h2 v0.16b,v4.16b,v1.16b
	add v25.2d,v25.2d,v21.2d
	ld1 {v24.2d},[x3],#16
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v2.16b,v3.16b,#8
	ext v6.16b,v1.16b,v2.16b,#8
	add v3.2d,v3.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xcec082d5 //sha512su0 v21.16b,v22.16b
	ext v7.16b,v17.16b,v18.16b,#8
	.inst 0xce6680a3 //sha512h v3.16b,v5.16b,v6.16b
	.inst 0xce678a95 //sha512su1 v21.16b,v20.16b,v7.16b
	add v4.2d,v1.2d,v3.2d // "D + T1"
	.inst 0xce608423 //sha512h2 v3.16b,v1.16b,v0.16b
	add v24.2d,v24.2d,v22.2d
	ld1 {v25.2d},[x3],#16
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v4.16b,v2.16b,#8
	ext v6.16b,v0.16b,v4.16b,#8
	add v2.2d,v2.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xcec082f6 //sha512su0 v22.16b,v23.16b
	ext v7.16b,v18.16b,v19.16b,#8
	.inst 0xce6680a2 //sha512h v2.16b,v5.16b,v6.16b
	.inst 0xce678ab6 //sha512su1 v22.16b,v21.16b,v7.16b
	add v1.2d,v0.2d,v2.2d // "D + T1"
	.inst 0xce638402 //sha512h2 v2.16b,v0.16b,v3.16b
	add v25.2d,v25.2d,v23.2d
	ld1 {v24.2d},[x3],#16
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v1.16b,v4.16b,#8
	ext v6.16b,v3.16b,v1.16b,#8
	add v4.2d,v4.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xcec08217 //sha512su0 v23.16b,v16.16b
	ext v7.16b,v19.16b,v20.16b,#8
	.inst 0xce6680a4 //sha512h v4.16b,v5.16b,v6.16b
	.inst 0xce678ad7 //sha512su1 v23.16b,v22.16b,v7.16b
	add v0.2d,v3.2d,v4.2d // "D + T1"
	.inst 0xce628464 //sha512h2 v4.16b,v3.16b,v2.16b
	add v24.2d,v24.2d,v16.2d
	ld1 {v25.2d},[x3],#16
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v0.16b,v1.16b,#8
	ext v6.16b,v2.16b,v0.16b,#8
	add v1.2d,v1.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xcec08230 //sha512su0 v16.16b,v17.16b
	ext v7.16b,v20.16b,v21.16b,#8
	.inst 0xce6680a1 //sha512h v1.16b,v5.16b,v6.16b
	.inst 0xce678af0 //sha512su1 v16.16b,v23.16b,v7.16b
	add v3.2d,v2.2d,v1.2d // "D + T1"
	.inst 0xce648441 //sha512h2 v1.16b,v2.16b,v4.16b
	add v25.2d,v25.2d,v17.2d
	ld1 {v24.2d},[x3],#16
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v3.16b,v0.16b,#8
	ext v6.16b,v4.16b,v3.16b,#8
	add v0.2d,v0.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xcec08251 //sha512su0 v17.16b,v18.16b
	ext v7.16b,v21.16b,v22.16b,#8
	.inst 0xce6680a0 //sha512h v0.16b,v5.16b,v6.16b
	.inst 0xce678a11 //sha512su1 v17.16b,v16.16b,v7.16b
	add v2.2d,v4.2d,v0.2d // "D + T1"
	.inst 0xce618480 //sha512h2 v0.16b,v4.16b,v1.16b
	add v24.2d,v24.2d,v18.2d
	ld1 {v25.2d},[x3],#16
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v2.16b,v3.16b,#8
	ext v6.16b,v1.16b,v2.16b,#8
	add v3.2d,v3.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xcec08272 //sha512su0 v18.16b,v19.16b
	ext v7.16b,v22.16b,v23.16b,#8
	.inst 0xce6680a3 //sha512h v3.16b,v5.16b,v6.16b
	.inst 0xce678a32 //sha512su1 v18.16b,v17.16b,v7.16b
	add v4.2d,v1.2d,v3.2d // "D + T1"
	.inst 0xce608423 //sha512h2 v3.16b,v1.16b,v0.16b
	add v25.2d,v25.2d,v19.2d
	ld1 {v24.2d},[x3],#16
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v4.16b,v2.16b,#8
	ext v6.16b,v0.16b,v4.16b,#8
	add v2.2d,v2.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xcec08293 //sha512su0 v19.16b,v20.16b
	ext v7.16b,v23.16b,v16.16b,#8
	.inst 0xce6680a2 //sha512h v2.16b,v5.16b,v6.16b
	.inst 0xce678a53 //sha512su1 v19.16b,v18.16b,v7.16b
	add v1.2d,v0.2d,v2.2d // "D + T1"
	.inst 0xce638402 //sha512h2 v2.16b,v0.16b,v3.16b
	add v24.2d,v24.2d,v20.2d
	ld1 {v25.2d},[x3],#16
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v1.16b,v4.16b,#8
	ext v6.16b,v3.16b,v1.16b,#8
	add v4.2d,v4.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xcec082b4 //sha512su0 v20.16b,v21.16b
	ext v7.16b,v16.16b,v17.16b,#8
	.inst 0xce6680a4 //sha512h v4.16b,v5.16b,v6.16b
	.inst 0xce678a74 //sha512su1 v20.16b,v19.16b,v7.16b
	add v0.2d,v3.2d,v4.2d // "D + T1"
	.inst 0xce628464 //sha512h2 v4.16b,v3.16b,v2.16b
	add v25.2d,v25.2d,v21.2d
	ld1 {v24.2d},[x3],#16
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v0.16b,v1.16b,#8
	ext v6.16b,v2.16b,v0.16b,#8
	add v1.2d,v1.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xcec082d5 //sha512su0 v21.16b,v22.16b
	ext v7.16b,v17.16b,v18.16b,#8
	.inst 0xce6680a1 //sha512h v1.16b,v5.16b,v6.16b
	.inst 0xce678a95 //sha512su1 v21.16b,v20.16b,v7.16b
	add v3.2d,v2.2d,v1.2d // "D + T1"
	.inst 0xce648441 //sha512h2 v1.16b,v2.16b,v4.16b
	add v24.2d,v24.2d,v22.2d
	ld1 {v25.2d},[x3],#16
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v3.16b,v0.16b,#8
	ext v6.16b,v4.16b,v3.16b,#8
	add v0.2d,v0.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xcec082f6 //sha512su0 v22.16b,v23.16b
	ext v7.16b,v18.16b,v19.16b,#8
	.inst 0xce6680a0 //sha512h v0.16b,v5.16b,v6.16b
	.inst 0xce678ab6 //sha512su1 v22.16b,v21.16b,v7.16b
	add v2.2d,v4.2d,v0.2d // "D + T1"
	.inst 0xce618480 //sha512h2 v0.16b,v4.16b,v1.16b
	add v25.2d,v25.2d,v23.2d
	ld1 {v24.2d},[x3],#16
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v2.16b,v3.16b,#8
	ext v6.16b,v1.16b,v2.16b,#8
	add v3.2d,v3.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xcec08217 //sha512su0 v23.16b,v16.16b
	ext v7.16b,v19.16b,v20.16b,#8
	.inst 0xce6680a3 //sha512h v3.16b,v5.16b,v6.16b
	.inst 0xce678ad7 //sha512su1 v23.16b,v22.16b,v7.16b
	add v4.2d,v1.2d,v3.2d // "D + T1"
	.inst 0xce608423 //sha512h2 v3.16b,v1.16b,v0.16b
	add v24.2d,v24.2d,v16.2d
	ld1 {v25.2d},[x3],#16
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v4.16b,v2.16b,#8
	ext v6.16b,v0.16b,v4.16b,#8
	add v2.2d,v2.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xcec08230 //sha512su0 v16.16b,v17.16b
	ext v7.16b,v20.16b,v21.16b,#8
	.inst 0xce6680a2 //sha512h v2.16b,v5.16b,v6.16b
	.inst 0xce678af0 //sha512su1 v16.16b,v23.16b,v7.16b
	add v1.2d,v0.2d,v2.2d // "D + T1"
	.inst 0xce638402 //sha512h2 v2.16b,v0.16b,v3.16b
	add v25.2d,v25.2d,v17.2d
	ld1 {v24.2d},[x3],#16
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v1.16b,v4.16b,#8
	ext v6.16b,v3.16b,v1.16b,#8
	add v4.2d,v4.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xcec08251 //sha512su0 v17.16b,v18.16b
	ext v7.16b,v21.16b,v22.16b,#8
	.inst 0xce6680a4 //sha512h v4.16b,v5.16b,v6.16b
	.inst 0xce678a11 //sha512su1 v17.16b,v16.16b,v7.16b
	add v0.2d,v3.2d,v4.2d // "D + T1"
	.inst 0xce628464 //sha512h2 v4.16b,v3.16b,v2.16b
	add v24.2d,v24.2d,v18.2d
	ld1 {v25.2d},[x3],#16
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v0.16b,v1.16b,#8
	ext v6.16b,v2.16b,v0.16b,#8
	add v1.2d,v1.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xcec08272 //sha512su0 v18.16b,v19.16b
	ext v7.16b,v22.16b,v23.16b,#8
	.inst 0xce6680a1 //sha512h v1.16b,v5.16b,v6.16b
	.inst 0xce678a32 //sha512su1 v18.16b,v17.16b,v7.16b
	add v3.2d,v2.2d,v1.2d // "D + T1"
	.inst 0xce648441 //sha512h2 v1.16b,v2.16b,v4.16b
	add v25.2d,v25.2d,v19.2d
	ld1 {v24.2d},[x3],#16
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v3.16b,v0.16b,#8
	ext v6.16b,v4.16b,v3.16b,#8
	add v0.2d,v0.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xcec08293 //sha512su0 v19.16b,v20.16b
	ext v7.16b,v23.16b,v16.16b,#8
	.inst 0xce6680a0 //sha512h v0.16b,v5.16b,v6.16b
	.inst 0xce678a53 //sha512su1 v19.16b,v18.16b,v7.16b
	add v2.2d,v4.2d,v0.2d // "D + T1"
	.inst 0xce618480 //sha512h2 v0.16b,v4.16b,v1.16b
	add v24.2d,v24.2d,v20.2d
	ld1 {v25.2d},[x3],#16
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v2.16b,v3.16b,#8
	ext v6.16b,v1.16b,v2.16b,#8
	add v3.2d,v3.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xcec082b4 //sha512su0 v20.16b,v21.16b
	ext v7.16b,v16.16b,v17.16b,#8
	.inst 0xce6680a3 //sha512h v3.16b,v5.16b,v6.16b
	.inst 0xce678a74 //sha512su1 v20.16b,v19.16b,v7.16b
	add v4.2d,v1.2d,v3.2d // "D + T1"
	.inst 0xce608423 //sha512h2 v3.16b,v1.16b,v0.16b
	add v25.2d,v25.2d,v21.2d
	ld1 {v24.2d},[x3],#16
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v4.16b,v2.16b,#8
	ext v6.16b,v0.16b,v4.16b,#8
	add v2.2d,v2.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xcec082d5 //sha512su0 v21.16b,v22.16b
	ext v7.16b,v17.16b,v18.16b,#8
	.inst 0xce6680a2 //sha512h v2.16b,v5.16b,v6.16b
	.inst 0xce678a95 //sha512su1 v21.16b,v20.16b,v7.16b
	add v1.2d,v0.2d,v2.2d // "D + T1"
	.inst 0xce638402 //sha512h2 v2.16b,v0.16b,v3.16b
	add v24.2d,v24.2d,v22.2d
	ld1 {v25.2d},[x3],#16
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v1.16b,v4.16b,#8
	ext v6.16b,v3.16b,v1.16b,#8
	add v4.2d,v4.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xcec082f6 //sha512su0 v22.16b,v23.16b
	ext v7.16b,v18.16b,v19.16b,#8
	.inst 0xce6680a4 //sha512h v4.16b,v5.16b,v6.16b
	.inst 0xce678ab6 //sha512su1 v22.16b,v21.16b,v7.16b
	add v0.2d,v3.2d,v4.2d // "D + T1"
	.inst 0xce628464 //sha512h2 v4.16b,v3.16b,v2.16b
	add v25.2d,v25.2d,v23.2d
	ld1 {v24.2d},[x3],#16
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v0.16b,v1.16b,#8
	ext v6.16b,v2.16b,v0.16b,#8
	add v1.2d,v1.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xcec08217 //sha512su0 v23.16b,v16.16b
	ext v7.16b,v19.16b,v20.16b,#8
	.inst 0xce6680a1 //sha512h v1.16b,v5.16b,v6.16b
	.inst 0xce678ad7 //sha512su1 v23.16b,v22.16b,v7.16b
	add v3.2d,v2.2d,v1.2d // "D + T1"
	.inst 0xce648441 //sha512h2 v1.16b,v2.16b,v4.16b
	add v24.2d,v24.2d,v16.2d
	ld1 {v25.2d},[x3],#16
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v3.16b,v0.16b,#8
	ext v6.16b,v4.16b,v3.16b,#8
	add v0.2d,v0.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xcec08230 //sha512su0 v16.16b,v17.16b
	ext v7.16b,v20.16b,v21.16b,#8
	.inst 0xce6680a0 //sha512h v0.16b,v5.16b,v6.16b
	.inst 0xce678af0 //sha512su1 v16.16b,v23.16b,v7.16b
	add v2.2d,v4.2d,v0.2d // "D + T1"
	.inst 0xce618480 //sha512h2 v0.16b,v4.16b,v1.16b
	add v25.2d,v25.2d,v17.2d
	ld1 {v24.2d},[x3],#16
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v2.16b,v3.16b,#8
	ext v6.16b,v1.16b,v2.16b,#8
	add v3.2d,v3.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xcec08251 //sha512su0 v17.16b,v18.16b
	ext v7.16b,v21.16b,v22.16b,#8
	.inst 0xce6680a3 //sha512h v3.16b,v5.16b,v6.16b
	.inst 0xce678a11 //sha512su1 v17.16b,v16.16b,v7.16b
	add v4.2d,v1.2d,v3.2d // "D + T1"
	.inst 0xce608423 //sha512h2 v3.16b,v1.16b,v0.16b
	add v24.2d,v24.2d,v18.2d
	ld1 {v25.2d},[x3],#16
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v4.16b,v2.16b,#8
	ext v6.16b,v0.16b,v4.16b,#8
	add v2.2d,v2.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xcec08272 //sha512su0 v18.16b,v19.16b
	ext v7.16b,v22.16b,v23.16b,#8
	.inst 0xce6680a2 //sha512h v2.16b,v5.16b,v6.16b
	.inst 0xce678a32 //sha512su1 v18.16b,v17.16b,v7.16b
	add v1.2d,v0.2d,v2.2d // "D + T1"
	.inst 0xce638402 //sha512h2 v2.16b,v0.16b,v3.16b
	add v25.2d,v25.2d,v19.2d
	ld1 {v24.2d},[x3],#16
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v1.16b,v4.16b,#8
	ext v6.16b,v3.16b,v1.16b,#8
	add v4.2d,v4.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xcec08293 //sha512su0 v19.16b,v20.16b
	ext v7.16b,v23.16b,v16.16b,#8
	.inst 0xce6680a4 //sha512h v4.16b,v5.16b,v6.16b
	.inst 0xce678a53 //sha512su1 v19.16b,v18.16b,v7.16b
	add v0.2d,v3.2d,v4.2d // "D + T1"
	.inst 0xce628464 //sha512h2 v4.16b,v3.16b,v2.16b
	add v24.2d,v24.2d,v20.2d
	ld1 {v25.2d},[x3],#16
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v0.16b,v1.16b,#8
	ext v6.16b,v2.16b,v0.16b,#8
	add v1.2d,v1.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xcec082b4 //sha512su0 v20.16b,v21.16b
	ext v7.16b,v16.16b,v17.16b,#8
	.inst 0xce6680a1 //sha512h v1.16b,v5.16b,v6.16b
	.inst 0xce678a74 //sha512su1 v20.16b,v19.16b,v7.16b
	add v3.2d,v2.2d,v1.2d // "D + T1"
	.inst 0xce648441 //sha512h2 v1.16b,v2.16b,v4.16b
	add v25.2d,v25.2d,v21.2d
	ld1 {v24.2d},[x3],#16
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v3.16b,v0.16b,#8
	ext v6.16b,v4.16b,v3.16b,#8
	add v0.2d,v0.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xcec082d5 //sha512su0 v21.16b,v22.16b
	ext v7.16b,v17.16b,v18.16b,#8
	.inst 0xce6680a0 //sha512h v0.16b,v5.16b,v6.16b
	.inst 0xce678a95 //sha512su1 v21.16b,v20.16b,v7.16b
	add v2.2d,v4.2d,v0.2d // "D + T1"
	.inst 0xce618480 //sha512h2 v0.16b,v4.16b,v1.16b
	add v24.2d,v24.2d,v22.2d
	ld1 {v25.2d},[x3],#16
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v2.16b,v3.16b,#8
	ext v6.16b,v1.16b,v2.16b,#8
	add v3.2d,v3.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xcec082f6 //sha512su0 v22.16b,v23.16b
	ext v7.16b,v18.16b,v19.16b,#8
	.inst 0xce6680a3 //sha512h v3.16b,v5.16b,v6.16b
	.inst 0xce678ab6 //sha512su1 v22.16b,v21.16b,v7.16b
	add v4.2d,v1.2d,v3.2d // "D + T1"
	.inst 0xce608423 //sha512h2 v3.16b,v1.16b,v0.16b
	add v25.2d,v25.2d,v23.2d
	ld1 {v24.2d},[x3],#16
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v4.16b,v2.16b,#8
	ext v6.16b,v0.16b,v4.16b,#8
	add v2.2d,v2.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xcec08217 //sha512su0 v23.16b,v16.16b
	ext v7.16b,v19.16b,v20.16b,#8
	.inst 0xce6680a2 //sha512h v2.16b,v5.16b,v6.16b
	.inst 0xce678ad7 //sha512su1 v23.16b,v22.16b,v7.16b
	add v1.2d,v0.2d,v2.2d // "D + T1"
	.inst 0xce638402 //sha512h2 v2.16b,v0.16b,v3.16b
	ld1 {v25.2d},[x3],#16
	add v24.2d,v24.2d,v16.2d
	ld1 {v16.16b},[x1],#16 // load next input
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v1.16b,v4.16b,#8
	ext v6.16b,v3.16b,v1.16b,#8
	add v4.2d,v4.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xce6680a4 //sha512h v4.16b,v5.16b,v6.16b
	rev64 v16.16b,v16.16b
	add v0.2d,v3.2d,v4.2d // "D + T1"
	.inst 0xce628464 //sha512h2 v4.16b,v3.16b,v2.16b
	ld1 {v24.2d},[x3],#16
	add v25.2d,v25.2d,v17.2d
	ld1 {v17.16b},[x1],#16 // load next input
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v0.16b,v1.16b,#8
	ext v6.16b,v2.16b,v0.16b,#8
	add v1.2d,v1.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xce6680a1 //sha512h v1.16b,v5.16b,v6.16b
	rev64 v17.16b,v17.16b
	add v3.2d,v2.2d,v1.2d // "D + T1"
	.inst 0xce648441 //sha512h2 v1.16b,v2.16b,v4.16b
	ld1 {v25.2d},[x3],#16
	add v24.2d,v24.2d,v18.2d
	ld1 {v18.16b},[x1],#16 // load next input
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v3.16b,v0.16b,#8
	ext v6.16b,v4.16b,v3.16b,#8
	add v0.2d,v0.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xce6680a0 //sha512h v0.16b,v5.16b,v6.16b
	rev64 v18.16b,v18.16b
	add v2.2d,v4.2d,v0.2d // "D + T1"
	.inst 0xce618480 //sha512h2 v0.16b,v4.16b,v1.16b
	ld1 {v24.2d},[x3],#16
	add v25.2d,v25.2d,v19.2d
	ld1 {v19.16b},[x1],#16 // load next input
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v2.16b,v3.16b,#8
	ext v6.16b,v1.16b,v2.16b,#8
	add v3.2d,v3.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xce6680a3 //sha512h v3.16b,v5.16b,v6.16b
	rev64 v19.16b,v19.16b
	add v4.2d,v1.2d,v3.2d // "D + T1"
	.inst 0xce608423 //sha512h2 v3.16b,v1.16b,v0.16b
	ld1 {v25.2d},[x3],#16
	add v24.2d,v24.2d,v20.2d
	ld1 {v20.16b},[x1],#16 // load next input
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v4.16b,v2.16b,#8
	ext v6.16b,v0.16b,v4.16b,#8
	add v2.2d,v2.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xce6680a2 //sha512h v2.16b,v5.16b,v6.16b
	rev64 v20.16b,v20.16b
	add v1.2d,v0.2d,v2.2d // "D + T1"
	.inst 0xce638402 //sha512h2 v2.16b,v0.16b,v3.16b
	ld1 {v24.2d},[x3],#16
	add v25.2d,v25.2d,v21.2d
	ld1 {v21.16b},[x1],#16 // load next input
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v1.16b,v4.16b,#8
	ext v6.16b,v3.16b,v1.16b,#8
	add v4.2d,v4.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xce6680a4 //sha512h v4.16b,v5.16b,v6.16b
	rev64 v21.16b,v21.16b
	add v0.2d,v3.2d,v4.2d // "D + T1"
	.inst 0xce628464 //sha512h2 v4.16b,v3.16b,v2.16b
	ld1 {v25.2d},[x3],#16
	add v24.2d,v24.2d,v22.2d
	ld1 {v22.16b},[x1],#16 // load next input
	ext v24.16b,v24.16b,v24.16b,#8
	ext v5.16b,v0.16b,v1.16b,#8
	ext v6.16b,v2.16b,v0.16b,#8
	add v1.2d,v1.2d,v24.2d // "T1 + H + K512[i]"
	.inst 0xce6680a1 //sha512h v1.16b,v5.16b,v6.16b
	rev64 v22.16b,v22.16b
	add v3.2d,v2.2d,v1.2d // "D + T1"
	.inst 0xce648441 //sha512h2 v1.16b,v2.16b,v4.16b
	sub x3,x3,#80*8 // rewind
	add v25.2d,v25.2d,v23.2d
	ld1 {v23.16b},[x1],#16 // load next input
	ext v25.16b,v25.16b,v25.16b,#8
	ext v5.16b,v3.16b,v0.16b,#8
	ext v6.16b,v4.16b,v3.16b,#8
	add v0.2d,v0.2d,v25.2d // "T1 + H + K512[i]"
	.inst 0xce6680a0 //sha512h v0.16b,v5.16b,v6.16b
	rev64 v23.16b,v23.16b
	add v2.2d,v4.2d,v0.2d // "D + T1"
	.inst 0xce618480 //sha512h2 v0.16b,v4.16b,v1.16b
	add v0.2d,v0.2d,v26.2d // accumulate
	add v1.2d,v1.2d,v27.2d
	add v2.2d,v2.2d,v28.2d
	add v3.2d,v3.2d,v29.2d

	cbnz x2,.Loop_hw

	st1 {v0.2d-v3.2d},[x0] // store context

	ldr x29,[sp],#16
	ret
	.size zfs_sha512_block_armv8,.-zfs_sha512_block_armv8
	#endif
	diff --git a/sys/contrib/openzfs/module/nvpair/nvpair.c b/sys/contrib/openzfs/module/nvpair/nvpair.c
	index d9449e47e87a..887f7d32df4a 100644
	--- a/sys/contrib/openzfs/module/nvpair/nvpair.c
	+++ b/sys/contrib/openzfs/module/nvpair/nvpair.c
	@@ -1,3796 +1,3797 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2015, 2017 by Delphix. All rights reserved.
	* Copyright 2018 RackTop Systems.
	*/

	/*
	* Links to Illumos.org for more information on Interface Libraries:
	* [1] https://illumos.org/man/3lib/libnvpair
	* [2] https://illumos.org/man/3nvpair/nvlist_alloc
	* [3] https://illumos.org/man/9f/nvlist_alloc
	* [4] https://illumos.org/man/9f/nvlist_next_nvpair
	* [5] https://illumos.org/man/9f/nvpair_value_byte
	*/

	#include <sys/debug.h>
	#include <sys/isa_defs.h>
	#include <sys/nvpair.h>
	#include <sys/nvpair_impl.h>
	#include <sys/types.h>
	#include <sys/param.h>
	#include <sys/string.h>
	+#include <rpc/types.h>
	#include <rpc/xdr.h>
	#include <sys/mod.h>

	#if defined(_KERNEL)
	#include <sys/sunddi.h>
	#include <sys/sysmacros.h>
	#else
	#include <stdarg.h>
	#include <stdlib.h>
	#include <stddef.h>
	#endif

	#define skip_whitespace(p) while (((p) == ' ') \|\| ((p) == '\t')) (p)++

	/*
	* nvpair.c - Provides kernel & userland interfaces for manipulating
	* name-value pairs.
	*
	* Overview Diagram
	*
	* +--------------+
	* \| nvlist_t \|
	* \|--------------\|
	* \| nvl_version \|
	* \| nvl_nvflag \|
	* \| nvl_priv -+-+
	* \| nvl_flag \| \|
	* \| nvl_pad \| \|
	* +--------------+ \|
	* V
	* +--------------+ last i_nvp in list
	* \| nvpriv_t \| +--------------------->
	* \|--------------\| \|
	* +--+- nvp_list \| \| +------------+
	* \| \| nvp_last -+--+ + nv_alloc_t \|
	* \| \| nvp_curr \| \|------------\|
	* \| \| nvp_nva -+----> \| nva_ops \|
	* \| \| nvp_stat \| \| nva_arg \|
	* \| +--------------+ +------------+
	* \|
	* +-------+
	* V
	* +---------------------+ +-------------------+
	* \| i_nvp_t \| +-->\| i_nvp_t \| +-->
	* \|---------------------\| \| \|-------------------\| \|
	* \| nvi_next -+--+ \| nvi_next -+--+
	* \| nvi_prev (NULL) \| <----+ nvi_prev \|
	* \| . . . . . . . . . . \| \| . . . . . . . . . \|
	* \| nvp (nvpair_t) \| \| nvp (nvpair_t) \|
	* \| - nvp_size \| \| - nvp_size \|
	* \| - nvp_name_sz \| \| - nvp_name_sz \|
	* \| - nvp_value_elem \| \| - nvp_value_elem \|
	* \| - nvp_type \| \| - nvp_type \|
	* \| - data ... \| \| - data ... \|
	* +---------------------+ +-------------------+
	*
	*
	*
	* +---------------------+ +---------------------+
	* \| i_nvp_t \| +--> +-->\| i_nvp_t (last) \|
	* \|---------------------\| \| \| \|---------------------\|
	* \| nvi_next -+--+ ... --+ \| nvi_next (NULL) \|
	* <-+- nvi_prev \|<-- ... <----+ nvi_prev \|
	* \| . . . . . . . . . \| \| . . . . . . . . . \|
	* \| nvp (nvpair_t) \| \| nvp (nvpair_t) \|
	* \| - nvp_size \| \| - nvp_size \|
	* \| - nvp_name_sz \| \| - nvp_name_sz \|
	* \| - nvp_value_elem \| \| - nvp_value_elem \|
	* \| - DATA_TYPE_NVLIST \| \| - nvp_type \|
	* \| - data (embedded) \| \| - data ... \|
	* \| nvlist name \| +---------------------+
	* \| +--------------+ \|
	* \| \| nvlist_t \| \|
	* \| \|--------------\| \|
	* \| \| nvl_version \| \|
	* \| \| nvl_nvflag \| \|
	* \| \| nvl_priv --+---+---->
	* \| \| nvl_flag \| \|
	* \| \| nvl_pad \| \|
	* \| +--------------+ \|
	* +---------------------+
	*
	*
	* N.B. nvpair_t may be aligned on 4 byte boundary, so +4 will
	* allow value to be aligned on 8 byte boundary
	*
	* name_len is the length of the name string including the null terminator
	* so it must be >= 1
	*/
	#define NVP_SIZE_CALC(name_len, data_len) \
	(NV_ALIGN((sizeof (nvpair_t)) + name_len) + NV_ALIGN(data_len))

	static int i_get_value_size(data_type_t type, const void *data, uint_t nelem);
	static int nvlist_add_common(nvlist_t nvl, const char name, data_type_t type,
	uint_t nelem, const void *data);

	#define NV_STAT_EMBEDDED 0x1
	#define EMBEDDED_NVL(nvp) ((nvlist_t )(void )NVP_VALUE(nvp))
	#define EMBEDDED_NVL_ARRAY(nvp) ((nvlist_t *)(void )NVP_VALUE(nvp))

	#define NVP_VALOFF(nvp) (NV_ALIGN(sizeof (nvpair_t) + (nvp)->nvp_name_sz))
	#define NVPAIR2I_NVP(nvp) \
	((i_nvp_t *)((size_t)(nvp) - offsetof(i_nvp_t, nvi_nvp)))

	#ifdef _KERNEL
	static const int nvpair_max_recursion = 20;
	#else
	static const int nvpair_max_recursion = 100;
	#endif

	static const uint64_t nvlist_hashtable_init_size = (1 << 4);

	int
	nv_alloc_init(nv_alloc_t nva, const nv_alloc_ops_t nvo, /* args */ ...)
	{
	va_list valist;
	int err = 0;

	nva->nva_ops = nvo;
	nva->nva_arg = NULL;

	va_start(valist, nvo);
	if (nva->nva_ops->nv_ao_init != NULL)
	err = nva->nva_ops->nv_ao_init(nva, valist);
	va_end(valist);

	return (err);
	}

	void
	nv_alloc_reset(nv_alloc_t *nva)
	{
	if (nva->nva_ops->nv_ao_reset != NULL)
	nva->nva_ops->nv_ao_reset(nva);
	}

	void
	nv_alloc_fini(nv_alloc_t *nva)
	{
	if (nva->nva_ops->nv_ao_fini != NULL)
	nva->nva_ops->nv_ao_fini(nva);
	}

	nv_alloc_t *
	nvlist_lookup_nv_alloc(nvlist_t *nvl)
	{
	nvpriv_t *priv;

	if (nvl == NULL \|\|
	(priv = (nvpriv_t *)(uintptr_t)nvl->nvl_priv) == NULL)
	return (NULL);

	return (priv->nvp_nva);
	}

	static void *
	nv_mem_zalloc(nvpriv_t *nvp, size_t size)
	{
	nv_alloc_t *nva = nvp->nvp_nva;
	void *buf;

	if ((buf = nva->nva_ops->nv_ao_alloc(nva, size)) != NULL)
	memset(buf, 0, size);

	return (buf);
	}

	static void
	nv_mem_free(nvpriv_t nvp, void buf, size_t size)
	{
	nv_alloc_t *nva = nvp->nvp_nva;

	nva->nva_ops->nv_ao_free(nva, buf, size);
	}

	static void
	nv_priv_init(nvpriv_t priv, nv_alloc_t nva, uint32_t stat)
	{
	memset(priv, 0, sizeof (nvpriv_t));

	priv->nvp_nva = nva;
	priv->nvp_stat = stat;
	}

	static nvpriv_t *
	nv_priv_alloc(nv_alloc_t *nva)
	{
	nvpriv_t *priv;

	/*
	* nv_mem_alloc() cannot called here because it needs the priv
	* argument.
	*/
	if ((priv = nva->nva_ops->nv_ao_alloc(nva, sizeof (nvpriv_t))) == NULL)
	return (NULL);

	nv_priv_init(priv, nva, 0);

	return (priv);
	}

	/*
	* Embedded lists need their own nvpriv_t's. We create a new
	* nvpriv_t using the parameters and allocator from the parent
	* list's nvpriv_t.
	*/
	static nvpriv_t *
	nv_priv_alloc_embedded(nvpriv_t *priv)
	{
	nvpriv_t *emb_priv;

	if ((emb_priv = nv_mem_zalloc(priv, sizeof (nvpriv_t))) == NULL)
	return (NULL);

	nv_priv_init(emb_priv, priv->nvp_nva, NV_STAT_EMBEDDED);

	return (emb_priv);
	}

	static int
	nvt_tab_alloc(nvpriv_t *priv, uint64_t buckets)
	{
	ASSERT3P(priv->nvp_hashtable, ==, NULL);
	ASSERT0(priv->nvp_nbuckets);
	ASSERT0(priv->nvp_nentries);

	i_nvp_t *tab = nv_mem_zalloc(priv, buckets sizeof (i_nvp_t *));
	if (tab == NULL)
	return (ENOMEM);

	priv->nvp_hashtable = tab;
	priv->nvp_nbuckets = buckets;
	return (0);
	}

	static void
	nvt_tab_free(nvpriv_t *priv)
	{
	i_nvp_t **tab = priv->nvp_hashtable;
	if (tab == NULL) {
	ASSERT0(priv->nvp_nbuckets);
	ASSERT0(priv->nvp_nentries);
	return;
	}

	nv_mem_free(priv, tab, priv->nvp_nbuckets * sizeof (i_nvp_t *));

	priv->nvp_hashtable = NULL;
	priv->nvp_nbuckets = 0;
	priv->nvp_nentries = 0;
	}

	static uint32_t
	nvt_hash(const char *p)
	{
	uint32_t g, hval = 0;

	while (*p) {
	hval = (hval << 4) + *p++;
	if ((g = (hval & 0xf0000000)) != 0)
	hval ^= g >> 24;
	hval &= ~g;
	}
	return (hval);
	}

	static boolean_t
	nvt_nvpair_match(const nvpair_t nvp1, const nvpair_t nvp2, uint32_t nvflag)
	{
	boolean_t match = B_FALSE;
	if (nvflag & NV_UNIQUE_NAME_TYPE) {
	if (strcmp(NVP_NAME(nvp1), NVP_NAME(nvp2)) == 0 &&
	NVP_TYPE(nvp1) == NVP_TYPE(nvp2))
	match = B_TRUE;
	} else {
	ASSERT(nvflag == 0 \|\| nvflag & NV_UNIQUE_NAME);
	if (strcmp(NVP_NAME(nvp1), NVP_NAME(nvp2)) == 0)
	match = B_TRUE;
	}
	return (match);
	}

	static nvpair_t *
	nvt_lookup_name_type(const nvlist_t nvl, const char name, data_type_t type)
	{
	const nvpriv_t priv = (const nvpriv_t )(uintptr_t)nvl->nvl_priv;
	ASSERT(priv != NULL);

	i_nvp_t **tab = priv->nvp_hashtable;

	if (tab == NULL) {
	ASSERT3P(priv->nvp_list, ==, NULL);
	ASSERT0(priv->nvp_nbuckets);
	ASSERT0(priv->nvp_nentries);
	return (NULL);
	} else {
	ASSERT(priv->nvp_nbuckets != 0);
	}

	uint64_t hash = nvt_hash(name);
	uint64_t index = hash & (priv->nvp_nbuckets - 1);

	ASSERT3U(index, <, priv->nvp_nbuckets);
	i_nvp_t *entry = tab[index];

	for (i_nvp_t *e = entry; e != NULL; e = e->nvi_hashtable_next) {
	if (strcmp(NVP_NAME(&e->nvi_nvp), name) == 0 &&
	(type == DATA_TYPE_DONTCARE \|\|
	NVP_TYPE(&e->nvi_nvp) == type))
	return (&e->nvi_nvp);
	}
	return (NULL);
	}

	static nvpair_t *
	nvt_lookup_name(const nvlist_t nvl, const char name)
	{
	return (nvt_lookup_name_type(nvl, name, DATA_TYPE_DONTCARE));
	}

	static int
	nvt_resize(nvpriv_t *priv, uint32_t new_size)
	{
	i_nvp_t **tab = priv->nvp_hashtable;

	/*
	* Migrate all the entries from the current table
	* to a newly-allocated table with the new size by
	* re-adjusting the pointers of their entries.
	*/
	uint32_t size = priv->nvp_nbuckets;
	uint32_t new_mask = new_size - 1;
	ASSERT(ISP2(new_size));

	i_nvp_t *new_tab = nv_mem_zalloc(priv, new_size sizeof (i_nvp_t *));
	if (new_tab == NULL)
	return (ENOMEM);

	uint32_t nentries = 0;
	for (uint32_t i = 0; i < size; i++) {
	i_nvp_t next, e = tab[i];

	while (e != NULL) {
	next = e->nvi_hashtable_next;

	uint32_t hash = nvt_hash(NVP_NAME(&e->nvi_nvp));
	uint32_t index = hash & new_mask;

	e->nvi_hashtable_next = new_tab[index];
	new_tab[index] = e;
	nentries++;

	e = next;
	}
	tab[i] = NULL;
	}
	ASSERT3U(nentries, ==, priv->nvp_nentries);

	nvt_tab_free(priv);

	priv->nvp_hashtable = new_tab;
	priv->nvp_nbuckets = new_size;
	priv->nvp_nentries = nentries;

	return (0);
	}

	static boolean_t
	nvt_needs_togrow(nvpriv_t *priv)
	{
	/*
	* Grow only when we have more elements than buckets
	* and the # of buckets doesn't overflow.
	*/
	return (priv->nvp_nentries > priv->nvp_nbuckets &&
	(UINT32_MAX >> 1) >= priv->nvp_nbuckets);
	}

	/*
	* Allocate a new table that's twice the size of the old one,
	* and migrate all the entries from the old one to the new
	* one by re-adjusting their pointers.
	*/
	static int
	nvt_grow(nvpriv_t *priv)
	{
	uint32_t current_size = priv->nvp_nbuckets;
	/* ensure we won't overflow */
	ASSERT3U(UINT32_MAX >> 1, >=, current_size);
	return (nvt_resize(priv, current_size << 1));
	}

	static boolean_t
	nvt_needs_toshrink(nvpriv_t *priv)
	{
	/*
	* Shrink only when the # of elements is less than or
	* equal to 1/4 the # of buckets. Never shrink less than
	* nvlist_hashtable_init_size.
	*/
	ASSERT3U(priv->nvp_nbuckets, >=, nvlist_hashtable_init_size);
	if (priv->nvp_nbuckets == nvlist_hashtable_init_size)
	return (B_FALSE);
	return (priv->nvp_nentries <= (priv->nvp_nbuckets >> 2));
	}

	/*
	* Allocate a new table that's half the size of the old one,
	* and migrate all the entries from the old one to the new
	* one by re-adjusting their pointers.
	*/
	static int
	nvt_shrink(nvpriv_t *priv)
	{
	uint32_t current_size = priv->nvp_nbuckets;
	/* ensure we won't overflow */
	ASSERT3U(current_size, >=, nvlist_hashtable_init_size);
	return (nvt_resize(priv, current_size >> 1));
	}

	static int
	nvt_remove_nvpair(nvlist_t nvl, const nvpair_t nvp)
	{
	nvpriv_t priv = (nvpriv_t )(uintptr_t)nvl->nvl_priv;

	if (nvt_needs_toshrink(priv)) {
	int err = nvt_shrink(priv);
	if (err != 0)
	return (err);
	}
	i_nvp_t **tab = priv->nvp_hashtable;

	const char *name = NVP_NAME(nvp);
	uint64_t hash = nvt_hash(name);
	uint64_t index = hash & (priv->nvp_nbuckets - 1);

	ASSERT3U(index, <, priv->nvp_nbuckets);
	i_nvp_t *bucket = tab[index];

	for (i_nvp_t prev = NULL, e = bucket;
	e != NULL; prev = e, e = e->nvi_hashtable_next) {
	if (nvt_nvpair_match(&e->nvi_nvp, nvp, nvl->nvl_nvflag)) {
	if (prev != NULL) {
	prev->nvi_hashtable_next =
	e->nvi_hashtable_next;
	} else {
	ASSERT3P(e, ==, bucket);
	tab[index] = e->nvi_hashtable_next;
	}
	e->nvi_hashtable_next = NULL;
	priv->nvp_nentries--;
	break;
	}
	}

	return (0);
	}

	static int
	nvt_add_nvpair(nvlist_t nvl, nvpair_t nvp)
	{
	nvpriv_t priv = (nvpriv_t )(uintptr_t)nvl->nvl_priv;

	/* initialize nvpair table now if it doesn't exist. */
	if (priv->nvp_hashtable == NULL) {
	int err = nvt_tab_alloc(priv, nvlist_hashtable_init_size);
	if (err != 0)
	return (err);
	}

	/*
	* if we don't allow duplicate entries, make sure to
	* unlink any existing entries from the table.
	*/
	if (nvl->nvl_nvflag != 0) {
	int err = nvt_remove_nvpair(nvl, nvp);
	if (err != 0)
	return (err);
	}

	if (nvt_needs_togrow(priv)) {
	int err = nvt_grow(priv);
	if (err != 0)
	return (err);
	}
	i_nvp_t **tab = priv->nvp_hashtable;

	const char *name = NVP_NAME(nvp);
	uint64_t hash = nvt_hash(name);
	uint64_t index = hash & (priv->nvp_nbuckets - 1);

	ASSERT3U(index, <, priv->nvp_nbuckets);
	// cppcheck-suppress nullPointerRedundantCheck
	i_nvp_t *bucket = tab[index];

	/* insert link at the beginning of the bucket */
	i_nvp_t *new_entry = NVPAIR2I_NVP(nvp);
	ASSERT3P(new_entry->nvi_hashtable_next, ==, NULL);
	new_entry->nvi_hashtable_next = bucket;
	// cppcheck-suppress nullPointerRedundantCheck
	tab[index] = new_entry;

	priv->nvp_nentries++;
	return (0);
	}

	static void
	nvlist_init(nvlist_t nvl, uint32_t nvflag, nvpriv_t priv)
	{
	nvl->nvl_version = NV_VERSION;
	nvl->nvl_nvflag = nvflag & (NV_UNIQUE_NAME\|NV_UNIQUE_NAME_TYPE);
	nvl->nvl_priv = (uint64_t)(uintptr_t)priv;
	nvl->nvl_flag = 0;
	nvl->nvl_pad = 0;
	}

	uint_t
	nvlist_nvflag(nvlist_t *nvl)
	{
	return (nvl->nvl_nvflag);
	}

	static nv_alloc_t *
	nvlist_nv_alloc(int kmflag)
	{
	#if defined(_KERNEL)
	switch (kmflag) {
	case KM_SLEEP:
	return (nv_alloc_sleep);
	case KM_NOSLEEP:
	return (nv_alloc_nosleep);
	default:
	return (nv_alloc_pushpage);
	}
	#else
	(void) kmflag;
	return (nv_alloc_nosleep);
	#endif /* _KERNEL */
	}

	/*
	* nvlist_alloc - Allocate nvlist.
	*/
	int
	nvlist_alloc(nvlist_t **nvlp, uint_t nvflag, int kmflag)
	{
	return (nvlist_xalloc(nvlp, nvflag, nvlist_nv_alloc(kmflag)));
	}

	int
	nvlist_xalloc(nvlist_t *nvlp, uint_t nvflag, nv_alloc_t nva)
	{
	nvpriv_t *priv;

	if (nvlp == NULL \|\| nva == NULL)
	return (EINVAL);

	if ((priv = nv_priv_alloc(nva)) == NULL)
	return (ENOMEM);

	if ((*nvlp = nv_mem_zalloc(priv,
	NV_ALIGN(sizeof (nvlist_t)))) == NULL) {
	nv_mem_free(priv, priv, sizeof (nvpriv_t));
	return (ENOMEM);
	}

	nvlist_init(*nvlp, nvflag, priv);

	return (0);
	}

	/*
	* nvp_buf_alloc - Allocate i_nvp_t for storing a new nv pair.
	*/
	static nvpair_t *
	nvp_buf_alloc(nvlist_t *nvl, size_t len)
	{
	nvpriv_t priv = (nvpriv_t )(uintptr_t)nvl->nvl_priv;
	i_nvp_t *buf;
	nvpair_t *nvp;
	size_t nvsize;

	/*
	* Allocate the buffer
	*/
	nvsize = len + offsetof(i_nvp_t, nvi_nvp);

	if ((buf = nv_mem_zalloc(priv, nvsize)) == NULL)
	return (NULL);

	nvp = &buf->nvi_nvp;
	nvp->nvp_size = len;

	return (nvp);
	}

	/*
	* nvp_buf_free - de-Allocate an i_nvp_t.
	*/
	static void
	nvp_buf_free(nvlist_t nvl, nvpair_t nvp)
	{
	nvpriv_t priv = (nvpriv_t )(uintptr_t)nvl->nvl_priv;
	size_t nvsize = nvp->nvp_size + offsetof(i_nvp_t, nvi_nvp);

	nv_mem_free(priv, NVPAIR2I_NVP(nvp), nvsize);
	}

	/*
	* nvp_buf_link - link a new nv pair into the nvlist.
	*/
	static void
	nvp_buf_link(nvlist_t nvl, nvpair_t nvp)
	{
	nvpriv_t priv = (nvpriv_t )(uintptr_t)nvl->nvl_priv;
	i_nvp_t *curr = NVPAIR2I_NVP(nvp);

	/* Put element at end of nvlist */
	if (priv->nvp_list == NULL) {
	priv->nvp_list = priv->nvp_last = curr;
	} else {
	curr->nvi_prev = priv->nvp_last;
	priv->nvp_last->nvi_next = curr;
	priv->nvp_last = curr;
	}
	}

	/*
	* nvp_buf_unlink - unlink an removed nvpair out of the nvlist.
	*/
	static void
	nvp_buf_unlink(nvlist_t nvl, nvpair_t nvp)
	{
	nvpriv_t priv = (nvpriv_t )(uintptr_t)nvl->nvl_priv;
	i_nvp_t *curr = NVPAIR2I_NVP(nvp);

	/*
	* protect nvlist_next_nvpair() against walking on freed memory.
	*/
	if (priv->nvp_curr == curr)
	priv->nvp_curr = curr->nvi_next;

	if (curr == priv->nvp_list)
	priv->nvp_list = curr->nvi_next;
	else
	curr->nvi_prev->nvi_next = curr->nvi_next;

	if (curr == priv->nvp_last)
	priv->nvp_last = curr->nvi_prev;
	else
	curr->nvi_next->nvi_prev = curr->nvi_prev;
	}

	/*
	* take a nvpair type and number of elements and make sure the are valid
	*/
	static int
	i_validate_type_nelem(data_type_t type, uint_t nelem)
	{
	switch (type) {
	case DATA_TYPE_BOOLEAN:
	if (nelem != 0)
	return (EINVAL);
	break;
	case DATA_TYPE_BOOLEAN_VALUE:
	case DATA_TYPE_BYTE:
	case DATA_TYPE_INT8:
	case DATA_TYPE_UINT8:
	case DATA_TYPE_INT16:
	case DATA_TYPE_UINT16:
	case DATA_TYPE_INT32:
	case DATA_TYPE_UINT32:
	case DATA_TYPE_INT64:
	case DATA_TYPE_UINT64:
	case DATA_TYPE_STRING:
	case DATA_TYPE_HRTIME:
	case DATA_TYPE_NVLIST:
	#if !defined(_KERNEL)
	case DATA_TYPE_DOUBLE:
	#endif
	if (nelem != 1)
	return (EINVAL);
	break;
	case DATA_TYPE_BOOLEAN_ARRAY:
	case DATA_TYPE_BYTE_ARRAY:
	case DATA_TYPE_INT8_ARRAY:
	case DATA_TYPE_UINT8_ARRAY:
	case DATA_TYPE_INT16_ARRAY:
	case DATA_TYPE_UINT16_ARRAY:
	case DATA_TYPE_INT32_ARRAY:
	case DATA_TYPE_UINT32_ARRAY:
	case DATA_TYPE_INT64_ARRAY:
	case DATA_TYPE_UINT64_ARRAY:
	case DATA_TYPE_STRING_ARRAY:
	case DATA_TYPE_NVLIST_ARRAY:
	/* we allow arrays with 0 elements */
	break;
	default:
	return (EINVAL);
	}
	return (0);
	}

	/*
	* Verify nvp_name_sz and check the name string length.
	*/
	static int
	i_validate_nvpair_name(nvpair_t *nvp)
	{
	if ((nvp->nvp_name_sz <= 0) \|\|
	(nvp->nvp_size < NVP_SIZE_CALC(nvp->nvp_name_sz, 0)))
	return (EFAULT);

	/* verify the name string, make sure its terminated */
	if (NVP_NAME(nvp)[nvp->nvp_name_sz - 1] != '\0')
	return (EFAULT);

	return (strlen(NVP_NAME(nvp)) == nvp->nvp_name_sz - 1 ? 0 : EFAULT);
	}

	static int
	i_validate_nvpair_value(data_type_t type, uint_t nelem, const void *data)
	{
	switch (type) {
	case DATA_TYPE_BOOLEAN_VALUE:
	if ((boolean_t )data != B_TRUE &&
	(boolean_t )data != B_FALSE)
	return (EINVAL);
	break;
	case DATA_TYPE_BOOLEAN_ARRAY: {
	int i;

	for (i = 0; i < nelem; i++)
	if (((boolean_t *)data)[i] != B_TRUE &&
	((boolean_t *)data)[i] != B_FALSE)
	return (EINVAL);
	break;
	}
	default:
	break;
	}

	return (0);
	}

	/*
	* This function takes a pointer to what should be a nvpair and it's size
	* and then verifies that all the nvpair fields make sense and can be
	* trusted. This function is used when decoding packed nvpairs.
	*/
	static int
	i_validate_nvpair(nvpair_t *nvp)
	{
	data_type_t type = NVP_TYPE(nvp);
	int size1, size2;

	/* verify nvp_name_sz, check the name string length */
	if (i_validate_nvpair_name(nvp) != 0)
	return (EFAULT);

	if (i_validate_nvpair_value(type, NVP_NELEM(nvp), NVP_VALUE(nvp)) != 0)
	return (EFAULT);

	/*
	* verify nvp_type, nvp_value_elem, and also possibly
	* verify string values and get the value size.
	*/
	size2 = i_get_value_size(type, NVP_VALUE(nvp), NVP_NELEM(nvp));
	size1 = nvp->nvp_size - NVP_VALOFF(nvp);
	if (size2 < 0 \|\| size1 != NV_ALIGN(size2))
	return (EFAULT);

	return (0);
	}

	static int
	nvlist_copy_pairs(const nvlist_t snvl, nvlist_t dnvl)
	{
	const nvpriv_t *priv;
	const i_nvp_t *curr;

	if ((priv = (const nvpriv_t *)(uintptr_t)snvl->nvl_priv) == NULL)
	return (EINVAL);

	for (curr = priv->nvp_list; curr != NULL; curr = curr->nvi_next) {
	const nvpair_t *nvp = &curr->nvi_nvp;
	int err;

	if ((err = nvlist_add_common(dnvl, NVP_NAME(nvp), NVP_TYPE(nvp),
	NVP_NELEM(nvp), NVP_VALUE(nvp))) != 0)
	return (err);
	}

	return (0);
	}

	/*
	* Frees all memory allocated for an nvpair (like embedded lists) with
	* the exception of the nvpair buffer itself.
	*/
	static void
	nvpair_free(nvpair_t *nvp)
	{
	switch (NVP_TYPE(nvp)) {
	case DATA_TYPE_NVLIST:
	nvlist_free(EMBEDDED_NVL(nvp));
	break;
	case DATA_TYPE_NVLIST_ARRAY: {
	nvlist_t **nvlp = EMBEDDED_NVL_ARRAY(nvp);
	int i;

	for (i = 0; i < NVP_NELEM(nvp); i++)
	if (nvlp[i] != NULL)
	nvlist_free(nvlp[i]);
	break;
	}
	default:
	break;
	}
	}

	/*
	* nvlist_free - free an unpacked nvlist
	*/
	void
	nvlist_free(nvlist_t *nvl)
	{
	nvpriv_t *priv;
	i_nvp_t *curr;

	if (nvl == NULL \|\|
	(priv = (nvpriv_t *)(uintptr_t)nvl->nvl_priv) == NULL)
	return;

	/*
	* Unpacked nvlist are linked through i_nvp_t
	*/
	curr = priv->nvp_list;
	while (curr != NULL) {
	nvpair_t *nvp = &curr->nvi_nvp;
	curr = curr->nvi_next;

	nvpair_free(nvp);
	nvp_buf_free(nvl, nvp);
	}

	if (!(priv->nvp_stat & NV_STAT_EMBEDDED))
	nv_mem_free(priv, nvl, NV_ALIGN(sizeof (nvlist_t)));
	else
	nvl->nvl_priv = 0;

	nvt_tab_free(priv);
	nv_mem_free(priv, priv, sizeof (nvpriv_t));
	}

	static int
	nvlist_contains_nvp(const nvlist_t nvl, const nvpair_t nvp)
	{
	const nvpriv_t priv = (const nvpriv_t )(uintptr_t)nvl->nvl_priv;
	const i_nvp_t *curr;

	if (nvp == NULL)
	return (0);

	for (curr = priv->nvp_list; curr != NULL; curr = curr->nvi_next)
	if (&curr->nvi_nvp == nvp)
	return (1);

	return (0);
	}

	/*
	* Make a copy of nvlist
	*/
	int
	nvlist_dup(const nvlist_t nvl, nvlist_t *nvlp, int kmflag)
	{
	return (nvlist_xdup(nvl, nvlp, nvlist_nv_alloc(kmflag)));
	}

	int
	nvlist_xdup(const nvlist_t nvl, nvlist_t nvlp, nv_alloc_t nva)
	{
	int err;
	nvlist_t *ret;

	if (nvl == NULL \|\| nvlp == NULL)
	return (EINVAL);

	if ((err = nvlist_xalloc(&ret, nvl->nvl_nvflag, nva)) != 0)
	return (err);

	if ((err = nvlist_copy_pairs(nvl, ret)) != 0)
	nvlist_free(ret);
	else
	*nvlp = ret;

	return (err);
	}

	/*
	* Remove all with matching name
	*/
	int
	nvlist_remove_all(nvlist_t nvl, const char name)
	{
	int error = ENOENT;

	if (nvl == NULL \|\| name == NULL \|\| nvl->nvl_priv == 0)
	return (EINVAL);

	nvpair_t *nvp;
	while ((nvp = nvt_lookup_name(nvl, name)) != NULL) {
	VERIFY0(nvlist_remove_nvpair(nvl, nvp));
	error = 0;
	}

	return (error);
	}

	/*
	* Remove first one with matching name and type
	*/
	int
	nvlist_remove(nvlist_t nvl, const char name, data_type_t type)
	{
	if (nvl == NULL \|\| name == NULL \|\| nvl->nvl_priv == 0)
	return (EINVAL);

	nvpair_t *nvp = nvt_lookup_name_type(nvl, name, type);
	if (nvp == NULL)
	return (ENOENT);

	return (nvlist_remove_nvpair(nvl, nvp));
	}

	int
	nvlist_remove_nvpair(nvlist_t nvl, nvpair_t nvp)
	{
	if (nvl == NULL \|\| nvp == NULL)
	return (EINVAL);

	int err = nvt_remove_nvpair(nvl, nvp);
	if (err != 0)
	return (err);

	nvp_buf_unlink(nvl, nvp);
	nvpair_free(nvp);
	nvp_buf_free(nvl, nvp);
	return (0);
	}

	/*
	* This function calculates the size of an nvpair value.
	*
	* The data argument controls the behavior in case of the data types
	* DATA_TYPE_STRING and
	* DATA_TYPE_STRING_ARRAY
	* Is data == NULL then the size of the string(s) is excluded.
	*/
	static int
	i_get_value_size(data_type_t type, const void *data, uint_t nelem)
	{
	uint64_t value_sz;

	if (i_validate_type_nelem(type, nelem) != 0)
	return (-1);

	/* Calculate required size for holding value */
	switch (type) {
	case DATA_TYPE_BOOLEAN:
	value_sz = 0;
	break;
	case DATA_TYPE_BOOLEAN_VALUE:
	value_sz = sizeof (boolean_t);
	break;
	case DATA_TYPE_BYTE:
	value_sz = sizeof (uchar_t);
	break;
	case DATA_TYPE_INT8:
	value_sz = sizeof (int8_t);
	break;
	case DATA_TYPE_UINT8:
	value_sz = sizeof (uint8_t);
	break;
	case DATA_TYPE_INT16:
	value_sz = sizeof (int16_t);
	break;
	case DATA_TYPE_UINT16:
	value_sz = sizeof (uint16_t);
	break;
	case DATA_TYPE_INT32:
	value_sz = sizeof (int32_t);
	break;
	case DATA_TYPE_UINT32:
	value_sz = sizeof (uint32_t);
	break;
	case DATA_TYPE_INT64:
	value_sz = sizeof (int64_t);
	break;
	case DATA_TYPE_UINT64:
	value_sz = sizeof (uint64_t);
	break;
	#if !defined(_KERNEL)
	case DATA_TYPE_DOUBLE:
	value_sz = sizeof (double);
	break;
	#endif
	case DATA_TYPE_STRING:
	if (data == NULL)
	value_sz = 0;
	else
	value_sz = strlen(data) + 1;
	break;
	case DATA_TYPE_BOOLEAN_ARRAY:
	value_sz = (uint64_t)nelem * sizeof (boolean_t);
	break;
	case DATA_TYPE_BYTE_ARRAY:
	value_sz = (uint64_t)nelem * sizeof (uchar_t);
	break;
	case DATA_TYPE_INT8_ARRAY:
	value_sz = (uint64_t)nelem * sizeof (int8_t);
	break;
	case DATA_TYPE_UINT8_ARRAY:
	value_sz = (uint64_t)nelem * sizeof (uint8_t);
	break;
	case DATA_TYPE_INT16_ARRAY:
	value_sz = (uint64_t)nelem * sizeof (int16_t);
	break;
	case DATA_TYPE_UINT16_ARRAY:
	value_sz = (uint64_t)nelem * sizeof (uint16_t);
	break;
	case DATA_TYPE_INT32_ARRAY:
	value_sz = (uint64_t)nelem * sizeof (int32_t);
	break;
	case DATA_TYPE_UINT32_ARRAY:
	value_sz = (uint64_t)nelem * sizeof (uint32_t);
	break;
	case DATA_TYPE_INT64_ARRAY:
	value_sz = (uint64_t)nelem * sizeof (int64_t);
	break;
	case DATA_TYPE_UINT64_ARRAY:
	value_sz = (uint64_t)nelem * sizeof (uint64_t);
	break;
	case DATA_TYPE_STRING_ARRAY:
	value_sz = (uint64_t)nelem * sizeof (uint64_t);

	if (data != NULL) {
	char const strs = data;
	uint_t i;

	/* no alignment requirement for strings */
	for (i = 0; i < nelem; i++) {
	if (strs[i] == NULL)
	return (-1);
	value_sz += strlen(strs[i]) + 1;
	}
	}
	break;
	case DATA_TYPE_HRTIME:
	value_sz = sizeof (hrtime_t);
	break;
	case DATA_TYPE_NVLIST:
	value_sz = NV_ALIGN(sizeof (nvlist_t));
	break;
	case DATA_TYPE_NVLIST_ARRAY:
	value_sz = (uint64_t)nelem * sizeof (uint64_t) +
	(uint64_t)nelem * NV_ALIGN(sizeof (nvlist_t));
	break;
	default:
	return (-1);
	}

	return (value_sz > INT32_MAX ? -1 : (int)value_sz);
	}

	static int
	nvlist_copy_embedded(nvlist_t nvl, nvlist_t onvl, nvlist_t *emb_nvl)
	{
	nvpriv_t *priv;
	int err;

	if ((priv = nv_priv_alloc_embedded((nvpriv_t *)(uintptr_t)
	nvl->nvl_priv)) == NULL)
	return (ENOMEM);

	nvlist_init(emb_nvl, onvl->nvl_nvflag, priv);

	if ((err = nvlist_copy_pairs(onvl, emb_nvl)) != 0) {
	nvlist_free(emb_nvl);
	emb_nvl->nvl_priv = 0;
	}

	return (err);
	}

	/*
	* nvlist_add_common - Add new <name,value> pair to nvlist
	*/
	static int
	nvlist_add_common(nvlist_t nvl, const char name,
	data_type_t type, uint_t nelem, const void *data)
	{
	nvpair_t *nvp;
	uint_t i;

	int nvp_sz, name_sz, value_sz;
	int err = 0;

	if (name == NULL \|\| nvl == NULL \|\| nvl->nvl_priv == 0)
	return (EINVAL);

	if (nelem != 0 && data == NULL)
	return (EINVAL);

	/*
	* Verify type and nelem and get the value size.
	* In case of data types DATA_TYPE_STRING and DATA_TYPE_STRING_ARRAY
	* is the size of the string(s) included.
	*/
	if ((value_sz = i_get_value_size(type, data, nelem)) < 0)
	return (EINVAL);

	if (i_validate_nvpair_value(type, nelem, data) != 0)
	return (EINVAL);

	/*
	* If we're adding an nvlist or nvlist array, ensure that we are not
	* adding the input nvlist to itself, which would cause recursion,
	* and ensure that no NULL nvlist pointers are present.
	*/
	switch (type) {
	case DATA_TYPE_NVLIST:
	if (data == nvl \|\| data == NULL)
	return (EINVAL);
	break;
	case DATA_TYPE_NVLIST_ARRAY: {
	nvlist_t onvlp = (nvlist_t )data;
	for (i = 0; i < nelem; i++) {
	if (onvlp[i] == nvl \|\| onvlp[i] == NULL)
	return (EINVAL);
	}
	break;
	}
	default:
	break;
	}

	/* calculate sizes of the nvpair elements and the nvpair itself */
	name_sz = strlen(name) + 1;
	if (name_sz >= 1ULL << (sizeof (nvp->nvp_name_sz) * NBBY - 1))
	return (EINVAL);

	nvp_sz = NVP_SIZE_CALC(name_sz, value_sz);

	if ((nvp = nvp_buf_alloc(nvl, nvp_sz)) == NULL)
	return (ENOMEM);

	ASSERT(nvp->nvp_size == nvp_sz);
	nvp->nvp_name_sz = name_sz;
	nvp->nvp_value_elem = nelem;
	nvp->nvp_type = type;
	memcpy(NVP_NAME(nvp), name, name_sz);

	switch (type) {
	case DATA_TYPE_BOOLEAN:
	break;
	case DATA_TYPE_STRING_ARRAY: {
	char const strs = data;
	char *buf = NVP_VALUE(nvp);
	char *cstrs = (void )buf;

	/* skip pre-allocated space for pointer array */
	buf += nelem * sizeof (uint64_t);
	for (i = 0; i < nelem; i++) {
	int slen = strlen(strs[i]) + 1;
	memcpy(buf, strs[i], slen);
	cstrs[i] = buf;
	buf += slen;
	}
	break;
	}
	case DATA_TYPE_NVLIST: {
	nvlist_t *nnvl = EMBEDDED_NVL(nvp);
	nvlist_t onvl = (nvlist_t )data;

	if ((err = nvlist_copy_embedded(nvl, onvl, nnvl)) != 0) {
	nvp_buf_free(nvl, nvp);
	return (err);
	}
	break;
	}
	case DATA_TYPE_NVLIST_ARRAY: {
	nvlist_t onvlp = (nvlist_t )data;
	nvlist_t **nvlp = EMBEDDED_NVL_ARRAY(nvp);
	nvlist_t embedded = (nvlist_t )
	((uintptr_t)nvlp + nelem * sizeof (uint64_t));

	for (i = 0; i < nelem; i++) {
	if ((err = nvlist_copy_embedded(nvl,
	onvlp[i], embedded)) != 0) {
	/*
	* Free any successfully created lists
	*/
	nvpair_free(nvp);
	nvp_buf_free(nvl, nvp);
	return (err);
	}

	nvlp[i] = embedded++;
	}
	break;
	}
	default:
	memcpy(NVP_VALUE(nvp), data, value_sz);
	}

	/* if unique name, remove before add */
	if (nvl->nvl_nvflag & NV_UNIQUE_NAME)
	(void) nvlist_remove_all(nvl, name);
	else if (nvl->nvl_nvflag & NV_UNIQUE_NAME_TYPE)
	(void) nvlist_remove(nvl, name, type);

	err = nvt_add_nvpair(nvl, nvp);
	if (err != 0) {
	nvpair_free(nvp);
	nvp_buf_free(nvl, nvp);
	return (err);
	}
	nvp_buf_link(nvl, nvp);

	return (0);
	}

	int
	nvlist_add_boolean(nvlist_t nvl, const char name)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_BOOLEAN, 0, NULL));
	}

	int
	nvlist_add_boolean_value(nvlist_t nvl, const char name, boolean_t val)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_BOOLEAN_VALUE, 1, &val));
	}

	int
	nvlist_add_byte(nvlist_t nvl, const char name, uchar_t val)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_BYTE, 1, &val));
	}

	int
	nvlist_add_int8(nvlist_t nvl, const char name, int8_t val)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_INT8, 1, &val));
	}

	int
	nvlist_add_uint8(nvlist_t nvl, const char name, uint8_t val)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_UINT8, 1, &val));
	}

	int
	nvlist_add_int16(nvlist_t nvl, const char name, int16_t val)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_INT16, 1, &val));
	}

	int
	nvlist_add_uint16(nvlist_t nvl, const char name, uint16_t val)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_UINT16, 1, &val));
	}

	int
	nvlist_add_int32(nvlist_t nvl, const char name, int32_t val)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_INT32, 1, &val));
	}

	int
	nvlist_add_uint32(nvlist_t nvl, const char name, uint32_t val)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_UINT32, 1, &val));
	}

	int
	nvlist_add_int64(nvlist_t nvl, const char name, int64_t val)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_INT64, 1, &val));
	}

	int
	nvlist_add_uint64(nvlist_t nvl, const char name, uint64_t val)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_UINT64, 1, &val));
	}

	#if !defined(_KERNEL)
	int
	nvlist_add_double(nvlist_t nvl, const char name, double val)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_DOUBLE, 1, &val));
	}
	#endif

	int
	nvlist_add_string(nvlist_t nvl, const char name, const char *val)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_STRING, 1, (void *)val));
	}

	int
	nvlist_add_boolean_array(nvlist_t nvl, const char name,
	const boolean_t *a, uint_t n)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_BOOLEAN_ARRAY, n, a));
	}

	int
	nvlist_add_byte_array(nvlist_t nvl, const char name, const uchar_t *a,
	uint_t n)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_BYTE_ARRAY, n, a));
	}

	int
	nvlist_add_int8_array(nvlist_t nvl, const char name, const int8_t *a,
	uint_t n)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_INT8_ARRAY, n, a));
	}

	int
	nvlist_add_uint8_array(nvlist_t nvl, const char name, const uint8_t *a,
	uint_t n)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_UINT8_ARRAY, n, a));
	}

	int
	nvlist_add_int16_array(nvlist_t nvl, const char name, const int16_t *a,
	uint_t n)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_INT16_ARRAY, n, a));
	}

	int
	nvlist_add_uint16_array(nvlist_t nvl, const char name, const uint16_t *a,
	uint_t n)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_UINT16_ARRAY, n, a));
	}

	int
	nvlist_add_int32_array(nvlist_t nvl, const char name, const int32_t *a,
	uint_t n)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_INT32_ARRAY, n, a));
	}

	int
	nvlist_add_uint32_array(nvlist_t nvl, const char name, const uint32_t *a,
	uint_t n)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_UINT32_ARRAY, n, a));
	}

	int
	nvlist_add_int64_array(nvlist_t nvl, const char name, const int64_t *a,
	uint_t n)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_INT64_ARRAY, n, a));
	}

	int
	nvlist_add_uint64_array(nvlist_t nvl, const char name, const uint64_t *a,
	uint_t n)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_UINT64_ARRAY, n, a));
	}

	int
	nvlist_add_string_array(nvlist_t nvl, const char name,
	const char const a, uint_t n)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_STRING_ARRAY, n, a));
	}

	int
	nvlist_add_hrtime(nvlist_t nvl, const char name, hrtime_t val)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_HRTIME, 1, &val));
	}

	int
	nvlist_add_nvlist(nvlist_t nvl, const char name, const nvlist_t *val)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_NVLIST, 1, val));
	}

	int
	nvlist_add_nvlist_array(nvlist_t nvl, const char name,
	const nvlist_t * const *a, uint_t n)
	{
	return (nvlist_add_common(nvl, name, DATA_TYPE_NVLIST_ARRAY, n, a));
	}

	/* reading name-value pairs */
	nvpair_t *
	nvlist_next_nvpair(nvlist_t nvl, const nvpair_t nvp)
	{
	nvpriv_t *priv;
	i_nvp_t *curr;

	if (nvl == NULL \|\|
	(priv = (nvpriv_t *)(uintptr_t)nvl->nvl_priv) == NULL)
	return (NULL);

	curr = NVPAIR2I_NVP(nvp);

	/*
	* Ensure that nvp is a valid nvpair on this nvlist.
	* NB: nvp_curr is used only as a hint so that we don't always
	* have to walk the list to determine if nvp is still on the list.
	*/
	if (nvp == NULL)
	curr = priv->nvp_list;
	else if (priv->nvp_curr == curr \|\| nvlist_contains_nvp(nvl, nvp))
	curr = curr->nvi_next;
	else
	curr = NULL;

	priv->nvp_curr = curr;

	return (curr != NULL ? &curr->nvi_nvp : NULL);
	}

	nvpair_t *
	nvlist_prev_nvpair(nvlist_t nvl, const nvpair_t nvp)
	{
	nvpriv_t *priv;
	i_nvp_t *curr;

	if (nvl == NULL \|\|
	(priv = (nvpriv_t *)(uintptr_t)nvl->nvl_priv) == NULL)
	return (NULL);

	curr = NVPAIR2I_NVP(nvp);

	if (nvp == NULL)
	curr = priv->nvp_last;
	else if (priv->nvp_curr == curr \|\| nvlist_contains_nvp(nvl, nvp))
	curr = curr->nvi_prev;
	else
	curr = NULL;

	priv->nvp_curr = curr;

	return (curr != NULL ? &curr->nvi_nvp : NULL);
	}

	boolean_t
	nvlist_empty(const nvlist_t *nvl)
	{
	const nvpriv_t *priv;

	if (nvl == NULL \|\|
	(priv = (const nvpriv_t *)(uintptr_t)nvl->nvl_priv) == NULL)
	return (B_TRUE);

	return (priv->nvp_list == NULL);
	}

	const char *
	nvpair_name(const nvpair_t *nvp)
	{
	return (NVP_NAME(nvp));
	}

	data_type_t
	nvpair_type(const nvpair_t *nvp)
	{
	return (NVP_TYPE(nvp));
	}

	int
	nvpair_type_is_array(const nvpair_t *nvp)
	{
	data_type_t type = NVP_TYPE(nvp);

	if ((type == DATA_TYPE_BYTE_ARRAY) \|\|
	(type == DATA_TYPE_INT8_ARRAY) \|\|
	(type == DATA_TYPE_UINT8_ARRAY) \|\|
	(type == DATA_TYPE_INT16_ARRAY) \|\|
	(type == DATA_TYPE_UINT16_ARRAY) \|\|
	(type == DATA_TYPE_INT32_ARRAY) \|\|
	(type == DATA_TYPE_UINT32_ARRAY) \|\|
	(type == DATA_TYPE_INT64_ARRAY) \|\|
	(type == DATA_TYPE_UINT64_ARRAY) \|\|
	(type == DATA_TYPE_BOOLEAN_ARRAY) \|\|
	(type == DATA_TYPE_STRING_ARRAY) \|\|
	(type == DATA_TYPE_NVLIST_ARRAY))
	return (1);
	return (0);

	}

	static int
	nvpair_value_common(const nvpair_t nvp, data_type_t type, uint_t nelem,
	void *data)
	{
	int value_sz;

	if (nvp == NULL \|\| nvpair_type(nvp) != type)
	return (EINVAL);

	/*
	* For non-array types, we copy the data.
	* For array types (including string), we set a pointer.
	*/
	switch (type) {
	case DATA_TYPE_BOOLEAN:
	if (nelem != NULL)
	*nelem = 0;
	break;

	case DATA_TYPE_BOOLEAN_VALUE:
	case DATA_TYPE_BYTE:
	case DATA_TYPE_INT8:
	case DATA_TYPE_UINT8:
	case DATA_TYPE_INT16:
	case DATA_TYPE_UINT16:
	case DATA_TYPE_INT32:
	case DATA_TYPE_UINT32:
	case DATA_TYPE_INT64:
	case DATA_TYPE_UINT64:
	case DATA_TYPE_HRTIME:
	#if !defined(_KERNEL)
	case DATA_TYPE_DOUBLE:
	#endif
	if (data == NULL)
	return (EINVAL);
	if ((value_sz = i_get_value_size(type, NULL, 1)) < 0)
	return (EINVAL);
	memcpy(data, NVP_VALUE(nvp), (size_t)value_sz);
	if (nelem != NULL)
	*nelem = 1;
	break;

	case DATA_TYPE_NVLIST:
	case DATA_TYPE_STRING:
	if (data == NULL)
	return (EINVAL);
	/*
	* This discards the const from nvp, so all callers for these
	* types must not accept const nvpairs.
	*/
	(void )data = (void )NVP_VALUE(nvp);
	if (nelem != NULL)
	*nelem = 1;
	break;

	case DATA_TYPE_BOOLEAN_ARRAY:
	case DATA_TYPE_BYTE_ARRAY:
	case DATA_TYPE_INT8_ARRAY:
	case DATA_TYPE_UINT8_ARRAY:
	case DATA_TYPE_INT16_ARRAY:
	case DATA_TYPE_UINT16_ARRAY:
	case DATA_TYPE_INT32_ARRAY:
	case DATA_TYPE_UINT32_ARRAY:
	case DATA_TYPE_INT64_ARRAY:
	case DATA_TYPE_UINT64_ARRAY:
	case DATA_TYPE_STRING_ARRAY:
	case DATA_TYPE_NVLIST_ARRAY:
	if (nelem == NULL \|\| data == NULL)
	return (EINVAL);
	/*
	* This discards the const from nvp, so all callers for these
	* types must not accept const nvpairs.
	*/
	if ((*nelem = NVP_NELEM(nvp)) != 0)
	(void )data = (void )NVP_VALUE(nvp);
	else
	(void *)data = NULL;
	break;

	default:
	return (ENOTSUP);
	}

	return (0);
	}

	static int
	nvlist_lookup_common(const nvlist_t nvl, const char name, data_type_t type,
	uint_t nelem, void data)
	{
	if (name == NULL \|\| nvl == NULL \|\| nvl->nvl_priv == 0)
	return (EINVAL);

	if (!(nvl->nvl_nvflag & (NV_UNIQUE_NAME \| NV_UNIQUE_NAME_TYPE)))
	return (ENOTSUP);

	nvpair_t *nvp = nvt_lookup_name_type(nvl, name, type);
	if (nvp == NULL)
	return (ENOENT);

	return (nvpair_value_common(nvp, type, nelem, data));
	}

	int
	nvlist_lookup_boolean(const nvlist_t nvl, const char name)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_BOOLEAN, NULL, NULL));
	}

	int
	nvlist_lookup_boolean_value(const nvlist_t nvl, const char name,
	boolean_t *val)
	{
	return (nvlist_lookup_common(nvl, name,
	DATA_TYPE_BOOLEAN_VALUE, NULL, val));
	}

	int
	nvlist_lookup_byte(const nvlist_t nvl, const char name, uchar_t *val)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_BYTE, NULL, val));
	}

	int
	nvlist_lookup_int8(const nvlist_t nvl, const char name, int8_t *val)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_INT8, NULL, val));
	}

	int
	nvlist_lookup_uint8(const nvlist_t nvl, const char name, uint8_t *val)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_UINT8, NULL, val));
	}

	int
	nvlist_lookup_int16(const nvlist_t nvl, const char name, int16_t *val)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_INT16, NULL, val));
	}

	int
	nvlist_lookup_uint16(const nvlist_t nvl, const char name, uint16_t *val)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_UINT16, NULL, val));
	}

	int
	nvlist_lookup_int32(const nvlist_t nvl, const char name, int32_t *val)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_INT32, NULL, val));
	}

	int
	nvlist_lookup_uint32(const nvlist_t nvl, const char name, uint32_t *val)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_UINT32, NULL, val));
	}

	int
	nvlist_lookup_int64(const nvlist_t nvl, const char name, int64_t *val)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_INT64, NULL, val));
	}

	int
	nvlist_lookup_uint64(const nvlist_t nvl, const char name, uint64_t *val)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_UINT64, NULL, val));
	}

	#if !defined(_KERNEL)
	int
	nvlist_lookup_double(const nvlist_t nvl, const char name, double *val)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_DOUBLE, NULL, val));
	}
	#endif

	int
	nvlist_lookup_string(const nvlist_t nvl, const char name, const char **val)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_STRING, NULL, val));
	}

	int
	nvlist_lookup_nvlist(nvlist_t nvl, const char name, nvlist_t **val)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_NVLIST, NULL, val));
	}

	int
	nvlist_lookup_boolean_array(nvlist_t nvl, const char name,
	boolean_t *a, uint_t n)
	{
	return (nvlist_lookup_common(nvl, name,
	DATA_TYPE_BOOLEAN_ARRAY, n, a));
	}

	int
	nvlist_lookup_byte_array(nvlist_t nvl, const char name,
	uchar_t *a, uint_t n)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_BYTE_ARRAY, n, a));
	}

	int
	nvlist_lookup_int8_array(nvlist_t nvl, const char name, int8_t *a, uint_t n)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_INT8_ARRAY, n, a));
	}

	int
	nvlist_lookup_uint8_array(nvlist_t nvl, const char name,
	uint8_t *a, uint_t n)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_UINT8_ARRAY, n, a));
	}

	int
	nvlist_lookup_int16_array(nvlist_t nvl, const char name,
	int16_t *a, uint_t n)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_INT16_ARRAY, n, a));
	}

	int
	nvlist_lookup_uint16_array(nvlist_t nvl, const char name,
	uint16_t *a, uint_t n)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_UINT16_ARRAY, n, a));
	}

	int
	nvlist_lookup_int32_array(nvlist_t nvl, const char name,
	int32_t *a, uint_t n)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_INT32_ARRAY, n, a));
	}

	int
	nvlist_lookup_uint32_array(nvlist_t nvl, const char name,
	uint32_t *a, uint_t n)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_UINT32_ARRAY, n, a));
	}

	int
	nvlist_lookup_int64_array(nvlist_t nvl, const char name,
	int64_t *a, uint_t n)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_INT64_ARRAY, n, a));
	}

	int
	nvlist_lookup_uint64_array(nvlist_t nvl, const char name,
	uint64_t *a, uint_t n)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_UINT64_ARRAY, n, a));
	}

	int
	nvlist_lookup_string_array(nvlist_t nvl, const char name,
	char **a, uint_t n)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_STRING_ARRAY, n, a));
	}

	int
	nvlist_lookup_nvlist_array(nvlist_t nvl, const char name,
	nvlist_t **a, uint_t n)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_NVLIST_ARRAY, n, a));
	}

	int
	nvlist_lookup_hrtime(nvlist_t nvl, const char name, hrtime_t *val)
	{
	return (nvlist_lookup_common(nvl, name, DATA_TYPE_HRTIME, NULL, val));
	}

	int
	nvlist_lookup_pairs(nvlist_t *nvl, int flag, ...)
	{
	va_list ap;
	char *name;
	int noentok = (flag & NV_FLAG_NOENTOK ? 1 : 0);
	int ret = 0;

	va_start(ap, flag);
	while (ret == 0 && (name = va_arg(ap, char *)) != NULL) {
	data_type_t type;
	void *val;
	uint_t *nelem;

	switch (type = va_arg(ap, data_type_t)) {
	case DATA_TYPE_BOOLEAN:
	ret = nvlist_lookup_common(nvl, name, type, NULL, NULL);
	break;

	case DATA_TYPE_BOOLEAN_VALUE:
	case DATA_TYPE_BYTE:
	case DATA_TYPE_INT8:
	case DATA_TYPE_UINT8:
	case DATA_TYPE_INT16:
	case DATA_TYPE_UINT16:
	case DATA_TYPE_INT32:
	case DATA_TYPE_UINT32:
	case DATA_TYPE_INT64:
	case DATA_TYPE_UINT64:
	case DATA_TYPE_HRTIME:
	case DATA_TYPE_STRING:
	case DATA_TYPE_NVLIST:
	#if !defined(_KERNEL)
	case DATA_TYPE_DOUBLE:
	#endif
	val = va_arg(ap, void *);
	ret = nvlist_lookup_common(nvl, name, type, NULL, val);
	break;

	case DATA_TYPE_BYTE_ARRAY:
	case DATA_TYPE_BOOLEAN_ARRAY:
	case DATA_TYPE_INT8_ARRAY:
	case DATA_TYPE_UINT8_ARRAY:
	case DATA_TYPE_INT16_ARRAY:
	case DATA_TYPE_UINT16_ARRAY:
	case DATA_TYPE_INT32_ARRAY:
	case DATA_TYPE_UINT32_ARRAY:
	case DATA_TYPE_INT64_ARRAY:
	case DATA_TYPE_UINT64_ARRAY:
	case DATA_TYPE_STRING_ARRAY:
	case DATA_TYPE_NVLIST_ARRAY:
	val = va_arg(ap, void *);
	nelem = va_arg(ap, uint_t *);
	ret = nvlist_lookup_common(nvl, name, type, nelem, val);
	break;

	default:
	ret = EINVAL;
	}

	if (ret == ENOENT && noentok)
	ret = 0;
	}
	va_end(ap);

	return (ret);
	}

	/*
	* Find the 'name'ed nvpair in the nvlist 'nvl'. If 'name' found, the function
	* returns zero and a pointer to the matching nvpair is returned in '*ret'
	* (given 'ret' is non-NULL). If 'sep' is specified then 'name' will penitrate
	* multiple levels of embedded nvlists, with 'sep' as the separator. As an
	* example, if sep is '.', name might look like: "a" or "a.b" or "a.c[3]" or
	* "a.d[3].e[1]". This matches the C syntax for array embed (for convenience,
	* code also supports "a.d[3]e[1]" syntax).
	*
	* If 'ip' is non-NULL and the last name component is an array, return the
	* value of the "...[index]" array index in *ip. For an array reference that
	* is not indexed, *ip will be returned as -1. If there is a syntax error in
	* 'name', and 'ep' is non-NULL then *ep will be set to point to the location
	* inside the 'name' string where the syntax error was detected.
	*/
	static int
	nvlist_lookup_nvpair_ei_sep(nvlist_t nvl, const char name, const char sep,
	nvpair_t *ret, int ip, const char **ep)
	{
	nvpair_t *nvp;
	const char *np;
	char *sepp = NULL;
	char idxp, idxep;
	nvlist_t **nva;
	long idx = 0;
	int n;

	if (ip)
	ip = -1; / not indexed */
	if (ep)
	*ep = NULL;

	if ((nvl == NULL) \|\| (name == NULL))
	return (EINVAL);

	sepp = NULL;
	idx = 0;
	/* step through components of name */
	for (np = name; np && *np; np = sepp) {
	/* ensure unique names */
	if (!(nvl->nvl_nvflag & NV_UNIQUE_NAME))
	return (ENOTSUP);

	/* skip white space */
	skip_whitespace(np);
	if (*np == 0)
	break;

	/* set 'sepp' to end of current component 'np' */
	if (sep)
	sepp = strchr(np, sep);
	else
	sepp = NULL;

	/* find start of next "[ index ]..." */
	idxp = strchr(np, '[');

	/* if sepp comes first, set idxp to NULL */
	if (sepp && idxp && (sepp < idxp))
	idxp = NULL;

	/*
	* At this point 'idxp' is set if there is an index
	* expected for the current component.
	*/
	if (idxp) {
	/* set 'n' to length of current 'np' name component */
	n = idxp++ - np;

	/* keep sepp up to date for ep use as we advance /
	skip_whitespace(idxp);
	sepp = idxp;

	/* determine the index value */
	#if defined(_KERNEL)
	if (ddi_strtol(idxp, &idxep, 0, &idx))
	goto fail;
	#else
	idx = strtol(idxp, &idxep, 0);
	#endif
	if (idxep == idxp)
	goto fail;

	/* keep sepp up to date for ep use as we advance /
	sepp = idxep;

	/* skip white space index value and check for ']' */
	skip_whitespace(sepp);
	if (*sepp++ != ']')
	goto fail;

	/* for embedded arrays, support C syntax: "a[1].b" */
	skip_whitespace(sepp);
	if (sep && (*sepp == sep))
	sepp++;
	} else if (sepp) {
	n = sepp++ - np;
	} else {
	n = strlen(np);
	}

	/* trim trailing whitespace by reducing length of 'np' */
	if (n == 0)
	goto fail;
	for (n--; (np[n] == ' ') \|\| (np[n] == '\t'); n--)
	;
	n++;

	/* skip whitespace, and set sepp to NULL if complete */
	if (sepp) {
	skip_whitespace(sepp);
	if (*sepp == 0)
	sepp = NULL;
	}

	/*
	* At this point:
	* o 'n' is the length of current 'np' component.
	* o 'idxp' is set if there was an index, and value 'idx'.
	* o 'sepp' is set to the beginning of the next component,
	* and set to NULL if we have no more components.
	*
	* Search for nvpair with matching component name.
	*/
	for (nvp = nvlist_next_nvpair(nvl, NULL); nvp != NULL;
	nvp = nvlist_next_nvpair(nvl, nvp)) {

	/* continue if no match on name */
	if (strncmp(np, nvpair_name(nvp), n) \|\|
	(strlen(nvpair_name(nvp)) != n))
	continue;

	/* if indexed, verify type is array oriented */
	if (idxp && !nvpair_type_is_array(nvp))
	goto fail;

	/*
	* Full match found, return nvp and idx if this
	* was the last component.
	*/
	if (sepp == NULL) {
	if (ret)
	*ret = nvp;
	if (ip && idxp)
	ip = (int)idx; / return index */
	return (0); /* found */
	}

	/*
	* More components: current match must be
	* of DATA_TYPE_NVLIST or DATA_TYPE_NVLIST_ARRAY
	* to support going deeper.
	*/
	if (nvpair_type(nvp) == DATA_TYPE_NVLIST) {
	nvl = EMBEDDED_NVL(nvp);
	break;
	} else if (nvpair_type(nvp) == DATA_TYPE_NVLIST_ARRAY) {
	if (nvpair_value_nvlist_array(nvp,
	&nva, (uint_t *)&n) != 0)
	goto fail;
	if (nva == NULL)
	goto fail;
	if ((n < 0) \|\| (idx >= n))
	goto fail;
	nvl = nva[idx];
	break;
	}

	/* type does not support more levels */
	goto fail;
	}
	if (nvp == NULL)
	goto fail; /* 'name' not found */

	/* search for match of next component in embedded 'nvl' list */
	}

	fail: if (ep && sepp)
	*ep = sepp;
	return (EINVAL);
	}

	/*
	* Return pointer to nvpair with specified 'name'.
	*/
	int
	nvlist_lookup_nvpair(nvlist_t nvl, const char name, nvpair_t **ret)
	{
	return (nvlist_lookup_nvpair_ei_sep(nvl, name, 0, ret, NULL, NULL));
	}

	/*
	* Determine if named nvpair exists in nvlist (use embedded separator of '.'
	* and return array index). See nvlist_lookup_nvpair_ei_sep for more detailed
	* description.
	*/
	int nvlist_lookup_nvpair_embedded_index(nvlist_t *nvl,
	const char name, nvpair_t ret, int ip, const char **ep)
	{
	return (nvlist_lookup_nvpair_ei_sep(nvl, name, '.', ret, ip, ep));
	}

	boolean_t
	nvlist_exists(const nvlist_t nvl, const char name)
	{
	nvpriv_t *priv;
	nvpair_t *nvp;
	i_nvp_t *curr;

	if (name == NULL \|\| nvl == NULL \|\|
	(priv = (nvpriv_t *)(uintptr_t)nvl->nvl_priv) == NULL)
	return (B_FALSE);

	for (curr = priv->nvp_list; curr != NULL; curr = curr->nvi_next) {
	nvp = &curr->nvi_nvp;

	if (strcmp(name, NVP_NAME(nvp)) == 0)
	return (B_TRUE);
	}

	return (B_FALSE);
	}

	int
	nvpair_value_boolean_value(const nvpair_t nvp, boolean_t val)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_BOOLEAN_VALUE, NULL, val));
	}

	int
	nvpair_value_byte(const nvpair_t nvp, uchar_t val)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_BYTE, NULL, val));
	}

	int
	nvpair_value_int8(const nvpair_t nvp, int8_t val)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_INT8, NULL, val));
	}

	int
	nvpair_value_uint8(const nvpair_t nvp, uint8_t val)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_UINT8, NULL, val));
	}

	int
	nvpair_value_int16(const nvpair_t nvp, int16_t val)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_INT16, NULL, val));
	}

	int
	nvpair_value_uint16(const nvpair_t nvp, uint16_t val)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_UINT16, NULL, val));
	}

	int
	nvpair_value_int32(const nvpair_t nvp, int32_t val)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_INT32, NULL, val));
	}

	int
	nvpair_value_uint32(const nvpair_t nvp, uint32_t val)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_UINT32, NULL, val));
	}

	int
	nvpair_value_int64(const nvpair_t nvp, int64_t val)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_INT64, NULL, val));
	}

	int
	nvpair_value_uint64(const nvpair_t nvp, uint64_t val)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_UINT64, NULL, val));
	}

	#if !defined(_KERNEL)
	int
	nvpair_value_double(const nvpair_t nvp, double val)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_DOUBLE, NULL, val));
	}
	#endif

	int
	nvpair_value_string(const nvpair_t nvp, const char *val)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_STRING, NULL, val));
	}

	int
	nvpair_value_nvlist(nvpair_t nvp, nvlist_t *val)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_NVLIST, NULL, val));
	}

	int
	nvpair_value_boolean_array(nvpair_t nvp, boolean_t val, uint_t nelem)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_BOOLEAN_ARRAY, nelem, val));
	}

	int
	nvpair_value_byte_array(nvpair_t nvp, uchar_t val, uint_t nelem)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_BYTE_ARRAY, nelem, val));
	}

	int
	nvpair_value_int8_array(nvpair_t nvp, int8_t val, uint_t nelem)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_INT8_ARRAY, nelem, val));
	}

	int
	nvpair_value_uint8_array(nvpair_t nvp, uint8_t val, uint_t nelem)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_UINT8_ARRAY, nelem, val));
	}

	int
	nvpair_value_int16_array(nvpair_t nvp, int16_t val, uint_t nelem)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_INT16_ARRAY, nelem, val));
	}

	int
	nvpair_value_uint16_array(nvpair_t nvp, uint16_t val, uint_t nelem)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_UINT16_ARRAY, nelem, val));
	}

	int
	nvpair_value_int32_array(nvpair_t nvp, int32_t val, uint_t nelem)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_INT32_ARRAY, nelem, val));
	}

	int
	nvpair_value_uint32_array(nvpair_t nvp, uint32_t val, uint_t nelem)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_UINT32_ARRAY, nelem, val));
	}

	int
	nvpair_value_int64_array(nvpair_t nvp, int64_t val, uint_t nelem)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_INT64_ARRAY, nelem, val));
	}

	int
	nvpair_value_uint64_array(nvpair_t nvp, uint64_t val, uint_t nelem)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_UINT64_ARRAY, nelem, val));
	}

	int
	nvpair_value_string_array(nvpair_t nvp, const char *val, uint_t nelem)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_STRING_ARRAY, nelem, val));
	}

	int
	nvpair_value_nvlist_array(nvpair_t nvp, nvlist_t *val, uint_t nelem)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_NVLIST_ARRAY, nelem, val));
	}

	int
	nvpair_value_hrtime(nvpair_t nvp, hrtime_t val)
	{
	return (nvpair_value_common(nvp, DATA_TYPE_HRTIME, NULL, val));
	}

	/*
	* Add specified pair to the list.
	*/
	int
	nvlist_add_nvpair(nvlist_t nvl, nvpair_t nvp)
	{
	if (nvl == NULL \|\| nvp == NULL)
	return (EINVAL);

	return (nvlist_add_common(nvl, NVP_NAME(nvp), NVP_TYPE(nvp),
	NVP_NELEM(nvp), NVP_VALUE(nvp)));
	}

	/*
	* Merge the supplied nvlists and put the result in dst.
	* The merged list will contain all names specified in both lists,
	* the values are taken from nvl in the case of duplicates.
	* Return 0 on success.
	*/
	int
	nvlist_merge(nvlist_t dst, nvlist_t nvl, int flag)
	{
	(void) flag;

	if (nvl == NULL \|\| dst == NULL)
	return (EINVAL);

	if (dst != nvl)
	return (nvlist_copy_pairs(nvl, dst));

	return (0);
	}

	/*
	* Encoding related routines
	*/
	#define NVS_OP_ENCODE 0
	#define NVS_OP_DECODE 1
	#define NVS_OP_GETSIZE 2

	typedef struct nvs_ops nvs_ops_t;

	typedef struct {
	int nvs_op;
	const nvs_ops_t *nvs_ops;
	void *nvs_private;
	nvpriv_t *nvs_priv;
	int nvs_recursion;
	} nvstream_t;

	/*
	* nvs operations are:
	* - nvs_nvlist
	* encoding / decoding of an nvlist header (nvlist_t)
	* calculates the size used for header and end detection
	*
	* - nvs_nvpair
	* responsible for the first part of encoding / decoding of an nvpair
	* calculates the decoded size of an nvpair
	*
	* - nvs_nvp_op
	* second part of encoding / decoding of an nvpair
	*
	* - nvs_nvp_size
	* calculates the encoding size of an nvpair
	*
	* - nvs_nvl_fini
	* encodes the end detection mark (zeros).
	*/
	struct nvs_ops {
	int (nvs_nvlist)(nvstream_t , nvlist_t , size_t );
	int (nvs_nvpair)(nvstream_t , nvpair_t , size_t );
	int (nvs_nvp_op)(nvstream_t , nvpair_t *);
	int (nvs_nvp_size)(nvstream_t , nvpair_t , size_t );
	int (nvs_nvl_fini)(nvstream_t );
	};

	typedef struct {
	char nvh_encoding; /* nvs encoding method */
	char nvh_endian; /* nvs endian */
	char nvh_reserved1; /* reserved for future use */
	char nvh_reserved2; /* reserved for future use */
	} nvs_header_t;

	static int
	nvs_encode_pairs(nvstream_t nvs, nvlist_t nvl)
	{
	nvpriv_t priv = (nvpriv_t )(uintptr_t)nvl->nvl_priv;
	i_nvp_t *curr;

	/*
	* Walk nvpair in list and encode each nvpair
	*/
	for (curr = priv->nvp_list; curr != NULL; curr = curr->nvi_next)
	if (nvs->nvs_ops->nvs_nvpair(nvs, &curr->nvi_nvp, NULL) != 0)
	return (EFAULT);

	return (nvs->nvs_ops->nvs_nvl_fini(nvs));
	}

	static int
	nvs_decode_pairs(nvstream_t nvs, nvlist_t nvl)
	{
	nvpair_t *nvp;
	size_t nvsize;
	int err;

	/*
	* Get decoded size of next pair in stream, alloc
	* memory for nvpair_t, then decode the nvpair
	*/
	while ((err = nvs->nvs_ops->nvs_nvpair(nvs, NULL, &nvsize)) == 0) {
	if (nvsize == 0) /* end of list */
	break;

	/* make sure len makes sense */
	if (nvsize < NVP_SIZE_CALC(1, 0))
	return (EFAULT);

	if ((nvp = nvp_buf_alloc(nvl, nvsize)) == NULL)
	return (ENOMEM);

	if ((err = nvs->nvs_ops->nvs_nvp_op(nvs, nvp)) != 0) {
	nvp_buf_free(nvl, nvp);
	return (err);
	}

	if (i_validate_nvpair(nvp) != 0) {
	nvpair_free(nvp);
	nvp_buf_free(nvl, nvp);
	return (EFAULT);
	}

	err = nvt_add_nvpair(nvl, nvp);
	if (err != 0) {
	nvpair_free(nvp);
	nvp_buf_free(nvl, nvp);
	return (err);
	}
	nvp_buf_link(nvl, nvp);
	}
	return (err);
	}

	static int
	nvs_getsize_pairs(nvstream_t nvs, nvlist_t nvl, size_t *buflen)
	{
	nvpriv_t priv = (nvpriv_t )(uintptr_t)nvl->nvl_priv;
	i_nvp_t *curr;
	uint64_t nvsize = *buflen;
	size_t size;

	/*
	* Get encoded size of nvpairs in nvlist
	*/
	for (curr = priv->nvp_list; curr != NULL; curr = curr->nvi_next) {
	if (nvs->nvs_ops->nvs_nvp_size(nvs, &curr->nvi_nvp, &size) != 0)
	return (EINVAL);

	if ((nvsize += size) > INT32_MAX)
	return (EINVAL);
	}

	*buflen = nvsize;
	return (0);
	}

	static int
	nvs_operation(nvstream_t nvs, nvlist_t nvl, size_t *buflen)
	{
	int err;

	if (nvl->nvl_priv == 0)
	return (EFAULT);

	/*
	* Perform the operation, starting with header, then each nvpair
	*/
	if ((err = nvs->nvs_ops->nvs_nvlist(nvs, nvl, buflen)) != 0)
	return (err);

	switch (nvs->nvs_op) {
	case NVS_OP_ENCODE:
	err = nvs_encode_pairs(nvs, nvl);
	break;

	case NVS_OP_DECODE:
	err = nvs_decode_pairs(nvs, nvl);
	break;

	case NVS_OP_GETSIZE:
	err = nvs_getsize_pairs(nvs, nvl, buflen);
	break;

	default:
	err = EINVAL;
	}

	return (err);
	}

	static int
	nvs_embedded(nvstream_t nvs, nvlist_t embedded)
	{
	switch (nvs->nvs_op) {
	case NVS_OP_ENCODE: {
	int err;

	if (nvs->nvs_recursion >= nvpair_max_recursion)
	return (EINVAL);
	nvs->nvs_recursion++;
	err = nvs_operation(nvs, embedded, NULL);
	nvs->nvs_recursion--;
	return (err);
	}
	case NVS_OP_DECODE: {
	nvpriv_t *priv;
	int err;

	if (embedded->nvl_version != NV_VERSION)
	return (ENOTSUP);

	if ((priv = nv_priv_alloc_embedded(nvs->nvs_priv)) == NULL)
	return (ENOMEM);

	nvlist_init(embedded, embedded->nvl_nvflag, priv);

	if (nvs->nvs_recursion >= nvpair_max_recursion) {
	nvlist_free(embedded);
	return (EINVAL);
	}
	nvs->nvs_recursion++;
	if ((err = nvs_operation(nvs, embedded, NULL)) != 0)
	nvlist_free(embedded);
	nvs->nvs_recursion--;
	return (err);
	}
	default:
	break;
	}

	return (EINVAL);
	}

	static int
	nvs_embedded_nvl_array(nvstream_t nvs, nvpair_t nvp, size_t *size)
	{
	size_t nelem = NVP_NELEM(nvp);
	nvlist_t **nvlp = EMBEDDED_NVL_ARRAY(nvp);
	int i;

	switch (nvs->nvs_op) {
	case NVS_OP_ENCODE:
	for (i = 0; i < nelem; i++)
	if (nvs_embedded(nvs, nvlp[i]) != 0)
	return (EFAULT);
	break;

	case NVS_OP_DECODE: {
	size_t len = nelem * sizeof (uint64_t);
	nvlist_t embedded = (nvlist_t )((uintptr_t)nvlp + len);

	memset(nvlp, 0, len); /* don't trust packed data */
	for (i = 0; i < nelem; i++) {
	if (nvs_embedded(nvs, embedded) != 0) {
	nvpair_free(nvp);
	return (EFAULT);
	}

	nvlp[i] = embedded++;
	}
	break;
	}
	case NVS_OP_GETSIZE: {
	uint64_t nvsize = 0;

	for (i = 0; i < nelem; i++) {
	size_t nvp_sz = 0;

	if (nvs_operation(nvs, nvlp[i], &nvp_sz) != 0)
	return (EINVAL);

	if ((nvsize += nvp_sz) > INT32_MAX)
	return (EINVAL);
	}

	*size = nvsize;
	break;
	}
	default:
	return (EINVAL);
	}

	return (0);
	}

	static int nvs_native(nvstream_t , nvlist_t , char , size_t );
	static int nvs_xdr(nvstream_t , nvlist_t , char , size_t );

	/*
	* Common routine for nvlist operations:
	* encode, decode, getsize (encoded size).
	*/
	static int
	nvlist_common(nvlist_t nvl, char buf, size_t *buflen, int encoding,
	int nvs_op)
	{
	int err = 0;
	nvstream_t nvs;
	int nvl_endian;
	#if defined(_ZFS_LITTLE_ENDIAN)
	int host_endian = 1;
	#elif defined(_ZFS_BIG_ENDIAN)
	int host_endian = 0;
	#else
	#error "No endian defined!"
	#endif /* _ZFS_LITTLE_ENDIAN */
	nvs_header_t *nvh;

	if (buflen == NULL \|\| nvl == NULL \|\|
	(nvs.nvs_priv = (nvpriv_t *)(uintptr_t)nvl->nvl_priv) == NULL)
	return (EINVAL);

	nvs.nvs_op = nvs_op;
	nvs.nvs_recursion = 0;

	/*
	* For NVS_OP_ENCODE and NVS_OP_DECODE make sure an nvlist and
	* a buffer is allocated. The first 4 bytes in the buffer are
	* used for encoding method and host endian.
	*/
	switch (nvs_op) {
	case NVS_OP_ENCODE:
	if (buf == NULL \|\| *buflen < sizeof (nvs_header_t))
	return (EINVAL);

	nvh = (void *)buf;
	nvh->nvh_encoding = encoding;
	nvh->nvh_endian = nvl_endian = host_endian;
	nvh->nvh_reserved1 = 0;
	nvh->nvh_reserved2 = 0;
	break;

	case NVS_OP_DECODE:
	if (buf == NULL \|\| *buflen < sizeof (nvs_header_t))
	return (EINVAL);

	/* get method of encoding from first byte */
	nvh = (void *)buf;
	encoding = nvh->nvh_encoding;
	nvl_endian = nvh->nvh_endian;
	break;

	case NVS_OP_GETSIZE:
	nvl_endian = host_endian;

	/*
	* add the size for encoding
	*/
	*buflen = sizeof (nvs_header_t);
	break;

	default:
	return (ENOTSUP);
	}

	/*
	* Create an nvstream with proper encoding method
	*/
	switch (encoding) {
	case NV_ENCODE_NATIVE:
	/*
	* check endianness, in case we are unpacking
	* from a file
	*/
	if (nvl_endian != host_endian)
	return (ENOTSUP);
	err = nvs_native(&nvs, nvl, buf, buflen);
	break;
	case NV_ENCODE_XDR:
	err = nvs_xdr(&nvs, nvl, buf, buflen);
	break;
	default:
	err = ENOTSUP;
	break;
	}

	return (err);
	}

	int
	nvlist_size(nvlist_t nvl, size_t size, int encoding)
	{
	return (nvlist_common(nvl, NULL, size, encoding, NVS_OP_GETSIZE));
	}

	/*
	* Pack nvlist into contiguous memory
	*/
	int
	nvlist_pack(nvlist_t nvl, char bufp, size_t buflen, int encoding,
	int kmflag)
	{
	return (nvlist_xpack(nvl, bufp, buflen, encoding,
	nvlist_nv_alloc(kmflag)));
	}

	int
	nvlist_xpack(nvlist_t nvl, char bufp, size_t buflen, int encoding,
	nv_alloc_t *nva)
	{
	nvpriv_t nvpriv;
	size_t alloc_size;
	char *buf;
	int err;

	if (nva == NULL \|\| nvl == NULL \|\| bufp == NULL \|\| buflen == NULL)
	return (EINVAL);

	if (*bufp != NULL)
	return (nvlist_common(nvl, *bufp, buflen, encoding,
	NVS_OP_ENCODE));

	/*
	* Here is a difficult situation:
	* 1. The nvlist has fixed allocator properties.
	* All other nvlist routines (like nvlist_add_*, ...) use
	* these properties.
	* 2. When using nvlist_pack() the user can specify their own
	* allocator properties (e.g. by using KM_NOSLEEP).
	*
	* We use the user specified properties (2). A clearer solution
	* will be to remove the kmflag from nvlist_pack(), but we will
	* not change the interface.
	*/
	nv_priv_init(&nvpriv, nva, 0);

	if ((err = nvlist_size(nvl, &alloc_size, encoding)))
	return (err);

	if ((buf = nv_mem_zalloc(&nvpriv, alloc_size)) == NULL)
	return (ENOMEM);

	if ((err = nvlist_common(nvl, buf, &alloc_size, encoding,
	NVS_OP_ENCODE)) != 0) {
	nv_mem_free(&nvpriv, buf, alloc_size);
	} else {
	*buflen = alloc_size;
	*bufp = buf;
	}

	return (err);
	}

	/*
	* Unpack buf into an nvlist_t
	*/
	int
	nvlist_unpack(char buf, size_t buflen, nvlist_t *nvlp, int kmflag)
	{
	return (nvlist_xunpack(buf, buflen, nvlp, nvlist_nv_alloc(kmflag)));
	}

	int
	nvlist_xunpack(char buf, size_t buflen, nvlist_t nvlp, nv_alloc_t nva)
	{
	nvlist_t *nvl;
	int err;

	if (nvlp == NULL)
	return (EINVAL);

	if ((err = nvlist_xalloc(&nvl, 0, nva)) != 0)
	return (err);

	if ((err = nvlist_common(nvl, buf, &buflen, NV_ENCODE_NATIVE,
	NVS_OP_DECODE)) != 0)
	nvlist_free(nvl);
	else
	*nvlp = nvl;

	return (err);
	}

	/*
	* Native encoding functions
	*/
	typedef struct {
	/*
	* This structure is used when decoding a packed nvpair in
	* the native format. n_base points to a buffer containing the
	* packed nvpair. n_end is a pointer to the end of the buffer.
	* (n_end actually points to the first byte past the end of the
	* buffer.) n_curr is a pointer that lies between n_base and n_end.
	* It points to the current data that we are decoding.
	* The amount of data left in the buffer is equal to n_end - n_curr.
	* n_flag is used to recognize a packed embedded list.
	*/
	caddr_t n_base;
	caddr_t n_end;
	caddr_t n_curr;
	uint_t n_flag;
	} nvs_native_t;

	static int
	nvs_native_create(nvstream_t nvs, nvs_native_t native, char *buf,
	size_t buflen)
	{
	switch (nvs->nvs_op) {
	case NVS_OP_ENCODE:
	case NVS_OP_DECODE:
	nvs->nvs_private = native;
	native->n_curr = native->n_base = buf;
	native->n_end = buf + buflen;
	native->n_flag = 0;
	return (0);

	case NVS_OP_GETSIZE:
	nvs->nvs_private = native;
	native->n_curr = native->n_base = native->n_end = NULL;
	native->n_flag = 0;
	return (0);
	default:
	return (EINVAL);
	}
	}

	static void
	nvs_native_destroy(nvstream_t *nvs)
	{
	nvs->nvs_private = NULL;
	}

	static int
	native_cp(nvstream_t nvs, void buf, size_t size)
	{
	nvs_native_t native = (nvs_native_t )nvs->nvs_private;

	if (native->n_curr + size > native->n_end)
	return (EFAULT);

	/*
	* The memcpy() below eliminates alignment requirement
	* on the buffer (stream) and is preferred over direct access.
	*/
	switch (nvs->nvs_op) {
	case NVS_OP_ENCODE:
	memcpy(native->n_curr, buf, size);
	break;
	case NVS_OP_DECODE:
	memcpy(buf, native->n_curr, size);
	break;
	default:
	return (EINVAL);
	}

	native->n_curr += size;
	return (0);
	}

	/*
	* operate on nvlist_t header
	*/
	static int
	nvs_native_nvlist(nvstream_t nvs, nvlist_t nvl, size_t *size)
	{
	nvs_native_t *native = nvs->nvs_private;

	switch (nvs->nvs_op) {
	case NVS_OP_ENCODE:
	case NVS_OP_DECODE:
	if (native->n_flag)
	return (0); /* packed embedded list */

	native->n_flag = 1;

	/* copy version and nvflag of the nvlist_t */
	if (native_cp(nvs, &nvl->nvl_version, sizeof (int32_t)) != 0 \|\|
	native_cp(nvs, &nvl->nvl_nvflag, sizeof (int32_t)) != 0)
	return (EFAULT);

	return (0);

	case NVS_OP_GETSIZE:
	/*
	* if calculate for packed embedded list
	* 4 for end of the embedded list
	* else
	* 2 * sizeof (int32_t) for nvl_version and nvl_nvflag
	* and 4 for end of the entire list
	*/
	if (native->n_flag) {
	*size += 4;
	} else {
	native->n_flag = 1;
	size += 2 sizeof (int32_t) + 4;
	}

	return (0);

	default:
	return (EINVAL);
	}
	}

	static int
	nvs_native_nvl_fini(nvstream_t *nvs)
	{
	if (nvs->nvs_op == NVS_OP_ENCODE) {
	nvs_native_t native = (nvs_native_t )nvs->nvs_private;
	/*
	* Add 4 zero bytes at end of nvlist. They are used
	* for end detection by the decode routine.
	*/
	if (native->n_curr + sizeof (int) > native->n_end)
	return (EFAULT);

	memset(native->n_curr, 0, sizeof (int));
	native->n_curr += sizeof (int);
	}

	return (0);
	}

	static int
	nvpair_native_embedded(nvstream_t nvs, nvpair_t nvp)
	{
	if (nvs->nvs_op == NVS_OP_ENCODE) {
	nvs_native_t native = (nvs_native_t )nvs->nvs_private;
	nvlist_t packed = (void )
	(native->n_curr - nvp->nvp_size + NVP_VALOFF(nvp));
	/*
	* Null out the pointer that is meaningless in the packed
	* structure. The address may not be aligned, so we have
	* to use memset.
	*/
	memset((char *)packed + offsetof(nvlist_t, nvl_priv),
	0, sizeof (uint64_t));
	}

	return (nvs_embedded(nvs, EMBEDDED_NVL(nvp)));
	}

	static int
	nvpair_native_embedded_array(nvstream_t nvs, nvpair_t nvp)
	{
	if (nvs->nvs_op == NVS_OP_ENCODE) {
	nvs_native_t native = (nvs_native_t )nvs->nvs_private;
	char *value = native->n_curr - nvp->nvp_size + NVP_VALOFF(nvp);
	size_t len = NVP_NELEM(nvp) * sizeof (uint64_t);
	nvlist_t packed = (nvlist_t )((uintptr_t)value + len);
	int i;
	/*
	* Null out pointers that are meaningless in the packed
	* structure. The addresses may not be aligned, so we have
	* to use memset.
	*/
	memset(value, 0, len);

	for (i = 0; i < NVP_NELEM(nvp); i++, packed++)
	/*
	* Null out the pointer that is meaningless in the
	* packed structure. The address may not be aligned,
	* so we have to use memset.
	*/
	memset((char *)packed + offsetof(nvlist_t, nvl_priv),
	0, sizeof (uint64_t));
	}

	return (nvs_embedded_nvl_array(nvs, nvp, NULL));
	}

	static void
	nvpair_native_string_array(nvstream_t nvs, nvpair_t nvp)
	{
	switch (nvs->nvs_op) {
	case NVS_OP_ENCODE: {
	nvs_native_t native = (nvs_native_t )nvs->nvs_private;
	uint64_t strp = (void )
	(native->n_curr - nvp->nvp_size + NVP_VALOFF(nvp));
	/*
	* Null out pointers that are meaningless in the packed
	* structure. The addresses may not be aligned, so we have
	* to use memset.
	*/
	memset(strp, 0, NVP_NELEM(nvp) * sizeof (uint64_t));
	break;
	}
	case NVS_OP_DECODE: {
	char *strp = (void )NVP_VALUE(nvp);
	char buf = ((char )strp + NVP_NELEM(nvp) * sizeof (uint64_t));
	int i;

	for (i = 0; i < NVP_NELEM(nvp); i++) {
	strp[i] = buf;
	buf += strlen(buf) + 1;
	}
	break;
	}
	}
	}

	static int
	nvs_native_nvp_op(nvstream_t nvs, nvpair_t nvp)
	{
	data_type_t type;
	int value_sz;
	int ret = 0;

	/*
	* We do the initial memcpy of the data before we look at
	* the nvpair type, because when we're decoding, we won't
	* have the correct values for the pair until we do the memcpy.
	*/
	switch (nvs->nvs_op) {
	case NVS_OP_ENCODE:
	case NVS_OP_DECODE:
	if (native_cp(nvs, nvp, nvp->nvp_size) != 0)
	return (EFAULT);
	break;
	default:
	return (EINVAL);
	}

	/* verify nvp_name_sz, check the name string length */
	if (i_validate_nvpair_name(nvp) != 0)
	return (EFAULT);

	type = NVP_TYPE(nvp);

	/*
	* Verify type and nelem and get the value size.
	* In case of data types DATA_TYPE_STRING and DATA_TYPE_STRING_ARRAY
	* is the size of the string(s) excluded.
	*/
	if ((value_sz = i_get_value_size(type, NULL, NVP_NELEM(nvp))) < 0)
	return (EFAULT);

	if (NVP_SIZE_CALC(nvp->nvp_name_sz, value_sz) > nvp->nvp_size)
	return (EFAULT);

	switch (type) {
	case DATA_TYPE_NVLIST:
	ret = nvpair_native_embedded(nvs, nvp);
	break;
	case DATA_TYPE_NVLIST_ARRAY:
	ret = nvpair_native_embedded_array(nvs, nvp);
	break;
	case DATA_TYPE_STRING_ARRAY:
	nvpair_native_string_array(nvs, nvp);
	break;
	default:
	break;
	}

	return (ret);
	}

	static int
	nvs_native_nvp_size(nvstream_t nvs, nvpair_t nvp, size_t *size)
	{
	uint64_t nvp_sz = nvp->nvp_size;

	switch (NVP_TYPE(nvp)) {
	case DATA_TYPE_NVLIST: {
	size_t nvsize = 0;

	if (nvs_operation(nvs, EMBEDDED_NVL(nvp), &nvsize) != 0)
	return (EINVAL);

	nvp_sz += nvsize;
	break;
	}
	case DATA_TYPE_NVLIST_ARRAY: {
	size_t nvsize;

	if (nvs_embedded_nvl_array(nvs, nvp, &nvsize) != 0)
	return (EINVAL);

	nvp_sz += nvsize;
	break;
	}
	default:
	break;
	}

	if (nvp_sz > INT32_MAX)
	return (EINVAL);

	*size = nvp_sz;

	return (0);
	}

	static int
	nvs_native_nvpair(nvstream_t nvs, nvpair_t nvp, size_t *size)
	{
	switch (nvs->nvs_op) {
	case NVS_OP_ENCODE:
	return (nvs_native_nvp_op(nvs, nvp));

	case NVS_OP_DECODE: {
	nvs_native_t native = (nvs_native_t )nvs->nvs_private;
	int32_t decode_len;

	/* try to read the size value from the stream */
	if (native->n_curr + sizeof (int32_t) > native->n_end)
	return (EFAULT);
	memcpy(&decode_len, native->n_curr, sizeof (int32_t));

	/* sanity check the size value */
	if (decode_len < 0 \|\|
	decode_len > native->n_end - native->n_curr)
	return (EFAULT);

	*size = decode_len;

	/*
	* If at the end of the stream then move the cursor
	* forward, otherwise nvpair_native_op() will read
	* the entire nvpair at the same cursor position.
	*/
	if (*size == 0)
	native->n_curr += sizeof (int32_t);
	break;
	}

	default:
	return (EINVAL);
	}

	return (0);
	}

	static const nvs_ops_t nvs_native_ops = {
	.nvs_nvlist = nvs_native_nvlist,
	.nvs_nvpair = nvs_native_nvpair,
	.nvs_nvp_op = nvs_native_nvp_op,
	.nvs_nvp_size = nvs_native_nvp_size,
	.nvs_nvl_fini = nvs_native_nvl_fini
	};

	static int
	nvs_native(nvstream_t nvs, nvlist_t nvl, char buf, size_t buflen)
	{
	nvs_native_t native;
	int err;

	nvs->nvs_ops = &nvs_native_ops;

	if ((err = nvs_native_create(nvs, &native, buf + sizeof (nvs_header_t),
	*buflen - sizeof (nvs_header_t))) != 0)
	return (err);

	err = nvs_operation(nvs, nvl, buflen);

	nvs_native_destroy(nvs);

	return (err);
	}

	/*
	* XDR encoding functions
	*
	* An xdr packed nvlist is encoded as:
	*
	* - encoding method and host endian (4 bytes)
	* - nvl_version (4 bytes)
	* - nvl_nvflag (4 bytes)
	*
	* - encoded nvpairs, the format of one xdr encoded nvpair is:
	* - encoded size of the nvpair (4 bytes)
	* - decoded size of the nvpair (4 bytes)
	* - name string, (4 + sizeof(NV_ALIGN4(string))
	* a string is coded as size (4 bytes) and data
	* - data type (4 bytes)
	* - number of elements in the nvpair (4 bytes)
	* - data
	*
	* - 2 zero's for end of the entire list (8 bytes)
	*/
	static int
	nvs_xdr_create(nvstream_t nvs, XDR xdr, char *buf, size_t buflen)
	{
	/* xdr data must be 4 byte aligned */
	if ((ulong_t)buf % 4 != 0)
	return (EFAULT);

	switch (nvs->nvs_op) {
	case NVS_OP_ENCODE:
	xdrmem_create(xdr, buf, (uint_t)buflen, XDR_ENCODE);
	nvs->nvs_private = xdr;
	return (0);
	case NVS_OP_DECODE:
	xdrmem_create(xdr, buf, (uint_t)buflen, XDR_DECODE);
	nvs->nvs_private = xdr;
	return (0);
	case NVS_OP_GETSIZE:
	nvs->nvs_private = NULL;
	return (0);
	default:
	return (EINVAL);
	}
	}

	static void
	nvs_xdr_destroy(nvstream_t *nvs)
	{
	switch (nvs->nvs_op) {
	case NVS_OP_ENCODE:
	case NVS_OP_DECODE:
	nvs->nvs_private = NULL;
	break;
	default:
	break;
	}
	}

	static int
	nvs_xdr_nvlist(nvstream_t nvs, nvlist_t nvl, size_t *size)
	{
	switch (nvs->nvs_op) {
	case NVS_OP_ENCODE:
	case NVS_OP_DECODE: {
	XDR *xdr = nvs->nvs_private;

	if (!xdr_int(xdr, &nvl->nvl_version) \|\|
	!xdr_u_int(xdr, &nvl->nvl_nvflag))
	return (EFAULT);
	break;
	}
	case NVS_OP_GETSIZE: {
	/*
	* 2 * 4 for nvl_version + nvl_nvflag
	* and 8 for end of the entire list
	*/
	size += 2 4 + 8;
	break;
	}
	default:
	return (EINVAL);
	}
	return (0);
	}

	static int
	nvs_xdr_nvl_fini(nvstream_t *nvs)
	{
	if (nvs->nvs_op == NVS_OP_ENCODE) {
	XDR *xdr = nvs->nvs_private;
	int zero = 0;

	if (!xdr_int(xdr, &zero) \|\| !xdr_int(xdr, &zero))
	return (EFAULT);
	}

	return (0);
	}

	/*
	* xdrproc_t-compatible callbacks for xdr_array()
	*/

	#if defined(_KERNEL) && defined(__linux__) /* Linux kernel */

	#define NVS_BUILD_XDRPROC_T(type) \
	static bool_t \
	nvs_xdr_nvp_##type(XDR xdrs, void ptr) \
	{ \
	return (xdr_##type(xdrs, ptr)); \
	}

	#elif !defined(_KERNEL) && defined(XDR_CONTROL) /* tirpc */

	#define NVS_BUILD_XDRPROC_T(type) \
	static bool_t \
	nvs_xdr_nvp_##type(XDR *xdrs, ...) \
	{ \
	va_list args; \
	void *ptr; \
	\
	va_start(args, xdrs); \
	ptr = va_arg(args, void *); \
	va_end(args); \
	\
	return (xdr_##type(xdrs, ptr)); \
	}

	#else /* FreeBSD, sunrpc */

	#define NVS_BUILD_XDRPROC_T(type) \
	static bool_t \
	nvs_xdr_nvp_##type(XDR xdrs, void ptr, ...) \
	{ \
	return (xdr_##type(xdrs, ptr)); \
	}

	#endif

	/* BEGIN CSTYLED */
	NVS_BUILD_XDRPROC_T(char);
	NVS_BUILD_XDRPROC_T(short);
	NVS_BUILD_XDRPROC_T(u_short);
	NVS_BUILD_XDRPROC_T(int);
	NVS_BUILD_XDRPROC_T(u_int);
	NVS_BUILD_XDRPROC_T(longlong_t);
	NVS_BUILD_XDRPROC_T(u_longlong_t);
	/* END CSTYLED */

	/*
	* The format of xdr encoded nvpair is:
	* encode_size, decode_size, name string, data type, nelem, data
	*/
	static int
	nvs_xdr_nvp_op(nvstream_t nvs, nvpair_t nvp)
	{
	ASSERT(nvs != NULL && nvp != NULL);

	data_type_t type;
	char *buf;
	char buf_end = (char )nvp + nvp->nvp_size;
	int value_sz;
	uint_t nelem, buflen;
	bool_t ret = FALSE;
	XDR *xdr = nvs->nvs_private;

	ASSERT(xdr != NULL);

	/* name string */
	if ((buf = NVP_NAME(nvp)) >= buf_end)
	return (EFAULT);
	buflen = buf_end - buf;

	if (!xdr_string(xdr, &buf, buflen - 1))
	return (EFAULT);
	nvp->nvp_name_sz = strlen(buf) + 1;

	/* type and nelem */
	if (!xdr_int(xdr, (int *)&nvp->nvp_type) \|\|
	!xdr_int(xdr, &nvp->nvp_value_elem))
	return (EFAULT);

	type = NVP_TYPE(nvp);
	nelem = nvp->nvp_value_elem;

	/*
	* Verify type and nelem and get the value size.
	* In case of data types DATA_TYPE_STRING and DATA_TYPE_STRING_ARRAY
	* is the size of the string(s) excluded.
	*/
	if ((value_sz = i_get_value_size(type, NULL, nelem)) < 0)
	return (EFAULT);

	/* if there is no data to extract then return */
	if (nelem == 0)
	return (0);

	/* value */
	if ((buf = NVP_VALUE(nvp)) >= buf_end)
	return (EFAULT);
	buflen = buf_end - buf;

	if (buflen < value_sz)
	return (EFAULT);

	switch (type) {
	case DATA_TYPE_NVLIST:
	if (nvs_embedded(nvs, (void *)buf) == 0)
	return (0);
	break;

	case DATA_TYPE_NVLIST_ARRAY:
	if (nvs_embedded_nvl_array(nvs, nvp, NULL) == 0)
	return (0);
	break;

	case DATA_TYPE_BOOLEAN:
	ret = TRUE;
	break;

	case DATA_TYPE_BYTE:
	case DATA_TYPE_INT8:
	case DATA_TYPE_UINT8:
	ret = xdr_char(xdr, buf);
	break;

	case DATA_TYPE_INT16:
	ret = xdr_short(xdr, (void *)buf);
	break;

	case DATA_TYPE_UINT16:
	ret = xdr_u_short(xdr, (void *)buf);
	break;

	case DATA_TYPE_BOOLEAN_VALUE:
	case DATA_TYPE_INT32:
	ret = xdr_int(xdr, (void *)buf);
	break;

	case DATA_TYPE_UINT32:
	ret = xdr_u_int(xdr, (void *)buf);
	break;

	case DATA_TYPE_INT64:
	ret = xdr_longlong_t(xdr, (void *)buf);
	break;

	case DATA_TYPE_UINT64:
	ret = xdr_u_longlong_t(xdr, (void *)buf);
	break;

	case DATA_TYPE_HRTIME:
	/*
	* NOTE: must expose the definition of hrtime_t here
	*/
	ret = xdr_longlong_t(xdr, (void *)buf);
	break;
	#if !defined(_KERNEL)
	case DATA_TYPE_DOUBLE:
	ret = xdr_double(xdr, (void *)buf);
	break;
	#endif
	case DATA_TYPE_STRING:
	ret = xdr_string(xdr, &buf, buflen - 1);
	break;

	case DATA_TYPE_BYTE_ARRAY:
	ret = xdr_opaque(xdr, buf, nelem);
	break;

	case DATA_TYPE_INT8_ARRAY:
	case DATA_TYPE_UINT8_ARRAY:
	ret = xdr_array(xdr, &buf, &nelem, buflen, sizeof (int8_t),
	nvs_xdr_nvp_char);
	break;

	case DATA_TYPE_INT16_ARRAY:
	ret = xdr_array(xdr, &buf, &nelem, buflen / sizeof (int16_t),
	sizeof (int16_t), nvs_xdr_nvp_short);
	break;

	case DATA_TYPE_UINT16_ARRAY:
	ret = xdr_array(xdr, &buf, &nelem, buflen / sizeof (uint16_t),
	sizeof (uint16_t), nvs_xdr_nvp_u_short);
	break;

	case DATA_TYPE_BOOLEAN_ARRAY:
	case DATA_TYPE_INT32_ARRAY:
	ret = xdr_array(xdr, &buf, &nelem, buflen / sizeof (int32_t),
	sizeof (int32_t), nvs_xdr_nvp_int);
	break;

	case DATA_TYPE_UINT32_ARRAY:
	ret = xdr_array(xdr, &buf, &nelem, buflen / sizeof (uint32_t),
	sizeof (uint32_t), nvs_xdr_nvp_u_int);
	break;

	case DATA_TYPE_INT64_ARRAY:
	ret = xdr_array(xdr, &buf, &nelem, buflen / sizeof (int64_t),
	sizeof (int64_t), nvs_xdr_nvp_longlong_t);
	break;

	case DATA_TYPE_UINT64_ARRAY:
	ret = xdr_array(xdr, &buf, &nelem, buflen / sizeof (uint64_t),
	sizeof (uint64_t), nvs_xdr_nvp_u_longlong_t);
	break;

	case DATA_TYPE_STRING_ARRAY: {
	size_t len = nelem * sizeof (uint64_t);
	char *strp = (void )buf;
	int i;

	if (nvs->nvs_op == NVS_OP_DECODE)
	memset(buf, 0, len); /* don't trust packed data */

	for (i = 0; i < nelem; i++) {
	if (buflen <= len)
	return (EFAULT);

	buf += len;
	buflen -= len;

	if (xdr_string(xdr, &buf, buflen - 1) != TRUE)
	return (EFAULT);

	if (nvs->nvs_op == NVS_OP_DECODE)
	strp[i] = buf;
	len = strlen(buf) + 1;
	}
	ret = TRUE;
	break;
	}
	default:
	break;
	}

	return (ret == TRUE ? 0 : EFAULT);
	}

	static int
	nvs_xdr_nvp_size(nvstream_t nvs, nvpair_t nvp, size_t *size)
	{
	data_type_t type = NVP_TYPE(nvp);
	/*
	* encode_size + decode_size + name string size + data type + nelem
	* where name string size = 4 + NV_ALIGN4(strlen(NVP_NAME(nvp)))
	*/
	uint64_t nvp_sz = 4 + 4 + 4 + NV_ALIGN4(strlen(NVP_NAME(nvp))) + 4 + 4;

	switch (type) {
	case DATA_TYPE_BOOLEAN:
	break;

	case DATA_TYPE_BOOLEAN_VALUE:
	case DATA_TYPE_BYTE:
	case DATA_TYPE_INT8:
	case DATA_TYPE_UINT8:
	case DATA_TYPE_INT16:
	case DATA_TYPE_UINT16:
	case DATA_TYPE_INT32:
	case DATA_TYPE_UINT32:
	nvp_sz += 4; /* 4 is the minimum xdr unit */
	break;

	case DATA_TYPE_INT64:
	case DATA_TYPE_UINT64:
	case DATA_TYPE_HRTIME:
	#if !defined(_KERNEL)
	case DATA_TYPE_DOUBLE:
	#endif
	nvp_sz += 8;
	break;

	case DATA_TYPE_STRING:
	nvp_sz += 4 + NV_ALIGN4(strlen((char *)NVP_VALUE(nvp)));
	break;

	case DATA_TYPE_BYTE_ARRAY:
	nvp_sz += NV_ALIGN4(NVP_NELEM(nvp));
	break;

	case DATA_TYPE_BOOLEAN_ARRAY:
	case DATA_TYPE_INT8_ARRAY:
	case DATA_TYPE_UINT8_ARRAY:
	case DATA_TYPE_INT16_ARRAY:
	case DATA_TYPE_UINT16_ARRAY:
	case DATA_TYPE_INT32_ARRAY:
	case DATA_TYPE_UINT32_ARRAY:
	nvp_sz += 4 + 4 * (uint64_t)NVP_NELEM(nvp);
	break;

	case DATA_TYPE_INT64_ARRAY:
	case DATA_TYPE_UINT64_ARRAY:
	nvp_sz += 4 + 8 * (uint64_t)NVP_NELEM(nvp);
	break;

	case DATA_TYPE_STRING_ARRAY: {
	int i;
	char *strs = (void )NVP_VALUE(nvp);

	for (i = 0; i < NVP_NELEM(nvp); i++)
	nvp_sz += 4 + NV_ALIGN4(strlen(strs[i]));

	break;
	}

	case DATA_TYPE_NVLIST:
	case DATA_TYPE_NVLIST_ARRAY: {
	size_t nvsize = 0;
	int old_nvs_op = nvs->nvs_op;
	int err;

	nvs->nvs_op = NVS_OP_GETSIZE;
	if (type == DATA_TYPE_NVLIST)
	err = nvs_operation(nvs, EMBEDDED_NVL(nvp), &nvsize);
	else
	err = nvs_embedded_nvl_array(nvs, nvp, &nvsize);
	nvs->nvs_op = old_nvs_op;

	if (err != 0)
	return (EINVAL);

	nvp_sz += nvsize;
	break;
	}

	default:
	return (EINVAL);
	}

	if (nvp_sz > INT32_MAX)
	return (EINVAL);

	*size = nvp_sz;

	return (0);
	}


	/*
	* The NVS_XDR_MAX_LEN macro takes a packed xdr buffer of size x and estimates
	* the largest nvpair that could be encoded in the buffer.
	*
	* See comments above nvpair_xdr_op() for the format of xdr encoding.
	* The size of a xdr packed nvpair without any data is 5 words.
	*
	* Using the size of the data directly as an estimate would be ok
	* in all cases except one. If the data type is of DATA_TYPE_STRING_ARRAY
	* then the actual nvpair has space for an array of pointers to index
	* the strings. These pointers are not encoded into the packed xdr buffer.
	*
	* If the data is of type DATA_TYPE_STRING_ARRAY and all the strings are
	* of length 0, then each string is encoded in xdr format as a single word.
	* Therefore when expanded to an nvpair there will be 2.25 word used for
	* each string. (a int64_t allocated for pointer usage, and a single char
	* for the null termination.)
	*
	* This is the calculation performed by the NVS_XDR_MAX_LEN macro.
	*/
	#define NVS_XDR_HDR_LEN ((size_t)(5 * 4))
	#define NVS_XDR_DATA_LEN(y) (((size_t)(y) <= NVS_XDR_HDR_LEN) ? \
	0 : ((size_t)(y) - NVS_XDR_HDR_LEN))
	#define NVS_XDR_MAX_LEN(x) (NVP_SIZE_CALC(1, 0) + \
	(NVS_XDR_DATA_LEN(x) * 2) + \
	NV_ALIGN4((NVS_XDR_DATA_LEN(x) / 4)))

	static int
	nvs_xdr_nvpair(nvstream_t nvs, nvpair_t nvp, size_t *size)
	{
	XDR *xdr = nvs->nvs_private;
	int32_t encode_len, decode_len;

	switch (nvs->nvs_op) {
	case NVS_OP_ENCODE: {
	size_t nvsize;

	if (nvs_xdr_nvp_size(nvs, nvp, &nvsize) != 0)
	return (EFAULT);

	decode_len = nvp->nvp_size;
	encode_len = nvsize;
	if (!xdr_int(xdr, &encode_len) \|\| !xdr_int(xdr, &decode_len))
	return (EFAULT);

	return (nvs_xdr_nvp_op(nvs, nvp));
	}
	case NVS_OP_DECODE: {
	struct xdr_bytesrec bytesrec;

	/* get the encode and decode size */
	if (!xdr_int(xdr, &encode_len) \|\| !xdr_int(xdr, &decode_len))
	return (EFAULT);
	*size = decode_len;

	/* are we at the end of the stream? */
	if (*size == 0)
	return (0);

	/* sanity check the size parameter */
	if (!xdr_control(xdr, XDR_GET_BYTES_AVAIL, &bytesrec))
	return (EFAULT);

	if (*size > NVS_XDR_MAX_LEN(bytesrec.xc_num_avail))
	return (EFAULT);
	break;
	}

	default:
	return (EINVAL);
	}
	return (0);
	}

	static const struct nvs_ops nvs_xdr_ops = {
	.nvs_nvlist = nvs_xdr_nvlist,
	.nvs_nvpair = nvs_xdr_nvpair,
	.nvs_nvp_op = nvs_xdr_nvp_op,
	.nvs_nvp_size = nvs_xdr_nvp_size,
	.nvs_nvl_fini = nvs_xdr_nvl_fini
	};

	static int
	nvs_xdr(nvstream_t nvs, nvlist_t nvl, char buf, size_t buflen)
	{
	XDR xdr;
	int err;

	nvs->nvs_ops = &nvs_xdr_ops;

	if ((err = nvs_xdr_create(nvs, &xdr, buf + sizeof (nvs_header_t),
	*buflen - sizeof (nvs_header_t))) != 0)
	return (err);

	err = nvs_operation(nvs, nvl, buflen);

	nvs_xdr_destroy(nvs);

	return (err);
	}

	EXPORT_SYMBOL(nv_alloc_init);
	EXPORT_SYMBOL(nv_alloc_reset);
	EXPORT_SYMBOL(nv_alloc_fini);

	/* list management */
	EXPORT_SYMBOL(nvlist_alloc);
	EXPORT_SYMBOL(nvlist_free);
	EXPORT_SYMBOL(nvlist_size);
	EXPORT_SYMBOL(nvlist_pack);
	EXPORT_SYMBOL(nvlist_unpack);
	EXPORT_SYMBOL(nvlist_dup);
	EXPORT_SYMBOL(nvlist_merge);

	EXPORT_SYMBOL(nvlist_xalloc);
	EXPORT_SYMBOL(nvlist_xpack);
	EXPORT_SYMBOL(nvlist_xunpack);
	EXPORT_SYMBOL(nvlist_xdup);
	EXPORT_SYMBOL(nvlist_lookup_nv_alloc);

	EXPORT_SYMBOL(nvlist_add_nvpair);
	EXPORT_SYMBOL(nvlist_add_boolean);
	EXPORT_SYMBOL(nvlist_add_boolean_value);
	EXPORT_SYMBOL(nvlist_add_byte);
	EXPORT_SYMBOL(nvlist_add_int8);
	EXPORT_SYMBOL(nvlist_add_uint8);
	EXPORT_SYMBOL(nvlist_add_int16);
	EXPORT_SYMBOL(nvlist_add_uint16);
	EXPORT_SYMBOL(nvlist_add_int32);
	EXPORT_SYMBOL(nvlist_add_uint32);
	EXPORT_SYMBOL(nvlist_add_int64);
	EXPORT_SYMBOL(nvlist_add_uint64);
	EXPORT_SYMBOL(nvlist_add_string);
	EXPORT_SYMBOL(nvlist_add_nvlist);
	EXPORT_SYMBOL(nvlist_add_boolean_array);
	EXPORT_SYMBOL(nvlist_add_byte_array);
	EXPORT_SYMBOL(nvlist_add_int8_array);
	EXPORT_SYMBOL(nvlist_add_uint8_array);
	EXPORT_SYMBOL(nvlist_add_int16_array);
	EXPORT_SYMBOL(nvlist_add_uint16_array);
	EXPORT_SYMBOL(nvlist_add_int32_array);
	EXPORT_SYMBOL(nvlist_add_uint32_array);
	EXPORT_SYMBOL(nvlist_add_int64_array);
	EXPORT_SYMBOL(nvlist_add_uint64_array);
	EXPORT_SYMBOL(nvlist_add_string_array);
	EXPORT_SYMBOL(nvlist_add_nvlist_array);
	EXPORT_SYMBOL(nvlist_next_nvpair);
	EXPORT_SYMBOL(nvlist_prev_nvpair);
	EXPORT_SYMBOL(nvlist_empty);
	EXPORT_SYMBOL(nvlist_add_hrtime);

	EXPORT_SYMBOL(nvlist_remove);
	EXPORT_SYMBOL(nvlist_remove_nvpair);
	EXPORT_SYMBOL(nvlist_remove_all);

	EXPORT_SYMBOL(nvlist_lookup_boolean);
	EXPORT_SYMBOL(nvlist_lookup_boolean_value);
	EXPORT_SYMBOL(nvlist_lookup_byte);
	EXPORT_SYMBOL(nvlist_lookup_int8);
	EXPORT_SYMBOL(nvlist_lookup_uint8);
	EXPORT_SYMBOL(nvlist_lookup_int16);
	EXPORT_SYMBOL(nvlist_lookup_uint16);
	EXPORT_SYMBOL(nvlist_lookup_int32);
	EXPORT_SYMBOL(nvlist_lookup_uint32);
	EXPORT_SYMBOL(nvlist_lookup_int64);
	EXPORT_SYMBOL(nvlist_lookup_uint64);
	EXPORT_SYMBOL(nvlist_lookup_string);
	EXPORT_SYMBOL(nvlist_lookup_nvlist);
	EXPORT_SYMBOL(nvlist_lookup_boolean_array);
	EXPORT_SYMBOL(nvlist_lookup_byte_array);
	EXPORT_SYMBOL(nvlist_lookup_int8_array);
	EXPORT_SYMBOL(nvlist_lookup_uint8_array);
	EXPORT_SYMBOL(nvlist_lookup_int16_array);
	EXPORT_SYMBOL(nvlist_lookup_uint16_array);
	EXPORT_SYMBOL(nvlist_lookup_int32_array);
	EXPORT_SYMBOL(nvlist_lookup_uint32_array);
	EXPORT_SYMBOL(nvlist_lookup_int64_array);
	EXPORT_SYMBOL(nvlist_lookup_uint64_array);
	EXPORT_SYMBOL(nvlist_lookup_string_array);
	EXPORT_SYMBOL(nvlist_lookup_nvlist_array);
	EXPORT_SYMBOL(nvlist_lookup_hrtime);
	EXPORT_SYMBOL(nvlist_lookup_pairs);

	EXPORT_SYMBOL(nvlist_lookup_nvpair);
	EXPORT_SYMBOL(nvlist_exists);

	/* processing nvpair */
	EXPORT_SYMBOL(nvpair_name);
	EXPORT_SYMBOL(nvpair_type);
	EXPORT_SYMBOL(nvpair_value_boolean_value);
	EXPORT_SYMBOL(nvpair_value_byte);
	EXPORT_SYMBOL(nvpair_value_int8);
	EXPORT_SYMBOL(nvpair_value_uint8);
	EXPORT_SYMBOL(nvpair_value_int16);
	EXPORT_SYMBOL(nvpair_value_uint16);
	EXPORT_SYMBOL(nvpair_value_int32);
	EXPORT_SYMBOL(nvpair_value_uint32);
	EXPORT_SYMBOL(nvpair_value_int64);
	EXPORT_SYMBOL(nvpair_value_uint64);
	EXPORT_SYMBOL(nvpair_value_string);
	EXPORT_SYMBOL(nvpair_value_nvlist);
	EXPORT_SYMBOL(nvpair_value_boolean_array);
	EXPORT_SYMBOL(nvpair_value_byte_array);
	EXPORT_SYMBOL(nvpair_value_int8_array);
	EXPORT_SYMBOL(nvpair_value_uint8_array);
	EXPORT_SYMBOL(nvpair_value_int16_array);
	EXPORT_SYMBOL(nvpair_value_uint16_array);
	EXPORT_SYMBOL(nvpair_value_int32_array);
	EXPORT_SYMBOL(nvpair_value_uint32_array);
	EXPORT_SYMBOL(nvpair_value_int64_array);
	EXPORT_SYMBOL(nvpair_value_uint64_array);
	EXPORT_SYMBOL(nvpair_value_string_array);
	EXPORT_SYMBOL(nvpair_value_nvlist_array);
	EXPORT_SYMBOL(nvpair_value_hrtime);
	diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/abd_os.c b/sys/contrib/openzfs/module/os/freebsd/zfs/abd_os.c
	index 58a37df62b69..3b812271f98b 100644
	--- a/sys/contrib/openzfs/module/os/freebsd/zfs/abd_os.c
	+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/abd_os.c
	@@ -1,505 +1,503 @@
	/*
	* This file and its contents are supplied under the terms of the
	* Common Development and Distribution License ("CDDL"), version 1.0.
	* You may only use this file in accordance with the terms of version
	* 1.0 of the CDDL.
	*
	* A full copy of the text of the CDDL should have accompanied this
	* source. A copy of the CDDL is also available via the Internet at
	* http://www.illumos.org/license/CDDL.
	*/

	/*
	* Copyright (c) 2014 by Chunwei Chen. All rights reserved.
	* Copyright (c) 2016 by Delphix. All rights reserved.
	*/

	/*
	* See abd.c for a general overview of the arc buffered data (ABD).
	*
	* Using a large proportion of scattered ABDs decreases ARC fragmentation since
	* when we are at the limit of allocatable space, using equal-size chunks will
	* allow us to quickly reclaim enough space for a new large allocation (assuming
	* it is also scattered).
	*
	* ABDs are allocated scattered by default unless the caller uses
	* abd_alloc_linear() or zfs_abd_scatter_enabled is disabled.
	*/

	#include <sys/abd_impl.h>
	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/zio.h>
	#include <sys/zfs_context.h>
	#include <sys/zfs_znode.h>

	typedef struct abd_stats {
	kstat_named_t abdstat_struct_size;
	kstat_named_t abdstat_scatter_cnt;
	kstat_named_t abdstat_scatter_data_size;
	kstat_named_t abdstat_scatter_chunk_waste;
	kstat_named_t abdstat_linear_cnt;
	kstat_named_t abdstat_linear_data_size;
	} abd_stats_t;

	static abd_stats_t abd_stats = {
	/* Amount of memory occupied by all of the abd_t struct allocations */
	{ "struct_size", KSTAT_DATA_UINT64 },
	/*
	* The number of scatter ABDs which are currently allocated, excluding
	* ABDs which don't own their data (for instance the ones which were
	* allocated through abd_get_offset()).
	*/
	{ "scatter_cnt", KSTAT_DATA_UINT64 },
	/* Amount of data stored in all scatter ABDs tracked by scatter_cnt */
	{ "scatter_data_size", KSTAT_DATA_UINT64 },
	/*
	* The amount of space wasted at the end of the last chunk across all
	* scatter ABDs tracked by scatter_cnt.
	*/
	{ "scatter_chunk_waste", KSTAT_DATA_UINT64 },
	/*
	* The number of linear ABDs which are currently allocated, excluding
	* ABDs which don't own their data (for instance the ones which were
	* allocated through abd_get_offset() and abd_get_from_buf()). If an
	* ABD takes ownership of its buf then it will become tracked.
	*/
	{ "linear_cnt", KSTAT_DATA_UINT64 },
	/* Amount of data stored in all linear ABDs tracked by linear_cnt */
	{ "linear_data_size", KSTAT_DATA_UINT64 },
	};

	struct {
	wmsum_t abdstat_struct_size;
	wmsum_t abdstat_scatter_cnt;
	wmsum_t abdstat_scatter_data_size;
	wmsum_t abdstat_scatter_chunk_waste;
	wmsum_t abdstat_linear_cnt;
	wmsum_t abdstat_linear_data_size;
	} abd_sums;

	/*
	* zfs_abd_scatter_min_size is the minimum allocation size to use scatter
	* ABD's for. Smaller allocations will use linear ABD's which use
	* zio_[data_]buf_alloc().
	*
	* Scatter ABD's use at least one page each, so sub-page allocations waste
	* some space when allocated as scatter (e.g. 2KB scatter allocation wastes
	* half of each page). Using linear ABD's for small allocations means that
	* they will be put on slabs which contain many allocations.
	*
	* Linear ABDs for multi-page allocations are easier to use, and in some cases
	* it allows to avoid buffer copying. But allocation and especially free
	* of multi-page linear ABDs are expensive operations due to KVA mapping and
	* unmapping, and with time they cause KVA fragmentations.
	*/
	static size_t zfs_abd_scatter_min_size = PAGE_SIZE + 1;

	#if defined(_KERNEL)
	SYSCTL_DECL(_vfs_zfs);

	SYSCTL_INT(_vfs_zfs, OID_AUTO, abd_scatter_enabled, CTLFLAG_RWTUN,
	&zfs_abd_scatter_enabled, 0, "Enable scattered ARC data buffers");
	SYSCTL_ULONG(_vfs_zfs, OID_AUTO, abd_scatter_min_size, CTLFLAG_RWTUN,
	&zfs_abd_scatter_min_size, 0, "Minimum size of scatter allocations.");
	#endif

	kmem_cache_t *abd_chunk_cache;
	static kstat_t *abd_ksp;

	/*
	* We use a scattered SPA_MAXBLOCKSIZE sized ABD whose chunks are
	* just a single zero'd page-sized buffer. This allows us to conserve
	* memory by only using a single zero buffer for the scatter chunks.
	*/
	abd_t *abd_zero_scatter = NULL;

	static uint_t
	abd_chunkcnt_for_bytes(size_t size)
	{
	return ((size + PAGE_MASK) >> PAGE_SHIFT);
	}

	static inline uint_t
	abd_scatter_chunkcnt(abd_t *abd)
	{
	ASSERT(!abd_is_linear(abd));
	return (abd_chunkcnt_for_bytes(
	ABD_SCATTER(abd).abd_offset + abd->abd_size));
	}

	boolean_t
	abd_size_alloc_linear(size_t size)
	{
	return (!zfs_abd_scatter_enabled \|\| size < zfs_abd_scatter_min_size);
	}

	void
	abd_update_scatter_stats(abd_t *abd, abd_stats_op_t op)
	{
	uint_t n = abd_scatter_chunkcnt(abd);
	ASSERT(op == ABDSTAT_INCR \|\| op == ABDSTAT_DECR);
	int waste = (n << PAGE_SHIFT) - abd->abd_size;
	if (op == ABDSTAT_INCR) {
	ABDSTAT_BUMP(abdstat_scatter_cnt);
	ABDSTAT_INCR(abdstat_scatter_data_size, abd->abd_size);
	ABDSTAT_INCR(abdstat_scatter_chunk_waste, waste);
	arc_space_consume(waste, ARC_SPACE_ABD_CHUNK_WASTE);
	} else {
	ABDSTAT_BUMPDOWN(abdstat_scatter_cnt);
	ABDSTAT_INCR(abdstat_scatter_data_size, -(int)abd->abd_size);
	ABDSTAT_INCR(abdstat_scatter_chunk_waste, -waste);
	arc_space_return(waste, ARC_SPACE_ABD_CHUNK_WASTE);
	}
	}

	void
	abd_update_linear_stats(abd_t *abd, abd_stats_op_t op)
	{
	ASSERT(op == ABDSTAT_INCR \|\| op == ABDSTAT_DECR);
	if (op == ABDSTAT_INCR) {
	ABDSTAT_BUMP(abdstat_linear_cnt);
	ABDSTAT_INCR(abdstat_linear_data_size, abd->abd_size);
	} else {
	ABDSTAT_BUMPDOWN(abdstat_linear_cnt);
	ABDSTAT_INCR(abdstat_linear_data_size, -(int)abd->abd_size);
	}
	}

	void
	abd_verify_scatter(abd_t *abd)
	{
	uint_t i, n;

	/*
	* There is no scatter linear pages in FreeBSD so there is
	* an error if the ABD has been marked as a linear page.
	*/
	ASSERT(!abd_is_linear_page(abd));
	ASSERT3U(ABD_SCATTER(abd).abd_offset, <, PAGE_SIZE);
	n = abd_scatter_chunkcnt(abd);
	for (i = 0; i < n; i++) {
	ASSERT3P(ABD_SCATTER(abd).abd_chunks[i], !=, NULL);
	}
	}

	void
	abd_alloc_chunks(abd_t *abd, size_t size)
	{
	uint_t i, n;

	n = abd_chunkcnt_for_bytes(size);
	for (i = 0; i < n; i++) {
	ABD_SCATTER(abd).abd_chunks[i] =
	kmem_cache_alloc(abd_chunk_cache, KM_PUSHPAGE);
	}
	}

	void
	abd_free_chunks(abd_t *abd)
	{
	uint_t i, n;

	n = abd_scatter_chunkcnt(abd);
	for (i = 0; i < n; i++) {
	kmem_cache_free(abd_chunk_cache,
	ABD_SCATTER(abd).abd_chunks[i]);
	}
	}

	abd_t *
	abd_alloc_struct_impl(size_t size)
	{
	uint_t chunkcnt = abd_chunkcnt_for_bytes(size);
	/*
	* In the event we are allocating a gang ABD, the size passed in
	* will be 0. We must make sure to set abd_size to the size of an
	* ABD struct as opposed to an ABD scatter with 0 chunks. The gang
	* ABD struct allocation accounts for an additional 24 bytes over
	* a scatter ABD with 0 chunks.
	*/
	size_t abd_size = MAX(sizeof (abd_t),
	offsetof(abd_t, abd_u.abd_scatter.abd_chunks[chunkcnt]));
	abd_t *abd = kmem_alloc(abd_size, KM_PUSHPAGE);
	ASSERT3P(abd, !=, NULL);
	ABDSTAT_INCR(abdstat_struct_size, abd_size);

	return (abd);
	}

	void
	abd_free_struct_impl(abd_t *abd)
	{
	uint_t chunkcnt = abd_is_linear(abd) \|\| abd_is_gang(abd) ? 0 :
	abd_scatter_chunkcnt(abd);
	ssize_t size = MAX(sizeof (abd_t),
	offsetof(abd_t, abd_u.abd_scatter.abd_chunks[chunkcnt]));
	kmem_free(abd, size);
	ABDSTAT_INCR(abdstat_struct_size, -size);
	}

	/*
	* Allocate scatter ABD of size SPA_MAXBLOCKSIZE, where
	* each chunk in the scatterlist will be set to the same area.
	*/
	_Static_assert(ZERO_REGION_SIZE >= PAGE_SIZE, "zero_region too small");
	static void
	abd_alloc_zero_scatter(void)
	{
	uint_t i, n;

	n = abd_chunkcnt_for_bytes(SPA_MAXBLOCKSIZE);
	abd_zero_scatter = abd_alloc_struct(SPA_MAXBLOCKSIZE);
	abd_zero_scatter->abd_flags \|= ABD_FLAG_OWNER \| ABD_FLAG_ZEROS;
	abd_zero_scatter->abd_size = SPA_MAXBLOCKSIZE;

	ABD_SCATTER(abd_zero_scatter).abd_offset = 0;

	for (i = 0; i < n; i++) {
	ABD_SCATTER(abd_zero_scatter).abd_chunks[i] =
	__DECONST(void *, zero_region);
	}

	ABDSTAT_BUMP(abdstat_scatter_cnt);
	ABDSTAT_INCR(abdstat_scatter_data_size, PAGE_SIZE);
	}

	static void
	abd_free_zero_scatter(void)
	{
	ABDSTAT_BUMPDOWN(abdstat_scatter_cnt);
	ABDSTAT_INCR(abdstat_scatter_data_size, -(int)PAGE_SIZE);

	abd_free_struct(abd_zero_scatter);
	abd_zero_scatter = NULL;
	}

	static int
	abd_kstats_update(kstat_t *ksp, int rw)
	{
	abd_stats_t *as = ksp->ks_data;

	if (rw == KSTAT_WRITE)
	return (EACCES);
	as->abdstat_struct_size.value.ui64 =
	wmsum_value(&abd_sums.abdstat_struct_size);
	as->abdstat_scatter_cnt.value.ui64 =
	wmsum_value(&abd_sums.abdstat_scatter_cnt);
	as->abdstat_scatter_data_size.value.ui64 =
	wmsum_value(&abd_sums.abdstat_scatter_data_size);
	as->abdstat_scatter_chunk_waste.value.ui64 =
	wmsum_value(&abd_sums.abdstat_scatter_chunk_waste);
	as->abdstat_linear_cnt.value.ui64 =
	wmsum_value(&abd_sums.abdstat_linear_cnt);
	as->abdstat_linear_data_size.value.ui64 =
	wmsum_value(&abd_sums.abdstat_linear_data_size);
	return (0);
	}

	void
	abd_init(void)
	{
	abd_chunk_cache = kmem_cache_create("abd_chunk", PAGE_SIZE, 0,
	NULL, NULL, NULL, NULL, 0, KMC_NODEBUG);

	wmsum_init(&abd_sums.abdstat_struct_size, 0);
	wmsum_init(&abd_sums.abdstat_scatter_cnt, 0);
	wmsum_init(&abd_sums.abdstat_scatter_data_size, 0);
	wmsum_init(&abd_sums.abdstat_scatter_chunk_waste, 0);
	wmsum_init(&abd_sums.abdstat_linear_cnt, 0);
	wmsum_init(&abd_sums.abdstat_linear_data_size, 0);

	abd_ksp = kstat_create("zfs", 0, "abdstats", "misc", KSTAT_TYPE_NAMED,
	sizeof (abd_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
	if (abd_ksp != NULL) {
	abd_ksp->ks_data = &abd_stats;
	abd_ksp->ks_update = abd_kstats_update;
	kstat_install(abd_ksp);
	}

	abd_alloc_zero_scatter();
	}

	void
	abd_fini(void)
	{
	abd_free_zero_scatter();

	if (abd_ksp != NULL) {
	kstat_delete(abd_ksp);
	abd_ksp = NULL;
	}

	wmsum_fini(&abd_sums.abdstat_struct_size);
	wmsum_fini(&abd_sums.abdstat_scatter_cnt);
	wmsum_fini(&abd_sums.abdstat_scatter_data_size);
	wmsum_fini(&abd_sums.abdstat_scatter_chunk_waste);
	wmsum_fini(&abd_sums.abdstat_linear_cnt);
	wmsum_fini(&abd_sums.abdstat_linear_data_size);

	kmem_cache_destroy(abd_chunk_cache);
	abd_chunk_cache = NULL;
	}

	void
	abd_free_linear_page(abd_t *abd)
	{
	/*
	* FreeBSD does not have scatter linear pages
	* so there is an error.
	*/
	VERIFY(0);
	}

	/*
	* If we're going to use this ABD for doing I/O using the block layer, the
	* consumer of the ABD data doesn't care if it's scattered or not, and we don't
	* plan to store this ABD in memory for a long period of time, we should
	* allocate the ABD type that requires the least data copying to do the I/O.
	*
	* Currently this is linear ABDs, however if ldi_strategy() can ever issue I/Os
	* using a scatter/gather list we should switch to that and replace this call
	* with vanilla abd_alloc().
	*/
	abd_t *
	abd_alloc_for_io(size_t size, boolean_t is_metadata)
	{
	return (abd_alloc_linear(size, is_metadata));
	}

	abd_t *
	abd_get_offset_scatter(abd_t abd, abd_t sabd, size_t off,
	size_t size)
	{
	abd_verify(sabd);
	ASSERT3U(off, <=, sabd->abd_size);

	size_t new_offset = ABD_SCATTER(sabd).abd_offset + off;
	size_t chunkcnt = abd_chunkcnt_for_bytes(
	(new_offset & PAGE_MASK) + size);

	ASSERT3U(chunkcnt, <=, abd_scatter_chunkcnt(sabd));

	/*
	* If an abd struct is provided, it is only the minimum size. If we
	* need additional chunks, we need to allocate a new struct.
	*/
	if (abd != NULL &&
	offsetof(abd_t, abd_u.abd_scatter.abd_chunks[chunkcnt]) >
	sizeof (abd_t)) {
	abd = NULL;
	}

	if (abd == NULL)
	abd = abd_alloc_struct(chunkcnt << PAGE_SHIFT);

	/*
	* Even if this buf is filesystem metadata, we only track that
	* if we own the underlying data buffer, which is not true in
	* this case. Therefore, we don't ever use ABD_FLAG_META here.
	*/

	ABD_SCATTER(abd).abd_offset = new_offset & PAGE_MASK;

	/* Copy the scatterlist starting at the correct offset */
	(void) memcpy(&ABD_SCATTER(abd).abd_chunks,
	&ABD_SCATTER(sabd).abd_chunks[new_offset >> PAGE_SHIFT],
	chunkcnt * sizeof (void *));

	return (abd);
	}

	/*
	* Initialize the abd_iter.
	*/
	void
	abd_iter_init(struct abd_iter aiter, abd_t abd)
	{
	ASSERT(!abd_is_gang(abd));
	abd_verify(abd);
	+ memset(aiter, 0, sizeof (struct abd_iter));
	aiter->iter_abd = abd;
	- aiter->iter_pos = 0;
	- aiter->iter_mapaddr = NULL;
	- aiter->iter_mapsize = 0;
	}

	/*
	* This is just a helper function to see if we have exhausted the
	* abd_iter and reached the end.
	*/
	boolean_t
	abd_iter_at_end(struct abd_iter *aiter)
	{
	return (aiter->iter_pos == aiter->iter_abd->abd_size);
	}

	/*
	* Advance the iterator by a certain amount. Cannot be called when a chunk is
	* in use. This can be safely called when the aiter has already exhausted, in
	* which case this does nothing.
	*/
	void
	abd_iter_advance(struct abd_iter *aiter, size_t amount)
	{
	ASSERT3P(aiter->iter_mapaddr, ==, NULL);
	ASSERT0(aiter->iter_mapsize);

	/* There's nothing left to advance to, so do nothing */
	if (abd_iter_at_end(aiter))
	return;

	aiter->iter_pos += amount;
	}

	/*
	* Map the current chunk into aiter. This can be safely called when the aiter
	* has already exhausted, in which case this does nothing.
	*/
	void
	abd_iter_map(struct abd_iter *aiter)
	{
	void *paddr;

	ASSERT3P(aiter->iter_mapaddr, ==, NULL);
	ASSERT0(aiter->iter_mapsize);

	/* There's nothing left to iterate over, so do nothing */
	if (abd_iter_at_end(aiter))
	return;

	abd_t *abd = aiter->iter_abd;
	size_t offset = aiter->iter_pos;
	if (abd_is_linear(abd)) {
	aiter->iter_mapsize = abd->abd_size - offset;
	paddr = ABD_LINEAR_BUF(abd);
	} else {
	offset += ABD_SCATTER(abd).abd_offset;
	paddr = ABD_SCATTER(abd).abd_chunks[offset >> PAGE_SHIFT];
	offset &= PAGE_MASK;
	aiter->iter_mapsize = MIN(PAGE_SIZE - offset,
	abd->abd_size - aiter->iter_pos);
	}
	aiter->iter_mapaddr = (char *)paddr + offset;
	}

	/*
	* Unmap the current chunk from aiter. This can be safely called when the aiter
	* has already exhausted, in which case this does nothing.
	*/
	void
	abd_iter_unmap(struct abd_iter *aiter)
	{
	if (!abd_iter_at_end(aiter)) {
	ASSERT3P(aiter->iter_mapaddr, !=, NULL);
	ASSERT3U(aiter->iter_mapsize, >, 0);
	}

	aiter->iter_mapaddr = NULL;
	aiter->iter_mapsize = 0;
	}

	void
	abd_cache_reap_now(void)
	{
	kmem_cache_reap_soon(abd_chunk_cache);
	}
	diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c
	index 93bbfa8e3ef2..617e3bbbca48 100644
	--- a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c
	+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c
	@@ -1,6444 +1,6442 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2012, 2015 by Delphix. All rights reserved.
	* Copyright (c) 2014 Integros [integros.com]
	* Copyright 2017 Nexenta Systems, Inc.
	*/

	/* Portions Copyright 2007 Jeremy Teo */
	/* Portions Copyright 2010 Robert Milkowski */

	#include <sys/param.h>
	#include <sys/time.h>
	#include <sys/systm.h>
	#include <sys/sysmacros.h>
	#include <sys/resource.h>
	#include <security/mac/mac_framework.h>
	#include <sys/vfs.h>
	#include <sys/endian.h>
	#include <sys/vm.h>
	#include <sys/vnode.h>
	#if __FreeBSD_version >= 1300102
	#include <sys/smr.h>
	#endif
	#include <sys/dirent.h>
	#include <sys/file.h>
	#include <sys/stat.h>
	#include <sys/kmem.h>
	#include <sys/taskq.h>
	#include <sys/uio.h>
	#include <sys/atomic.h>
	#include <sys/namei.h>
	#include <sys/mman.h>
	#include <sys/cmn_err.h>
	#include <sys/kdb.h>
	#include <sys/sysproto.h>
	#include <sys/errno.h>
	#include <sys/unistd.h>
	#include <sys/zfs_dir.h>
	#include <sys/zfs_ioctl.h>
	#include <sys/fs/zfs.h>
	#include <sys/dmu.h>
	#include <sys/dmu_objset.h>
	#include <sys/spa.h>
	#include <sys/txg.h>
	#include <sys/dbuf.h>
	#include <sys/zap.h>
	#include <sys/sa.h>
	#include <sys/policy.h>
	#include <sys/sunddi.h>
	#include <sys/filio.h>
	#include <sys/sid.h>
	#include <sys/zfs_ctldir.h>
	#include <sys/zfs_fuid.h>
	#include <sys/zfs_quota.h>
	#include <sys/zfs_sa.h>
	#include <sys/zfs_rlock.h>
	#include <sys/bio.h>
	#include <sys/buf.h>
	#include <sys/sched.h>
	#include <sys/acl.h>
	#include <sys/vmmeter.h>
	#include <vm/vm_param.h>
	#include <sys/zil.h>
	#include <sys/zfs_vnops.h>
	#include <sys/module.h>
	#include <sys/sysent.h>
	#include <sys/dmu_impl.h>
	#include <sys/brt.h>
	#include <sys/zfeature.h>

	#include <vm/vm_object.h>

	#include <sys/extattr.h>
	#include <sys/priv.h>

	#ifndef VN_OPEN_INVFS
	#define VN_OPEN_INVFS 0x0
	#endif

	VFS_SMR_DECLARE;

	#if __FreeBSD_version < 1300103
	#define NDFREE_PNBUF(ndp) NDFREE((ndp), NDF_ONLY_PNBUF)
	#endif

	#if __FreeBSD_version >= 1300047
	#define vm_page_wire_lock(pp)
	#define vm_page_wire_unlock(pp)
	#else
	#define vm_page_wire_lock(pp) vm_page_lock(pp)
	#define vm_page_wire_unlock(pp) vm_page_unlock(pp)
	#endif

	#ifdef DEBUG_VFS_LOCKS
	#define VNCHECKREF(vp) \
	VNASSERT((vp)->v_holdcnt > 0 && (vp)->v_usecount > 0, vp, \
	("%s: wrong ref counts", __func__));
	#else
	#define VNCHECKREF(vp)
	#endif

	#if __FreeBSD_version >= 1400045
	typedef uint64_t cookie_t;
	#else
	typedef ulong_t cookie_t;
	#endif

	/*
	* Programming rules.
	*
	* Each vnode op performs some logical unit of work. To do this, the ZPL must
	* properly lock its in-core state, create a DMU transaction, do the work,
	* record this work in the intent log (ZIL), commit the DMU transaction,
	* and wait for the intent log to commit if it is a synchronous operation.
	* Moreover, the vnode ops must work in both normal and log replay context.
	* The ordering of events is important to avoid deadlocks and references
	* to freed memory. The example below illustrates the following Big Rules:
	*
	* (1) A check must be made in each zfs thread for a mounted file system.
	* This is done avoiding races using zfs_enter(zfsvfs).
	* A zfs_exit(zfsvfs) is needed before all returns. Any znodes
	* must be checked with zfs_verify_zp(zp). Both of these macros
	* can return EIO from the calling function.
	*
	* (2) VN_RELE() should always be the last thing except for zil_commit()
	* (if necessary) and zfs_exit(). This is for 3 reasons:
	* First, if it's the last reference, the vnode/znode
	* can be freed, so the zp may point to freed memory. Second, the last
	* reference will call zfs_zinactive(), which may induce a lot of work --
	* pushing cached pages (which acquires range locks) and syncing out
	* cached atime changes. Third, zfs_zinactive() may require a new tx,
	* which could deadlock the system if you were already holding one.
	* If you must call VN_RELE() within a tx then use VN_RELE_ASYNC().
	*
	* (3) All range locks must be grabbed before calling dmu_tx_assign(),
	* as they can span dmu_tx_assign() calls.
	*
	* (4) If ZPL locks are held, pass TXG_NOWAIT as the second argument to
	* dmu_tx_assign(). This is critical because we don't want to block
	* while holding locks.
	*
	* If no ZPL locks are held (aside from zfs_enter()), use TXG_WAIT. This
	* reduces lock contention and CPU usage when we must wait (note that if
	* throughput is constrained by the storage, nearly every transaction
	* must wait).
	*
	* Note, in particular, that if a lock is sometimes acquired before
	* the tx assigns, and sometimes after (e.g. z_lock), then failing
	* to use a non-blocking assign can deadlock the system. The scenario:
	*
	* Thread A has grabbed a lock before calling dmu_tx_assign().
	* Thread B is in an already-assigned tx, and blocks for this lock.
	* Thread A calls dmu_tx_assign(TXG_WAIT) and blocks in txg_wait_open()
	* forever, because the previous txg can't quiesce until B's tx commits.
	*
	* If dmu_tx_assign() returns ERESTART and zfsvfs->z_assign is TXG_NOWAIT,
	* then drop all locks, call dmu_tx_wait(), and try again. On subsequent
	* calls to dmu_tx_assign(), pass TXG_NOTHROTTLE in addition to TXG_NOWAIT,
	* to indicate that this operation has already called dmu_tx_wait().
	* This will ensure that we don't retry forever, waiting a short bit
	* each time.
	*
	* (5) If the operation succeeded, generate the intent log entry for it
	* before dropping locks. This ensures that the ordering of events
	* in the intent log matches the order in which they actually occurred.
	* During ZIL replay the zfs_log_* functions will update the sequence
	* number to indicate the zil transaction has replayed.
	*
	* (6) At the end of each vnode op, the DMU tx must always commit,
	* regardless of whether there were any errors.
	*
	* (7) After dropping all locks, invoke zil_commit(zilog, foid)
	* to ensure that synchronous semantics are provided when necessary.
	*
	* In general, this is how things should be ordered in each vnode op:
	*
	* zfs_enter(zfsvfs); // exit if unmounted
	* top:
	* zfs_dirent_lookup(&dl, ...) // lock directory entry (may VN_HOLD())
	* rw_enter(...); // grab any other locks you need
	* tx = dmu_tx_create(...); // get DMU tx
	* dmu_tx_hold_*(); // hold each object you might modify
	* error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) \| TXG_NOWAIT);
	* if (error) {
	* rw_exit(...); // drop locks
	* zfs_dirent_unlock(dl); // unlock directory entry
	* VN_RELE(...); // release held vnodes
	* if (error == ERESTART) {
	* waited = B_TRUE;
	* dmu_tx_wait(tx);
	* dmu_tx_abort(tx);
	* goto top;
	* }
	* dmu_tx_abort(tx); // abort DMU tx
	* zfs_exit(zfsvfs); // finished in zfs
	* return (error); // really out of space
	* }
	* error = do_real_work(); // do whatever this VOP does
	* if (error == 0)
	* zfs_log_*(...); // on success, make ZIL entry
	* dmu_tx_commit(tx); // commit DMU tx -- error or not
	* rw_exit(...); // drop locks
	* zfs_dirent_unlock(dl); // unlock directory entry
	* VN_RELE(...); // release held vnodes
	* zil_commit(zilog, foid); // synchronous when necessary
	* zfs_exit(zfsvfs); // finished in zfs
	* return (error); // done, report error
	*/
	static int
	zfs_open(vnode_t *vpp, int flag, cred_t cr)
	{
	(void) cr;
	znode_t zp = VTOZ(vpp);
	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
	int error;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	if ((flag & FWRITE) && (zp->z_pflags & ZFS_APPENDONLY) &&
	((flag & FAPPEND) == 0)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EPERM));
	}

	/* Keep a count of the synchronous opens in the znode */
	if (flag & O_SYNC)
	atomic_inc_32(&zp->z_sync_cnt);

	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	static int
	zfs_close(vnode_t vp, int flag, int count, offset_t offset, cred_t cr)
	{
	(void) offset, (void) cr;
	znode_t *zp = VTOZ(vp);
	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
	int error;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	/* Decrement the synchronous opens in the znode */
	if ((flag & O_SYNC) && (count == 1))
	atomic_dec_32(&zp->z_sync_cnt);

	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	static int
	zfs_ioctl(vnode_t vp, ulong_t com, intptr_t data, int flag, cred_t cred,
	int *rvalp)
	{
	(void) flag, (void) cred, (void) rvalp;
	loff_t off;
	int error;

	switch (com) {
	case _FIOFFS:
	{
	return (0);

	/*
	* The following two ioctls are used by bfu. Faking out,
	* necessary to avoid bfu errors.
	*/
	}
	case _FIOGDIO:
	case _FIOSDIO:
	{
	return (0);
	}

	case F_SEEK_DATA:
	case F_SEEK_HOLE:
	{
	off = (offset_t )data;
	/* offset parameter is in/out */
	error = zfs_holey(VTOZ(vp), com, &off);
	if (error)
	return (error);
	(offset_t )data = off;
	return (0);
	}
	}
	return (SET_ERROR(ENOTTY));
	}

	static vm_page_t
	page_busy(vnode_t *vp, int64_t start, int64_t off, int64_t nbytes)
	{
	vm_object_t obj;
	vm_page_t pp;
	int64_t end;

	/*
	* At present vm_page_clear_dirty extends the cleared range to DEV_BSIZE
	* aligned boundaries, if the range is not aligned. As a result a
	* DEV_BSIZE subrange with partially dirty data may get marked as clean.
	* It may happen that all DEV_BSIZE subranges are marked clean and thus
	* the whole page would be considered clean despite have some
	* dirty data.
	* For this reason we should shrink the range to DEV_BSIZE aligned
	* boundaries before calling vm_page_clear_dirty.
	*/
	end = rounddown2(off + nbytes, DEV_BSIZE);
	off = roundup2(off, DEV_BSIZE);
	nbytes = end - off;

	obj = vp->v_object;
	zfs_vmobject_assert_wlocked_12(obj);
	#if __FreeBSD_version < 1300050
	for (;;) {
	if ((pp = vm_page_lookup(obj, OFF_TO_IDX(start))) != NULL &&
	pp->valid) {
	if (vm_page_xbusied(pp)) {
	/*
	* Reference the page before unlocking and
	* sleeping so that the page daemon is less
	* likely to reclaim it.
	*/
	vm_page_reference(pp);
	vm_page_lock(pp);
	zfs_vmobject_wunlock(obj);
	vm_page_busy_sleep(pp, "zfsmwb", true);
	zfs_vmobject_wlock(obj);
	continue;
	}
	vm_page_sbusy(pp);
	} else if (pp != NULL) {
	ASSERT(!pp->valid);
	pp = NULL;
	}
	if (pp != NULL) {
	ASSERT3U(pp->valid, ==, VM_PAGE_BITS_ALL);
	vm_object_pip_add(obj, 1);
	pmap_remove_write(pp);
	if (nbytes != 0)
	vm_page_clear_dirty(pp, off, nbytes);
	}
	break;
	}
	#else
	vm_page_grab_valid_unlocked(&pp, obj, OFF_TO_IDX(start),
	VM_ALLOC_NOCREAT \| VM_ALLOC_SBUSY \| VM_ALLOC_NORMAL \|
	VM_ALLOC_IGN_SBUSY);
	if (pp != NULL) {
	ASSERT3U(pp->valid, ==, VM_PAGE_BITS_ALL);
	vm_object_pip_add(obj, 1);
	pmap_remove_write(pp);
	if (nbytes != 0)
	vm_page_clear_dirty(pp, off, nbytes);
	}
	#endif
	return (pp);
	}

	static void
	page_unbusy(vm_page_t pp)
	{

	vm_page_sunbusy(pp);
	#if __FreeBSD_version >= 1300041
	vm_object_pip_wakeup(pp->object);
	#else
	vm_object_pip_subtract(pp->object, 1);
	#endif
	}

	#if __FreeBSD_version > 1300051
	static vm_page_t
	page_hold(vnode_t *vp, int64_t start)
	{
	vm_object_t obj;
	vm_page_t m;

	obj = vp->v_object;
	vm_page_grab_valid_unlocked(&m, obj, OFF_TO_IDX(start),
	VM_ALLOC_NOCREAT \| VM_ALLOC_WIRED \| VM_ALLOC_IGN_SBUSY \|
	VM_ALLOC_NOBUSY);
	return (m);
	}
	#else
	static vm_page_t
	page_hold(vnode_t *vp, int64_t start)
	{
	vm_object_t obj;
	vm_page_t pp;

	obj = vp->v_object;
	zfs_vmobject_assert_wlocked(obj);

	for (;;) {
	if ((pp = vm_page_lookup(obj, OFF_TO_IDX(start))) != NULL &&
	pp->valid) {
	if (vm_page_xbusied(pp)) {
	/*
	* Reference the page before unlocking and
	* sleeping so that the page daemon is less
	* likely to reclaim it.
	*/
	vm_page_reference(pp);
	vm_page_lock(pp);
	zfs_vmobject_wunlock(obj);
	vm_page_busy_sleep(pp, "zfsmwb", true);
	zfs_vmobject_wlock(obj);
	continue;
	}

	ASSERT3U(pp->valid, ==, VM_PAGE_BITS_ALL);
	vm_page_wire_lock(pp);
	vm_page_hold(pp);
	vm_page_wire_unlock(pp);

	} else
	pp = NULL;
	break;
	}
	return (pp);
	}
	#endif

	static void
	page_unhold(vm_page_t pp)
	{

	vm_page_wire_lock(pp);
	#if __FreeBSD_version >= 1300035
	vm_page_unwire(pp, PQ_ACTIVE);
	#else
	vm_page_unhold(pp);
	#endif
	vm_page_wire_unlock(pp);
	}

	/*
	* When a file is memory mapped, we must keep the IO data synchronized
	* between the DMU cache and the memory mapped pages. What this means:
	*
	* On Write: If we find a memory mapped page, we write to both
	* the page and the dmu buffer.
	*/
	void
	update_pages(znode_t zp, int64_t start, int len, objset_t os)
	{
	vm_object_t obj;
	struct sf_buf *sf;
	vnode_t *vp = ZTOV(zp);
	caddr_t va;
	int off;

	ASSERT3P(vp->v_mount, !=, NULL);
	obj = vp->v_object;
	ASSERT3P(obj, !=, NULL);

	off = start & PAGEOFFSET;
	zfs_vmobject_wlock_12(obj);
	#if __FreeBSD_version >= 1300041
	vm_object_pip_add(obj, 1);
	#endif
	for (start &= PAGEMASK; len > 0; start += PAGESIZE) {
	vm_page_t pp;
	int nbytes = imin(PAGESIZE - off, len);

	if ((pp = page_busy(vp, start, off, nbytes)) != NULL) {
	zfs_vmobject_wunlock_12(obj);

	va = zfs_map_page(pp, &sf);
	(void) dmu_read(os, zp->z_id, start + off, nbytes,
	va + off, DMU_READ_PREFETCH);
	zfs_unmap_page(sf);

	zfs_vmobject_wlock_12(obj);
	page_unbusy(pp);
	}
	len -= nbytes;
	off = 0;
	}
	#if __FreeBSD_version >= 1300041
	vm_object_pip_wakeup(obj);
	#else
	vm_object_pip_wakeupn(obj, 0);
	#endif
	zfs_vmobject_wunlock_12(obj);
	}

	/*
	* Read with UIO_NOCOPY flag means that sendfile(2) requests
	* ZFS to populate a range of page cache pages with data.
	*
	* NOTE: this function could be optimized to pre-allocate
	* all pages in advance, drain exclusive busy on all of them,
	* map them into contiguous KVA region and populate them
	* in one single dmu_read() call.
	*/
	int
	mappedread_sf(znode_t zp, int nbytes, zfs_uio_t uio)
	{
	vnode_t *vp = ZTOV(zp);
	objset_t *os = zp->z_zfsvfs->z_os;
	struct sf_buf *sf;
	vm_object_t obj;
	vm_page_t pp;
	int64_t start;
	caddr_t va;
	int len = nbytes;
	int error = 0;

	ASSERT3U(zfs_uio_segflg(uio), ==, UIO_NOCOPY);
	ASSERT3P(vp->v_mount, !=, NULL);
	obj = vp->v_object;
	ASSERT3P(obj, !=, NULL);
	ASSERT0(zfs_uio_offset(uio) & PAGEOFFSET);

	zfs_vmobject_wlock_12(obj);
	for (start = zfs_uio_offset(uio); len > 0; start += PAGESIZE) {
	int bytes = MIN(PAGESIZE, len);

	pp = vm_page_grab_unlocked(obj, OFF_TO_IDX(start),
	VM_ALLOC_SBUSY \| VM_ALLOC_NORMAL \| VM_ALLOC_IGN_SBUSY);
	if (vm_page_none_valid(pp)) {
	zfs_vmobject_wunlock_12(obj);
	va = zfs_map_page(pp, &sf);
	error = dmu_read(os, zp->z_id, start, bytes, va,
	DMU_READ_PREFETCH);
	if (bytes != PAGESIZE && error == 0)
	memset(va + bytes, 0, PAGESIZE - bytes);
	zfs_unmap_page(sf);
	zfs_vmobject_wlock_12(obj);
	#if __FreeBSD_version >= 1300081
	if (error == 0) {
	vm_page_valid(pp);
	vm_page_activate(pp);
	vm_page_do_sunbusy(pp);
	} else {
	zfs_vmobject_wlock(obj);
	if (!vm_page_wired(pp) && pp->valid == 0 &&
	vm_page_busy_tryupgrade(pp))
	vm_page_free(pp);
	else
	vm_page_sunbusy(pp);
	zfs_vmobject_wunlock(obj);
	}
	#else
	vm_page_do_sunbusy(pp);
	vm_page_lock(pp);
	if (error) {
	if (pp->wire_count == 0 && pp->valid == 0 &&
	!vm_page_busied(pp))
	vm_page_free(pp);
	} else {
	pp->valid = VM_PAGE_BITS_ALL;
	vm_page_activate(pp);
	}
	vm_page_unlock(pp);
	#endif
	} else {
	ASSERT3U(pp->valid, ==, VM_PAGE_BITS_ALL);
	vm_page_do_sunbusy(pp);
	}
	if (error)
	break;
	zfs_uio_advance(uio, bytes);
	len -= bytes;
	}
	zfs_vmobject_wunlock_12(obj);
	return (error);
	}

	/*
	* When a file is memory mapped, we must keep the IO data synchronized
	* between the DMU cache and the memory mapped pages. What this means:
	*
	* On Read: We "read" preferentially from memory mapped pages,
	* else we default from the dmu buffer.
	*
	* NOTE: We will always "break up" the IO into PAGESIZE uiomoves when
	* the file is memory mapped.
	*/
	int
	mappedread(znode_t zp, int nbytes, zfs_uio_t uio)
	{
	vnode_t *vp = ZTOV(zp);
	vm_object_t obj;
	int64_t start;
	int len = nbytes;
	int off;
	int error = 0;

	ASSERT3P(vp->v_mount, !=, NULL);
	obj = vp->v_object;
	ASSERT3P(obj, !=, NULL);

	start = zfs_uio_offset(uio);
	off = start & PAGEOFFSET;
	zfs_vmobject_wlock_12(obj);
	for (start &= PAGEMASK; len > 0; start += PAGESIZE) {
	vm_page_t pp;
	uint64_t bytes = MIN(PAGESIZE - off, len);

	if ((pp = page_hold(vp, start))) {
	struct sf_buf *sf;
	caddr_t va;

	zfs_vmobject_wunlock_12(obj);
	va = zfs_map_page(pp, &sf);
	error = vn_io_fault_uiomove(va + off, bytes,
	GET_UIO_STRUCT(uio));
	zfs_unmap_page(sf);
	zfs_vmobject_wlock_12(obj);
	page_unhold(pp);
	} else {
	zfs_vmobject_wunlock_12(obj);
	error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl),
	uio, bytes);
	zfs_vmobject_wlock_12(obj);
	}
	len -= bytes;
	off = 0;
	if (error)
	break;
	}
	zfs_vmobject_wunlock_12(obj);
	return (error);
	}

	int
	zfs_write_simple(znode_t zp, const void data, size_t len,
	loff_t pos, size_t *presid)
	{
	int error = 0;
	ssize_t resid;

	error = vn_rdwr(UIO_WRITE, ZTOV(zp), __DECONST(void *, data), len, pos,
	UIO_SYSSPACE, IO_SYNC, kcred, NOCRED, &resid, curthread);

	if (error) {
	return (SET_ERROR(error));
	} else if (presid == NULL) {
	if (resid != 0) {
	error = SET_ERROR(EIO);
	}
	} else {
	*presid = resid;
	}
	return (error);
	}

	void
	zfs_zrele_async(znode_t *zp)
	{
	vnode_t *vp = ZTOV(zp);
	objset_t *os = ITOZSB(vp)->z_os;

	VN_RELE_ASYNC(vp, dsl_pool_zrele_taskq(dmu_objset_pool(os)));
	}

	static int
	zfs_dd_callback(struct mount mp, void arg, int lkflags, struct vnode **vpp)
	{
	int error;

	*vpp = arg;
	error = vn_lock(*vpp, lkflags);
	if (error != 0)
	vrele(*vpp);
	return (error);
	}

	static int
	zfs_lookup_lock(vnode_t dvp, vnode_t vp, const char *name, int lkflags)
	{
	znode_t *zdp = VTOZ(dvp);
	zfsvfs_t *zfsvfs __unused = zdp->z_zfsvfs;
	int error;
	int ltype;

	if (zfsvfs->z_replay == B_FALSE)
	ASSERT_VOP_LOCKED(dvp, __func__);

	if (name[0] == 0 \|\| (name[0] == '.' && name[1] == 0)) {
	ASSERT3P(dvp, ==, vp);
	vref(dvp);
	ltype = lkflags & LK_TYPE_MASK;
	if (ltype != VOP_ISLOCKED(dvp)) {
	if (ltype == LK_EXCLUSIVE)
	vn_lock(dvp, LK_UPGRADE \| LK_RETRY);
	else /* if (ltype == LK_SHARED) */
	vn_lock(dvp, LK_DOWNGRADE \| LK_RETRY);

	/*
	* Relock for the "." case could leave us with
	* reclaimed vnode.
	*/
	if (VN_IS_DOOMED(dvp)) {
	vrele(dvp);
	return (SET_ERROR(ENOENT));
	}
	}
	return (0);
	} else if (name[0] == '.' && name[1] == '.' && name[2] == 0) {
	/*
	* Note that in this case, dvp is the child vnode, and we
	* are looking up the parent vnode - exactly reverse from
	* normal operation. Unlocking dvp requires some rather
	* tricky unlock/relock dance to prevent mp from being freed;
	* use vn_vget_ino_gen() which takes care of all that.
	*
	* XXX Note that there is a time window when both vnodes are
	* unlocked. It is possible, although highly unlikely, that
	* during that window the parent-child relationship between
	* the vnodes may change, for example, get reversed.
	* In that case we would have a wrong lock order for the vnodes.
	* All other filesystems seem to ignore this problem, so we
	* do the same here.
	* A potential solution could be implemented as follows:
	* - using LK_NOWAIT when locking the second vnode and retrying
	* if necessary
	* - checking that the parent-child relationship still holds
	* after locking both vnodes and retrying if it doesn't
	*/
	error = vn_vget_ino_gen(dvp, zfs_dd_callback, vp, lkflags, &vp);
	return (error);
	} else {
	error = vn_lock(vp, lkflags);
	if (error != 0)
	vrele(vp);
	return (error);
	}
	}

	/*
	* Lookup an entry in a directory, or an extended attribute directory.
	* If it exists, return a held vnode reference for it.
	*
	* IN: dvp - vnode of directory to search.
	* nm - name of entry to lookup.
	* pnp - full pathname to lookup [UNUSED].
	* flags - LOOKUP_XATTR set if looking for an attribute.
	* rdir - root directory vnode [UNUSED].
	* cr - credentials of caller.
	* ct - caller context
	*
	* OUT: vpp - vnode of located entry, NULL if not found.
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* NA
	*/
	static int
	zfs_lookup(vnode_t dvp, const char nm, vnode_t **vpp,
	struct componentname cnp, int nameiop, cred_t cr, int flags,
	boolean_t cached)
	{
	znode_t *zdp = VTOZ(dvp);
	znode_t *zp;
	zfsvfs_t *zfsvfs = zdp->z_zfsvfs;
	#if __FreeBSD_version > 1300124
	seqc_t dvp_seqc;
	#endif
	int error = 0;

	/*
	* Fast path lookup, however we must skip DNLC lookup
	* for case folding or normalizing lookups because the
	* DNLC code only stores the passed in name. This means
	* creating 'a' and removing 'A' on a case insensitive
	* file system would work, but DNLC still thinks 'a'
	* exists and won't let you create it again on the next
	* pass through fast path.
	*/
	if (!(flags & LOOKUP_XATTR)) {
	if (dvp->v_type != VDIR) {
	return (SET_ERROR(ENOTDIR));
	} else if (zdp->z_sa_hdl == NULL) {
	return (SET_ERROR(EIO));
	}
	}

	DTRACE_PROBE2(zfs__fastpath__lookup__miss, vnode_t *, dvp,
	const char *, nm);

	if ((error = zfs_enter_verify_zp(zfsvfs, zdp, FTAG)) != 0)
	return (error);

	#if __FreeBSD_version > 1300124
	dvp_seqc = vn_seqc_read_notmodify(dvp);
	#endif

	*vpp = NULL;

	if (flags & LOOKUP_XATTR) {
	/*
	* If the xattr property is off, refuse the lookup request.
	*/
	if (!(zfsvfs->z_flags & ZSB_XATTR)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EOPNOTSUPP));
	}

	/*
	* We don't allow recursive attributes..
	* Maybe someday we will.
	*/
	if (zdp->z_pflags & ZFS_XATTR) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	if ((error = zfs_get_xattrdir(VTOZ(dvp), &zp, cr, flags))) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	*vpp = ZTOV(zp);

	/*
	* Do we have permission to get into attribute directory?
	*/
	error = zfs_zaccess(zp, ACE_EXECUTE, 0, B_FALSE, cr, NULL);
	if (error) {
	vrele(ZTOV(zp));
	}

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Check accessibility of directory if we're not coming in via
	* VOP_CACHEDLOOKUP.
	*/
	if (!cached) {
	#ifdef NOEXECCHECK
	if ((cnp->cn_flags & NOEXECCHECK) != 0) {
	cnp->cn_flags &= ~NOEXECCHECK;
	} else
	#endif
	if ((error = zfs_zaccess(zdp, ACE_EXECUTE, 0, B_FALSE, cr,
	NULL))) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	}

	if (zfsvfs->z_utf8 && u8_validate(nm, strlen(nm),
	NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EILSEQ));
	}


	/*
	* First handle the special cases.
	*/
	if ((cnp->cn_flags & ISDOTDOT) != 0) {
	/*
	* If we are a snapshot mounted under .zfs, return
	* the vp for the snapshot directory.
	*/
	if (zdp->z_id == zfsvfs->z_root && zfsvfs->z_parent != zfsvfs) {
	struct componentname cn;
	vnode_t *zfsctl_vp;
	int ltype;

	zfs_exit(zfsvfs, FTAG);
	ltype = VOP_ISLOCKED(dvp);
	VOP_UNLOCK1(dvp);
	error = zfsctl_root(zfsvfs->z_parent, LK_SHARED,
	&zfsctl_vp);
	if (error == 0) {
	cn.cn_nameptr = "snapshot";
	cn.cn_namelen = strlen(cn.cn_nameptr);
	cn.cn_nameiop = cnp->cn_nameiop;
	cn.cn_flags = cnp->cn_flags & ~ISDOTDOT;
	cn.cn_lkflags = cnp->cn_lkflags;
	error = VOP_LOOKUP(zfsctl_vp, vpp, &cn);
	vput(zfsctl_vp);
	}
	vn_lock(dvp, ltype \| LK_RETRY);
	return (error);
	}
	}
	if (zfs_has_ctldir(zdp) && strcmp(nm, ZFS_CTLDIR_NAME) == 0) {
	zfs_exit(zfsvfs, FTAG);
	if ((cnp->cn_flags & ISLASTCN) != 0 && nameiop != LOOKUP)
	return (SET_ERROR(ENOTSUP));
	error = zfsctl_root(zfsvfs, cnp->cn_lkflags, vpp);
	return (error);
	}

	/*
	* The loop is retry the lookup if the parent-child relationship
	* changes during the dot-dot locking complexities.
	*/
	for (;;) {
	uint64_t parent;

	error = zfs_dirlook(zdp, nm, &zp);
	if (error == 0)
	*vpp = ZTOV(zp);

	zfs_exit(zfsvfs, FTAG);
	if (error != 0)
	break;

	error = zfs_lookup_lock(dvp, *vpp, nm, cnp->cn_lkflags);
	if (error != 0) {
	/*
	* If we've got a locking error, then the vnode
	* got reclaimed because of a force unmount.
	* We never enter doomed vnodes into the name cache.
	*/
	*vpp = NULL;
	return (error);
	}

	if ((cnp->cn_flags & ISDOTDOT) == 0)
	break;

	if ((error = zfs_enter(zfsvfs, FTAG)) != 0) {
	vput(ZTOV(zp));
	*vpp = NULL;
	return (error);
	}
	if (zdp->z_sa_hdl == NULL) {
	error = SET_ERROR(EIO);
	} else {
	error = sa_lookup(zdp->z_sa_hdl, SA_ZPL_PARENT(zfsvfs),
	&parent, sizeof (parent));
	}
	if (error != 0) {
	zfs_exit(zfsvfs, FTAG);
	vput(ZTOV(zp));
	break;
	}
	if (zp->z_id == parent) {
	zfs_exit(zfsvfs, FTAG);
	break;
	}
	vput(ZTOV(zp));
	}

	if (error != 0)
	*vpp = NULL;

	/* Translate errors and add SAVENAME when needed. */
	if (cnp->cn_flags & ISLASTCN) {
	switch (nameiop) {
	case CREATE:
	case RENAME:
	if (error == ENOENT) {
	error = EJUSTRETURN;
	#if __FreeBSD_version < 1400068
	cnp->cn_flags \|= SAVENAME;
	#endif
	break;
	}
	zfs_fallthrough;
	case DELETE:
	#if __FreeBSD_version < 1400068
	if (error == 0)
	cnp->cn_flags \|= SAVENAME;
	#endif
	break;
	}
	}

	#if __FreeBSD_version > 1300124
	if ((cnp->cn_flags & ISDOTDOT) != 0) {
	/*
	* FIXME: zfs_lookup_lock relocks vnodes and does nothing to
	* handle races. In particular different callers may end up
	* with different vnodes and will try to add conflicting
	* entries to the namecache.
	*
	* While finding different result may be acceptable in face
	* of concurrent modification, adding conflicting entries
	* trips over an assert in the namecache.
	*
	* Ultimately let an entry through once everything settles.
	*/
	if (!vn_seqc_consistent(dvp, dvp_seqc)) {
	cnp->cn_flags &= ~MAKEENTRY;
	}
	}
	#endif

	/* Insert name into cache (as non-existent) if appropriate. */
	if (zfsvfs->z_use_namecache && !zfsvfs->z_replay &&
	error == ENOENT && (cnp->cn_flags & MAKEENTRY) != 0)
	cache_enter(dvp, NULL, cnp);

	/* Insert name into cache if appropriate. */
	if (zfsvfs->z_use_namecache && !zfsvfs->z_replay &&
	error == 0 && (cnp->cn_flags & MAKEENTRY)) {
	if (!(cnp->cn_flags & ISLASTCN) \|\|
	(nameiop != DELETE && nameiop != RENAME)) {
	cache_enter(dvp, *vpp, cnp);
	}
	}

	return (error);
	}

	/*
	* Attempt to create a new entry in a directory. If the entry
	* already exists, truncate the file if permissible, else return
	* an error. Return the vp of the created or trunc'd file.
	*
	* IN: dvp - vnode of directory to put new file entry in.
	* name - name of new file entry.
	* vap - attributes of new file.
	* excl - flag indicating exclusive or non-exclusive mode.
	* mode - mode to open file with.
	* cr - credentials of caller.
	* flag - large file flag [UNUSED].
	* ct - caller context
	* vsecp - ACL to be set
	* mnt_ns - Unused on FreeBSD
	*
	* OUT: vpp - vnode of created or trunc'd entry.
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* dvp - ctime\|mtime updated if new entry created
	* vp - ctime\|mtime always, atime if new
	*/
	int
	zfs_create(znode_t dzp, const char name, vattr_t *vap, int excl, int mode,
	znode_t *zpp, cred_t cr, int flag, vsecattr_t vsecp, zidmap_t mnt_ns)
	{
	(void) excl, (void) mode, (void) flag;
	znode_t *zp;
	zfsvfs_t *zfsvfs = dzp->z_zfsvfs;
	zilog_t *zilog;
	objset_t *os;
	dmu_tx_t *tx;
	int error;
	uid_t uid = crgetuid(cr);
	gid_t gid = crgetgid(cr);
	uint64_t projid = ZFS_DEFAULT_PROJID;
	zfs_acl_ids_t acl_ids;
	boolean_t fuid_dirtied;
	uint64_t txtype;
	#ifdef DEBUG_VFS_LOCKS
	vnode_t *dvp = ZTOV(dzp);
	#endif

	/*
	* If we have an ephemeral id, ACL, or XVATTR then
	* make sure file system is at proper version
	*/
	if (zfsvfs->z_use_fuids == B_FALSE &&
	(vsecp \|\| (vap->va_mask & AT_XVATTR) \|\|
	IS_EPHEMERAL(uid) \|\| IS_EPHEMERAL(gid)))
	return (SET_ERROR(EINVAL));

	if ((error = zfs_enter_verify_zp(zfsvfs, dzp, FTAG)) != 0)
	return (error);
	os = zfsvfs->z_os;
	zilog = zfsvfs->z_log;

	if (zfsvfs->z_utf8 && u8_validate(name, strlen(name),
	NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EILSEQ));
	}

	if (vap->va_mask & AT_XVATTR) {
	if ((error = secpolicy_xvattr(ZTOV(dzp), (xvattr_t *)vap,
	crgetuid(cr), cr, vap->va_type)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	}

	*zpp = NULL;

	if ((vap->va_mode & S_ISVTX) && secpolicy_vnode_stky_modify(cr))
	vap->va_mode &= ~S_ISVTX;

	error = zfs_dirent_lookup(dzp, name, &zp, ZNEW);
	if (error) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	ASSERT3P(zp, ==, NULL);

	/*
	* Create a new file object and update the directory
	* to reference it.
	*/
	if ((error = zfs_zaccess(dzp, ACE_ADD_FILE, 0, B_FALSE, cr, mnt_ns))) {
	goto out;
	}

	/*
	* We only support the creation of regular files in
	* extended attribute directories.
	*/

	if ((dzp->z_pflags & ZFS_XATTR) &&
	(vap->va_type != VREG)) {
	error = SET_ERROR(EINVAL);
	goto out;
	}

	if ((error = zfs_acl_ids_create(dzp, 0, vap,
	cr, vsecp, &acl_ids, NULL)) != 0)
	goto out;

	if (S_ISREG(vap->va_mode) \|\| S_ISDIR(vap->va_mode))
	projid = zfs_inherit_projid(dzp);
	if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, projid)) {
	zfs_acl_ids_free(&acl_ids);
	error = SET_ERROR(EDQUOT);
	goto out;
	}

	getnewvnode_reserve_();

	tx = dmu_tx_create(os);

	dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes +
	ZFS_SA_BASE_ATTR_SIZE);

	fuid_dirtied = zfsvfs->z_fuid_dirty;
	if (fuid_dirtied)
	zfs_fuid_txhold(zfsvfs, tx);
	dmu_tx_hold_zap(tx, dzp->z_id, TRUE, name);
	dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE);
	if (!zfsvfs->z_use_sa &&
	acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) {
	dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
	0, acl_ids.z_aclp->z_acl_bytes);
	}
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error) {
	zfs_acl_ids_free(&acl_ids);
	dmu_tx_abort(tx);
	getnewvnode_drop_reserve();
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	zfs_mknode(dzp, vap, tx, cr, 0, &zp, &acl_ids);
	if (fuid_dirtied)
	zfs_fuid_sync(zfsvfs, tx);

	(void) zfs_link_create(dzp, name, zp, tx, ZNEW);
	txtype = zfs_log_create_txtype(Z_FILE, vsecp, vap);
	zfs_log_create(zilog, tx, txtype, dzp, zp, name,
	vsecp, acl_ids.z_fuidp, vap);
	zfs_acl_ids_free(&acl_ids);
	dmu_tx_commit(tx);

	getnewvnode_drop_reserve();

	out:
	VNCHECKREF(dvp);
	if (error == 0) {
	*zpp = zp;
	}

	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Remove an entry from a directory.
	*
	* IN: dvp - vnode of directory to remove entry from.
	* name - name of entry to remove.
	* cr - credentials of caller.
	* ct - caller context
	* flags - case flags
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* dvp - ctime\|mtime
	* vp - ctime (if nlink > 0)
	*/
	static int
	zfs_remove_(vnode_t dvp, vnode_t vp, const char name, cred_t cr)
	{
	znode_t *dzp = VTOZ(dvp);
	znode_t *zp;
	znode_t *xzp;
	zfsvfs_t *zfsvfs = dzp->z_zfsvfs;
	zilog_t *zilog;
	uint64_t xattr_obj;
	uint64_t obj = 0;
	dmu_tx_t *tx;
	boolean_t unlinked;
	uint64_t txtype;
	int error;


	if ((error = zfs_enter_verify_zp(zfsvfs, dzp, FTAG)) != 0)
	return (error);
	zp = VTOZ(vp);
	if ((error = zfs_verify_zp(zp)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	zilog = zfsvfs->z_log;

	xattr_obj = 0;
	xzp = NULL;

	if ((error = zfs_zaccess_delete(dzp, zp, cr, NULL))) {
	goto out;
	}

	/*
	* Need to use rmdir for removing directories.
	*/
	if (vp->v_type == VDIR) {
	error = SET_ERROR(EPERM);
	goto out;
	}

	vnevent_remove(vp, dvp, name, ct);

	obj = zp->z_id;

	/* are there any extended attributes? */
	error = sa_lookup(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs),
	&xattr_obj, sizeof (xattr_obj));
	if (error == 0 && xattr_obj) {
	error = zfs_zget(zfsvfs, xattr_obj, &xzp);
	ASSERT0(error);
	}

	/*
	* We may delete the znode now, or we may put it in the unlinked set;
	* it depends on whether we're the last link, and on whether there are
	* other holds on the vnode. So we dmu_tx_hold() the right things to
	* allow for either case.
	*/
	tx = dmu_tx_create(zfsvfs->z_os);
	dmu_tx_hold_zap(tx, dzp->z_id, FALSE, name);
	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
	zfs_sa_upgrade_txholds(tx, zp);
	zfs_sa_upgrade_txholds(tx, dzp);

	if (xzp) {
	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE);
	dmu_tx_hold_sa(tx, xzp->z_sa_hdl, B_FALSE);
	}

	/* charge as an update -- would be nice not to charge at all */
	dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);

	/*
	* Mark this transaction as typically resulting in a net free of space
	*/
	dmu_tx_mark_netfree(tx);

	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error) {
	dmu_tx_abort(tx);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Remove the directory entry.
	*/
	error = zfs_link_destroy(dzp, name, zp, tx, ZEXISTS, &unlinked);

	if (error) {
	dmu_tx_commit(tx);
	goto out;
	}

	if (unlinked) {
	zfs_unlinked_add(zp, tx);
	vp->v_vflag \|= VV_NOSYNC;
	}
	/* XXX check changes to linux vnops */
	txtype = TX_REMOVE;
	zfs_log_remove(zilog, tx, txtype, dzp, name, obj, unlinked);

	dmu_tx_commit(tx);
	out:

	if (xzp)
	vrele(ZTOV(xzp));

	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);


	zfs_exit(zfsvfs, FTAG);
	return (error);
	}


	static int
	zfs_lookup_internal(znode_t dzp, const char name, vnode_t **vpp,
	struct componentname *cnp, int nameiop)
	{
	zfsvfs_t *zfsvfs = dzp->z_zfsvfs;
	int error;

	cnp->cn_nameptr = __DECONST(char *, name);
	cnp->cn_namelen = strlen(name);
	cnp->cn_nameiop = nameiop;
	cnp->cn_flags = ISLASTCN;
	#if __FreeBSD_version < 1400068
	cnp->cn_flags \|= SAVENAME;
	#endif
	cnp->cn_lkflags = LK_EXCLUSIVE \| LK_RETRY;
	cnp->cn_cred = kcred;
	#if __FreeBSD_version < 1400037
	cnp->cn_thread = curthread;
	#endif

	if (zfsvfs->z_use_namecache && !zfsvfs->z_replay) {
	struct vop_lookup_args a;

	a.a_gen.a_desc = &vop_lookup_desc;
	a.a_dvp = ZTOV(dzp);
	a.a_vpp = vpp;
	a.a_cnp = cnp;
	error = vfs_cache_lookup(&a);
	} else {
	error = zfs_lookup(ZTOV(dzp), name, vpp, cnp, nameiop, kcred, 0,
	B_FALSE);
	}
	#ifdef ZFS_DEBUG
	if (error) {
	printf("got error %d on name %s on op %d\n", error, name,
	nameiop);
	kdb_backtrace();
	}
	#endif
	return (error);
	}

	int
	zfs_remove(znode_t dzp, const char name, cred_t *cr, int flags)
	{
	vnode_t *vp;
	int error;
	struct componentname cn;

	if ((error = zfs_lookup_internal(dzp, name, &vp, &cn, DELETE)))
	return (error);

	error = zfs_remove_(ZTOV(dzp), vp, name, cr);
	vput(vp);
	return (error);
	}
	/*
	* Create a new directory and insert it into dvp using the name
	* provided. Return a pointer to the inserted directory.
	*
	* IN: dvp - vnode of directory to add subdir to.
	* dirname - name of new directory.
	* vap - attributes of new directory.
	* cr - credentials of caller.
	* ct - caller context
	* flags - case flags
	* vsecp - ACL to be set
	* mnt_ns - Unused on FreeBSD
	*
	* OUT: vpp - vnode of created directory.
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* dvp - ctime\|mtime updated
	* vp - ctime\|mtime\|atime updated
	*/
	int
	zfs_mkdir(znode_t dzp, const char dirname, vattr_t vap, znode_t *zpp,
	cred_t cr, int flags, vsecattr_t vsecp, zidmap_t *mnt_ns)
	{
	(void) flags, (void) vsecp;
	znode_t *zp;
	zfsvfs_t *zfsvfs = dzp->z_zfsvfs;
	zilog_t *zilog;
	uint64_t txtype;
	dmu_tx_t *tx;
	int error;
	uid_t uid = crgetuid(cr);
	gid_t gid = crgetgid(cr);
	zfs_acl_ids_t acl_ids;
	boolean_t fuid_dirtied;

	ASSERT3U(vap->va_type, ==, VDIR);

	/*
	* If we have an ephemeral id, ACL, or XVATTR then
	* make sure file system is at proper version
	*/
	if (zfsvfs->z_use_fuids == B_FALSE &&
	((vap->va_mask & AT_XVATTR) \|\|
	IS_EPHEMERAL(uid) \|\| IS_EPHEMERAL(gid)))
	return (SET_ERROR(EINVAL));

	if ((error = zfs_enter_verify_zp(zfsvfs, dzp, FTAG)) != 0)
	return (error);
	zilog = zfsvfs->z_log;

	if (dzp->z_pflags & ZFS_XATTR) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	if (zfsvfs->z_utf8 && u8_validate(dirname,
	strlen(dirname), NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EILSEQ));
	}

	if (vap->va_mask & AT_XVATTR) {
	if ((error = secpolicy_xvattr(ZTOV(dzp), (xvattr_t *)vap,
	crgetuid(cr), cr, vap->va_type)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	}

	if ((error = zfs_acl_ids_create(dzp, 0, vap, cr,
	NULL, &acl_ids, NULL)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* First make sure the new directory doesn't exist.
	*
	* Existence is checked first to make sure we don't return
	* EACCES instead of EEXIST which can cause some applications
	* to fail.
	*/
	*zpp = NULL;

	if ((error = zfs_dirent_lookup(dzp, dirname, &zp, ZNEW))) {
	zfs_acl_ids_free(&acl_ids);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	ASSERT3P(zp, ==, NULL);

	if ((error = zfs_zaccess(dzp, ACE_ADD_SUBDIRECTORY, 0, B_FALSE, cr,
	mnt_ns))) {
	zfs_acl_ids_free(&acl_ids);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, zfs_inherit_projid(dzp))) {
	zfs_acl_ids_free(&acl_ids);
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EDQUOT));
	}

	/*
	* Add a new entry to the directory.
	*/
	getnewvnode_reserve_();
	tx = dmu_tx_create(zfsvfs->z_os);
	dmu_tx_hold_zap(tx, dzp->z_id, TRUE, dirname);
	dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, FALSE, NULL);
	fuid_dirtied = zfsvfs->z_fuid_dirty;
	if (fuid_dirtied)
	zfs_fuid_txhold(zfsvfs, tx);
	if (!zfsvfs->z_use_sa && acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) {
	dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0,
	acl_ids.z_aclp->z_acl_bytes);
	}

	dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes +
	ZFS_SA_BASE_ATTR_SIZE);

	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error) {
	zfs_acl_ids_free(&acl_ids);
	dmu_tx_abort(tx);
	getnewvnode_drop_reserve();
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Create new node.
	*/
	zfs_mknode(dzp, vap, tx, cr, 0, &zp, &acl_ids);

	if (fuid_dirtied)
	zfs_fuid_sync(zfsvfs, tx);

	/*
	* Now put new name in parent dir.
	*/
	(void) zfs_link_create(dzp, dirname, zp, tx, ZNEW);

	*zpp = zp;

	txtype = zfs_log_create_txtype(Z_DIR, NULL, vap);
	zfs_log_create(zilog, tx, txtype, dzp, zp, dirname, NULL,
	acl_ids.z_fuidp, vap);

	zfs_acl_ids_free(&acl_ids);

	dmu_tx_commit(tx);

	getnewvnode_drop_reserve();

	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);

	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	#if __FreeBSD_version < 1300124
	static void
	cache_vop_rmdir(struct vnode dvp, struct vnode vp)
	{

	cache_purge(dvp);
	cache_purge(vp);
	}
	#endif

	/*
	* Remove a directory subdir entry. If the current working
	* directory is the same as the subdir to be removed, the
	* remove will fail.
	*
	* IN: dvp - vnode of directory to remove from.
	* name - name of directory to be removed.
	* cwd - vnode of current working directory.
	* cr - credentials of caller.
	* ct - caller context
	* flags - case flags
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* dvp - ctime\|mtime updated
	*/
	static int
	zfs_rmdir_(vnode_t dvp, vnode_t vp, const char name, cred_t cr)
	{
	znode_t *dzp = VTOZ(dvp);
	znode_t *zp = VTOZ(vp);
	zfsvfs_t *zfsvfs = dzp->z_zfsvfs;
	zilog_t *zilog;
	dmu_tx_t *tx;
	int error;

	if ((error = zfs_enter_verify_zp(zfsvfs, dzp, FTAG)) != 0)
	return (error);
	if ((error = zfs_verify_zp(zp)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	zilog = zfsvfs->z_log;


	if ((error = zfs_zaccess_delete(dzp, zp, cr, NULL))) {
	goto out;
	}

	if (vp->v_type != VDIR) {
	error = SET_ERROR(ENOTDIR);
	goto out;
	}

	vnevent_rmdir(vp, dvp, name, ct);

	tx = dmu_tx_create(zfsvfs->z_os);
	dmu_tx_hold_zap(tx, dzp->z_id, FALSE, name);
	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
	dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);
	zfs_sa_upgrade_txholds(tx, zp);
	zfs_sa_upgrade_txholds(tx, dzp);
	dmu_tx_mark_netfree(tx);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error) {
	dmu_tx_abort(tx);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	error = zfs_link_destroy(dzp, name, zp, tx, ZEXISTS, NULL);

	if (error == 0) {
	uint64_t txtype = TX_RMDIR;
	zfs_log_remove(zilog, tx, txtype, dzp, name,
	ZFS_NO_OBJECT, B_FALSE);
	}

	dmu_tx_commit(tx);

	if (zfsvfs->z_use_namecache)
	cache_vop_rmdir(dvp, vp);
	out:
	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	int
	zfs_rmdir(znode_t dzp, const char name, znode_t cwd, cred_t cr, int flags)
	{
	struct componentname cn;
	vnode_t *vp;
	int error;

	if ((error = zfs_lookup_internal(dzp, name, &vp, &cn, DELETE)))
	return (error);

	error = zfs_rmdir_(ZTOV(dzp), vp, name, cr);
	vput(vp);
	return (error);
	}

	/*
	* Read as many directory entries as will fit into the provided
	* buffer from the given directory cursor position (specified in
	* the uio structure).
	*
	* IN: vp - vnode of directory to read.
	* uio - structure supplying read location, range info,
	* and return buffer.
	* cr - credentials of caller.
	* ct - caller context
	*
	* OUT: uio - updated offset and range, buffer filled.
	* eofp - set to true if end-of-file detected.
	* ncookies- number of entries in cookies
	* cookies - offsets to directory entries
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* vp - atime updated
	*
	* Note that the low 4 bits of the cookie returned by zap is always zero.
	* This allows us to use the low range for "special" directory entries:
	* We use 0 for '.', and 1 for '..'. If this is the root of the filesystem,
	* we use the offset 2 for the '.zfs' directory.
	*/
	static int
	zfs_readdir(vnode_t vp, zfs_uio_t uio, cred_t cr, int eofp,
	int ncookies, cookie_t *cookies)
	{
	znode_t *zp = VTOZ(vp);
	iovec_t *iovp;
	dirent64_t *odp;
	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
	objset_t *os;
	caddr_t outbuf;
	size_t bufsize;
	zap_cursor_t zc;
	zap_attribute_t zap;
	uint_t bytes_wanted;
	uint64_t offset; /* must be unsigned; checks for < 1 */
	uint64_t parent;
	int local_eof;
	int outcount;
	int error;
	uint8_t prefetch;
	uint8_t type;
	int ncooks;
	cookie_t *cooks = NULL;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_PARENT(zfsvfs),
	&parent, sizeof (parent))) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* If we are not given an eof variable,
	* use a local one.
	*/
	if (eofp == NULL)
	eofp = &local_eof;

	/*
	* Check for valid iov_len.
	*/
	if (GET_UIO_STRUCT(uio)->uio_iov->iov_len <= 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	/*
	* Quit if directory has been removed (posix)
	*/
	if ((*eofp = zp->z_unlinked) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	error = 0;
	os = zfsvfs->z_os;
	offset = zfs_uio_offset(uio);
	prefetch = zp->z_zn_prefetch;

	/*
	* Initialize the iterator cursor.
	*/
	if (offset <= 3) {
	/*
	* Start iteration from the beginning of the directory.
	*/
	zap_cursor_init(&zc, os, zp->z_id);
	} else {
	/*
	* The offset is a serialized cursor.
	*/
	zap_cursor_init_serialized(&zc, os, zp->z_id, offset);
	}

	/*
	* Get space to change directory entries into fs independent format.
	*/
	iovp = GET_UIO_STRUCT(uio)->uio_iov;
	bytes_wanted = iovp->iov_len;
	if (zfs_uio_segflg(uio) != UIO_SYSSPACE \|\| zfs_uio_iovcnt(uio) != 1) {
	bufsize = bytes_wanted;
	outbuf = kmem_alloc(bufsize, KM_SLEEP);
	odp = (struct dirent64 *)outbuf;
	} else {
	bufsize = bytes_wanted;
	outbuf = NULL;
	odp = (struct dirent64 *)iovp->iov_base;
	}

	if (ncookies != NULL) {
	/*
	* Minimum entry size is dirent size and 1 byte for a file name.
	*/
	ncooks = zfs_uio_resid(uio) / (sizeof (struct dirent) -
	sizeof (((struct dirent *)NULL)->d_name) + 1);
	cooks = malloc(ncooks * sizeof (*cooks), M_TEMP, M_WAITOK);
	*cookies = cooks;
	*ncookies = ncooks;
	}

	/*
	* Transform to file-system independent format
	*/
	outcount = 0;
	while (outcount < bytes_wanted) {
	ino64_t objnum;
	ushort_t reclen;
	off64_t *next = NULL;

	/*
	* Special case `.', `..', and `.zfs'.
	*/
	if (offset == 0) {
	(void) strcpy(zap.za_name, ".");
	zap.za_normalization_conflict = 0;
	objnum = zp->z_id;
	type = DT_DIR;
	} else if (offset == 1) {
	(void) strcpy(zap.za_name, "..");
	zap.za_normalization_conflict = 0;
	objnum = parent;
	type = DT_DIR;
	} else if (offset == 2 && zfs_show_ctldir(zp)) {
	(void) strcpy(zap.za_name, ZFS_CTLDIR_NAME);
	zap.za_normalization_conflict = 0;
	objnum = ZFSCTL_INO_ROOT;
	type = DT_DIR;
	} else {
	/*
	* Grab next entry.
	*/
	if ((error = zap_cursor_retrieve(&zc, &zap))) {
	if ((*eofp = (error == ENOENT)) != 0)
	break;
	else
	goto update;
	}

	if (zap.za_integer_length != 8 \|\|
	zap.za_num_integers != 1) {
	cmn_err(CE_WARN, "zap_readdir: bad directory "
	"entry, obj = %lld, offset = %lld\n",
	(u_longlong_t)zp->z_id,
	(u_longlong_t)offset);
	error = SET_ERROR(ENXIO);
	goto update;
	}

	objnum = ZFS_DIRENT_OBJ(zap.za_first_integer);
	/*
	* MacOS X can extract the object type here such as:
	* uint8_t type = ZFS_DIRENT_TYPE(zap.za_first_integer);
	*/
	type = ZFS_DIRENT_TYPE(zap.za_first_integer);
	}

	reclen = DIRENT64_RECLEN(strlen(zap.za_name));

	/*
	* Will this entry fit in the buffer?
	*/
	if (outcount + reclen > bufsize) {
	/*
	* Did we manage to fit anything in the buffer?
	*/
	if (!outcount) {
	error = SET_ERROR(EINVAL);
	goto update;
	}
	break;
	}
	/*
	* Add normal entry:
	*/
	odp->d_ino = objnum;
	odp->d_reclen = reclen;
	odp->d_namlen = strlen(zap.za_name);
	/* NOTE: d_off is the offset for the next entry. */
	next = &odp->d_off;
	strlcpy(odp->d_name, zap.za_name, odp->d_namlen + 1);
	odp->d_type = type;
	dirent_terminate(odp);
	odp = (dirent64_t *)((intptr_t)odp + reclen);

	outcount += reclen;

	ASSERT3S(outcount, <=, bufsize);

	- /* Prefetch znode */
	if (prefetch)
	- dmu_prefetch(os, objnum, 0, 0, 0,
	- ZIO_PRIORITY_SYNC_READ);
	+ dmu_prefetch_dnode(os, objnum, ZIO_PRIORITY_SYNC_READ);

	/*
	* Move to the next entry, fill in the previous offset.
	*/
	if (offset > 2 \|\| (offset == 2 && !zfs_show_ctldir(zp))) {
	zap_cursor_advance(&zc);
	offset = zap_cursor_serialize(&zc);
	} else {
	offset += 1;
	}

	/* Fill the offset right after advancing the cursor. */
	if (next != NULL)
	*next = offset;
	if (cooks != NULL) {
	*cooks++ = offset;
	ncooks--;
	KASSERT(ncooks >= 0, ("ncookies=%d", ncooks));
	}
	}
	zp->z_zn_prefetch = B_FALSE; /* a lookup will re-enable pre-fetching */

	/* Subtract unused cookies */
	if (ncookies != NULL)
	*ncookies -= ncooks;

	if (zfs_uio_segflg(uio) == UIO_SYSSPACE && zfs_uio_iovcnt(uio) == 1) {
	iovp->iov_base += outcount;
	iovp->iov_len -= outcount;
	zfs_uio_resid(uio) -= outcount;
	} else if ((error =
	zfs_uiomove(outbuf, (long)outcount, UIO_READ, uio))) {
	/*
	* Reset the pointer.
	*/
	offset = zfs_uio_offset(uio);
	}

	update:
	zap_cursor_fini(&zc);
	if (zfs_uio_segflg(uio) != UIO_SYSSPACE \|\| zfs_uio_iovcnt(uio) != 1)
	kmem_free(outbuf, bufsize);

	if (error == ENOENT)
	error = 0;

	ZFS_ACCESSTIME_STAMP(zfsvfs, zp);

	zfs_uio_setoffset(uio, offset);
	zfs_exit(zfsvfs, FTAG);
	if (error != 0 && cookies != NULL) {
	free(*cookies, M_TEMP);
	*cookies = NULL;
	*ncookies = 0;
	}
	return (error);
	}

	/*
	* Get the requested file attributes and place them in the provided
	* vattr structure.
	*
	* IN: vp - vnode of file.
	* vap - va_mask identifies requested attributes.
	* If AT_XVATTR set, then optional attrs are requested
	* flags - ATTR_NOACLCHECK (CIFS server context)
	* cr - credentials of caller.
	*
	* OUT: vap - attribute values.
	*
	* RETURN: 0 (always succeeds).
	*/
	static int
	zfs_getattr(vnode_t vp, vattr_t vap, int flags, cred_t *cr)
	{
	znode_t *zp = VTOZ(vp);
	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
	int error = 0;
	uint32_t blksize;
	u_longlong_t nblocks;
	uint64_t mtime[2], ctime[2], crtime[2], rdev;
	xvattr_t xvap = (xvattr_t )vap; /* vap may be an xvattr_t * */
	xoptattr_t *xoap = NULL;
	boolean_t skipaclchk = (flags & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
	sa_bulk_attr_t bulk[4];
	int count = 0;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	zfs_fuid_map_ids(zp, cr, &vap->va_uid, &vap->va_gid);

	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CRTIME(zfsvfs), NULL, &crtime, 16);
	if (vp->v_type == VBLK \|\| vp->v_type == VCHR)
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_RDEV(zfsvfs), NULL,
	&rdev, 8);

	if ((error = sa_bulk_lookup(zp->z_sa_hdl, bulk, count)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* If ACL is trivial don't bother looking for ACE_READ_ATTRIBUTES.
	* Also, if we are the owner don't bother, since owner should
	* always be allowed to read basic attributes of file.
	*/
	if (!(zp->z_pflags & ZFS_ACL_TRIVIAL) &&
	(vap->va_uid != crgetuid(cr))) {
	if ((error = zfs_zaccess(zp, ACE_READ_ATTRIBUTES, 0,
	skipaclchk, cr, NULL))) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	}

	/*
	* Return all attributes. It's cheaper to provide the answer
	* than to determine whether we were asked the question.
	*/

	vap->va_type = IFTOVT(zp->z_mode);
	vap->va_mode = zp->z_mode & ~S_IFMT;
	vn_fsid(vp, vap);
	vap->va_nodeid = zp->z_id;
	vap->va_nlink = zp->z_links;
	if ((vp->v_flag & VROOT) && zfs_show_ctldir(zp) &&
	zp->z_links < ZFS_LINK_MAX)
	vap->va_nlink++;
	vap->va_size = zp->z_size;
	if (vp->v_type == VBLK \|\| vp->v_type == VCHR)
	vap->va_rdev = zfs_cmpldev(rdev);
	else
	vap->va_rdev = 0;
	vap->va_gen = zp->z_gen;
	vap->va_flags = 0; /* FreeBSD: Reset chflags(2) flags. */
	vap->va_filerev = zp->z_seq;

	/*
	* Add in any requested optional attributes and the create time.
	* Also set the corresponding bits in the returned attribute bitmap.
	*/
	if ((xoap = xva_getxoptattr(xvap)) != NULL && zfsvfs->z_use_fuids) {
	if (XVA_ISSET_REQ(xvap, XAT_ARCHIVE)) {
	xoap->xoa_archive =
	((zp->z_pflags & ZFS_ARCHIVE) != 0);
	XVA_SET_RTN(xvap, XAT_ARCHIVE);
	}

	if (XVA_ISSET_REQ(xvap, XAT_READONLY)) {
	xoap->xoa_readonly =
	((zp->z_pflags & ZFS_READONLY) != 0);
	XVA_SET_RTN(xvap, XAT_READONLY);
	}

	if (XVA_ISSET_REQ(xvap, XAT_SYSTEM)) {
	xoap->xoa_system =
	((zp->z_pflags & ZFS_SYSTEM) != 0);
	XVA_SET_RTN(xvap, XAT_SYSTEM);
	}

	if (XVA_ISSET_REQ(xvap, XAT_HIDDEN)) {
	xoap->xoa_hidden =
	((zp->z_pflags & ZFS_HIDDEN) != 0);
	XVA_SET_RTN(xvap, XAT_HIDDEN);
	}

	if (XVA_ISSET_REQ(xvap, XAT_NOUNLINK)) {
	xoap->xoa_nounlink =
	((zp->z_pflags & ZFS_NOUNLINK) != 0);
	XVA_SET_RTN(xvap, XAT_NOUNLINK);
	}

	if (XVA_ISSET_REQ(xvap, XAT_IMMUTABLE)) {
	xoap->xoa_immutable =
	((zp->z_pflags & ZFS_IMMUTABLE) != 0);
	XVA_SET_RTN(xvap, XAT_IMMUTABLE);
	}

	if (XVA_ISSET_REQ(xvap, XAT_APPENDONLY)) {
	xoap->xoa_appendonly =
	((zp->z_pflags & ZFS_APPENDONLY) != 0);
	XVA_SET_RTN(xvap, XAT_APPENDONLY);
	}

	if (XVA_ISSET_REQ(xvap, XAT_NODUMP)) {
	xoap->xoa_nodump =
	((zp->z_pflags & ZFS_NODUMP) != 0);
	XVA_SET_RTN(xvap, XAT_NODUMP);
	}

	if (XVA_ISSET_REQ(xvap, XAT_OPAQUE)) {
	xoap->xoa_opaque =
	((zp->z_pflags & ZFS_OPAQUE) != 0);
	XVA_SET_RTN(xvap, XAT_OPAQUE);
	}

	if (XVA_ISSET_REQ(xvap, XAT_AV_QUARANTINED)) {
	xoap->xoa_av_quarantined =
	((zp->z_pflags & ZFS_AV_QUARANTINED) != 0);
	XVA_SET_RTN(xvap, XAT_AV_QUARANTINED);
	}

	if (XVA_ISSET_REQ(xvap, XAT_AV_MODIFIED)) {
	xoap->xoa_av_modified =
	((zp->z_pflags & ZFS_AV_MODIFIED) != 0);
	XVA_SET_RTN(xvap, XAT_AV_MODIFIED);
	}

	if (XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP) &&
	vp->v_type == VREG) {
	zfs_sa_get_scanstamp(zp, xvap);
	}

	if (XVA_ISSET_REQ(xvap, XAT_REPARSE)) {
	xoap->xoa_reparse = ((zp->z_pflags & ZFS_REPARSE) != 0);
	XVA_SET_RTN(xvap, XAT_REPARSE);
	}
	if (XVA_ISSET_REQ(xvap, XAT_GEN)) {
	xoap->xoa_generation = zp->z_gen;
	XVA_SET_RTN(xvap, XAT_GEN);
	}

	if (XVA_ISSET_REQ(xvap, XAT_OFFLINE)) {
	xoap->xoa_offline =
	((zp->z_pflags & ZFS_OFFLINE) != 0);
	XVA_SET_RTN(xvap, XAT_OFFLINE);
	}

	if (XVA_ISSET_REQ(xvap, XAT_SPARSE)) {
	xoap->xoa_sparse =
	((zp->z_pflags & ZFS_SPARSE) != 0);
	XVA_SET_RTN(xvap, XAT_SPARSE);
	}

	if (XVA_ISSET_REQ(xvap, XAT_PROJINHERIT)) {
	xoap->xoa_projinherit =
	((zp->z_pflags & ZFS_PROJINHERIT) != 0);
	XVA_SET_RTN(xvap, XAT_PROJINHERIT);
	}

	if (XVA_ISSET_REQ(xvap, XAT_PROJID)) {
	xoap->xoa_projid = zp->z_projid;
	XVA_SET_RTN(xvap, XAT_PROJID);
	}
	}

	ZFS_TIME_DECODE(&vap->va_atime, zp->z_atime);
	ZFS_TIME_DECODE(&vap->va_mtime, mtime);
	ZFS_TIME_DECODE(&vap->va_ctime, ctime);
	ZFS_TIME_DECODE(&vap->va_birthtime, crtime);


	sa_object_size(zp->z_sa_hdl, &blksize, &nblocks);
	vap->va_blksize = blksize;
	vap->va_bytes = nblocks << 9; /* nblocks * 512 */

	if (zp->z_blksz == 0) {
	/*
	* Block size hasn't been set; suggest maximal I/O transfers.
	*/
	vap->va_blksize = zfsvfs->z_max_blksz;
	}

	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	/*
	* Set the file attributes to the values contained in the
	* vattr structure.
	*
	* IN: zp - znode of file to be modified.
	* vap - new attribute values.
	* If AT_XVATTR set, then optional attrs are being set
	* flags - ATTR_UTIME set if non-default time values provided.
	* - ATTR_NOACLCHECK (CIFS context only).
	* cr - credentials of caller.
	* mnt_ns - Unused on FreeBSD
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* vp - ctime updated, mtime updated if size changed.
	*/
	int
	zfs_setattr(znode_t zp, vattr_t vap, int flags, cred_t cr, zidmap_t mnt_ns)
	{
	vnode_t *vp = ZTOV(zp);
	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
	objset_t *os;
	zilog_t *zilog;
	dmu_tx_t *tx;
	vattr_t oldva;
	xvattr_t tmpxvattr;
	uint_t mask = vap->va_mask;
	uint_t saved_mask = 0;
	uint64_t saved_mode;
	int trim_mask = 0;
	uint64_t new_mode;
	uint64_t new_uid, new_gid;
	uint64_t xattr_obj;
	uint64_t mtime[2], ctime[2];
	uint64_t projid = ZFS_INVALID_PROJID;
	znode_t *attrzp;
	int need_policy = FALSE;
	int err, err2;
	zfs_fuid_info_t *fuidp = NULL;
	xvattr_t xvap = (xvattr_t )vap; /* vap may be an xvattr_t * */
	xoptattr_t *xoap;
	zfs_acl_t *aclp;
	boolean_t skipaclchk = (flags & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
	boolean_t fuid_dirtied = B_FALSE;
	sa_bulk_attr_t bulk[7], xattr_bulk[7];
	int count = 0, xattr_count = 0;

	if (mask == 0)
	return (0);

	if (mask & AT_NOSET)
	return (SET_ERROR(EINVAL));

	if ((err = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (err);

	os = zfsvfs->z_os;
	zilog = zfsvfs->z_log;

	/*
	* Make sure that if we have ephemeral uid/gid or xvattr specified
	* that file system is at proper version level
	*/

	if (zfsvfs->z_use_fuids == B_FALSE &&
	(((mask & AT_UID) && IS_EPHEMERAL(vap->va_uid)) \|\|
	((mask & AT_GID) && IS_EPHEMERAL(vap->va_gid)) \|\|
	(mask & AT_XVATTR))) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	if (mask & AT_SIZE && vp->v_type == VDIR) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EISDIR));
	}

	if (mask & AT_SIZE && vp->v_type != VREG && vp->v_type != VFIFO) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	/*
	* If this is an xvattr_t, then get a pointer to the structure of
	* optional attributes. If this is NULL, then we have a vattr_t.
	*/
	xoap = xva_getxoptattr(xvap);

	xva_init(&tmpxvattr);

	/*
	* Immutable files can only alter immutable bit and atime
	*/
	if ((zp->z_pflags & ZFS_IMMUTABLE) &&
	((mask & (AT_SIZE\|AT_UID\|AT_GID\|AT_MTIME\|AT_MODE)) \|\|
	((mask & AT_XVATTR) && XVA_ISSET_REQ(xvap, XAT_CREATETIME)))) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EPERM));
	}

	/*
	* Note: ZFS_READONLY is handled in zfs_zaccess_common.
	*/

	/*
	* Verify timestamps doesn't overflow 32 bits.
	* ZFS can handle large timestamps, but 32bit syscalls can't
	* handle times greater than 2039. This check should be removed
	* once large timestamps are fully supported.
	*/
	if (mask & (AT_ATIME \| AT_MTIME)) {
	if (((mask & AT_ATIME) && TIMESPEC_OVERFLOW(&vap->va_atime)) \|\|
	((mask & AT_MTIME) && TIMESPEC_OVERFLOW(&vap->va_mtime))) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EOVERFLOW));
	}
	}
	if (xoap != NULL && (mask & AT_XVATTR)) {
	if (XVA_ISSET_REQ(xvap, XAT_CREATETIME) &&
	TIMESPEC_OVERFLOW(&vap->va_birthtime)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EOVERFLOW));
	}

	if (XVA_ISSET_REQ(xvap, XAT_PROJID)) {
	if (!dmu_objset_projectquota_enabled(os) \|\|
	(!S_ISREG(zp->z_mode) && !S_ISDIR(zp->z_mode))) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EOPNOTSUPP));
	}

	projid = xoap->xoa_projid;
	if (unlikely(projid == ZFS_INVALID_PROJID)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	if (projid == zp->z_projid && zp->z_pflags & ZFS_PROJID)
	projid = ZFS_INVALID_PROJID;
	else
	need_policy = TRUE;
	}

	if (XVA_ISSET_REQ(xvap, XAT_PROJINHERIT) &&
	(xoap->xoa_projinherit !=
	((zp->z_pflags & ZFS_PROJINHERIT) != 0)) &&
	(!dmu_objset_projectquota_enabled(os) \|\|
	(!S_ISREG(zp->z_mode) && !S_ISDIR(zp->z_mode)))) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EOPNOTSUPP));
	}
	}

	attrzp = NULL;
	aclp = NULL;

	if (zfsvfs->z_vfs->vfs_flag & VFS_RDONLY) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EROFS));
	}

	/*
	* First validate permissions
	*/

	if (mask & AT_SIZE) {
	/*
	* XXX - Note, we are not providing any open
	* mode flags here (like FNDELAY), so we may
	* block if there are locks present... this
	* should be addressed in openat().
	*/
	/* XXX - would it be OK to generate a log record here? */
	err = zfs_freesp(zp, vap->va_size, 0, 0, FALSE);
	if (err) {
	zfs_exit(zfsvfs, FTAG);
	return (err);
	}
	}

	if (mask & (AT_ATIME\|AT_MTIME) \|\|
	((mask & AT_XVATTR) && (XVA_ISSET_REQ(xvap, XAT_HIDDEN) \|\|
	XVA_ISSET_REQ(xvap, XAT_READONLY) \|\|
	XVA_ISSET_REQ(xvap, XAT_ARCHIVE) \|\|
	XVA_ISSET_REQ(xvap, XAT_OFFLINE) \|\|
	XVA_ISSET_REQ(xvap, XAT_SPARSE) \|\|
	XVA_ISSET_REQ(xvap, XAT_CREATETIME) \|\|
	XVA_ISSET_REQ(xvap, XAT_SYSTEM)))) {
	need_policy = zfs_zaccess(zp, ACE_WRITE_ATTRIBUTES, 0,
	skipaclchk, cr, mnt_ns);
	}

	if (mask & (AT_UID\|AT_GID)) {
	int idmask = (mask & (AT_UID\|AT_GID));
	int take_owner;
	int take_group;

	/*
	* NOTE: even if a new mode is being set,
	* we may clear S_ISUID/S_ISGID bits.
	*/

	if (!(mask & AT_MODE))
	vap->va_mode = zp->z_mode;

	/*
	* Take ownership or chgrp to group we are a member of
	*/

	take_owner = (mask & AT_UID) && (vap->va_uid == crgetuid(cr));
	take_group = (mask & AT_GID) &&
	zfs_groupmember(zfsvfs, vap->va_gid, cr);

	/*
	* If both AT_UID and AT_GID are set then take_owner and
	* take_group must both be set in order to allow taking
	* ownership.
	*
	* Otherwise, send the check through secpolicy_vnode_setattr()
	*
	*/

	if (((idmask == (AT_UID\|AT_GID)) && take_owner && take_group) \|\|
	((idmask == AT_UID) && take_owner) \|\|
	((idmask == AT_GID) && take_group)) {
	if (zfs_zaccess(zp, ACE_WRITE_OWNER, 0,
	skipaclchk, cr, mnt_ns) == 0) {
	/*
	* Remove setuid/setgid for non-privileged users
	*/
	secpolicy_setid_clear(vap, vp, cr);
	trim_mask = (mask & (AT_UID\|AT_GID));
	} else {
	need_policy = TRUE;
	}
	} else {
	need_policy = TRUE;
	}
	}

	oldva.va_mode = zp->z_mode;
	zfs_fuid_map_ids(zp, cr, &oldva.va_uid, &oldva.va_gid);
	if (mask & AT_XVATTR) {
	/*
	* Update xvattr mask to include only those attributes
	* that are actually changing.
	*
	* the bits will be restored prior to actually setting
	* the attributes so the caller thinks they were set.
	*/
	if (XVA_ISSET_REQ(xvap, XAT_APPENDONLY)) {
	if (xoap->xoa_appendonly !=
	((zp->z_pflags & ZFS_APPENDONLY) != 0)) {
	need_policy = TRUE;
	} else {
	XVA_CLR_REQ(xvap, XAT_APPENDONLY);
	XVA_SET_REQ(&tmpxvattr, XAT_APPENDONLY);
	}
	}

	if (XVA_ISSET_REQ(xvap, XAT_PROJINHERIT)) {
	if (xoap->xoa_projinherit !=
	((zp->z_pflags & ZFS_PROJINHERIT) != 0)) {
	need_policy = TRUE;
	} else {
	XVA_CLR_REQ(xvap, XAT_PROJINHERIT);
	XVA_SET_REQ(&tmpxvattr, XAT_PROJINHERIT);
	}
	}

	if (XVA_ISSET_REQ(xvap, XAT_NOUNLINK)) {
	if (xoap->xoa_nounlink !=
	((zp->z_pflags & ZFS_NOUNLINK) != 0)) {
	need_policy = TRUE;
	} else {
	XVA_CLR_REQ(xvap, XAT_NOUNLINK);
	XVA_SET_REQ(&tmpxvattr, XAT_NOUNLINK);
	}
	}

	if (XVA_ISSET_REQ(xvap, XAT_IMMUTABLE)) {
	if (xoap->xoa_immutable !=
	((zp->z_pflags & ZFS_IMMUTABLE) != 0)) {
	need_policy = TRUE;
	} else {
	XVA_CLR_REQ(xvap, XAT_IMMUTABLE);
	XVA_SET_REQ(&tmpxvattr, XAT_IMMUTABLE);
	}
	}

	if (XVA_ISSET_REQ(xvap, XAT_NODUMP)) {
	if (xoap->xoa_nodump !=
	((zp->z_pflags & ZFS_NODUMP) != 0)) {
	need_policy = TRUE;
	} else {
	XVA_CLR_REQ(xvap, XAT_NODUMP);
	XVA_SET_REQ(&tmpxvattr, XAT_NODUMP);
	}
	}

	if (XVA_ISSET_REQ(xvap, XAT_AV_MODIFIED)) {
	if (xoap->xoa_av_modified !=
	((zp->z_pflags & ZFS_AV_MODIFIED) != 0)) {
	need_policy = TRUE;
	} else {
	XVA_CLR_REQ(xvap, XAT_AV_MODIFIED);
	XVA_SET_REQ(&tmpxvattr, XAT_AV_MODIFIED);
	}
	}

	if (XVA_ISSET_REQ(xvap, XAT_AV_QUARANTINED)) {
	if ((vp->v_type != VREG &&
	xoap->xoa_av_quarantined) \|\|
	xoap->xoa_av_quarantined !=
	((zp->z_pflags & ZFS_AV_QUARANTINED) != 0)) {
	need_policy = TRUE;
	} else {
	XVA_CLR_REQ(xvap, XAT_AV_QUARANTINED);
	XVA_SET_REQ(&tmpxvattr, XAT_AV_QUARANTINED);
	}
	}

	if (XVA_ISSET_REQ(xvap, XAT_REPARSE)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EPERM));
	}

	if (need_policy == FALSE &&
	(XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP) \|\|
	XVA_ISSET_REQ(xvap, XAT_OPAQUE))) {
	need_policy = TRUE;
	}
	}

	if (mask & AT_MODE) {
	if (zfs_zaccess(zp, ACE_WRITE_ACL, 0, skipaclchk, cr,
	mnt_ns) == 0) {
	err = secpolicy_setid_setsticky_clear(vp, vap,
	&oldva, cr);
	if (err) {
	zfs_exit(zfsvfs, FTAG);
	return (err);
	}
	trim_mask \|= AT_MODE;
	} else {
	need_policy = TRUE;
	}
	}

	if (need_policy) {
	/*
	* If trim_mask is set then take ownership
	* has been granted or write_acl is present and user
	* has the ability to modify mode. In that case remove
	* UID\|GID and or MODE from mask so that
	* secpolicy_vnode_setattr() doesn't revoke it.
	*/

	if (trim_mask) {
	saved_mask = vap->va_mask;
	vap->va_mask &= ~trim_mask;
	if (trim_mask & AT_MODE) {
	/*
	* Save the mode, as secpolicy_vnode_setattr()
	* will overwrite it with ova.va_mode.
	*/
	saved_mode = vap->va_mode;
	}
	}
	err = secpolicy_vnode_setattr(cr, vp, vap, &oldva, flags,
	(int ()(void , int, cred_t *))zfs_zaccess_unix, zp);
	if (err) {
	zfs_exit(zfsvfs, FTAG);
	return (err);
	}

	if (trim_mask) {
	vap->va_mask \|= saved_mask;
	if (trim_mask & AT_MODE) {
	/*
	* Recover the mode after
	* secpolicy_vnode_setattr().
	*/
	vap->va_mode = saved_mode;
	}
	}
	}

	/*
	* secpolicy_vnode_setattr, or take ownership may have
	* changed va_mask
	*/
	mask = vap->va_mask;

	if ((mask & (AT_UID \| AT_GID)) \|\| projid != ZFS_INVALID_PROJID) {
	err = sa_lookup(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs),
	&xattr_obj, sizeof (xattr_obj));

	if (err == 0 && xattr_obj) {
	err = zfs_zget(zp->z_zfsvfs, xattr_obj, &attrzp);
	if (err == 0) {
	err = vn_lock(ZTOV(attrzp), LK_EXCLUSIVE);
	if (err != 0)
	vrele(ZTOV(attrzp));
	}
	if (err)
	goto out2;
	}
	if (mask & AT_UID) {
	new_uid = zfs_fuid_create(zfsvfs,
	(uint64_t)vap->va_uid, cr, ZFS_OWNER, &fuidp);
	if (new_uid != zp->z_uid &&
	zfs_id_overquota(zfsvfs, DMU_USERUSED_OBJECT,
	new_uid)) {
	if (attrzp)
	vput(ZTOV(attrzp));
	err = SET_ERROR(EDQUOT);
	goto out2;
	}
	}

	if (mask & AT_GID) {
	new_gid = zfs_fuid_create(zfsvfs, (uint64_t)vap->va_gid,
	cr, ZFS_GROUP, &fuidp);
	if (new_gid != zp->z_gid &&
	zfs_id_overquota(zfsvfs, DMU_GROUPUSED_OBJECT,
	new_gid)) {
	if (attrzp)
	vput(ZTOV(attrzp));
	err = SET_ERROR(EDQUOT);
	goto out2;
	}
	}

	if (projid != ZFS_INVALID_PROJID &&
	zfs_id_overquota(zfsvfs, DMU_PROJECTUSED_OBJECT, projid)) {
	if (attrzp)
	vput(ZTOV(attrzp));
	err = SET_ERROR(EDQUOT);
	goto out2;
	}
	}
	tx = dmu_tx_create(os);

	if (mask & AT_MODE) {
	uint64_t pmode = zp->z_mode;
	uint64_t acl_obj;
	new_mode = (pmode & S_IFMT) \| (vap->va_mode & ~S_IFMT);

	if (zp->z_zfsvfs->z_acl_mode == ZFS_ACL_RESTRICTED &&
	!(zp->z_pflags & ZFS_ACL_TRIVIAL)) {
	err = SET_ERROR(EPERM);
	goto out;
	}

	if ((err = zfs_acl_chmod_setattr(zp, &aclp, new_mode)))
	goto out;

	if (!zp->z_is_sa && ((acl_obj = zfs_external_acl(zp)) != 0)) {
	/*
	* Are we upgrading ACL from old V0 format
	* to V1 format?
	*/
	if (zfsvfs->z_version >= ZPL_VERSION_FUID &&
	zfs_znode_acl_version(zp) ==
	ZFS_ACL_VERSION_INITIAL) {
	dmu_tx_hold_free(tx, acl_obj, 0,
	DMU_OBJECT_END);
	dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
	0, aclp->z_acl_bytes);
	} else {
	dmu_tx_hold_write(tx, acl_obj, 0,
	aclp->z_acl_bytes);
	}
	} else if (!zp->z_is_sa && aclp->z_acl_bytes > ZFS_ACE_SPACE) {
	dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
	0, aclp->z_acl_bytes);
	}
	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE);
	} else {
	if (((mask & AT_XVATTR) &&
	XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP)) \|\|
	(projid != ZFS_INVALID_PROJID &&
	!(zp->z_pflags & ZFS_PROJID)))
	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE);
	else
	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
	}

	if (attrzp) {
	dmu_tx_hold_sa(tx, attrzp->z_sa_hdl, B_FALSE);
	}

	fuid_dirtied = zfsvfs->z_fuid_dirty;
	if (fuid_dirtied)
	zfs_fuid_txhold(zfsvfs, tx);

	zfs_sa_upgrade_txholds(tx, zp);

	err = dmu_tx_assign(tx, TXG_WAIT);
	if (err)
	goto out;

	count = 0;
	/*
	* Set each attribute requested.
	* We group settings according to the locks they need to acquire.
	*
	* Note: you cannot set ctime directly, although it will be
	* updated as a side-effect of calling this function.
	*/

	if (projid != ZFS_INVALID_PROJID && !(zp->z_pflags & ZFS_PROJID)) {
	/*
	* For the existed object that is upgraded from old system,
	* its on-disk layout has no slot for the project ID attribute.
	* But quota accounting logic needs to access related slots by
	* offset directly. So we need to adjust old objects' layout
	* to make the project ID to some unified and fixed offset.
	*/
	if (attrzp)
	err = sa_add_projid(attrzp->z_sa_hdl, tx, projid);
	if (err == 0)
	err = sa_add_projid(zp->z_sa_hdl, tx, projid);

	if (unlikely(err == EEXIST))
	err = 0;
	else if (err != 0)
	goto out;
	else
	projid = ZFS_INVALID_PROJID;
	}

	if (mask & (AT_UID\|AT_GID\|AT_MODE))
	mutex_enter(&zp->z_acl_lock);

	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
	&zp->z_pflags, sizeof (zp->z_pflags));

	if (attrzp) {
	if (mask & (AT_UID\|AT_GID\|AT_MODE))
	mutex_enter(&attrzp->z_acl_lock);
	SA_ADD_BULK_ATTR(xattr_bulk, xattr_count,
	SA_ZPL_FLAGS(zfsvfs), NULL, &attrzp->z_pflags,
	sizeof (attrzp->z_pflags));
	if (projid != ZFS_INVALID_PROJID) {
	attrzp->z_projid = projid;
	SA_ADD_BULK_ATTR(xattr_bulk, xattr_count,
	SA_ZPL_PROJID(zfsvfs), NULL, &attrzp->z_projid,
	sizeof (attrzp->z_projid));
	}
	}

	if (mask & (AT_UID\|AT_GID)) {

	if (mask & AT_UID) {
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zfsvfs), NULL,
	&new_uid, sizeof (new_uid));
	zp->z_uid = new_uid;
	if (attrzp) {
	SA_ADD_BULK_ATTR(xattr_bulk, xattr_count,
	SA_ZPL_UID(zfsvfs), NULL, &new_uid,
	sizeof (new_uid));
	attrzp->z_uid = new_uid;
	}
	}

	if (mask & AT_GID) {
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zfsvfs),
	NULL, &new_gid, sizeof (new_gid));
	zp->z_gid = new_gid;
	if (attrzp) {
	SA_ADD_BULK_ATTR(xattr_bulk, xattr_count,
	SA_ZPL_GID(zfsvfs), NULL, &new_gid,
	sizeof (new_gid));
	attrzp->z_gid = new_gid;
	}
	}
	if (!(mask & AT_MODE)) {
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs),
	NULL, &new_mode, sizeof (new_mode));
	new_mode = zp->z_mode;
	}
	err = zfs_acl_chown_setattr(zp);
	ASSERT0(err);
	if (attrzp) {
	vn_seqc_write_begin(ZTOV(attrzp));
	err = zfs_acl_chown_setattr(attrzp);
	vn_seqc_write_end(ZTOV(attrzp));
	ASSERT0(err);
	}
	}

	if (mask & AT_MODE) {
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs), NULL,
	&new_mode, sizeof (new_mode));
	zp->z_mode = new_mode;
	ASSERT3P(aclp, !=, NULL);
	err = zfs_aclset_common(zp, aclp, cr, tx);
	ASSERT0(err);
	if (zp->z_acl_cached)
	zfs_acl_free(zp->z_acl_cached);
	zp->z_acl_cached = aclp;
	aclp = NULL;
	}


	if (mask & AT_ATIME) {
	ZFS_TIME_ENCODE(&vap->va_atime, zp->z_atime);
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ATIME(zfsvfs), NULL,
	&zp->z_atime, sizeof (zp->z_atime));
	}

	if (mask & AT_MTIME) {
	ZFS_TIME_ENCODE(&vap->va_mtime, mtime);
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL,
	mtime, sizeof (mtime));
	}

	if (projid != ZFS_INVALID_PROJID) {
	zp->z_projid = projid;
	SA_ADD_BULK_ATTR(bulk, count,
	SA_ZPL_PROJID(zfsvfs), NULL, &zp->z_projid,
	sizeof (zp->z_projid));
	}

	/* XXX - shouldn't this be done before the ATIME/MTIME checks? */
	if (mask & AT_SIZE && !(mask & AT_MTIME)) {
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs),
	NULL, mtime, sizeof (mtime));
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL,
	&ctime, sizeof (ctime));
	zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);
	} else if (mask != 0) {
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL,
	&ctime, sizeof (ctime));
	zfs_tstamp_update_setup(zp, STATE_CHANGED, mtime, ctime);
	if (attrzp) {
	SA_ADD_BULK_ATTR(xattr_bulk, xattr_count,
	SA_ZPL_CTIME(zfsvfs), NULL,
	&ctime, sizeof (ctime));
	zfs_tstamp_update_setup(attrzp, STATE_CHANGED,
	mtime, ctime);
	}
	}

	/*
	* Do this after setting timestamps to prevent timestamp
	* update from toggling bit
	*/

	if (xoap && (mask & AT_XVATTR)) {

	if (XVA_ISSET_REQ(xvap, XAT_CREATETIME))
	xoap->xoa_createtime = vap->va_birthtime;
	/*
	* restore trimmed off masks
	* so that return masks can be set for caller.
	*/

	if (XVA_ISSET_REQ(&tmpxvattr, XAT_APPENDONLY)) {
	XVA_SET_REQ(xvap, XAT_APPENDONLY);
	}
	if (XVA_ISSET_REQ(&tmpxvattr, XAT_NOUNLINK)) {
	XVA_SET_REQ(xvap, XAT_NOUNLINK);
	}
	if (XVA_ISSET_REQ(&tmpxvattr, XAT_IMMUTABLE)) {
	XVA_SET_REQ(xvap, XAT_IMMUTABLE);
	}
	if (XVA_ISSET_REQ(&tmpxvattr, XAT_NODUMP)) {
	XVA_SET_REQ(xvap, XAT_NODUMP);
	}
	if (XVA_ISSET_REQ(&tmpxvattr, XAT_AV_MODIFIED)) {
	XVA_SET_REQ(xvap, XAT_AV_MODIFIED);
	}
	if (XVA_ISSET_REQ(&tmpxvattr, XAT_AV_QUARANTINED)) {
	XVA_SET_REQ(xvap, XAT_AV_QUARANTINED);
	}
	if (XVA_ISSET_REQ(&tmpxvattr, XAT_PROJINHERIT)) {
	XVA_SET_REQ(xvap, XAT_PROJINHERIT);
	}

	if (XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP))
	ASSERT3S(vp->v_type, ==, VREG);

	zfs_xvattr_set(zp, xvap, tx);
	}

	if (fuid_dirtied)
	zfs_fuid_sync(zfsvfs, tx);

	if (mask != 0)
	zfs_log_setattr(zilog, tx, TX_SETATTR, zp, vap, mask, fuidp);

	if (mask & (AT_UID\|AT_GID\|AT_MODE))
	mutex_exit(&zp->z_acl_lock);

	if (attrzp) {
	if (mask & (AT_UID\|AT_GID\|AT_MODE))
	mutex_exit(&attrzp->z_acl_lock);
	}
	out:
	if (err == 0 && attrzp) {
	err2 = sa_bulk_update(attrzp->z_sa_hdl, xattr_bulk,
	xattr_count, tx);
	ASSERT0(err2);
	}

	if (attrzp)
	vput(ZTOV(attrzp));

	if (aclp)
	zfs_acl_free(aclp);

	if (fuidp) {
	zfs_fuid_info_free(fuidp);
	fuidp = NULL;
	}

	if (err) {
	dmu_tx_abort(tx);
	} else {
	err2 = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
	dmu_tx_commit(tx);
	}

	out2:
	if (os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);

	zfs_exit(zfsvfs, FTAG);
	return (err);
	}

	/*
	* Look up the directory entries corresponding to the source and target
	* directory/name pairs.
	*/
	static int
	zfs_rename_relock_lookup(znode_t sdzp, const struct componentname scnp,
	znode_t *szpp, znode_t tdzp, const struct componentname *tcnp,
	znode_t **tzpp)
	{
	zfsvfs_t *zfsvfs;
	znode_t szp, tzp;
	int error;

	/*
	* Before using sdzp and tdzp we must ensure that they are live.
	* As a porting legacy from illumos we have two things to worry
	* about. One is typical for FreeBSD and it is that the vnode is
	* not reclaimed (doomed). The other is that the znode is live.
	* The current code can invalidate the znode without acquiring the
	* corresponding vnode lock if the object represented by the znode
	* and vnode is no longer valid after a rollback or receive operation.
	* z_teardown_lock hidden behind zfs_enter and zfs_exit is the lock
	* that protects the znodes from the invalidation.
	*/
	zfsvfs = sdzp->z_zfsvfs;
	ASSERT3P(zfsvfs, ==, tdzp->z_zfsvfs);
	if ((error = zfs_enter_verify_zp(zfsvfs, sdzp, FTAG)) != 0)
	return (error);
	if ((error = zfs_verify_zp(tdzp)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Re-resolve svp to be certain it still exists and fetch the
	* correct vnode.
	*/
	error = zfs_dirent_lookup(sdzp, scnp->cn_nameptr, &szp, ZEXISTS);
	if (error != 0) {
	/* Source entry invalid or not there. */
	if ((scnp->cn_flags & ISDOTDOT) != 0 \|\|
	(scnp->cn_namelen == 1 && scnp->cn_nameptr[0] == '.'))
	error = SET_ERROR(EINVAL);
	goto out;
	}
	*szpp = szp;

	/*
	* Re-resolve tvp, if it disappeared we just carry on.
	*/
	error = zfs_dirent_lookup(tdzp, tcnp->cn_nameptr, &tzp, 0);
	if (error != 0) {
	vrele(ZTOV(szp));
	if ((tcnp->cn_flags & ISDOTDOT) != 0)
	error = SET_ERROR(EINVAL);
	goto out;
	}
	*tzpp = tzp;
	out:
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* We acquire all but fdvp locks using non-blocking acquisitions. If we
	* fail to acquire any lock in the path we will drop all held locks,
	* acquire the new lock in a blocking fashion, and then release it and
	* restart the rename. This acquire/release step ensures that we do not
	* spin on a lock waiting for release. On error release all vnode locks
	* and decrement references the way tmpfs_rename() would do.
	*/
	static int
	zfs_rename_relock(struct vnode sdvp, struct vnode *svpp,
	struct vnode tdvp, struct vnode *tvpp,
	const struct componentname scnp, const struct componentname tcnp)
	{
	struct vnode nvp, svp, *tvp;
	znode_t sdzp, tdzp, szp, tzp;
	int error;

	VOP_UNLOCK1(tdvp);
	if (tvpp != NULL && tvpp != tdvp)
	VOP_UNLOCK1(*tvpp);

	relock:
	error = vn_lock(sdvp, LK_EXCLUSIVE);
	if (error)
	goto out;
	error = vn_lock(tdvp, LK_EXCLUSIVE \| LK_NOWAIT);
	if (error != 0) {
	VOP_UNLOCK1(sdvp);
	if (error != EBUSY)
	goto out;
	error = vn_lock(tdvp, LK_EXCLUSIVE);
	if (error)
	goto out;
	VOP_UNLOCK1(tdvp);
	goto relock;
	}
	tdzp = VTOZ(tdvp);
	sdzp = VTOZ(sdvp);

	error = zfs_rename_relock_lookup(sdzp, scnp, &szp, tdzp, tcnp, &tzp);
	if (error != 0) {
	VOP_UNLOCK1(sdvp);
	VOP_UNLOCK1(tdvp);
	goto out;
	}
	svp = ZTOV(szp);
	tvp = tzp != NULL ? ZTOV(tzp) : NULL;

	/*
	* Now try acquire locks on svp and tvp.
	*/
	nvp = svp;
	error = vn_lock(nvp, LK_EXCLUSIVE \| LK_NOWAIT);
	if (error != 0) {
	VOP_UNLOCK1(sdvp);
	VOP_UNLOCK1(tdvp);
	if (tvp != NULL)
	vrele(tvp);
	if (error != EBUSY) {
	vrele(nvp);
	goto out;
	}
	error = vn_lock(nvp, LK_EXCLUSIVE);
	if (error != 0) {
	vrele(nvp);
	goto out;
	}
	VOP_UNLOCK1(nvp);
	/*
	* Concurrent rename race.
	* XXX ?
	*/
	if (nvp == tdvp) {
	vrele(nvp);
	error = SET_ERROR(EINVAL);
	goto out;
	}
	vrele(*svpp);
	*svpp = nvp;
	goto relock;
	}
	vrele(*svpp);
	*svpp = nvp;

	if (*tvpp != NULL)
	vrele(*tvpp);
	*tvpp = NULL;
	if (tvp != NULL) {
	nvp = tvp;
	error = vn_lock(nvp, LK_EXCLUSIVE \| LK_NOWAIT);
	if (error != 0) {
	VOP_UNLOCK1(sdvp);
	VOP_UNLOCK1(tdvp);
	VOP_UNLOCK1(*svpp);
	if (error != EBUSY) {
	vrele(nvp);
	goto out;
	}
	error = vn_lock(nvp, LK_EXCLUSIVE);
	if (error != 0) {
	vrele(nvp);
	goto out;
	}
	vput(nvp);
	goto relock;
	}
	*tvpp = nvp;
	}

	return (0);

	out:
	return (error);
	}

	/*
	* Note that we must use VRELE_ASYNC in this function as it walks
	* up the directory tree and vrele may need to acquire an exclusive
	* lock if a last reference to a vnode is dropped.
	*/
	static int
	zfs_rename_check(znode_t szp, znode_t sdzp, znode_t *tdzp)
	{
	zfsvfs_t *zfsvfs;
	znode_t zp, zp1;
	uint64_t parent;
	int error;

	zfsvfs = tdzp->z_zfsvfs;
	if (tdzp == szp)
	return (SET_ERROR(EINVAL));
	if (tdzp == sdzp)
	return (0);
	if (tdzp->z_id == zfsvfs->z_root)
	return (0);
	zp = tdzp;
	for (;;) {
	ASSERT(!zp->z_unlinked);
	if ((error = sa_lookup(zp->z_sa_hdl,
	SA_ZPL_PARENT(zfsvfs), &parent, sizeof (parent))) != 0)
	break;

	if (parent == szp->z_id) {
	error = SET_ERROR(EINVAL);
	break;
	}
	if (parent == zfsvfs->z_root)
	break;
	if (parent == sdzp->z_id)
	break;

	error = zfs_zget(zfsvfs, parent, &zp1);
	if (error != 0)
	break;

	if (zp != tdzp)
	VN_RELE_ASYNC(ZTOV(zp),
	dsl_pool_zrele_taskq(
	dmu_objset_pool(zfsvfs->z_os)));
	zp = zp1;
	}

	if (error == ENOTDIR)
	panic("checkpath: .. not a directory\n");
	if (zp != tdzp)
	VN_RELE_ASYNC(ZTOV(zp),
	dsl_pool_zrele_taskq(dmu_objset_pool(zfsvfs->z_os)));
	return (error);
	}

	#if __FreeBSD_version < 1300124
	static void
	cache_vop_rename(struct vnode fdvp, struct vnode fvp, struct vnode *tdvp,
	struct vnode tvp, struct componentname fcnp, struct componentname *tcnp)
	{

	cache_purge(fvp);
	if (tvp != NULL)
	cache_purge(tvp);
	cache_purge_negative(tdvp);
	}
	#endif

	static int
	zfs_do_rename_impl(vnode_t sdvp, vnode_t svpp, struct componentname scnp,
	vnode_t tdvp, vnode_t tvpp, struct componentname tcnp,
	cred_t *cr);

	/*
	* Move an entry from the provided source directory to the target
	* directory. Change the entry name as indicated.
	*
	* IN: sdvp - Source directory containing the "old entry".
	* scnp - Old entry name.
	* tdvp - Target directory to contain the "new entry".
	* tcnp - New entry name.
	* cr - credentials of caller.
	* INOUT: svpp - Source file
	* tvpp - Target file, may point to NULL initially
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* sdvp,tdvp - ctime\|mtime updated
	*/
	static int
	zfs_do_rename(vnode_t sdvp, vnode_t svpp, struct componentname scnp,
	vnode_t tdvp, vnode_t tvpp, struct componentname tcnp,
	cred_t *cr)
	{
	int error;

	ASSERT_VOP_ELOCKED(tdvp, __func__);
	if (*tvpp != NULL)
	ASSERT_VOP_ELOCKED(*tvpp, __func__);

	/* Reject renames across filesystems. */
	if ((*svpp)->v_mount != tdvp->v_mount \|\|
	((tvpp) != NULL && (svpp)->v_mount != (*tvpp)->v_mount)) {
	error = SET_ERROR(EXDEV);
	goto out;
	}

	if (zfsctl_is_node(tdvp)) {
	error = SET_ERROR(EXDEV);
	goto out;
	}

	/*
	* Lock all four vnodes to ensure safety and semantics of renaming.
	*/
	error = zfs_rename_relock(sdvp, svpp, tdvp, tvpp, scnp, tcnp);
	if (error != 0) {
	/* no vnodes are locked in the case of error here */
	return (error);
	}

	error = zfs_do_rename_impl(sdvp, svpp, scnp, tdvp, tvpp, tcnp, cr);
	VOP_UNLOCK1(sdvp);
	VOP_UNLOCK1(*svpp);
	out:
	if (*tvpp != NULL)
	VOP_UNLOCK1(*tvpp);
	if (tdvp != *tvpp)
	VOP_UNLOCK1(tdvp);

	return (error);
	}

	static int
	zfs_do_rename_impl(vnode_t sdvp, vnode_t svpp, struct componentname scnp,
	vnode_t tdvp, vnode_t tvpp, struct componentname tcnp,
	cred_t *cr)
	{
	dmu_tx_t *tx;
	zfsvfs_t *zfsvfs;
	zilog_t *zilog;
	znode_t tdzp, sdzp, tzp, szp;
	const char *snm = scnp->cn_nameptr;
	const char *tnm = tcnp->cn_nameptr;
	int error;

	tdzp = VTOZ(tdvp);
	sdzp = VTOZ(sdvp);
	zfsvfs = tdzp->z_zfsvfs;

	if ((error = zfs_enter_verify_zp(zfsvfs, tdzp, FTAG)) != 0)
	return (error);
	if ((error = zfs_verify_zp(sdzp)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	zilog = zfsvfs->z_log;

	if (zfsvfs->z_utf8 && u8_validate(tnm,
	strlen(tnm), NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
	error = SET_ERROR(EILSEQ);
	goto out;
	}

	/* If source and target are the same file, there is nothing to do. */
	if ((svpp) == (tvpp)) {
	error = 0;
	goto out;
	}

	if (((svpp)->v_type == VDIR && (svpp)->v_mountedhere != NULL) \|\|
	((tvpp) != NULL && (tvpp)->v_type == VDIR &&
	(*tvpp)->v_mountedhere != NULL)) {
	error = SET_ERROR(EXDEV);
	goto out;
	}

	szp = VTOZ(*svpp);
	if ((error = zfs_verify_zp(szp)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	tzp = tvpp == NULL ? NULL : VTOZ(tvpp);
	if (tzp != NULL) {
	if ((error = zfs_verify_zp(tzp)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	}

	/*
	* This is to prevent the creation of links into attribute space
	* by renaming a linked file into/outof an attribute directory.
	* See the comment in zfs_link() for why this is considered bad.
	*/
	if ((tdzp->z_pflags & ZFS_XATTR) != (sdzp->z_pflags & ZFS_XATTR)) {
	error = SET_ERROR(EINVAL);
	goto out;
	}

	/*
	* If we are using project inheritance, means if the directory has
	* ZFS_PROJINHERIT set, then its descendant directories will inherit
	* not only the project ID, but also the ZFS_PROJINHERIT flag. Under
	* such case, we only allow renames into our tree when the project
	* IDs are the same.
	*/
	if (tdzp->z_pflags & ZFS_PROJINHERIT &&
	tdzp->z_projid != szp->z_projid) {
	error = SET_ERROR(EXDEV);
	goto out;
	}

	/*
	* Must have write access at the source to remove the old entry
	* and write access at the target to create the new entry.
	* Note that if target and source are the same, this can be
	* done in a single check.
	*/
	if ((error = zfs_zaccess_rename(sdzp, szp, tdzp, tzp, cr, NULL)))
	goto out;

	if ((*svpp)->v_type == VDIR) {
	/*
	* Avoid ".", "..", and aliases of "." for obvious reasons.
	*/
	if ((scnp->cn_namelen == 1 && scnp->cn_nameptr[0] == '.') \|\|
	sdzp == szp \|\|
	(scnp->cn_flags \| tcnp->cn_flags) & ISDOTDOT) {
	error = EINVAL;
	goto out;
	}

	/*
	* Check to make sure rename is valid.
	* Can't do a move like this: /usr/a/b to /usr/a/b/c/d
	*/
	if ((error = zfs_rename_check(szp, sdzp, tdzp)))
	goto out;
	}

	/*
	* Does target exist?
	*/
	if (tzp) {
	/*
	* Source and target must be the same type.
	*/
	if ((*svpp)->v_type == VDIR) {
	if ((*tvpp)->v_type != VDIR) {
	error = SET_ERROR(ENOTDIR);
	goto out;
	} else {
	cache_purge(tdvp);
	if (sdvp != tdvp)
	cache_purge(sdvp);
	}
	} else {
	if ((*tvpp)->v_type == VDIR) {
	error = SET_ERROR(EISDIR);
	goto out;
	}
	}
	}

	vn_seqc_write_begin(*svpp);
	vn_seqc_write_begin(sdvp);
	if (*tvpp != NULL)
	vn_seqc_write_begin(*tvpp);
	if (tdvp != *tvpp)
	vn_seqc_write_begin(tdvp);

	vnevent_rename_src(*svpp, sdvp, scnp->cn_nameptr, ct);
	if (tzp)
	vnevent_rename_dest(*tvpp, tdvp, tnm, ct);

	/*
	* notify the target directory if it is not the same
	* as source directory.
	*/
	if (tdvp != sdvp) {
	vnevent_rename_dest_dir(tdvp, ct);
	}

	tx = dmu_tx_create(zfsvfs->z_os);
	dmu_tx_hold_sa(tx, szp->z_sa_hdl, B_FALSE);
	dmu_tx_hold_sa(tx, sdzp->z_sa_hdl, B_FALSE);
	dmu_tx_hold_zap(tx, sdzp->z_id, FALSE, snm);
	dmu_tx_hold_zap(tx, tdzp->z_id, TRUE, tnm);
	if (sdzp != tdzp) {
	dmu_tx_hold_sa(tx, tdzp->z_sa_hdl, B_FALSE);
	zfs_sa_upgrade_txholds(tx, tdzp);
	}
	if (tzp) {
	dmu_tx_hold_sa(tx, tzp->z_sa_hdl, B_FALSE);
	zfs_sa_upgrade_txholds(tx, tzp);
	}

	zfs_sa_upgrade_txholds(tx, szp);
	dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error) {
	dmu_tx_abort(tx);
	goto out_seq;
	}

	if (tzp) /* Attempt to remove the existing target */
	error = zfs_link_destroy(tdzp, tnm, tzp, tx, 0, NULL);

	if (error == 0) {
	error = zfs_link_create(tdzp, tnm, szp, tx, ZRENAMING);
	if (error == 0) {
	szp->z_pflags \|= ZFS_AV_MODIFIED;

	error = sa_update(szp->z_sa_hdl, SA_ZPL_FLAGS(zfsvfs),
	(void *)&szp->z_pflags, sizeof (uint64_t), tx);
	ASSERT0(error);

	error = zfs_link_destroy(sdzp, snm, szp, tx, ZRENAMING,
	NULL);
	if (error == 0) {
	zfs_log_rename(zilog, tx, TX_RENAME, sdzp,
	snm, tdzp, tnm, szp);
	} else {
	/*
	* At this point, we have successfully created
	* the target name, but have failed to remove
	* the source name. Since the create was done
	* with the ZRENAMING flag, there are
	* complications; for one, the link count is
	* wrong. The easiest way to deal with this
	* is to remove the newly created target, and
	* return the original error. This must
	* succeed; fortunately, it is very unlikely to
	* fail, since we just created it.
	*/
	VERIFY0(zfs_link_destroy(tdzp, tnm, szp, tx,
	ZRENAMING, NULL));
	}
	}
	if (error == 0) {
	cache_vop_rename(sdvp, svpp, tdvp, tvpp, scnp, tcnp);
	}
	}

	dmu_tx_commit(tx);

	out_seq:
	vn_seqc_write_end(*svpp);
	vn_seqc_write_end(sdvp);
	if (*tvpp != NULL)
	vn_seqc_write_end(*tvpp);
	if (tdvp != *tvpp)
	vn_seqc_write_end(tdvp);

	out:
	if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);
	zfs_exit(zfsvfs, FTAG);

	return (error);
	}

	int
	zfs_rename(znode_t sdzp, const char sname, znode_t tdzp, const char tname,
	cred_t cr, int flags, uint64_t rflags, vattr_t wo_vap, zidmap_t *mnt_ns)
	{
	struct componentname scn, tcn;
	vnode_t sdvp, tdvp;
	vnode_t svp, tvp;
	int error;
	svp = tvp = NULL;

	if (rflags != 0 \|\| wo_vap != NULL)
	return (SET_ERROR(EINVAL));

	sdvp = ZTOV(sdzp);
	tdvp = ZTOV(tdzp);
	error = zfs_lookup_internal(sdzp, sname, &svp, &scn, DELETE);
	if (sdzp->z_zfsvfs->z_replay == B_FALSE)
	VOP_UNLOCK1(sdvp);
	if (error != 0)
	goto fail;
	VOP_UNLOCK1(svp);

	vn_lock(tdvp, LK_EXCLUSIVE \| LK_RETRY);
	error = zfs_lookup_internal(tdzp, tname, &tvp, &tcn, RENAME);
	if (error == EJUSTRETURN)
	tvp = NULL;
	else if (error != 0) {
	VOP_UNLOCK1(tdvp);
	goto fail;
	}

	error = zfs_do_rename(sdvp, &svp, &scn, tdvp, &tvp, &tcn, cr);
	fail:
	if (svp != NULL)
	vrele(svp);
	if (tvp != NULL)
	vrele(tvp);

	return (error);
	}

	/*
	* Insert the indicated symbolic reference entry into the directory.
	*
	* IN: dvp - Directory to contain new symbolic link.
	* link - Name for new symlink entry.
	* vap - Attributes of new entry.
	* cr - credentials of caller.
	* ct - caller context
	* flags - case flags
	* mnt_ns - Unused on FreeBSD
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* dvp - ctime\|mtime updated
	*/
	int
	zfs_symlink(znode_t dzp, const char name, vattr_t *vap,
	const char link, znode_t zpp, cred_t cr, int flags, zidmap_t *mnt_ns)
	{
	(void) flags;
	znode_t *zp;
	dmu_tx_t *tx;
	zfsvfs_t *zfsvfs = dzp->z_zfsvfs;
	zilog_t *zilog;
	uint64_t len = strlen(link);
	int error;
	zfs_acl_ids_t acl_ids;
	boolean_t fuid_dirtied;
	uint64_t txtype = TX_SYMLINK;

	ASSERT3S(vap->va_type, ==, VLNK);

	if ((error = zfs_enter_verify_zp(zfsvfs, dzp, FTAG)) != 0)
	return (error);
	zilog = zfsvfs->z_log;

	if (zfsvfs->z_utf8 && u8_validate(name, strlen(name),
	NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EILSEQ));
	}

	if (len > MAXPATHLEN) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(ENAMETOOLONG));
	}

	if ((error = zfs_acl_ids_create(dzp, 0,
	vap, cr, NULL, &acl_ids, NULL)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Attempt to lock directory; fail if entry already exists.
	*/
	error = zfs_dirent_lookup(dzp, name, &zp, ZNEW);
	if (error) {
	zfs_acl_ids_free(&acl_ids);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	if ((error = zfs_zaccess(dzp, ACE_ADD_FILE, 0, B_FALSE, cr, mnt_ns))) {
	zfs_acl_ids_free(&acl_ids);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	if (zfs_acl_ids_overquota(zfsvfs, &acl_ids,
	0 /* projid */)) {
	zfs_acl_ids_free(&acl_ids);
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EDQUOT));
	}

	getnewvnode_reserve_();
	tx = dmu_tx_create(zfsvfs->z_os);
	fuid_dirtied = zfsvfs->z_fuid_dirty;
	dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0, MAX(1, len));
	dmu_tx_hold_zap(tx, dzp->z_id, TRUE, name);
	dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes +
	ZFS_SA_BASE_ATTR_SIZE + len);
	dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE);
	if (!zfsvfs->z_use_sa && acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) {
	dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0,
	acl_ids.z_aclp->z_acl_bytes);
	}
	if (fuid_dirtied)
	zfs_fuid_txhold(zfsvfs, tx);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error) {
	zfs_acl_ids_free(&acl_ids);
	dmu_tx_abort(tx);
	getnewvnode_drop_reserve();
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Create a new object for the symlink.
	* for version 4 ZPL datasets the symlink will be an SA attribute
	*/
	zfs_mknode(dzp, vap, tx, cr, 0, &zp, &acl_ids);

	if (fuid_dirtied)
	zfs_fuid_sync(zfsvfs, tx);

	if (zp->z_is_sa)
	error = sa_update(zp->z_sa_hdl, SA_ZPL_SYMLINK(zfsvfs),
	__DECONST(void *, link), len, tx);
	else
	zfs_sa_symlink(zp, __DECONST(char *, link), len, tx);

	zp->z_size = len;
	(void) sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(zfsvfs),
	&zp->z_size, sizeof (zp->z_size), tx);
	/*
	* Insert the new object into the directory.
	*/
	(void) zfs_link_create(dzp, name, zp, tx, ZNEW);

	zfs_log_symlink(zilog, tx, txtype, dzp, zp, name, link);
	*zpp = zp;

	zfs_acl_ids_free(&acl_ids);

	dmu_tx_commit(tx);

	getnewvnode_drop_reserve();

	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Return, in the buffer contained in the provided uio structure,
	* the symbolic path referred to by vp.
	*
	* IN: vp - vnode of symbolic link.
	* uio - structure to contain the link path.
	* cr - credentials of caller.
	* ct - caller context
	*
	* OUT: uio - structure containing the link path.
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* vp - atime updated
	*/
	static int
	zfs_readlink(vnode_t vp, zfs_uio_t uio, cred_t cr, caller_context_t ct)
	{
	(void) cr, (void) ct;
	znode_t *zp = VTOZ(vp);
	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
	int error;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	if (zp->z_is_sa)
	error = sa_lookup_uio(zp->z_sa_hdl,
	SA_ZPL_SYMLINK(zfsvfs), uio);
	else
	error = zfs_sa_readlink(zp, uio);

	ZFS_ACCESSTIME_STAMP(zfsvfs, zp);

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Insert a new entry into directory tdvp referencing svp.
	*
	* IN: tdvp - Directory to contain new entry.
	* svp - vnode of new entry.
	* name - name of new entry.
	* cr - credentials of caller.
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* tdvp - ctime\|mtime updated
	* svp - ctime updated
	*/
	int
	zfs_link(znode_t tdzp, znode_t szp, const char name, cred_t cr,
	int flags)
	{
	(void) flags;
	znode_t *tzp;
	zfsvfs_t *zfsvfs = tdzp->z_zfsvfs;
	zilog_t *zilog;
	dmu_tx_t *tx;
	int error;
	uint64_t parent;
	uid_t owner;

	ASSERT3S(ZTOV(tdzp)->v_type, ==, VDIR);

	if ((error = zfs_enter_verify_zp(zfsvfs, tdzp, FTAG)) != 0)
	return (error);
	zilog = zfsvfs->z_log;

	/*
	* POSIX dictates that we return EPERM here.
	* Better choices include ENOTSUP or EISDIR.
	*/
	if (ZTOV(szp)->v_type == VDIR) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EPERM));
	}

	if ((error = zfs_verify_zp(szp)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* If we are using project inheritance, means if the directory has
	* ZFS_PROJINHERIT set, then its descendant directories will inherit
	* not only the project ID, but also the ZFS_PROJINHERIT flag. Under
	* such case, we only allow hard link creation in our tree when the
	* project IDs are the same.
	*/
	if (tdzp->z_pflags & ZFS_PROJINHERIT &&
	tdzp->z_projid != szp->z_projid) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EXDEV));
	}

	if (szp->z_pflags & (ZFS_APPENDONLY \|
	ZFS_IMMUTABLE \| ZFS_READONLY)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EPERM));
	}

	/* Prevent links to .zfs/shares files */

	if ((error = sa_lookup(szp->z_sa_hdl, SA_ZPL_PARENT(zfsvfs),
	&parent, sizeof (uint64_t))) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	if (parent == zfsvfs->z_shares_dir) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EPERM));
	}

	if (zfsvfs->z_utf8 && u8_validate(name,
	strlen(name), NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EILSEQ));
	}

	/*
	* We do not support links between attributes and non-attributes
	* because of the potential security risk of creating links
	* into "normal" file space in order to circumvent restrictions
	* imposed in attribute space.
	*/
	if ((szp->z_pflags & ZFS_XATTR) != (tdzp->z_pflags & ZFS_XATTR)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}


	owner = zfs_fuid_map_id(zfsvfs, szp->z_uid, cr, ZFS_OWNER);
	if (owner != crgetuid(cr) && secpolicy_basic_link(ZTOV(szp), cr) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EPERM));
	}

	if ((error = zfs_zaccess(tdzp, ACE_ADD_FILE, 0, B_FALSE, cr, NULL))) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Attempt to lock directory; fail if entry already exists.
	*/
	error = zfs_dirent_lookup(tdzp, name, &tzp, ZNEW);
	if (error) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	tx = dmu_tx_create(zfsvfs->z_os);
	dmu_tx_hold_sa(tx, szp->z_sa_hdl, B_FALSE);
	dmu_tx_hold_zap(tx, tdzp->z_id, TRUE, name);
	zfs_sa_upgrade_txholds(tx, szp);
	zfs_sa_upgrade_txholds(tx, tdzp);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error) {
	dmu_tx_abort(tx);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	error = zfs_link_create(tdzp, name, szp, tx, 0);

	if (error == 0) {
	uint64_t txtype = TX_LINK;
	zfs_log_link(zilog, tx, txtype, tdzp, szp, name);
	}

	dmu_tx_commit(tx);

	if (error == 0) {
	vnevent_link(ZTOV(szp), ct);
	}

	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Free or allocate space in a file. Currently, this function only
	* supports the `F_FREESP' command. However, this command is somewhat
	* misnamed, as its functionality includes the ability to allocate as
	* well as free space.
	*
	* IN: ip - inode of file to free data in.
	* cmd - action to take (only F_FREESP supported).
	* bfp - section of file to free/alloc.
	* flag - current file open mode flags.
	* offset - current file offset.
	* cr - credentials of caller.
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* ip - ctime\|mtime updated
	*/
	int
	zfs_space(znode_t zp, int cmd, flock64_t bfp, int flag,
	offset_t offset, cred_t *cr)
	{
	(void) offset;
	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	uint64_t off, len;
	int error;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	if (cmd != F_FREESP) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	/*
	* Callers might not be able to detect properly that we are read-only,
	* so check it explicitly here.
	*/
	if (zfs_is_readonly(zfsvfs)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EROFS));
	}

	if (bfp->l_len < 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	/*
	* Permissions aren't checked on Solaris because on this OS
	* zfs_space() can only be called with an opened file handle.
	* On Linux we can get here through truncate_range() which
	* operates directly on inodes, so we need to check access rights.
	*/
	if ((error = zfs_zaccess(zp, ACE_WRITE_DATA, 0, B_FALSE, cr, NULL))) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	off = bfp->l_start;
	len = bfp->l_len; /* 0 means from off to end of file */

	error = zfs_freesp(zp, off, len, flag, TRUE);

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	static void
	zfs_inactive(vnode_t vp, cred_t cr, caller_context_t *ct)
	{
	(void) cr, (void) ct;
	znode_t *zp = VTOZ(vp);
	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
	int error;

	ZFS_TEARDOWN_INACTIVE_ENTER_READ(zfsvfs);
	if (zp->z_sa_hdl == NULL) {
	/*
	* The fs has been unmounted, or we did a
	* suspend/resume and this file no longer exists.
	*/
	ZFS_TEARDOWN_INACTIVE_EXIT_READ(zfsvfs);
	vrecycle(vp);
	return;
	}

	if (zp->z_unlinked) {
	/*
	* Fast path to recycle a vnode of a removed file.
	*/
	ZFS_TEARDOWN_INACTIVE_EXIT_READ(zfsvfs);
	vrecycle(vp);
	return;
	}

	if (zp->z_atime_dirty && zp->z_unlinked == 0) {
	dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os);

	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
	zfs_sa_upgrade_txholds(tx, zp);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error) {
	dmu_tx_abort(tx);
	} else {
	(void) sa_update(zp->z_sa_hdl, SA_ZPL_ATIME(zfsvfs),
	(void *)&zp->z_atime, sizeof (zp->z_atime), tx);
	zp->z_atime_dirty = 0;
	dmu_tx_commit(tx);
	}
	}
	ZFS_TEARDOWN_INACTIVE_EXIT_READ(zfsvfs);
	}


	_Static_assert(sizeof (struct zfid_short) <= sizeof (struct fid),
	"struct zfid_short bigger than struct fid");
	_Static_assert(sizeof (struct zfid_long) <= sizeof (struct fid),
	"struct zfid_long bigger than struct fid");

	static int
	zfs_fid(vnode_t vp, fid_t fidp, caller_context_t *ct)
	{
	(void) ct;
	znode_t *zp = VTOZ(vp);
	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
	uint32_t gen;
	uint64_t gen64;
	uint64_t object = zp->z_id;
	zfid_short_t *zfid;
	int size, i, error;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_GEN(zfsvfs),
	&gen64, sizeof (uint64_t))) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	gen = (uint32_t)gen64;

	size = (zfsvfs->z_parent != zfsvfs) ? LONG_FID_LEN : SHORT_FID_LEN;
	fidp->fid_len = size;

	zfid = (zfid_short_t *)fidp;

	zfid->zf_len = size;

	for (i = 0; i < sizeof (zfid->zf_object); i++)
	zfid->zf_object[i] = (uint8_t)(object >> (8 * i));

	/* Must have a non-zero generation number to distinguish from .zfs */
	if (gen == 0)
	gen = 1;
	for (i = 0; i < sizeof (zfid->zf_gen); i++)
	zfid->zf_gen[i] = (uint8_t)(gen >> (8 * i));

	if (size == LONG_FID_LEN) {
	uint64_t objsetid = dmu_objset_id(zfsvfs->z_os);
	zfid_long_t *zlfid;

	zlfid = (zfid_long_t *)fidp;

	for (i = 0; i < sizeof (zlfid->zf_setid); i++)
	zlfid->zf_setid[i] = (uint8_t)(objsetid >> (8 * i));

	/* XXX - this should be the generation number for the objset */
	for (i = 0; i < sizeof (zlfid->zf_setgen); i++)
	zlfid->zf_setgen[i] = 0;
	}

	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	static int
	zfs_pathconf(vnode_t vp, int cmd, ulong_t valp, cred_t *cr,
	caller_context_t *ct)
	{
	znode_t *zp;
	zfsvfs_t *zfsvfs;
	int error;

	switch (cmd) {
	case _PC_LINK_MAX:
	*valp = MIN(LONG_MAX, ZFS_LINK_MAX);
	return (0);

	case _PC_FILESIZEBITS:
	*valp = 64;
	return (0);
	case _PC_MIN_HOLE_SIZE:
	*valp = (int)SPA_MINBLOCKSIZE;
	return (0);
	case _PC_ACL_EXTENDED:
	#if 0 /* POSIX ACLs are not implemented for ZFS on FreeBSD yet. */
	zp = VTOZ(vp);
	zfsvfs = zp->z_zfsvfs;
	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);
	*valp = zfsvfs->z_acl_type == ZFSACLTYPE_POSIX ? 1 : 0;
	zfs_exit(zfsvfs, FTAG);
	#else
	*valp = 0;
	#endif
	return (0);

	case _PC_ACL_NFS4:
	zp = VTOZ(vp);
	zfsvfs = zp->z_zfsvfs;
	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);
	*valp = zfsvfs->z_acl_type == ZFS_ACLTYPE_NFSV4 ? 1 : 0;
	zfs_exit(zfsvfs, FTAG);
	return (0);

	case _PC_ACL_PATH_MAX:
	*valp = ACL_MAX_ENTRIES;
	return (0);

	default:
	return (EOPNOTSUPP);
	}
	}

	static int
	zfs_getpages(struct vnode vp, vm_page_t ma, int count, int *rbehind,
	int *rahead)
	{
	znode_t *zp = VTOZ(vp);
	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
	zfs_locked_range_t *lr;
	vm_object_t object;
	off_t start, end, obj_size;
	uint_t blksz;
	int pgsin_b, pgsin_a;
	int error;

	if (zfs_enter_verify_zp(zfsvfs, zp, FTAG) != 0)
	return (zfs_vm_pagerret_error);

	start = IDX_TO_OFF(ma[0]->pindex);
	end = IDX_TO_OFF(ma[count - 1]->pindex + 1);

	/*
	* Lock a range covering all required and optional pages.
	* Note that we need to handle the case of the block size growing.
	*/
	for (;;) {
	blksz = zp->z_blksz;
	lr = zfs_rangelock_tryenter(&zp->z_rangelock,
	rounddown(start, blksz),
	roundup(end, blksz) - rounddown(start, blksz), RL_READER);
	if (lr == NULL) {
	if (rahead != NULL) {
	*rahead = 0;
	rahead = NULL;
	}
	if (rbehind != NULL) {
	*rbehind = 0;
	rbehind = NULL;
	}
	break;
	}
	if (blksz == zp->z_blksz)
	break;
	zfs_rangelock_exit(lr);
	}

	object = ma[0]->object;
	zfs_vmobject_wlock(object);
	obj_size = object->un_pager.vnp.vnp_size;
	zfs_vmobject_wunlock(object);
	if (IDX_TO_OFF(ma[count - 1]->pindex) >= obj_size) {
	if (lr != NULL)
	zfs_rangelock_exit(lr);
	zfs_exit(zfsvfs, FTAG);
	return (zfs_vm_pagerret_bad);
	}

	pgsin_b = 0;
	if (rbehind != NULL) {
	pgsin_b = OFF_TO_IDX(start - rounddown(start, blksz));
	pgsin_b = MIN(*rbehind, pgsin_b);
	}

	pgsin_a = 0;
	if (rahead != NULL) {
	pgsin_a = OFF_TO_IDX(roundup(end, blksz) - end);
	if (end + IDX_TO_OFF(pgsin_a) >= obj_size)
	pgsin_a = OFF_TO_IDX(round_page(obj_size) - end);
	pgsin_a = MIN(*rahead, pgsin_a);
	}

	/*
	* NB: we need to pass the exact byte size of the data that we expect
	* to read after accounting for the file size. This is required because
	* ZFS will panic if we request DMU to read beyond the end of the last
	* allocated block.
	*/
	error = dmu_read_pages(zfsvfs->z_os, zp->z_id, ma, count, &pgsin_b,
	&pgsin_a, MIN(end, obj_size) - (end - PAGE_SIZE));

	if (lr != NULL)
	zfs_rangelock_exit(lr);
	ZFS_ACCESSTIME_STAMP(zfsvfs, zp);

	dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, count*PAGE_SIZE);

	zfs_exit(zfsvfs, FTAG);

	if (error != 0)
	return (zfs_vm_pagerret_error);

	VM_CNT_INC(v_vnodein);
	VM_CNT_ADD(v_vnodepgsin, count + pgsin_b + pgsin_a);
	if (rbehind != NULL)
	*rbehind = pgsin_b;
	if (rahead != NULL)
	*rahead = pgsin_a;
	return (zfs_vm_pagerret_ok);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_getpages_args {
	struct vnode *a_vp;
	vm_page_t *a_m;
	int a_count;
	int *a_rbehind;
	int *a_rahead;
	};
	#endif

	static int
	zfs_freebsd_getpages(struct vop_getpages_args *ap)
	{

	return (zfs_getpages(ap->a_vp, ap->a_m, ap->a_count, ap->a_rbehind,
	ap->a_rahead));
	}

	static int
	zfs_putpages(struct vnode vp, vm_page_t ma, size_t len, int flags,
	int *rtvals)
	{
	znode_t *zp = VTOZ(vp);
	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
	zfs_locked_range_t *lr;
	dmu_tx_t *tx;
	struct sf_buf *sf;
	vm_object_t object;
	vm_page_t m;
	caddr_t va;
	size_t tocopy;
	size_t lo_len;
	vm_ooffset_t lo_off;
	vm_ooffset_t off;
	uint_t blksz;
	int ncount;
	int pcount;
	int err;
	int i;

	object = vp->v_object;
	KASSERT(ma[0]->object == object, ("mismatching object"));
	KASSERT(len > 0 && (len & PAGE_MASK) == 0, ("unexpected length"));

	pcount = btoc(len);
	ncount = pcount;
	for (i = 0; i < pcount; i++)
	rtvals[i] = zfs_vm_pagerret_error;

	if (zfs_enter_verify_zp(zfsvfs, zp, FTAG) != 0)
	return (zfs_vm_pagerret_error);

	off = IDX_TO_OFF(ma[0]->pindex);
	blksz = zp->z_blksz;
	lo_off = rounddown(off, blksz);
	lo_len = roundup(len + (off - lo_off), blksz);
	lr = zfs_rangelock_enter(&zp->z_rangelock, lo_off, lo_len, RL_WRITER);

	zfs_vmobject_wlock(object);
	if (len + off > object->un_pager.vnp.vnp_size) {
	if (object->un_pager.vnp.vnp_size > off) {
	int pgoff;

	len = object->un_pager.vnp.vnp_size - off;
	ncount = btoc(len);
	if ((pgoff = (int)len & PAGE_MASK) != 0) {
	/*
	* If the object is locked and the following
	* conditions hold, then the page's dirty
	* field cannot be concurrently changed by a
	* pmap operation.
	*/
	m = ma[ncount - 1];
	vm_page_assert_sbusied(m);
	KASSERT(!pmap_page_is_write_mapped(m),
	("zfs_putpages: page %p is not read-only",
	m));
	vm_page_clear_dirty(m, pgoff, PAGE_SIZE -
	pgoff);
	}
	} else {
	len = 0;
	ncount = 0;
	}
	if (ncount < pcount) {
	for (i = ncount; i < pcount; i++) {
	rtvals[i] = zfs_vm_pagerret_bad;
	}
	}
	}
	zfs_vmobject_wunlock(object);

	if (ncount == 0)
	goto out;

	if (zfs_id_overblockquota(zfsvfs, DMU_USERUSED_OBJECT, zp->z_uid) \|\|
	zfs_id_overblockquota(zfsvfs, DMU_GROUPUSED_OBJECT, zp->z_gid) \|\|
	(zp->z_projid != ZFS_DEFAULT_PROJID &&
	zfs_id_overblockquota(zfsvfs, DMU_PROJECTUSED_OBJECT,
	zp->z_projid))) {
	goto out;
	}

	tx = dmu_tx_create(zfsvfs->z_os);
	dmu_tx_hold_write(tx, zp->z_id, off, len);

	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
	zfs_sa_upgrade_txholds(tx, zp);
	err = dmu_tx_assign(tx, TXG_WAIT);
	if (err != 0) {
	dmu_tx_abort(tx);
	goto out;
	}

	if (zp->z_blksz < PAGE_SIZE) {
	for (i = 0; len > 0; off += tocopy, len -= tocopy, i++) {
	tocopy = len > PAGE_SIZE ? PAGE_SIZE : len;
	va = zfs_map_page(ma[i], &sf);
	dmu_write(zfsvfs->z_os, zp->z_id, off, tocopy, va, tx);
	zfs_unmap_page(sf);
	}
	} else {
	err = dmu_write_pages(zfsvfs->z_os, zp->z_id, off, len, ma, tx);
	}

	if (err == 0) {
	uint64_t mtime[2], ctime[2];
	sa_bulk_attr_t bulk[3];
	int count = 0;

	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL,
	&mtime, 16);
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL,
	&ctime, 16);
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
	&zp->z_pflags, 8);
	zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);
	err = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
	ASSERT0(err);
	/*
	* XXX we should be passing a callback to undirty
	* but that would make the locking messier
	*/
	zfs_log_write(zfsvfs->z_log, tx, TX_WRITE, zp, off,
	len, 0, NULL, NULL);

	zfs_vmobject_wlock(object);
	for (i = 0; i < ncount; i++) {
	rtvals[i] = zfs_vm_pagerret_ok;
	vm_page_undirty(ma[i]);
	}
	zfs_vmobject_wunlock(object);
	VM_CNT_INC(v_vnodeout);
	VM_CNT_ADD(v_vnodepgsout, ncount);
	}
	dmu_tx_commit(tx);

	out:
	zfs_rangelock_exit(lr);
	if ((flags & (zfs_vm_pagerput_sync \| zfs_vm_pagerput_inval)) != 0 \|\|
	zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zfsvfs->z_log, zp->z_id);

	dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, len);

	zfs_exit(zfsvfs, FTAG);
	return (rtvals[0]);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_putpages_args {
	struct vnode *a_vp;
	vm_page_t *a_m;
	int a_count;
	int a_sync;
	int *a_rtvals;
	};
	#endif

	static int
	zfs_freebsd_putpages(struct vop_putpages_args *ap)
	{

	return (zfs_putpages(ap->a_vp, ap->a_m, ap->a_count, ap->a_sync,
	ap->a_rtvals));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_bmap_args {
	struct vnode *a_vp;
	daddr_t a_bn;
	struct bufobj **a_bop;
	daddr_t *a_bnp;
	int *a_runp;
	int *a_runb;
	};
	#endif

	static int
	zfs_freebsd_bmap(struct vop_bmap_args *ap)
	{

	if (ap->a_bop != NULL)
	*ap->a_bop = &ap->a_vp->v_bufobj;
	if (ap->a_bnp != NULL)
	*ap->a_bnp = ap->a_bn;
	if (ap->a_runp != NULL)
	*ap->a_runp = 0;
	if (ap->a_runb != NULL)
	*ap->a_runb = 0;

	return (0);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_open_args {
	struct vnode *a_vp;
	int a_mode;
	struct ucred *a_cred;
	struct thread *a_td;
	};
	#endif

	static int
	zfs_freebsd_open(struct vop_open_args *ap)
	{
	vnode_t *vp = ap->a_vp;
	znode_t *zp = VTOZ(vp);
	int error;

	error = zfs_open(&vp, ap->a_mode, ap->a_cred);
	if (error == 0)
	vnode_create_vobject(vp, zp->z_size, ap->a_td);
	return (error);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_close_args {
	struct vnode *a_vp;
	int a_fflag;
	struct ucred *a_cred;
	struct thread *a_td;
	};
	#endif

	static int
	zfs_freebsd_close(struct vop_close_args *ap)
	{

	return (zfs_close(ap->a_vp, ap->a_fflag, 1, 0, ap->a_cred));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_ioctl_args {
	struct vnode *a_vp;
	ulong_t a_command;
	caddr_t a_data;
	int a_fflag;
	struct ucred *cred;
	struct thread *td;
	};
	#endif

	static int
	zfs_freebsd_ioctl(struct vop_ioctl_args *ap)
	{

	return (zfs_ioctl(ap->a_vp, ap->a_command, (intptr_t)ap->a_data,
	ap->a_fflag, ap->a_cred, NULL));
	}

	static int
	ioflags(int ioflags)
	{
	int flags = 0;

	if (ioflags & IO_APPEND)
	flags \|= O_APPEND;
	if (ioflags & IO_NDELAY)
	flags \|= O_NONBLOCK;
	if (ioflags & IO_SYNC)
	flags \|= O_SYNC;

	return (flags);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_read_args {
	struct vnode *a_vp;
	struct uio *a_uio;
	int a_ioflag;
	struct ucred *a_cred;
	};
	#endif

	static int
	zfs_freebsd_read(struct vop_read_args *ap)
	{
	zfs_uio_t uio;
	zfs_uio_init(&uio, ap->a_uio);
	return (zfs_read(VTOZ(ap->a_vp), &uio, ioflags(ap->a_ioflag),
	ap->a_cred));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_write_args {
	struct vnode *a_vp;
	struct uio *a_uio;
	int a_ioflag;
	struct ucred *a_cred;
	};
	#endif

	static int
	zfs_freebsd_write(struct vop_write_args *ap)
	{
	zfs_uio_t uio;
	zfs_uio_init(&uio, ap->a_uio);
	return (zfs_write(VTOZ(ap->a_vp), &uio, ioflags(ap->a_ioflag),
	ap->a_cred));
	}

	#if __FreeBSD_version >= 1300102
	/*
	* VOP_FPLOOKUP_VEXEC routines are subject to special circumstances, see
	* the comment above cache_fplookup for details.
	*/
	static int
	zfs_freebsd_fplookup_vexec(struct vop_fplookup_vexec_args *v)
	{
	vnode_t *vp;
	znode_t *zp;
	uint64_t pflags;

	vp = v->a_vp;
	zp = VTOZ_SMR(vp);
	if (__predict_false(zp == NULL))
	return (EAGAIN);
	pflags = atomic_load_64(&zp->z_pflags);
	if (pflags & ZFS_AV_QUARANTINED)
	return (EAGAIN);
	if (pflags & ZFS_XATTR)
	return (EAGAIN);
	if ((pflags & ZFS_NO_EXECS_DENIED) == 0)
	return (EAGAIN);
	return (0);
	}
	#endif

	#if __FreeBSD_version >= 1300139
	static int
	zfs_freebsd_fplookup_symlink(struct vop_fplookup_symlink_args *v)
	{
	vnode_t *vp;
	znode_t *zp;
	char *target;

	vp = v->a_vp;
	zp = VTOZ_SMR(vp);
	if (__predict_false(zp == NULL)) {
	return (EAGAIN);
	}

	target = atomic_load_consume_ptr(&zp->z_cached_symlink);
	if (target == NULL) {
	return (EAGAIN);
	}
	return (cache_symlink_resolve(v->a_fpl, target, strlen(target)));
	}
	#endif

	#ifndef _SYS_SYSPROTO_H_
	struct vop_access_args {
	struct vnode *a_vp;
	accmode_t a_accmode;
	struct ucred *a_cred;
	struct thread *a_td;
	};
	#endif

	static int
	zfs_freebsd_access(struct vop_access_args *ap)
	{
	vnode_t *vp = ap->a_vp;
	znode_t *zp = VTOZ(vp);
	accmode_t accmode;
	int error = 0;


	if (ap->a_accmode == VEXEC) {
	if (zfs_fastaccesschk_execute(zp, ap->a_cred) == 0)
	return (0);
	}

	/*
	* ZFS itself only knowns about VREAD, VWRITE, VEXEC and VAPPEND,
	*/
	accmode = ap->a_accmode & (VREAD\|VWRITE\|VEXEC\|VAPPEND);
	if (accmode != 0)
	error = zfs_access(zp, accmode, 0, ap->a_cred);

	/*
	* VADMIN has to be handled by vaccess().
	*/
	if (error == 0) {
	accmode = ap->a_accmode & ~(VREAD\|VWRITE\|VEXEC\|VAPPEND);
	if (accmode != 0) {
	#if __FreeBSD_version >= 1300105
	error = vaccess(vp->v_type, zp->z_mode, zp->z_uid,
	zp->z_gid, accmode, ap->a_cred);
	#else
	error = vaccess(vp->v_type, zp->z_mode, zp->z_uid,
	zp->z_gid, accmode, ap->a_cred, NULL);
	#endif
	}
	}

	/*
	* For VEXEC, ensure that at least one execute bit is set for
	* non-directories.
	*/
	if (error == 0 && (ap->a_accmode & VEXEC) != 0 && vp->v_type != VDIR &&
	(zp->z_mode & (S_IXUSR \| S_IXGRP \| S_IXOTH)) == 0) {
	error = EACCES;
	}

	return (error);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_lookup_args {
	struct vnode *a_dvp;
	struct vnode **a_vpp;
	struct componentname *a_cnp;
	};
	#endif

	static int
	zfs_freebsd_lookup(struct vop_lookup_args *ap, boolean_t cached)
	{
	struct componentname *cnp = ap->a_cnp;
	char nm[NAME_MAX + 1];

	ASSERT3U(cnp->cn_namelen, <, sizeof (nm));
	strlcpy(nm, cnp->cn_nameptr, MIN(cnp->cn_namelen + 1, sizeof (nm)));

	return (zfs_lookup(ap->a_dvp, nm, ap->a_vpp, cnp, cnp->cn_nameiop,
	cnp->cn_cred, 0, cached));
	}

	static int
	zfs_freebsd_cachedlookup(struct vop_cachedlookup_args *ap)
	{

	return (zfs_freebsd_lookup((struct vop_lookup_args *)ap, B_TRUE));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_lookup_args {
	struct vnode *a_dvp;
	struct vnode **a_vpp;
	struct componentname *a_cnp;
	};
	#endif

	static int
	zfs_cache_lookup(struct vop_lookup_args *ap)
	{
	zfsvfs_t *zfsvfs;

	zfsvfs = ap->a_dvp->v_mount->mnt_data;
	if (zfsvfs->z_use_namecache)
	return (vfs_cache_lookup(ap));
	else
	return (zfs_freebsd_lookup(ap, B_FALSE));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_create_args {
	struct vnode *a_dvp;
	struct vnode **a_vpp;
	struct componentname *a_cnp;
	struct vattr *a_vap;
	};
	#endif

	static int
	zfs_freebsd_create(struct vop_create_args *ap)
	{
	zfsvfs_t *zfsvfs;
	struct componentname *cnp = ap->a_cnp;
	vattr_t *vap = ap->a_vap;
	znode_t *zp = NULL;
	int rc, mode;

	#if __FreeBSD_version < 1400068
	ASSERT(cnp->cn_flags & SAVENAME);
	#endif

	vattr_init_mask(vap);
	mode = vap->va_mode & ALLPERMS;
	zfsvfs = ap->a_dvp->v_mount->mnt_data;
	*ap->a_vpp = NULL;

	rc = zfs_create(VTOZ(ap->a_dvp), cnp->cn_nameptr, vap, 0, mode,
	&zp, cnp->cn_cred, 0 /* flag /, NULL / vsecattr */, NULL);
	if (rc == 0)
	*ap->a_vpp = ZTOV(zp);
	if (zfsvfs->z_use_namecache &&
	rc == 0 && (cnp->cn_flags & MAKEENTRY) != 0)
	cache_enter(ap->a_dvp, *ap->a_vpp, cnp);

	return (rc);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_remove_args {
	struct vnode *a_dvp;
	struct vnode *a_vp;
	struct componentname *a_cnp;
	};
	#endif

	static int
	zfs_freebsd_remove(struct vop_remove_args *ap)
	{

	#if __FreeBSD_version < 1400068
	ASSERT(ap->a_cnp->cn_flags & SAVENAME);
	#endif

	return (zfs_remove_(ap->a_dvp, ap->a_vp, ap->a_cnp->cn_nameptr,
	ap->a_cnp->cn_cred));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_mkdir_args {
	struct vnode *a_dvp;
	struct vnode **a_vpp;
	struct componentname *a_cnp;
	struct vattr *a_vap;
	};
	#endif

	static int
	zfs_freebsd_mkdir(struct vop_mkdir_args *ap)
	{
	vattr_t *vap = ap->a_vap;
	znode_t *zp = NULL;
	int rc;

	#if __FreeBSD_version < 1400068
	ASSERT(ap->a_cnp->cn_flags & SAVENAME);
	#endif

	vattr_init_mask(vap);
	*ap->a_vpp = NULL;

	rc = zfs_mkdir(VTOZ(ap->a_dvp), ap->a_cnp->cn_nameptr, vap, &zp,
	ap->a_cnp->cn_cred, 0, NULL, NULL);

	if (rc == 0)
	*ap->a_vpp = ZTOV(zp);
	return (rc);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_rmdir_args {
	struct vnode *a_dvp;
	struct vnode *a_vp;
	struct componentname *a_cnp;
	};
	#endif

	static int
	zfs_freebsd_rmdir(struct vop_rmdir_args *ap)
	{
	struct componentname *cnp = ap->a_cnp;

	#if __FreeBSD_version < 1400068
	ASSERT(cnp->cn_flags & SAVENAME);
	#endif

	return (zfs_rmdir_(ap->a_dvp, ap->a_vp, cnp->cn_nameptr, cnp->cn_cred));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_readdir_args {
	struct vnode *a_vp;
	struct uio *a_uio;
	struct ucred *a_cred;
	int *a_eofflag;
	int *a_ncookies;
	cookie_t **a_cookies;
	};
	#endif

	static int
	zfs_freebsd_readdir(struct vop_readdir_args *ap)
	{
	zfs_uio_t uio;
	zfs_uio_init(&uio, ap->a_uio);
	return (zfs_readdir(ap->a_vp, &uio, ap->a_cred, ap->a_eofflag,
	ap->a_ncookies, ap->a_cookies));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_fsync_args {
	struct vnode *a_vp;
	int a_waitfor;
	struct thread *a_td;
	};
	#endif

	static int
	zfs_freebsd_fsync(struct vop_fsync_args *ap)
	{

	return (zfs_fsync(VTOZ(ap->a_vp), 0, ap->a_td->td_ucred));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_getattr_args {
	struct vnode *a_vp;
	struct vattr *a_vap;
	struct ucred *a_cred;
	};
	#endif

	static int
	zfs_freebsd_getattr(struct vop_getattr_args *ap)
	{
	vattr_t *vap = ap->a_vap;
	xvattr_t xvap;
	ulong_t fflags = 0;
	int error;

	xva_init(&xvap);
	xvap.xva_vattr = *vap;
	xvap.xva_vattr.va_mask \|= AT_XVATTR;

	/* Convert chflags into ZFS-type flags. */
	/* XXX: what about SF_SETTABLE?. */
	XVA_SET_REQ(&xvap, XAT_IMMUTABLE);
	XVA_SET_REQ(&xvap, XAT_APPENDONLY);
	XVA_SET_REQ(&xvap, XAT_NOUNLINK);
	XVA_SET_REQ(&xvap, XAT_NODUMP);
	XVA_SET_REQ(&xvap, XAT_READONLY);
	XVA_SET_REQ(&xvap, XAT_ARCHIVE);
	XVA_SET_REQ(&xvap, XAT_SYSTEM);
	XVA_SET_REQ(&xvap, XAT_HIDDEN);
	XVA_SET_REQ(&xvap, XAT_REPARSE);
	XVA_SET_REQ(&xvap, XAT_OFFLINE);
	XVA_SET_REQ(&xvap, XAT_SPARSE);

	error = zfs_getattr(ap->a_vp, (vattr_t *)&xvap, 0, ap->a_cred);
	if (error != 0)
	return (error);

	/* Convert ZFS xattr into chflags. */
	#define FLAG_CHECK(fflag, xflag, xfield) do { \
	if (XVA_ISSET_RTN(&xvap, (xflag)) && (xfield) != 0) \
	fflags \|= (fflag); \
	} while (0)
	FLAG_CHECK(SF_IMMUTABLE, XAT_IMMUTABLE,
	xvap.xva_xoptattrs.xoa_immutable);
	FLAG_CHECK(SF_APPEND, XAT_APPENDONLY,
	xvap.xva_xoptattrs.xoa_appendonly);
	FLAG_CHECK(SF_NOUNLINK, XAT_NOUNLINK,
	xvap.xva_xoptattrs.xoa_nounlink);
	FLAG_CHECK(UF_ARCHIVE, XAT_ARCHIVE,
	xvap.xva_xoptattrs.xoa_archive);
	FLAG_CHECK(UF_NODUMP, XAT_NODUMP,
	xvap.xva_xoptattrs.xoa_nodump);
	FLAG_CHECK(UF_READONLY, XAT_READONLY,
	xvap.xva_xoptattrs.xoa_readonly);
	FLAG_CHECK(UF_SYSTEM, XAT_SYSTEM,
	xvap.xva_xoptattrs.xoa_system);
	FLAG_CHECK(UF_HIDDEN, XAT_HIDDEN,
	xvap.xva_xoptattrs.xoa_hidden);
	FLAG_CHECK(UF_REPARSE, XAT_REPARSE,
	xvap.xva_xoptattrs.xoa_reparse);
	FLAG_CHECK(UF_OFFLINE, XAT_OFFLINE,
	xvap.xva_xoptattrs.xoa_offline);
	FLAG_CHECK(UF_SPARSE, XAT_SPARSE,
	xvap.xva_xoptattrs.xoa_sparse);

	#undef FLAG_CHECK
	*vap = xvap.xva_vattr;
	vap->va_flags = fflags;
	return (0);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_setattr_args {
	struct vnode *a_vp;
	struct vattr *a_vap;
	struct ucred *a_cred;
	};
	#endif

	static int
	zfs_freebsd_setattr(struct vop_setattr_args *ap)
	{
	vnode_t *vp = ap->a_vp;
	vattr_t *vap = ap->a_vap;
	cred_t *cred = ap->a_cred;
	xvattr_t xvap;
	ulong_t fflags;
	uint64_t zflags;

	vattr_init_mask(vap);
	vap->va_mask &= ~AT_NOSET;

	xva_init(&xvap);
	xvap.xva_vattr = *vap;

	zflags = VTOZ(vp)->z_pflags;

	if (vap->va_flags != VNOVAL) {
	zfsvfs_t *zfsvfs = VTOZ(vp)->z_zfsvfs;
	int error;

	if (zfsvfs->z_use_fuids == B_FALSE)
	return (EOPNOTSUPP);

	fflags = vap->va_flags;
	/*
	* XXX KDM
	* We need to figure out whether it makes sense to allow
	* UF_REPARSE through, since we don't really have other
	* facilities to handle reparse points and zfs_setattr()
	* doesn't currently allow setting that attribute anyway.
	*/
	if ((fflags & ~(SF_IMMUTABLE\|SF_APPEND\|SF_NOUNLINK\|UF_ARCHIVE\|
	UF_NODUMP\|UF_SYSTEM\|UF_HIDDEN\|UF_READONLY\|UF_REPARSE\|
	UF_OFFLINE\|UF_SPARSE)) != 0)
	return (EOPNOTSUPP);
	/*
	* Unprivileged processes are not permitted to unset system
	* flags, or modify flags if any system flags are set.
	* Privileged non-jail processes may not modify system flags
	* if securelevel > 0 and any existing system flags are set.
	* Privileged jail processes behave like privileged non-jail
	* processes if the PR_ALLOW_CHFLAGS permission bit is set;
	* otherwise, they behave like unprivileged processes.
	*/
	if (secpolicy_fs_owner(vp->v_mount, cred) == 0 \|\|
	spl_priv_check_cred(cred, PRIV_VFS_SYSFLAGS) == 0) {
	if (zflags &
	(ZFS_IMMUTABLE \| ZFS_APPENDONLY \| ZFS_NOUNLINK)) {
	error = securelevel_gt(cred, 0);
	if (error != 0)
	return (error);
	}
	} else {
	/*
	* Callers may only modify the file flags on
	* objects they have VADMIN rights for.
	*/
	if ((error = VOP_ACCESS(vp, VADMIN, cred,
	curthread)) != 0)
	return (error);
	if (zflags &
	(ZFS_IMMUTABLE \| ZFS_APPENDONLY \|
	ZFS_NOUNLINK)) {
	return (EPERM);
	}
	if (fflags &
	(SF_IMMUTABLE \| SF_APPEND \| SF_NOUNLINK)) {
	return (EPERM);
	}
	}

	#define FLAG_CHANGE(fflag, zflag, xflag, xfield) do { \
	if (((fflags & (fflag)) && !(zflags & (zflag))) \|\| \
	((zflags & (zflag)) && !(fflags & (fflag)))) { \
	XVA_SET_REQ(&xvap, (xflag)); \
	(xfield) = ((fflags & (fflag)) != 0); \
	} \
	} while (0)
	/* Convert chflags into ZFS-type flags. */
	/* XXX: what about SF_SETTABLE?. */
	FLAG_CHANGE(SF_IMMUTABLE, ZFS_IMMUTABLE, XAT_IMMUTABLE,
	xvap.xva_xoptattrs.xoa_immutable);
	FLAG_CHANGE(SF_APPEND, ZFS_APPENDONLY, XAT_APPENDONLY,
	xvap.xva_xoptattrs.xoa_appendonly);
	FLAG_CHANGE(SF_NOUNLINK, ZFS_NOUNLINK, XAT_NOUNLINK,
	xvap.xva_xoptattrs.xoa_nounlink);
	FLAG_CHANGE(UF_ARCHIVE, ZFS_ARCHIVE, XAT_ARCHIVE,
	xvap.xva_xoptattrs.xoa_archive);
	FLAG_CHANGE(UF_NODUMP, ZFS_NODUMP, XAT_NODUMP,
	xvap.xva_xoptattrs.xoa_nodump);
	FLAG_CHANGE(UF_READONLY, ZFS_READONLY, XAT_READONLY,
	xvap.xva_xoptattrs.xoa_readonly);
	FLAG_CHANGE(UF_SYSTEM, ZFS_SYSTEM, XAT_SYSTEM,
	xvap.xva_xoptattrs.xoa_system);
	FLAG_CHANGE(UF_HIDDEN, ZFS_HIDDEN, XAT_HIDDEN,
	xvap.xva_xoptattrs.xoa_hidden);
	FLAG_CHANGE(UF_REPARSE, ZFS_REPARSE, XAT_REPARSE,
	xvap.xva_xoptattrs.xoa_reparse);
	FLAG_CHANGE(UF_OFFLINE, ZFS_OFFLINE, XAT_OFFLINE,
	xvap.xva_xoptattrs.xoa_offline);
	FLAG_CHANGE(UF_SPARSE, ZFS_SPARSE, XAT_SPARSE,
	xvap.xva_xoptattrs.xoa_sparse);
	#undef FLAG_CHANGE
	}
	if (vap->va_birthtime.tv_sec != VNOVAL) {
	xvap.xva_vattr.va_mask \|= AT_XVATTR;
	XVA_SET_REQ(&xvap, XAT_CREATETIME);
	}
	return (zfs_setattr(VTOZ(vp), (vattr_t *)&xvap, 0, cred, NULL));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_rename_args {
	struct vnode *a_fdvp;
	struct vnode *a_fvp;
	struct componentname *a_fcnp;
	struct vnode *a_tdvp;
	struct vnode *a_tvp;
	struct componentname *a_tcnp;
	};
	#endif

	static int
	zfs_freebsd_rename(struct vop_rename_args *ap)
	{
	vnode_t *fdvp = ap->a_fdvp;
	vnode_t *fvp = ap->a_fvp;
	vnode_t *tdvp = ap->a_tdvp;
	vnode_t *tvp = ap->a_tvp;
	int error;

	#if __FreeBSD_version < 1400068
	ASSERT(ap->a_fcnp->cn_flags & (SAVENAME\|SAVESTART));
	ASSERT(ap->a_tcnp->cn_flags & (SAVENAME\|SAVESTART));
	#endif

	error = zfs_do_rename(fdvp, &fvp, ap->a_fcnp, tdvp, &tvp,
	ap->a_tcnp, ap->a_fcnp->cn_cred);

	vrele(fdvp);
	vrele(fvp);
	vrele(tdvp);
	if (tvp != NULL)
	vrele(tvp);

	return (error);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_symlink_args {
	struct vnode *a_dvp;
	struct vnode **a_vpp;
	struct componentname *a_cnp;
	struct vattr *a_vap;
	char *a_target;
	};
	#endif

	static int
	zfs_freebsd_symlink(struct vop_symlink_args *ap)
	{
	struct componentname *cnp = ap->a_cnp;
	vattr_t *vap = ap->a_vap;
	znode_t *zp = NULL;
	#if __FreeBSD_version >= 1300139
	char *symlink;
	size_t symlink_len;
	#endif
	int rc;

	#if __FreeBSD_version < 1400068
	ASSERT(cnp->cn_flags & SAVENAME);
	#endif

	vap->va_type = VLNK; /* FreeBSD: Syscall only sets va_mode. */
	vattr_init_mask(vap);
	*ap->a_vpp = NULL;

	rc = zfs_symlink(VTOZ(ap->a_dvp), cnp->cn_nameptr, vap,
	ap->a_target, &zp, cnp->cn_cred, 0 /* flags */, NULL);
	if (rc == 0) {
	*ap->a_vpp = ZTOV(zp);
	ASSERT_VOP_ELOCKED(ZTOV(zp), __func__);
	#if __FreeBSD_version >= 1300139
	MPASS(zp->z_cached_symlink == NULL);
	symlink_len = strlen(ap->a_target);
	symlink = cache_symlink_alloc(symlink_len + 1, M_WAITOK);
	if (symlink != NULL) {
	memcpy(symlink, ap->a_target, symlink_len);
	symlink[symlink_len] = '\0';
	atomic_store_rel_ptr((uintptr_t *)&zp->z_cached_symlink,
	(uintptr_t)symlink);
	}
	#endif
	}
	return (rc);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_readlink_args {
	struct vnode *a_vp;
	struct uio *a_uio;
	struct ucred *a_cred;
	};
	#endif

	static int
	zfs_freebsd_readlink(struct vop_readlink_args *ap)
	{
	zfs_uio_t uio;
	int error;
	#if __FreeBSD_version >= 1300139
	znode_t *zp = VTOZ(ap->a_vp);
	char symlink, base;
	size_t symlink_len;
	bool trycache;
	#endif

	zfs_uio_init(&uio, ap->a_uio);
	#if __FreeBSD_version >= 1300139
	trycache = false;
	if (zfs_uio_segflg(&uio) == UIO_SYSSPACE &&
	zfs_uio_iovcnt(&uio) == 1) {
	base = zfs_uio_iovbase(&uio, 0);
	symlink_len = zfs_uio_iovlen(&uio, 0);
	trycache = true;
	}
	#endif
	error = zfs_readlink(ap->a_vp, &uio, ap->a_cred, NULL);
	#if __FreeBSD_version >= 1300139
	if (atomic_load_ptr(&zp->z_cached_symlink) != NULL \|\|
	error != 0 \|\| !trycache) {
	return (error);
	}
	symlink_len -= zfs_uio_resid(&uio);
	symlink = cache_symlink_alloc(symlink_len + 1, M_WAITOK);
	if (symlink != NULL) {
	memcpy(symlink, base, symlink_len);
	symlink[symlink_len] = '\0';
	if (!atomic_cmpset_rel_ptr((uintptr_t *)&zp->z_cached_symlink,
	(uintptr_t)NULL, (uintptr_t)symlink)) {
	cache_symlink_free(symlink, symlink_len + 1);
	}
	}
	#endif
	return (error);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_link_args {
	struct vnode *a_tdvp;
	struct vnode *a_vp;
	struct componentname *a_cnp;
	};
	#endif

	static int
	zfs_freebsd_link(struct vop_link_args *ap)
	{
	struct componentname *cnp = ap->a_cnp;
	vnode_t *vp = ap->a_vp;
	vnode_t *tdvp = ap->a_tdvp;

	if (tdvp->v_mount != vp->v_mount)
	return (EXDEV);

	#if __FreeBSD_version < 1400068
	ASSERT(cnp->cn_flags & SAVENAME);
	#endif

	return (zfs_link(VTOZ(tdvp), VTOZ(vp),
	cnp->cn_nameptr, cnp->cn_cred, 0));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_inactive_args {
	struct vnode *a_vp;
	struct thread *a_td;
	};
	#endif

	static int
	zfs_freebsd_inactive(struct vop_inactive_args *ap)
	{
	vnode_t *vp = ap->a_vp;

	#if __FreeBSD_version >= 1300123
	zfs_inactive(vp, curthread->td_ucred, NULL);
	#else
	zfs_inactive(vp, ap->a_td->td_ucred, NULL);
	#endif
	return (0);
	}

	#if __FreeBSD_version >= 1300042
	#ifndef _SYS_SYSPROTO_H_
	struct vop_need_inactive_args {
	struct vnode *a_vp;
	struct thread *a_td;
	};
	#endif

	static int
	zfs_freebsd_need_inactive(struct vop_need_inactive_args *ap)
	{
	vnode_t *vp = ap->a_vp;
	znode_t *zp = VTOZ(vp);
	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
	int need;

	if (vn_need_pageq_flush(vp))
	return (1);

	if (!ZFS_TEARDOWN_INACTIVE_TRY_ENTER_READ(zfsvfs))
	return (1);
	need = (zp->z_sa_hdl == NULL \|\| zp->z_unlinked \|\| zp->z_atime_dirty);
	ZFS_TEARDOWN_INACTIVE_EXIT_READ(zfsvfs);

	return (need);
	}
	#endif

	#ifndef _SYS_SYSPROTO_H_
	struct vop_reclaim_args {
	struct vnode *a_vp;
	struct thread *a_td;
	};
	#endif

	static int
	zfs_freebsd_reclaim(struct vop_reclaim_args *ap)
	{
	vnode_t *vp = ap->a_vp;
	znode_t *zp = VTOZ(vp);
	zfsvfs_t *zfsvfs = zp->z_zfsvfs;

	ASSERT3P(zp, !=, NULL);

	#if __FreeBSD_version < 1300042
	/* Destroy the vm object and flush associated pages. */
	vnode_destroy_vobject(vp);
	#endif
	/*
	* z_teardown_inactive_lock protects from a race with
	* zfs_znode_dmu_fini in zfsvfs_teardown during
	* force unmount.
	*/
	ZFS_TEARDOWN_INACTIVE_ENTER_READ(zfsvfs);
	if (zp->z_sa_hdl == NULL)
	zfs_znode_free(zp);
	else
	zfs_zinactive(zp);
	ZFS_TEARDOWN_INACTIVE_EXIT_READ(zfsvfs);

	vp->v_data = NULL;
	return (0);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_fid_args {
	struct vnode *a_vp;
	struct fid *a_fid;
	};
	#endif

	static int
	zfs_freebsd_fid(struct vop_fid_args *ap)
	{

	return (zfs_fid(ap->a_vp, (void *)ap->a_fid, NULL));
	}


	#ifndef _SYS_SYSPROTO_H_
	struct vop_pathconf_args {
	struct vnode *a_vp;
	int a_name;
	register_t *a_retval;
	} *ap;
	#endif

	static int
	zfs_freebsd_pathconf(struct vop_pathconf_args *ap)
	{
	ulong_t val;
	int error;

	error = zfs_pathconf(ap->a_vp, ap->a_name, &val,
	curthread->td_ucred, NULL);
	if (error == 0) {
	*ap->a_retval = val;
	return (error);
	}
	if (error != EOPNOTSUPP)
	return (error);

	switch (ap->a_name) {
	case _PC_NAME_MAX:
	*ap->a_retval = NAME_MAX;
	return (0);
	#if __FreeBSD_version >= 1400032
	case _PC_DEALLOC_PRESENT:
	*ap->a_retval = 1;
	return (0);
	#endif
	case _PC_PIPE_BUF:
	if (ap->a_vp->v_type == VDIR \|\| ap->a_vp->v_type == VFIFO) {
	*ap->a_retval = PIPE_BUF;
	return (0);
	}
	return (EINVAL);
	default:
	return (vop_stdpathconf(ap));
	}
	}

	static int zfs_xattr_compat = 1;

	static int
	zfs_check_attrname(const char *name)
	{
	/* We don't allow '/' character in attribute name. */
	if (strchr(name, '/') != NULL)
	return (SET_ERROR(EINVAL));
	/* We don't allow attribute names that start with a namespace prefix. */
	if (ZFS_XA_NS_PREFIX_FORBIDDEN(name))
	return (SET_ERROR(EINVAL));
	return (0);
	}

	/*
	* FreeBSD's extended attributes namespace defines file name prefix for ZFS'
	* extended attribute name:
	*
	* NAMESPACE XATTR_COMPAT PREFIX
	* system * freebsd:system:
	* user 1 (none, can be used to access ZFS
	* fsattr(5) attributes created on Solaris)
	* user 0 user.
	*/
	static int
	zfs_create_attrname(int attrnamespace, const char name, char attrname,
	size_t size, boolean_t compat)
	{
	const char namespace, prefix, *suffix;

	memset(attrname, 0, size);

	switch (attrnamespace) {
	case EXTATTR_NAMESPACE_USER:
	if (compat) {
	/*
	* This is the default namespace by which we can access
	* all attributes created on Solaris.
	*/
	prefix = namespace = suffix = "";
	} else {
	/*
	* This is compatible with the user namespace encoding
	* on Linux prior to xattr_compat, but nothing
	* else.
	*/
	prefix = "";
	namespace = "user";
	suffix = ".";
	}
	break;
	case EXTATTR_NAMESPACE_SYSTEM:
	prefix = "freebsd:";
	namespace = EXTATTR_NAMESPACE_SYSTEM_STRING;
	suffix = ":";
	break;
	case EXTATTR_NAMESPACE_EMPTY:
	default:
	return (SET_ERROR(EINVAL));
	}
	if (snprintf(attrname, size, "%s%s%s%s", prefix, namespace, suffix,
	name) >= size) {
	return (SET_ERROR(ENAMETOOLONG));
	}
	return (0);
	}

	static int
	zfs_ensure_xattr_cached(znode_t *zp)
	{
	int error = 0;

	ASSERT(RW_LOCK_HELD(&zp->z_xattr_lock));

	if (zp->z_xattr_cached != NULL)
	return (0);

	if (rw_write_held(&zp->z_xattr_lock))
	return (zfs_sa_get_xattr(zp));

	if (!rw_tryupgrade(&zp->z_xattr_lock)) {
	rw_exit(&zp->z_xattr_lock);
	rw_enter(&zp->z_xattr_lock, RW_WRITER);
	}
	if (zp->z_xattr_cached == NULL)
	error = zfs_sa_get_xattr(zp);
	rw_downgrade(&zp->z_xattr_lock);
	return (error);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_getextattr {
	IN struct vnode *a_vp;
	IN int a_attrnamespace;
	IN const char *a_name;
	INOUT struct uio *a_uio;
	OUT size_t *a_size;
	IN struct ucred *a_cred;
	IN struct thread *a_td;
	};
	#endif

	static int
	zfs_getextattr_dir(struct vop_getextattr_args ap, const char attrname)
	{
	struct thread *td = ap->a_td;
	struct nameidata nd;
	struct vattr va;
	vnode_t xvp = NULL, vp;
	int error, flags;

	error = zfs_lookup(ap->a_vp, NULL, &xvp, NULL, 0, ap->a_cred,
	LOOKUP_XATTR, B_FALSE);
	if (error != 0)
	return (error);

	flags = FREAD;
	#if __FreeBSD_version < 1400043
	NDINIT_ATVP(&nd, LOOKUP, NOFOLLOW, UIO_SYSSPACE, attrname,
	xvp, td);
	#else
	NDINIT_ATVP(&nd, LOOKUP, NOFOLLOW, UIO_SYSSPACE, attrname, xvp);
	#endif
	error = vn_open_cred(&nd, &flags, 0, VN_OPEN_INVFS, ap->a_cred, NULL);
	if (error != 0)
	return (SET_ERROR(error));
	vp = nd.ni_vp;
	NDFREE_PNBUF(&nd);

	if (ap->a_size != NULL) {
	error = VOP_GETATTR(vp, &va, ap->a_cred);
	if (error == 0)
	*ap->a_size = (size_t)va.va_size;
	} else if (ap->a_uio != NULL)
	error = VOP_READ(vp, ap->a_uio, IO_UNIT, ap->a_cred);

	VOP_UNLOCK1(vp);
	vn_close(vp, flags, ap->a_cred, td);
	return (error);
	}

	static int
	zfs_getextattr_sa(struct vop_getextattr_args ap, const char attrname)
	{
	znode_t *zp = VTOZ(ap->a_vp);
	uchar_t *nv_value;
	uint_t nv_size;
	int error;

	error = zfs_ensure_xattr_cached(zp);
	if (error != 0)
	return (error);

	ASSERT(RW_LOCK_HELD(&zp->z_xattr_lock));
	ASSERT3P(zp->z_xattr_cached, !=, NULL);

	error = nvlist_lookup_byte_array(zp->z_xattr_cached, attrname,
	&nv_value, &nv_size);
	if (error != 0)
	return (SET_ERROR(error));

	if (ap->a_size != NULL)
	*ap->a_size = nv_size;
	else if (ap->a_uio != NULL)
	error = uiomove(nv_value, nv_size, ap->a_uio);
	if (error != 0)
	return (SET_ERROR(error));

	return (0);
	}

	static int
	zfs_getextattr_impl(struct vop_getextattr_args *ap, boolean_t compat)
	{
	znode_t *zp = VTOZ(ap->a_vp);
	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	char attrname[EXTATTR_MAXNAMELEN+1];
	int error;

	error = zfs_create_attrname(ap->a_attrnamespace, ap->a_name, attrname,
	sizeof (attrname), compat);
	if (error != 0)
	return (error);

	error = ENOENT;
	if (zfsvfs->z_use_sa && zp->z_is_sa)
	error = zfs_getextattr_sa(ap, attrname);
	if (error == ENOENT)
	error = zfs_getextattr_dir(ap, attrname);
	return (error);
	}

	/*
	* Vnode operation to retrieve a named extended attribute.
	*/
	static int
	zfs_getextattr(struct vop_getextattr_args *ap)
	{
	znode_t *zp = VTOZ(ap->a_vp);
	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	int error;

	/*
	* If the xattr property is off, refuse the request.
	*/
	if (!(zfsvfs->z_flags & ZSB_XATTR))
	return (SET_ERROR(EOPNOTSUPP));

	error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace,
	ap->a_cred, ap->a_td, VREAD);
	if (error != 0)
	return (SET_ERROR(error));

	error = zfs_check_attrname(ap->a_name);
	if (error != 0)
	return (error);

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);
	error = ENOENT;
	rw_enter(&zp->z_xattr_lock, RW_READER);

	error = zfs_getextattr_impl(ap, zfs_xattr_compat);
	if ((error == ENOENT \|\| error == ENOATTR) &&
	ap->a_attrnamespace == EXTATTR_NAMESPACE_USER) {
	/*
	* Fall back to the alternate namespace format if we failed to
	* find a user xattr.
	*/
	error = zfs_getextattr_impl(ap, !zfs_xattr_compat);
	}

	rw_exit(&zp->z_xattr_lock);
	zfs_exit(zfsvfs, FTAG);
	if (error == ENOENT)
	error = SET_ERROR(ENOATTR);
	return (error);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_deleteextattr {
	IN struct vnode *a_vp;
	IN int a_attrnamespace;
	IN const char *a_name;
	IN struct ucred *a_cred;
	IN struct thread *a_td;
	};
	#endif

	static int
	zfs_deleteextattr_dir(struct vop_deleteextattr_args ap, const char attrname)
	{
	struct nameidata nd;
	vnode_t xvp = NULL, vp;
	int error;

	error = zfs_lookup(ap->a_vp, NULL, &xvp, NULL, 0, ap->a_cred,
	LOOKUP_XATTR, B_FALSE);
	if (error != 0)
	return (error);

	#if __FreeBSD_version < 1400043
	NDINIT_ATVP(&nd, DELETE, NOFOLLOW \| LOCKPARENT \| LOCKLEAF,
	UIO_SYSSPACE, attrname, xvp, ap->a_td);
	#else
	NDINIT_ATVP(&nd, DELETE, NOFOLLOW \| LOCKPARENT \| LOCKLEAF,
	UIO_SYSSPACE, attrname, xvp);
	#endif
	error = namei(&nd);
	if (error != 0)
	return (SET_ERROR(error));

	vp = nd.ni_vp;
	error = VOP_REMOVE(nd.ni_dvp, vp, &nd.ni_cnd);
	NDFREE_PNBUF(&nd);

	vput(nd.ni_dvp);
	if (vp == nd.ni_dvp)
	vrele(vp);
	else
	vput(vp);

	return (error);
	}

	static int
	zfs_deleteextattr_sa(struct vop_deleteextattr_args ap, const char attrname)
	{
	znode_t *zp = VTOZ(ap->a_vp);
	nvlist_t *nvl;
	int error;

	error = zfs_ensure_xattr_cached(zp);
	if (error != 0)
	return (error);

	ASSERT(RW_WRITE_HELD(&zp->z_xattr_lock));
	ASSERT3P(zp->z_xattr_cached, !=, NULL);

	nvl = zp->z_xattr_cached;
	error = nvlist_remove(nvl, attrname, DATA_TYPE_BYTE_ARRAY);
	if (error != 0)
	error = SET_ERROR(error);
	else
	error = zfs_sa_set_xattr(zp, attrname, NULL, 0);
	if (error != 0) {
	zp->z_xattr_cached = NULL;
	nvlist_free(nvl);
	}
	return (error);
	}

	static int
	zfs_deleteextattr_impl(struct vop_deleteextattr_args *ap, boolean_t compat)
	{
	znode_t *zp = VTOZ(ap->a_vp);
	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	char attrname[EXTATTR_MAXNAMELEN+1];
	int error;

	error = zfs_create_attrname(ap->a_attrnamespace, ap->a_name, attrname,
	sizeof (attrname), compat);
	if (error != 0)
	return (error);

	error = ENOENT;
	if (zfsvfs->z_use_sa && zp->z_is_sa)
	error = zfs_deleteextattr_sa(ap, attrname);
	if (error == ENOENT)
	error = zfs_deleteextattr_dir(ap, attrname);
	return (error);
	}

	/*
	* Vnode operation to remove a named attribute.
	*/
	static int
	zfs_deleteextattr(struct vop_deleteextattr_args *ap)
	{
	znode_t *zp = VTOZ(ap->a_vp);
	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	int error;

	/*
	* If the xattr property is off, refuse the request.
	*/
	if (!(zfsvfs->z_flags & ZSB_XATTR))
	return (SET_ERROR(EOPNOTSUPP));

	error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace,
	ap->a_cred, ap->a_td, VWRITE);
	if (error != 0)
	return (SET_ERROR(error));

	error = zfs_check_attrname(ap->a_name);
	if (error != 0)
	return (error);

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);
	rw_enter(&zp->z_xattr_lock, RW_WRITER);

	error = zfs_deleteextattr_impl(ap, zfs_xattr_compat);
	if ((error == ENOENT \|\| error == ENOATTR) &&
	ap->a_attrnamespace == EXTATTR_NAMESPACE_USER) {
	/*
	* Fall back to the alternate namespace format if we failed to
	* find a user xattr.
	*/
	error = zfs_deleteextattr_impl(ap, !zfs_xattr_compat);
	}

	rw_exit(&zp->z_xattr_lock);
	zfs_exit(zfsvfs, FTAG);
	if (error == ENOENT)
	error = SET_ERROR(ENOATTR);
	return (error);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_setextattr {
	IN struct vnode *a_vp;
	IN int a_attrnamespace;
	IN const char *a_name;
	INOUT struct uio *a_uio;
	IN struct ucred *a_cred;
	IN struct thread *a_td;
	};
	#endif

	static int
	zfs_setextattr_dir(struct vop_setextattr_args ap, const char attrname)
	{
	struct thread *td = ap->a_td;
	struct nameidata nd;
	struct vattr va;
	vnode_t xvp = NULL, vp;
	int error, flags;

	error = zfs_lookup(ap->a_vp, NULL, &xvp, NULL, 0, ap->a_cred,
	LOOKUP_XATTR \| CREATE_XATTR_DIR, B_FALSE);
	if (error != 0)
	return (error);

	flags = FFLAGS(O_WRONLY \| O_CREAT);
	#if __FreeBSD_version < 1400043
	NDINIT_ATVP(&nd, LOOKUP, NOFOLLOW, UIO_SYSSPACE, attrname, xvp, td);
	#else
	NDINIT_ATVP(&nd, LOOKUP, NOFOLLOW, UIO_SYSSPACE, attrname, xvp);
	#endif
	error = vn_open_cred(&nd, &flags, 0600, VN_OPEN_INVFS, ap->a_cred,
	NULL);
	if (error != 0)
	return (SET_ERROR(error));
	vp = nd.ni_vp;
	NDFREE_PNBUF(&nd);

	VATTR_NULL(&va);
	va.va_size = 0;
	error = VOP_SETATTR(vp, &va, ap->a_cred);
	if (error == 0)
	VOP_WRITE(vp, ap->a_uio, IO_UNIT, ap->a_cred);

	VOP_UNLOCK1(vp);
	vn_close(vp, flags, ap->a_cred, td);
	return (error);
	}

	static int
	zfs_setextattr_sa(struct vop_setextattr_args ap, const char attrname)
	{
	znode_t *zp = VTOZ(ap->a_vp);
	nvlist_t *nvl;
	size_t sa_size;
	int error;

	error = zfs_ensure_xattr_cached(zp);
	if (error != 0)
	return (error);

	ASSERT(RW_WRITE_HELD(&zp->z_xattr_lock));
	ASSERT3P(zp->z_xattr_cached, !=, NULL);

	nvl = zp->z_xattr_cached;
	size_t entry_size = ap->a_uio->uio_resid;
	if (entry_size > DXATTR_MAX_ENTRY_SIZE)
	return (SET_ERROR(EFBIG));
	error = nvlist_size(nvl, &sa_size, NV_ENCODE_XDR);
	if (error != 0)
	return (SET_ERROR(error));
	if (sa_size > DXATTR_MAX_SA_SIZE)
	return (SET_ERROR(EFBIG));
	uchar_t *buf = kmem_alloc(entry_size, KM_SLEEP);
	error = uiomove(buf, entry_size, ap->a_uio);
	if (error != 0) {
	error = SET_ERROR(error);
	} else {
	error = nvlist_add_byte_array(nvl, attrname, buf, entry_size);
	if (error != 0)
	error = SET_ERROR(error);
	}
	if (error == 0)
	error = zfs_sa_set_xattr(zp, attrname, buf, entry_size);
	kmem_free(buf, entry_size);
	if (error != 0) {
	zp->z_xattr_cached = NULL;
	nvlist_free(nvl);
	}
	return (error);
	}

	static int
	zfs_setextattr_impl(struct vop_setextattr_args *ap, boolean_t compat)
	{
	znode_t *zp = VTOZ(ap->a_vp);
	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	char attrname[EXTATTR_MAXNAMELEN+1];
	int error;

	error = zfs_create_attrname(ap->a_attrnamespace, ap->a_name, attrname,
	sizeof (attrname), compat);
	if (error != 0)
	return (error);

	struct vop_deleteextattr_args vda = {
	.a_vp = ap->a_vp,
	.a_attrnamespace = ap->a_attrnamespace,
	.a_name = ap->a_name,
	.a_cred = ap->a_cred,
	.a_td = ap->a_td,
	};
	error = ENOENT;
	if (zfsvfs->z_use_sa && zp->z_is_sa && zfsvfs->z_xattr_sa) {
	error = zfs_setextattr_sa(ap, attrname);
	if (error == 0) {
	/*
	* Successfully put into SA, we need to clear the one
	* in dir if present.
	*/
	zfs_deleteextattr_dir(&vda, attrname);
	}
	}
	if (error != 0) {
	error = zfs_setextattr_dir(ap, attrname);
	if (error == 0 && zp->z_is_sa) {
	/*
	* Successfully put into dir, we need to clear the one
	* in SA if present.
	*/
	zfs_deleteextattr_sa(&vda, attrname);
	}
	}
	if (error == 0 && ap->a_attrnamespace == EXTATTR_NAMESPACE_USER) {
	/*
	* Also clear all versions of the alternate compat name.
	*/
	zfs_deleteextattr_impl(&vda, !compat);
	}
	return (error);
	}

	/*
	* Vnode operation to set a named attribute.
	*/
	static int
	zfs_setextattr(struct vop_setextattr_args *ap)
	{
	znode_t *zp = VTOZ(ap->a_vp);
	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	int error;

	/*
	* If the xattr property is off, refuse the request.
	*/
	if (!(zfsvfs->z_flags & ZSB_XATTR))
	return (SET_ERROR(EOPNOTSUPP));

	error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace,
	ap->a_cred, ap->a_td, VWRITE);
	if (error != 0)
	return (SET_ERROR(error));

	error = zfs_check_attrname(ap->a_name);
	if (error != 0)
	return (error);

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);
	rw_enter(&zp->z_xattr_lock, RW_WRITER);

	error = zfs_setextattr_impl(ap, zfs_xattr_compat);

	rw_exit(&zp->z_xattr_lock);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_listextattr {
	IN struct vnode *a_vp;
	IN int a_attrnamespace;
	INOUT struct uio *a_uio;
	OUT size_t *a_size;
	IN struct ucred *a_cred;
	IN struct thread *a_td;
	};
	#endif

	static int
	zfs_listextattr_dir(struct vop_listextattr_args ap, const char attrprefix)
	{
	struct thread *td = ap->a_td;
	struct nameidata nd;
	uint8_t dirbuf[sizeof (struct dirent)];
	struct iovec aiov;
	struct uio auio;
	vnode_t xvp = NULL, vp;
	int error, eof;

	error = zfs_lookup(ap->a_vp, NULL, &xvp, NULL, 0, ap->a_cred,
	LOOKUP_XATTR, B_FALSE);
	if (error != 0) {
	/*
	* ENOATTR means that the EA directory does not yet exist,
	* i.e. there are no extended attributes there.
	*/
	if (error == ENOATTR)
	error = 0;
	return (error);
	}

	#if __FreeBSD_version < 1400043
	NDINIT_ATVP(&nd, LOOKUP, NOFOLLOW \| LOCKLEAF \| LOCKSHARED,
	UIO_SYSSPACE, ".", xvp, td);
	#else
	NDINIT_ATVP(&nd, LOOKUP, NOFOLLOW \| LOCKLEAF \| LOCKSHARED,
	UIO_SYSSPACE, ".", xvp);
	#endif
	error = namei(&nd);
	if (error != 0)
	return (SET_ERROR(error));
	vp = nd.ni_vp;
	NDFREE_PNBUF(&nd);

	auio.uio_iov = &aiov;
	auio.uio_iovcnt = 1;
	auio.uio_segflg = UIO_SYSSPACE;
	auio.uio_td = td;
	auio.uio_rw = UIO_READ;
	auio.uio_offset = 0;

	size_t plen = strlen(attrprefix);

	do {
	aiov.iov_base = (void *)dirbuf;
	aiov.iov_len = sizeof (dirbuf);
	auio.uio_resid = sizeof (dirbuf);
	error = VOP_READDIR(vp, &auio, ap->a_cred, &eof, NULL, NULL);
	if (error != 0)
	break;
	int done = sizeof (dirbuf) - auio.uio_resid;
	for (int pos = 0; pos < done; ) {
	struct dirent dp = (struct dirent )(dirbuf + pos);
	pos += dp->d_reclen;
	/*
	* XXX: Temporarily we also accept DT_UNKNOWN, as this
	* is what we get when attribute was created on Solaris.
	*/
	if (dp->d_type != DT_REG && dp->d_type != DT_UNKNOWN)
	continue;
	else if (plen == 0 &&
	ZFS_XA_NS_PREFIX_FORBIDDEN(dp->d_name))
	continue;
	else if (strncmp(dp->d_name, attrprefix, plen) != 0)
	continue;
	uint8_t nlen = dp->d_namlen - plen;
	if (ap->a_size != NULL) {
	*ap->a_size += 1 + nlen;
	} else if (ap->a_uio != NULL) {
	/*
	* Format of extattr name entry is one byte for
	* length and the rest for name.
	*/
	error = uiomove(&nlen, 1, ap->a_uio);
	if (error == 0) {
	char *namep = dp->d_name + plen;
	error = uiomove(namep, nlen, ap->a_uio);
	}
	if (error != 0) {
	error = SET_ERROR(error);
	break;
	}
	}
	}
	} while (!eof && error == 0);

	vput(vp);
	return (error);
	}

	static int
	zfs_listextattr_sa(struct vop_listextattr_args ap, const char attrprefix)
	{
	znode_t *zp = VTOZ(ap->a_vp);
	int error;

	error = zfs_ensure_xattr_cached(zp);
	if (error != 0)
	return (error);

	ASSERT(RW_LOCK_HELD(&zp->z_xattr_lock));
	ASSERT3P(zp->z_xattr_cached, !=, NULL);

	size_t plen = strlen(attrprefix);
	nvpair_t *nvp = NULL;
	while ((nvp = nvlist_next_nvpair(zp->z_xattr_cached, nvp)) != NULL) {
	ASSERT3U(nvpair_type(nvp), ==, DATA_TYPE_BYTE_ARRAY);

	const char *name = nvpair_name(nvp);
	if (plen == 0 && ZFS_XA_NS_PREFIX_FORBIDDEN(name))
	continue;
	else if (strncmp(name, attrprefix, plen) != 0)
	continue;
	uint8_t nlen = strlen(name) - plen;
	if (ap->a_size != NULL) {
	*ap->a_size += 1 + nlen;
	} else if (ap->a_uio != NULL) {
	/*
	* Format of extattr name entry is one byte for
	* length and the rest for name.
	*/
	error = uiomove(&nlen, 1, ap->a_uio);
	if (error == 0) {
	char namep = __DECONST(char , name) + plen;
	error = uiomove(namep, nlen, ap->a_uio);
	}
	if (error != 0) {
	error = SET_ERROR(error);
	break;
	}
	}
	}

	return (error);
	}

	static int
	zfs_listextattr_impl(struct vop_listextattr_args *ap, boolean_t compat)
	{
	znode_t *zp = VTOZ(ap->a_vp);
	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	char attrprefix[16];
	int error;

	error = zfs_create_attrname(ap->a_attrnamespace, "", attrprefix,
	sizeof (attrprefix), compat);
	if (error != 0)
	return (error);

	if (zfsvfs->z_use_sa && zp->z_is_sa)
	error = zfs_listextattr_sa(ap, attrprefix);
	if (error == 0)
	error = zfs_listextattr_dir(ap, attrprefix);
	return (error);
	}

	/*
	* Vnode operation to retrieve extended attributes on a vnode.
	*/
	static int
	zfs_listextattr(struct vop_listextattr_args *ap)
	{
	znode_t *zp = VTOZ(ap->a_vp);
	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	int error;

	if (ap->a_size != NULL)
	*ap->a_size = 0;

	/*
	* If the xattr property is off, refuse the request.
	*/
	if (!(zfsvfs->z_flags & ZSB_XATTR))
	return (SET_ERROR(EOPNOTSUPP));

	error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace,
	ap->a_cred, ap->a_td, VREAD);
	if (error != 0)
	return (SET_ERROR(error));

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);
	rw_enter(&zp->z_xattr_lock, RW_READER);

	error = zfs_listextattr_impl(ap, zfs_xattr_compat);
	if (error == 0 && ap->a_attrnamespace == EXTATTR_NAMESPACE_USER) {
	/* Also list user xattrs with the alternate format. */
	error = zfs_listextattr_impl(ap, !zfs_xattr_compat);
	}

	rw_exit(&zp->z_xattr_lock);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_getacl_args {
	struct vnode *vp;
	acl_type_t type;
	struct acl *aclp;
	struct ucred *cred;
	struct thread *td;
	};
	#endif

	static int
	zfs_freebsd_getacl(struct vop_getacl_args *ap)
	{
	int error;
	vsecattr_t vsecattr;

	if (ap->a_type != ACL_TYPE_NFS4)
	return (EINVAL);

	vsecattr.vsa_mask = VSA_ACE \| VSA_ACECNT;
	if ((error = zfs_getsecattr(VTOZ(ap->a_vp),
	&vsecattr, 0, ap->a_cred)))
	return (error);

	error = acl_from_aces(ap->a_aclp, vsecattr.vsa_aclentp,
	vsecattr.vsa_aclcnt);
	if (vsecattr.vsa_aclentp != NULL)
	kmem_free(vsecattr.vsa_aclentp, vsecattr.vsa_aclentsz);

	return (error);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_setacl_args {
	struct vnode *vp;
	acl_type_t type;
	struct acl *aclp;
	struct ucred *cred;
	struct thread *td;
	};
	#endif

	static int
	zfs_freebsd_setacl(struct vop_setacl_args *ap)
	{
	int error;
	vsecattr_t vsecattr;
	int aclbsize; /* size of acl list in bytes */
	aclent_t *aaclp;

	if (ap->a_type != ACL_TYPE_NFS4)
	return (EINVAL);

	if (ap->a_aclp == NULL)
	return (EINVAL);

	if (ap->a_aclp->acl_cnt < 1 \|\| ap->a_aclp->acl_cnt > MAX_ACL_ENTRIES)
	return (EINVAL);

	/*
	* With NFSv4 ACLs, chmod(2) may need to add additional entries,
	* splitting every entry into two and appending "canonical six"
	* entries at the end. Don't allow for setting an ACL that would
	* cause chmod(2) to run out of ACL entries.
	*/
	if (ap->a_aclp->acl_cnt * 2 + 6 > ACL_MAX_ENTRIES)
	return (ENOSPC);

	error = acl_nfs4_check(ap->a_aclp, ap->a_vp->v_type == VDIR);
	if (error != 0)
	return (error);

	vsecattr.vsa_mask = VSA_ACE;
	aclbsize = ap->a_aclp->acl_cnt * sizeof (ace_t);
	vsecattr.vsa_aclentp = kmem_alloc(aclbsize, KM_SLEEP);
	aaclp = vsecattr.vsa_aclentp;
	vsecattr.vsa_aclentsz = aclbsize;

	aces_from_acl(vsecattr.vsa_aclentp, &vsecattr.vsa_aclcnt, ap->a_aclp);
	error = zfs_setsecattr(VTOZ(ap->a_vp), &vsecattr, 0, ap->a_cred);
	kmem_free(aaclp, aclbsize);

	return (error);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct vop_aclcheck_args {
	struct vnode *vp;
	acl_type_t type;
	struct acl *aclp;
	struct ucred *cred;
	struct thread *td;
	};
	#endif

	static int
	zfs_freebsd_aclcheck(struct vop_aclcheck_args *ap)
	{

	return (EOPNOTSUPP);
	}

	static int
	zfs_vptocnp(struct vop_vptocnp_args *ap)
	{
	vnode_t *covered_vp;
	vnode_t *vp = ap->a_vp;
	zfsvfs_t *zfsvfs = vp->v_vfsp->vfs_data;
	znode_t *zp = VTOZ(vp);
	int ltype;
	int error;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	/*
	* If we are a snapshot mounted under .zfs, run the operation
	* on the covered vnode.
	*/
	if (zp->z_id != zfsvfs->z_root \|\| zfsvfs->z_parent == zfsvfs) {
	char name[MAXNAMLEN + 1];
	znode_t *dzp;
	size_t len;

	error = zfs_znode_parent_and_name(zp, &dzp, name);
	if (error == 0) {
	len = strlen(name);
	if (*ap->a_buflen < len)
	error = SET_ERROR(ENOMEM);
	}
	if (error == 0) {
	*ap->a_buflen -= len;
	memcpy(ap->a_buf + *ap->a_buflen, name, len);
	*ap->a_vpp = ZTOV(dzp);
	}
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	zfs_exit(zfsvfs, FTAG);

	covered_vp = vp->v_mount->mnt_vnodecovered;
	#if __FreeBSD_version >= 1300045
	enum vgetstate vs = vget_prep(covered_vp);
	#else
	vhold(covered_vp);
	#endif
	ltype = VOP_ISLOCKED(vp);
	VOP_UNLOCK1(vp);
	#if __FreeBSD_version >= 1300045
	error = vget_finish(covered_vp, LK_SHARED, vs);
	#else
	error = vget(covered_vp, LK_SHARED \| LK_VNHELD, curthread);
	#endif
	if (error == 0) {
	#if __FreeBSD_version >= 1300123
	error = VOP_VPTOCNP(covered_vp, ap->a_vpp, ap->a_buf,
	ap->a_buflen);
	#else
	error = VOP_VPTOCNP(covered_vp, ap->a_vpp, ap->a_cred,
	ap->a_buf, ap->a_buflen);
	#endif
	vput(covered_vp);
	}
	vn_lock(vp, ltype \| LK_RETRY);
	if (VN_IS_DOOMED(vp))
	error = SET_ERROR(ENOENT);
	return (error);
	}

	#if __FreeBSD_version >= 1400032
	static int
	zfs_deallocate(struct vop_deallocate_args *ap)
	{
	znode_t *zp = VTOZ(ap->a_vp);
	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
	zilog_t *zilog;
	off_t off, len, file_sz;
	int error;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	/*
	* Callers might not be able to detect properly that we are read-only,
	* so check it explicitly here.
	*/
	if (zfs_is_readonly(zfsvfs)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EROFS));
	}

	zilog = zfsvfs->z_log;
	off = *ap->a_offset;
	len = *ap->a_len;
	file_sz = zp->z_size;
	if (off + len > file_sz)
	len = file_sz - off;
	/* Fast path for out-of-range request. */
	if (len <= 0) {
	*ap->a_len = 0;
	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	error = zfs_freesp(zp, off, len, O_RDWR, TRUE);
	if (error == 0) {
	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS \|\|
	(ap->a_ioflag & IO_SYNC) != 0)
	zil_commit(zilog, zp->z_id);
	*ap->a_offset = off + len;
	*ap->a_len = 0;
	}

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	#endif

	#if __FreeBSD_version >= 1300039
	#ifndef _SYS_SYSPROTO_H_
	struct vop_copy_file_range_args {
	struct vnode *a_invp;
	off_t *a_inoffp;
	struct vnode *a_outvp;
	off_t *a_outoffp;
	size_t *a_lenp;
	unsigned int a_flags;
	struct ucred *a_incred;
	struct ucred *a_outcred;
	struct thread *a_fsizetd;
	}
	#endif
	/*
	* TODO: FreeBSD will only call file system-specific copy_file_range() if both
	* files resides under the same mountpoint. In case of ZFS we want to be called
	* even is files are in different datasets (but on the same pools, but we need
	* to check that ourselves).
	*/
	static int
	zfs_freebsd_copy_file_range(struct vop_copy_file_range_args *ap)
	{
	zfsvfs_t *outzfsvfs;
	struct vnode *invp = ap->a_invp;
	struct vnode *outvp = ap->a_outvp;
	struct mount *mp;
	struct uio io;
	int error;
	uint64_t len = *ap->a_lenp;

	if (!zfs_bclone_enabled) {
	mp = NULL;
	goto bad_write_fallback;
	}

	/*
	* TODO: If offset/length is not aligned to recordsize, use
	* vn_generic_copy_file_range() on this fragment.
	* It would be better to do this after we lock the vnodes, but then we
	* need something else than vn_generic_copy_file_range().
	*/

	vn_start_write(outvp, &mp, V_WAIT);
	if (__predict_true(mp == outvp->v_mount)) {
	outzfsvfs = (zfsvfs_t *)mp->mnt_data;
	if (!spa_feature_is_enabled(dmu_objset_spa(outzfsvfs->z_os),
	SPA_FEATURE_BLOCK_CLONING)) {
	goto bad_write_fallback;
	}
	}
	if (invp == outvp) {
	if (vn_lock(outvp, LK_EXCLUSIVE) != 0) {
	goto bad_write_fallback;
	}
	} else {
	#if (__FreeBSD_version >= 1302506 && __FreeBSD_version < 1400000) \|\| \
	__FreeBSD_version >= 1400086
	vn_lock_pair(invp, false, LK_EXCLUSIVE, outvp, false,
	LK_EXCLUSIVE);
	#else
	vn_lock_pair(invp, false, outvp, false);
	#endif
	if (VN_IS_DOOMED(invp) \|\| VN_IS_DOOMED(outvp)) {
	goto bad_locked_fallback;
	}
	}

	#ifdef MAC
	error = mac_vnode_check_write(curthread->td_ucred, ap->a_outcred,
	outvp);
	if (error != 0)
	goto out_locked;
	#endif

	io.uio_offset = *ap->a_outoffp;
	io.uio_resid = *ap->a_lenp;
	error = vn_rlimit_fsize(outvp, &io, ap->a_fsizetd);
	if (error != 0)
	goto out_locked;

	error = zfs_clone_range(VTOZ(invp), ap->a_inoffp, VTOZ(outvp),
	ap->a_outoffp, &len, ap->a_outcred);
	if (error == EXDEV \|\| error == EAGAIN \|\| error == EINVAL \|\|
	error == EOPNOTSUPP)
	goto bad_locked_fallback;
	*ap->a_lenp = (size_t)len;
	out_locked:
	if (invp != outvp)
	VOP_UNLOCK(invp);
	VOP_UNLOCK(outvp);
	if (mp != NULL)
	vn_finished_write(mp);
	return (error);

	bad_locked_fallback:
	if (invp != outvp)
	VOP_UNLOCK(invp);
	VOP_UNLOCK(outvp);
	bad_write_fallback:
	if (mp != NULL)
	vn_finished_write(mp);
	error = ENOSYS;
	return (error);
	}
	#endif

	struct vop_vector zfs_vnodeops;
	struct vop_vector zfs_fifoops;
	struct vop_vector zfs_shareops;

	struct vop_vector zfs_vnodeops = {
	.vop_default = &default_vnodeops,
	.vop_inactive = zfs_freebsd_inactive,
	#if __FreeBSD_version >= 1300042
	.vop_need_inactive = zfs_freebsd_need_inactive,
	#endif
	.vop_reclaim = zfs_freebsd_reclaim,
	#if __FreeBSD_version >= 1300102
	.vop_fplookup_vexec = zfs_freebsd_fplookup_vexec,
	#endif
	#if __FreeBSD_version >= 1300139
	.vop_fplookup_symlink = zfs_freebsd_fplookup_symlink,
	#endif
	.vop_access = zfs_freebsd_access,
	.vop_allocate = VOP_EINVAL,
	#if __FreeBSD_version >= 1400032
	.vop_deallocate = zfs_deallocate,
	#endif
	.vop_lookup = zfs_cache_lookup,
	.vop_cachedlookup = zfs_freebsd_cachedlookup,
	.vop_getattr = zfs_freebsd_getattr,
	.vop_setattr = zfs_freebsd_setattr,
	.vop_create = zfs_freebsd_create,
	.vop_mknod = (vop_mknod_t *)zfs_freebsd_create,
	.vop_mkdir = zfs_freebsd_mkdir,
	.vop_readdir = zfs_freebsd_readdir,
	.vop_fsync = zfs_freebsd_fsync,
	.vop_open = zfs_freebsd_open,
	.vop_close = zfs_freebsd_close,
	.vop_rmdir = zfs_freebsd_rmdir,
	.vop_ioctl = zfs_freebsd_ioctl,
	.vop_link = zfs_freebsd_link,
	.vop_symlink = zfs_freebsd_symlink,
	.vop_readlink = zfs_freebsd_readlink,
	.vop_read = zfs_freebsd_read,
	.vop_write = zfs_freebsd_write,
	.vop_remove = zfs_freebsd_remove,
	.vop_rename = zfs_freebsd_rename,
	.vop_pathconf = zfs_freebsd_pathconf,
	.vop_bmap = zfs_freebsd_bmap,
	.vop_fid = zfs_freebsd_fid,
	.vop_getextattr = zfs_getextattr,
	.vop_deleteextattr = zfs_deleteextattr,
	.vop_setextattr = zfs_setextattr,
	.vop_listextattr = zfs_listextattr,
	.vop_getacl = zfs_freebsd_getacl,
	.vop_setacl = zfs_freebsd_setacl,
	.vop_aclcheck = zfs_freebsd_aclcheck,
	.vop_getpages = zfs_freebsd_getpages,
	.vop_putpages = zfs_freebsd_putpages,
	.vop_vptocnp = zfs_vptocnp,
	#if __FreeBSD_version >= 1300064
	.vop_lock1 = vop_lock,
	.vop_unlock = vop_unlock,
	.vop_islocked = vop_islocked,
	#endif
	#if __FreeBSD_version >= 1400043
	.vop_add_writecount = vop_stdadd_writecount_nomsync,
	#endif
	#if __FreeBSD_version >= 1300039
	.vop_copy_file_range = zfs_freebsd_copy_file_range,
	#endif
	};
	VFS_VOP_VECTOR_REGISTER(zfs_vnodeops);

	struct vop_vector zfs_fifoops = {
	.vop_default = &fifo_specops,
	.vop_fsync = zfs_freebsd_fsync,
	#if __FreeBSD_version >= 1300102
	.vop_fplookup_vexec = zfs_freebsd_fplookup_vexec,
	#endif
	#if __FreeBSD_version >= 1300139
	.vop_fplookup_symlink = zfs_freebsd_fplookup_symlink,
	#endif
	.vop_access = zfs_freebsd_access,
	.vop_getattr = zfs_freebsd_getattr,
	.vop_inactive = zfs_freebsd_inactive,
	.vop_read = VOP_PANIC,
	.vop_reclaim = zfs_freebsd_reclaim,
	.vop_setattr = zfs_freebsd_setattr,
	.vop_write = VOP_PANIC,
	.vop_pathconf = zfs_freebsd_pathconf,
	.vop_fid = zfs_freebsd_fid,
	.vop_getacl = zfs_freebsd_getacl,
	.vop_setacl = zfs_freebsd_setacl,
	.vop_aclcheck = zfs_freebsd_aclcheck,
	#if __FreeBSD_version >= 1400043
	.vop_add_writecount = vop_stdadd_writecount_nomsync,
	#endif
	};
	VFS_VOP_VECTOR_REGISTER(zfs_fifoops);

	/*
	* special share hidden files vnode operations template
	*/
	struct vop_vector zfs_shareops = {
	.vop_default = &default_vnodeops,
	#if __FreeBSD_version >= 1300121
	.vop_fplookup_vexec = VOP_EAGAIN,
	#endif
	#if __FreeBSD_version >= 1300139
	.vop_fplookup_symlink = VOP_EAGAIN,
	#endif
	.vop_access = zfs_freebsd_access,
	.vop_inactive = zfs_freebsd_inactive,
	.vop_reclaim = zfs_freebsd_reclaim,
	.vop_fid = zfs_freebsd_fid,
	.vop_pathconf = zfs_freebsd_pathconf,
	#if __FreeBSD_version >= 1400043
	.vop_add_writecount = vop_stdadd_writecount_nomsync,
	#endif
	};
	VFS_VOP_VECTOR_REGISTER(zfs_shareops);

	ZFS_MODULE_PARAM(zfs, zfs_, xattr_compat, INT, ZMOD_RW,
	"Use legacy ZFS xattr naming for writing new user namespace xattrs");
	diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zvol_os.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zvol_os.c
	index b6edac434dea..7c418f26fc14 100644
	--- a/sys/contrib/openzfs/module/os/freebsd/zfs/zvol_os.c
	+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zvol_os.c
	@@ -1,1627 +1,1627 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	*
	* Copyright (c) 2006-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
	* All rights reserved.
	*
	* Portions Copyright 2010 Robert Milkowski
	*
	* Copyright 2011 Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2012, 2017 by Delphix. All rights reserved.
	* Copyright (c) 2013, Joyent, Inc. All rights reserved.
	* Copyright (c) 2014 Integros [integros.com]
	*/

	/* Portions Copyright 2011 Martin Matuska <mm@FreeBSD.org> */

	/*
	* ZFS volume emulation driver.
	*
	* Makes a DMU object look like a volume of arbitrary size, up to 2^64 bytes.
	* Volumes are accessed through the symbolic links named:
	*
	* /dev/zvol/<pool_name>/<dataset_name>
	*
	* Volumes are persistent through reboot. No user command needs to be
	* run before opening and using a device.
	*
	* On FreeBSD ZVOLs are simply GEOM providers like any other storage device
	* in the system. Except when they're simply character devices (volmode=dev).
	*/

	#include <sys/types.h>
	#include <sys/param.h>
	#include <sys/kernel.h>
	#include <sys/errno.h>
	#include <sys/uio.h>
	#include <sys/bio.h>
	#include <sys/buf.h>
	#include <sys/kmem.h>
	#include <sys/conf.h>
	#include <sys/cmn_err.h>
	#include <sys/stat.h>
	#include <sys/proc.h>
	#include <sys/zap.h>
	#include <sys/spa.h>
	#include <sys/spa_impl.h>
	#include <sys/zio.h>
	#include <sys/disk.h>
	#include <sys/dmu_traverse.h>
	#include <sys/dnode.h>
	#include <sys/dsl_dataset.h>
	#include <sys/dsl_prop.h>
	#include <sys/dsl_dir.h>
	#include <sys/byteorder.h>
	#include <sys/sunddi.h>
	#include <sys/dirent.h>
	#include <sys/policy.h>
	#include <sys/queue.h>
	#include <sys/fs/zfs.h>
	#include <sys/zfs_ioctl.h>
	#include <sys/zil.h>
	#include <sys/zfs_znode.h>
	#include <sys/zfs_rlock.h>
	#include <sys/vdev_impl.h>
	#include <sys/vdev_raidz.h>
	#include <sys/zvol.h>
	#include <sys/zil_impl.h>
	#include <sys/dataset_kstats.h>
	#include <sys/dbuf.h>
	#include <sys/dmu_tx.h>
	#include <sys/zfeature.h>
	#include <sys/zio_checksum.h>
	#include <sys/zil_impl.h>
	#include <sys/filio.h>
	#include <sys/freebsd_event.h>

	#include <geom/geom.h>
	#include <sys/zvol.h>
	#include <sys/zvol_impl.h>

	#include "zfs_namecheck.h"

	#define ZVOL_DUMPSIZE "dumpsize"

	#ifdef ZVOL_LOCK_DEBUG
	#define ZVOL_RW_READER RW_WRITER
	#define ZVOL_RW_READ_HELD RW_WRITE_HELD
	#else
	#define ZVOL_RW_READER RW_READER
	#define ZVOL_RW_READ_HELD RW_READ_HELD
	#endif

	enum zvol_geom_state {
	ZVOL_GEOM_UNINIT,
	ZVOL_GEOM_STOPPED,
	ZVOL_GEOM_RUNNING,
	};

	struct zvol_state_os {
	#define zso_dev _zso_state._zso_dev
	#define zso_geom _zso_state._zso_geom
	union {
	/* volmode=dev */
	struct zvol_state_dev {
	struct cdev *zsd_cdev;
	uint64_t zsd_sync_cnt;
	struct selinfo zsd_selinfo;
	} _zso_dev;

	/* volmode=geom */
	struct zvol_state_geom {
	struct g_provider *zsg_provider;
	struct bio_queue_head zsg_queue;
	struct mtx zsg_queue_mtx;
	enum zvol_geom_state zsg_state;
	} _zso_geom;
	} _zso_state;
	int zso_dying;
	};

	static uint32_t zvol_minors;

	SYSCTL_DECL(_vfs_zfs);
	SYSCTL_NODE(_vfs_zfs, OID_AUTO, vol, CTLFLAG_RW, 0, "ZFS VOLUME");
	SYSCTL_INT(_vfs_zfs_vol, OID_AUTO, mode, CTLFLAG_RWTUN, &zvol_volmode, 0,
	"Expose as GEOM providers (1), device files (2) or neither");
	static boolean_t zpool_on_zvol = B_FALSE;
	SYSCTL_INT(_vfs_zfs_vol, OID_AUTO, recursive, CTLFLAG_RWTUN, &zpool_on_zvol, 0,
	"Allow zpools to use zvols as vdevs (DANGEROUS)");

	/*
	* Toggle unmap functionality.
	*/
	boolean_t zvol_unmap_enabled = B_TRUE;

	SYSCTL_INT(_vfs_zfs_vol, OID_AUTO, unmap_enabled, CTLFLAG_RWTUN,
	&zvol_unmap_enabled, 0, "Enable UNMAP functionality");

	/*
	* zvol maximum transfer in one DMU tx.
	*/
	int zvol_maxphys = DMU_MAX_ACCESS / 2;

	static void zvol_ensure_zilog(zvol_state_t *zv);

	static d_open_t zvol_cdev_open;
	static d_close_t zvol_cdev_close;
	static d_ioctl_t zvol_cdev_ioctl;
	static d_read_t zvol_cdev_read;
	static d_write_t zvol_cdev_write;
	static d_strategy_t zvol_geom_bio_strategy;
	static d_kqfilter_t zvol_cdev_kqfilter;

	static struct cdevsw zvol_cdevsw = {
	.d_name = "zvol",
	.d_version = D_VERSION,
	.d_flags = D_DISK \| D_TRACKCLOSE,
	.d_open = zvol_cdev_open,
	.d_close = zvol_cdev_close,
	.d_ioctl = zvol_cdev_ioctl,
	.d_read = zvol_cdev_read,
	.d_write = zvol_cdev_write,
	.d_strategy = zvol_geom_bio_strategy,
	.d_kqfilter = zvol_cdev_kqfilter,
	};

	static void zvol_filter_detach(struct knote *kn);
	static int zvol_filter_vnode(struct knote *kn, long hint);

	static struct filterops zvol_filterops_vnode = {
	.f_isfd = 1,
	.f_detach = zvol_filter_detach,
	.f_event = zvol_filter_vnode,
	};

	extern uint_t zfs_geom_probe_vdev_key;

	struct g_class zfs_zvol_class = {
	.name = "ZFS::ZVOL",
	.version = G_VERSION,
	};

	DECLARE_GEOM_CLASS(zfs_zvol_class, zfs_zvol);

	static int zvol_geom_open(struct g_provider *pp, int flag, int count);
	static int zvol_geom_close(struct g_provider *pp, int flag, int count);
	static void zvol_geom_run(zvol_state_t *zv);
	static void zvol_geom_destroy(zvol_state_t *zv);
	static int zvol_geom_access(struct g_provider *pp, int acr, int acw, int ace);
	static void zvol_geom_worker(void *arg);
	static void zvol_geom_bio_start(struct bio *bp);
	static int zvol_geom_bio_getattr(struct bio *bp);
	/* static d_strategy_t zvol_geom_bio_strategy; (declared elsewhere) */

	/*
	* GEOM mode implementation
	*/

	static int
	zvol_geom_open(struct g_provider *pp, int flag, int count)
	{
	zvol_state_t *zv;
	int err = 0;
	boolean_t drop_suspend = B_FALSE;

	if (!zpool_on_zvol && tsd_get(zfs_geom_probe_vdev_key) != NULL) {
	/*
	* If zfs_geom_probe_vdev_key is set, that means that zfs is
	* attempting to probe geom providers while looking for a
	* replacement for a missing VDEV. In this case, the
	* spa_namespace_lock will not be held, but it is still illegal
	* to use a zvol as a vdev. Deadlocks can result if another
	* thread has spa_namespace_lock.
	*/
	return (SET_ERROR(EOPNOTSUPP));
	}

	retry:
	rw_enter(&zvol_state_lock, ZVOL_RW_READER);
	/*
	* Obtain a copy of private under zvol_state_lock to make sure either
	* the result of zvol free code setting private to NULL is observed,
	* or the zv is protected from being freed because of the positive
	* zv_open_count.
	*/
	zv = pp->private;
	if (zv == NULL) {
	rw_exit(&zvol_state_lock);
	err = SET_ERROR(ENXIO);
	goto out_locked;
	}

	mutex_enter(&zv->zv_state_lock);
	if (zv->zv_zso->zso_dying) {
	rw_exit(&zvol_state_lock);
	err = SET_ERROR(ENXIO);
	goto out_zv_locked;
	}
	ASSERT3S(zv->zv_volmode, ==, ZFS_VOLMODE_GEOM);

	/*
	* Make sure zvol is not suspended during first open
	* (hold zv_suspend_lock) and respect proper lock acquisition
	* ordering - zv_suspend_lock before zv_state_lock.
	*/
	if (zv->zv_open_count == 0) {
	drop_suspend = B_TRUE;
	if (!rw_tryenter(&zv->zv_suspend_lock, ZVOL_RW_READER)) {
	mutex_exit(&zv->zv_state_lock);
	rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);
	mutex_enter(&zv->zv_state_lock);
	/* Check to see if zv_suspend_lock is needed. */
	if (zv->zv_open_count != 0) {
	rw_exit(&zv->zv_suspend_lock);
	drop_suspend = B_FALSE;
	}
	}
	}
	rw_exit(&zvol_state_lock);

	ASSERT(MUTEX_HELD(&zv->zv_state_lock));

	if (zv->zv_open_count == 0) {
	boolean_t drop_namespace = B_FALSE;

	ASSERT(ZVOL_RW_READ_HELD(&zv->zv_suspend_lock));

	/*
	* Take spa_namespace_lock to prevent lock inversion when
	* zvols from one pool are opened as vdevs in another.
	*/
	if (!mutex_owned(&spa_namespace_lock)) {
	if (!mutex_tryenter(&spa_namespace_lock)) {
	mutex_exit(&zv->zv_state_lock);
	rw_exit(&zv->zv_suspend_lock);
	kern_yield(PRI_USER);
	goto retry;
	} else {
	drop_namespace = B_TRUE;
	}
	}
	err = zvol_first_open(zv, !(flag & FWRITE));
	if (drop_namespace)
	mutex_exit(&spa_namespace_lock);
	if (err)
	goto out_zv_locked;
	pp->mediasize = zv->zv_volsize;
	pp->stripeoffset = 0;
	pp->stripesize = zv->zv_volblocksize;
	}

	ASSERT(MUTEX_HELD(&zv->zv_state_lock));

	/*
	* Check for a bad on-disk format version now since we
	* lied about owning the dataset readonly before.
	*/
	if ((flag & FWRITE) && ((zv->zv_flags & ZVOL_RDONLY) \|\|
	dmu_objset_incompatible_encryption_version(zv->zv_objset))) {
	err = SET_ERROR(EROFS);
	goto out_opened;
	}
	if (zv->zv_flags & ZVOL_EXCL) {
	err = SET_ERROR(EBUSY);
	goto out_opened;
	}
	if (flag & O_EXCL) {
	if (zv->zv_open_count != 0) {
	err = SET_ERROR(EBUSY);
	goto out_opened;
	}
	zv->zv_flags \|= ZVOL_EXCL;
	}

	zv->zv_open_count += count;
	out_opened:
	if (zv->zv_open_count == 0) {
	zvol_last_close(zv);
	wakeup(zv);
	}
	out_zv_locked:
	mutex_exit(&zv->zv_state_lock);
	out_locked:
	if (drop_suspend)
	rw_exit(&zv->zv_suspend_lock);
	return (err);
	}

	static int
	zvol_geom_close(struct g_provider *pp, int flag, int count)
	{
	(void) flag;
	zvol_state_t *zv;
	boolean_t drop_suspend = B_TRUE;
	int new_open_count;

	rw_enter(&zvol_state_lock, ZVOL_RW_READER);
	zv = pp->private;
	if (zv == NULL) {
	rw_exit(&zvol_state_lock);
	return (SET_ERROR(ENXIO));
	}

	mutex_enter(&zv->zv_state_lock);
	if (zv->zv_flags & ZVOL_EXCL) {
	ASSERT3U(zv->zv_open_count, ==, 1);
	zv->zv_flags &= ~ZVOL_EXCL;
	}

	ASSERT3S(zv->zv_volmode, ==, ZFS_VOLMODE_GEOM);

	/*
	* If the open count is zero, this is a spurious close.
	* That indicates a bug in the kernel / DDI framework.
	*/
	ASSERT3U(zv->zv_open_count, >, 0);

	/*
	* Make sure zvol is not suspended during last close
	* (hold zv_suspend_lock) and respect proper lock acquisition
	* ordering - zv_suspend_lock before zv_state_lock.
	*/
	new_open_count = zv->zv_open_count - count;
	if (new_open_count == 0) {
	if (!rw_tryenter(&zv->zv_suspend_lock, ZVOL_RW_READER)) {
	mutex_exit(&zv->zv_state_lock);
	rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);
	mutex_enter(&zv->zv_state_lock);
	/* Check to see if zv_suspend_lock is needed. */
	new_open_count = zv->zv_open_count - count;
	if (new_open_count != 0) {
	rw_exit(&zv->zv_suspend_lock);
	drop_suspend = B_FALSE;
	}
	}
	} else {
	drop_suspend = B_FALSE;
	}
	rw_exit(&zvol_state_lock);

	ASSERT(MUTEX_HELD(&zv->zv_state_lock));

	/*
	* You may get multiple opens, but only one close.
	*/
	zv->zv_open_count = new_open_count;
	if (zv->zv_open_count == 0) {
	ASSERT(ZVOL_RW_READ_HELD(&zv->zv_suspend_lock));
	zvol_last_close(zv);
	wakeup(zv);
	}

	mutex_exit(&zv->zv_state_lock);

	if (drop_suspend)
	rw_exit(&zv->zv_suspend_lock);
	return (0);
	}

	static void
	zvol_geom_run(zvol_state_t *zv)
	{
	struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom;
	struct g_provider *pp = zsg->zsg_provider;

	ASSERT3S(zv->zv_volmode, ==, ZFS_VOLMODE_GEOM);

	g_error_provider(pp, 0);

	kproc_kthread_add(zvol_geom_worker, zv, &system_proc, NULL, 0, 0,
	"zfskern", "zvol %s", pp->name + sizeof (ZVOL_DRIVER));
	}

	static void
	zvol_geom_destroy(zvol_state_t *zv)
	{
	struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom;
	struct g_provider *pp = zsg->zsg_provider;

	ASSERT3S(zv->zv_volmode, ==, ZFS_VOLMODE_GEOM);

	g_topology_assert();

	mutex_enter(&zv->zv_state_lock);
	VERIFY3S(zsg->zsg_state, ==, ZVOL_GEOM_RUNNING);
	mutex_exit(&zv->zv_state_lock);
	zsg->zsg_provider = NULL;
	g_wither_geom(pp->geom, ENXIO);
	}

	void
	zvol_wait_close(zvol_state_t *zv)
	{

	if (zv->zv_volmode != ZFS_VOLMODE_GEOM)
	return;
	mutex_enter(&zv->zv_state_lock);
	zv->zv_zso->zso_dying = B_TRUE;

	if (zv->zv_open_count)
	msleep(zv, &zv->zv_state_lock,
	PRIBIO, "zvol:dying", 10*hz);
	mutex_exit(&zv->zv_state_lock);
	}


	static int
	zvol_geom_access(struct g_provider *pp, int acr, int acw, int ace)
	{
	int count, error, flags;

	g_topology_assert();

	/*
	* To make it easier we expect either open or close, but not both
	* at the same time.
	*/
	KASSERT((acr >= 0 && acw >= 0 && ace >= 0) \|\|
	(acr <= 0 && acw <= 0 && ace <= 0),
	("Unsupported access request to %s (acr=%d, acw=%d, ace=%d).",
	pp->name, acr, acw, ace));

	if (pp->private == NULL) {
	if (acr <= 0 && acw <= 0 && ace <= 0)
	return (0);
	return (pp->error);
	}

	/*
	* We don't pass FEXCL flag to zvol_geom_open()/zvol_geom_close() if
	* ace != 0, because GEOM already handles that and handles it a bit
	* differently. GEOM allows for multiple read/exclusive consumers and
	* ZFS allows only one exclusive consumer, no matter if it is reader or
	* writer. I like better the way GEOM works so I'll leave it for GEOM
	* to decide what to do.
	*/

	count = acr + acw + ace;
	if (count == 0)
	return (0);

	flags = 0;
	if (acr != 0 \|\| ace != 0)
	flags \|= FREAD;
	if (acw != 0)
	flags \|= FWRITE;

	g_topology_unlock();
	if (count > 0)
	error = zvol_geom_open(pp, flags, count);
	else
	error = zvol_geom_close(pp, flags, -count);
	g_topology_lock();
	return (error);
	}

	static void
	zvol_geom_worker(void *arg)
	{
	zvol_state_t *zv = arg;
	struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom;
	struct bio *bp;

	ASSERT3S(zv->zv_volmode, ==, ZFS_VOLMODE_GEOM);

	thread_lock(curthread);
	sched_prio(curthread, PRIBIO);
	thread_unlock(curthread);

	for (;;) {
	mtx_lock(&zsg->zsg_queue_mtx);
	bp = bioq_takefirst(&zsg->zsg_queue);
	if (bp == NULL) {
	if (zsg->zsg_state == ZVOL_GEOM_STOPPED) {
	zsg->zsg_state = ZVOL_GEOM_RUNNING;
	wakeup(&zsg->zsg_state);
	mtx_unlock(&zsg->zsg_queue_mtx);
	kthread_exit();
	}
	msleep(&zsg->zsg_queue, &zsg->zsg_queue_mtx,
	PRIBIO \| PDROP, "zvol:io", 0);
	continue;
	}
	mtx_unlock(&zsg->zsg_queue_mtx);
	zvol_geom_bio_strategy(bp);
	}
	}

	static void
	zvol_geom_bio_start(struct bio *bp)
	{
	zvol_state_t *zv = bp->bio_to->private;
	struct zvol_state_geom *zsg;
	boolean_t first;

	if (zv == NULL) {
	g_io_deliver(bp, ENXIO);
	return;
	}
	if (bp->bio_cmd == BIO_GETATTR) {
	if (zvol_geom_bio_getattr(bp))
	g_io_deliver(bp, EOPNOTSUPP);
	return;
	}

	if (!THREAD_CAN_SLEEP()) {
	zsg = &zv->zv_zso->zso_geom;
	mtx_lock(&zsg->zsg_queue_mtx);
	first = (bioq_first(&zsg->zsg_queue) == NULL);
	bioq_insert_tail(&zsg->zsg_queue, bp);
	mtx_unlock(&zsg->zsg_queue_mtx);
	if (first)
	wakeup_one(&zsg->zsg_queue);
	return;
	}

	zvol_geom_bio_strategy(bp);
	}

	static int
	zvol_geom_bio_getattr(struct bio *bp)
	{
	zvol_state_t *zv;

	zv = bp->bio_to->private;
	ASSERT3P(zv, !=, NULL);

	spa_t *spa = dmu_objset_spa(zv->zv_objset);
	uint64_t refd, avail, usedobjs, availobjs;

	if (g_handleattr_int(bp, "GEOM::candelete", 1))
	return (0);
	if (strcmp(bp->bio_attribute, "blocksavail") == 0) {
	dmu_objset_space(zv->zv_objset, &refd, &avail,
	&usedobjs, &availobjs);
	if (g_handleattr_off_t(bp, "blocksavail", avail / DEV_BSIZE))
	return (0);
	} else if (strcmp(bp->bio_attribute, "blocksused") == 0) {
	dmu_objset_space(zv->zv_objset, &refd, &avail,
	&usedobjs, &availobjs);
	if (g_handleattr_off_t(bp, "blocksused", refd / DEV_BSIZE))
	return (0);
	} else if (strcmp(bp->bio_attribute, "poolblocksavail") == 0) {
	avail = metaslab_class_get_space(spa_normal_class(spa));
	avail -= metaslab_class_get_alloc(spa_normal_class(spa));
	if (g_handleattr_off_t(bp, "poolblocksavail",
	avail / DEV_BSIZE))
	return (0);
	} else if (strcmp(bp->bio_attribute, "poolblocksused") == 0) {
	refd = metaslab_class_get_alloc(spa_normal_class(spa));
	if (g_handleattr_off_t(bp, "poolblocksused", refd / DEV_BSIZE))
	return (0);
	}
	return (1);
	}

	static void
	zvol_filter_detach(struct knote *kn)
	{
	zvol_state_t *zv;
	struct zvol_state_dev *zsd;

	zv = kn->kn_hook;
	zsd = &zv->zv_zso->zso_dev;

	knlist_remove(&zsd->zsd_selinfo.si_note, kn, 0);
	}

	static int
	zvol_filter_vnode(struct knote *kn, long hint)
	{
	kn->kn_fflags \|= kn->kn_sfflags & hint;

	return (kn->kn_fflags != 0);
	}

	static int
	zvol_cdev_kqfilter(struct cdev dev, struct knote kn)
	{
	zvol_state_t *zv;
	struct zvol_state_dev *zsd;

	zv = dev->si_drv2;
	zsd = &zv->zv_zso->zso_dev;

	if (kn->kn_filter != EVFILT_VNODE)
	return (EINVAL);

	/* XXX: extend support for other NOTE_* events */
	if (kn->kn_sfflags != NOTE_ATTRIB)
	return (EINVAL);

	kn->kn_fop = &zvol_filterops_vnode;
	kn->kn_hook = zv;
	knlist_add(&zsd->zsd_selinfo.si_note, kn, 0);

	return (0);
	}

	static void
	zvol_geom_bio_strategy(struct bio *bp)
	{
	zvol_state_t *zv;
	uint64_t off, volsize;
	size_t resid;
	char *addr;
	objset_t *os;
	zfs_locked_range_t *lr;
	int error = 0;
	boolean_t doread = B_FALSE;
	boolean_t is_dumpified;
	boolean_t sync;

	if (bp->bio_to)
	zv = bp->bio_to->private;
	else
	zv = bp->bio_dev->si_drv2;

	if (zv == NULL) {
	error = SET_ERROR(ENXIO);
	goto out;
	}

	rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);

	switch (bp->bio_cmd) {
	case BIO_READ:
	doread = B_TRUE;
	break;
	case BIO_WRITE:
	case BIO_FLUSH:
	case BIO_DELETE:
	if (zv->zv_flags & ZVOL_RDONLY) {
	error = SET_ERROR(EROFS);
	goto resume;
	}
	zvol_ensure_zilog(zv);
	if (bp->bio_cmd == BIO_FLUSH)
	goto sync;
	break;
	default:
	error = SET_ERROR(EOPNOTSUPP);
	goto resume;
	}

	off = bp->bio_offset;
	volsize = zv->zv_volsize;

	os = zv->zv_objset;
	ASSERT3P(os, !=, NULL);

	addr = bp->bio_data;
	resid = bp->bio_length;

	if (resid > 0 && off >= volsize) {
	error = SET_ERROR(EIO);
	goto resume;
	}

	is_dumpified = B_FALSE;
	sync = !doread && !is_dumpified &&
	zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS;

	/*
	* There must be no buffer changes when doing a dmu_sync() because
	* we can't change the data whilst calculating the checksum.
	*/
	lr = zfs_rangelock_enter(&zv->zv_rangelock, off, resid,
	doread ? RL_READER : RL_WRITER);

	if (bp->bio_cmd == BIO_DELETE) {
	dmu_tx_t *tx = dmu_tx_create(zv->zv_objset);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error != 0) {
	dmu_tx_abort(tx);
	} else {
	zvol_log_truncate(zv, tx, off, resid, sync);
	dmu_tx_commit(tx);
	error = dmu_free_long_range(zv->zv_objset, ZVOL_OBJ,
	off, resid);
	resid = 0;
	}
	goto unlock;
	}
	while (resid != 0 && off < volsize) {
	size_t size = MIN(resid, zvol_maxphys);
	if (doread) {
	error = dmu_read(os, ZVOL_OBJ, off, size, addr,
	DMU_READ_PREFETCH);
	} else {
	dmu_tx_t *tx = dmu_tx_create(os);
	dmu_tx_hold_write_by_dnode(tx, zv->zv_dn, off, size);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error) {
	dmu_tx_abort(tx);
	} else {
	dmu_write(os, ZVOL_OBJ, off, size, addr, tx);
	zvol_log_write(zv, tx, off, size, sync);
	dmu_tx_commit(tx);
	}
	}
	if (error) {
	/* Convert checksum errors into IO errors. */
	if (error == ECKSUM)
	error = SET_ERROR(EIO);
	break;
	}
	off += size;
	addr += size;
	resid -= size;
	}
	unlock:
	zfs_rangelock_exit(lr);

	bp->bio_completed = bp->bio_length - resid;
	if (bp->bio_completed < bp->bio_length && off > volsize)
	error = SET_ERROR(EINVAL);

	switch (bp->bio_cmd) {
	case BIO_FLUSH:
	break;
	case BIO_READ:
	dataset_kstats_update_read_kstats(&zv->zv_kstat,
	bp->bio_completed);
	break;
	case BIO_WRITE:
	dataset_kstats_update_write_kstats(&zv->zv_kstat,
	bp->bio_completed);
	break;
	case BIO_DELETE:
	break;
	default:
	break;
	}

	if (sync) {
	sync:
	zil_commit(zv->zv_zilog, ZVOL_OBJ);
	}
	resume:
	rw_exit(&zv->zv_suspend_lock);
	out:
	if (bp->bio_to)
	g_io_deliver(bp, error);
	else
	biofinish(bp, NULL, error);
	}

	/*
	* Character device mode implementation
	*/

	static int
	zvol_cdev_read(struct cdev dev, struct uio uio_s, int ioflag)
	{
	zvol_state_t *zv;
	uint64_t volsize;
	zfs_locked_range_t *lr;
	int error = 0;
	zfs_uio_t uio;

	zfs_uio_init(&uio, uio_s);

	zv = dev->si_drv2;

	volsize = zv->zv_volsize;
	/*
	* uio_loffset == volsize isn't an error as
	* it's required for EOF processing.
	*/
	if (zfs_uio_resid(&uio) > 0 &&
	(zfs_uio_offset(&uio) < 0 \|\| zfs_uio_offset(&uio) > volsize))
	return (SET_ERROR(EIO));

	rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);
	ssize_t start_resid = zfs_uio_resid(&uio);
	lr = zfs_rangelock_enter(&zv->zv_rangelock, zfs_uio_offset(&uio),
	zfs_uio_resid(&uio), RL_READER);
	while (zfs_uio_resid(&uio) > 0 && zfs_uio_offset(&uio) < volsize) {
	uint64_t bytes = MIN(zfs_uio_resid(&uio), DMU_MAX_ACCESS >> 1);

	/* Don't read past the end. */
	if (bytes > volsize - zfs_uio_offset(&uio))
	bytes = volsize - zfs_uio_offset(&uio);

	error = dmu_read_uio_dnode(zv->zv_dn, &uio, bytes);
	if (error) {
	/* Convert checksum errors into IO errors. */
	if (error == ECKSUM)
	error = SET_ERROR(EIO);
	break;
	}
	}
	zfs_rangelock_exit(lr);
	int64_t nread = start_resid - zfs_uio_resid(&uio);
	dataset_kstats_update_read_kstats(&zv->zv_kstat, nread);
	rw_exit(&zv->zv_suspend_lock);

	return (error);
	}

	static int
	zvol_cdev_write(struct cdev dev, struct uio uio_s, int ioflag)
	{
	zvol_state_t *zv;
	uint64_t volsize;
	zfs_locked_range_t *lr;
	int error = 0;
	boolean_t sync;
	zfs_uio_t uio;

	zv = dev->si_drv2;

	volsize = zv->zv_volsize;

	zfs_uio_init(&uio, uio_s);

	if (zfs_uio_resid(&uio) > 0 &&
	(zfs_uio_offset(&uio) < 0 \|\| zfs_uio_offset(&uio) > volsize))
	return (SET_ERROR(EIO));

	ssize_t start_resid = zfs_uio_resid(&uio);
	sync = (ioflag & IO_SYNC) \|\|
	(zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS);

	rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);
	zvol_ensure_zilog(zv);

	lr = zfs_rangelock_enter(&zv->zv_rangelock, zfs_uio_offset(&uio),
	zfs_uio_resid(&uio), RL_WRITER);
	while (zfs_uio_resid(&uio) > 0 && zfs_uio_offset(&uio) < volsize) {
	uint64_t bytes = MIN(zfs_uio_resid(&uio), DMU_MAX_ACCESS >> 1);
	uint64_t off = zfs_uio_offset(&uio);
	dmu_tx_t *tx = dmu_tx_create(zv->zv_objset);

	if (bytes > volsize - off) /* Don't write past the end. */
	bytes = volsize - off;

	dmu_tx_hold_write_by_dnode(tx, zv->zv_dn, off, bytes);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error) {
	dmu_tx_abort(tx);
	break;
	}
	error = dmu_write_uio_dnode(zv->zv_dn, &uio, bytes, tx);
	if (error == 0)
	zvol_log_write(zv, tx, off, bytes, sync);
	dmu_tx_commit(tx);

	if (error)
	break;
	}
	zfs_rangelock_exit(lr);
	int64_t nwritten = start_resid - zfs_uio_resid(&uio);
	dataset_kstats_update_write_kstats(&zv->zv_kstat, nwritten);
	if (sync)
	zil_commit(zv->zv_zilog, ZVOL_OBJ);
	rw_exit(&zv->zv_suspend_lock);
	return (error);
	}

	static int
	zvol_cdev_open(struct cdev dev, int flags, int fmt, struct thread td)
	{
	zvol_state_t *zv;
	struct zvol_state_dev *zsd;
	int err = 0;
	boolean_t drop_suspend = B_FALSE;

	retry:
	rw_enter(&zvol_state_lock, ZVOL_RW_READER);
	/*
	* Obtain a copy of si_drv2 under zvol_state_lock to make sure either
	* the result of zvol free code setting si_drv2 to NULL is observed,
	* or the zv is protected from being freed because of the positive
	* zv_open_count.
	*/
	zv = dev->si_drv2;
	if (zv == NULL) {
	rw_exit(&zvol_state_lock);
	err = SET_ERROR(ENXIO);
	goto out_locked;
	}

	mutex_enter(&zv->zv_state_lock);
	if (zv->zv_zso->zso_dying) {
	rw_exit(&zvol_state_lock);
	err = SET_ERROR(ENXIO);
	goto out_zv_locked;
	}
	ASSERT3S(zv->zv_volmode, ==, ZFS_VOLMODE_DEV);

	/*
	* Make sure zvol is not suspended during first open
	* (hold zv_suspend_lock) and respect proper lock acquisition
	* ordering - zv_suspend_lock before zv_state_lock.
	*/
	if (zv->zv_open_count == 0) {
	drop_suspend = B_TRUE;
	if (!rw_tryenter(&zv->zv_suspend_lock, ZVOL_RW_READER)) {
	mutex_exit(&zv->zv_state_lock);
	rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);
	mutex_enter(&zv->zv_state_lock);
	/* Check to see if zv_suspend_lock is needed. */
	if (zv->zv_open_count != 0) {
	rw_exit(&zv->zv_suspend_lock);
	drop_suspend = B_FALSE;
	}
	}
	}
	rw_exit(&zvol_state_lock);

	ASSERT(MUTEX_HELD(&zv->zv_state_lock));

	if (zv->zv_open_count == 0) {
	boolean_t drop_namespace = B_FALSE;

	ASSERT(ZVOL_RW_READ_HELD(&zv->zv_suspend_lock));

	/*
	* Take spa_namespace_lock to prevent lock inversion when
	* zvols from one pool are opened as vdevs in another.
	*/
	if (!mutex_owned(&spa_namespace_lock)) {
	if (!mutex_tryenter(&spa_namespace_lock)) {
	mutex_exit(&zv->zv_state_lock);
	rw_exit(&zv->zv_suspend_lock);
	kern_yield(PRI_USER);
	goto retry;
	} else {
	drop_namespace = B_TRUE;
	}
	}
	err = zvol_first_open(zv, !(flags & FWRITE));
	if (drop_namespace)
	mutex_exit(&spa_namespace_lock);
	if (err)
	goto out_zv_locked;
	}

	ASSERT(MUTEX_HELD(&zv->zv_state_lock));

	if ((flags & FWRITE) && (zv->zv_flags & ZVOL_RDONLY)) {
	err = SET_ERROR(EROFS);
	goto out_opened;
	}
	if (zv->zv_flags & ZVOL_EXCL) {
	err = SET_ERROR(EBUSY);
	goto out_opened;
	}
	if (flags & O_EXCL) {
	if (zv->zv_open_count != 0) {
	err = SET_ERROR(EBUSY);
	goto out_opened;
	}
	zv->zv_flags \|= ZVOL_EXCL;
	}

	zv->zv_open_count++;
	if (flags & O_SYNC) {
	zsd = &zv->zv_zso->zso_dev;
	zsd->zsd_sync_cnt++;
	if (zsd->zsd_sync_cnt == 1 &&
	(zv->zv_flags & ZVOL_WRITTEN_TO) != 0)
	zil_async_to_sync(zv->zv_zilog, ZVOL_OBJ);
	}
	out_opened:
	if (zv->zv_open_count == 0) {
	zvol_last_close(zv);
	wakeup(zv);
	}
	out_zv_locked:
	mutex_exit(&zv->zv_state_lock);
	out_locked:
	if (drop_suspend)
	rw_exit(&zv->zv_suspend_lock);
	return (err);
	}

	static int
	zvol_cdev_close(struct cdev dev, int flags, int fmt, struct thread td)
	{
	zvol_state_t *zv;
	struct zvol_state_dev *zsd;
	boolean_t drop_suspend = B_TRUE;

	rw_enter(&zvol_state_lock, ZVOL_RW_READER);
	zv = dev->si_drv2;
	if (zv == NULL) {
	rw_exit(&zvol_state_lock);
	return (SET_ERROR(ENXIO));
	}

	mutex_enter(&zv->zv_state_lock);
	if (zv->zv_flags & ZVOL_EXCL) {
	ASSERT3U(zv->zv_open_count, ==, 1);
	zv->zv_flags &= ~ZVOL_EXCL;
	}

	ASSERT3S(zv->zv_volmode, ==, ZFS_VOLMODE_DEV);

	/*
	* If the open count is zero, this is a spurious close.
	* That indicates a bug in the kernel / DDI framework.
	*/
	ASSERT3U(zv->zv_open_count, >, 0);
	/*
	* Make sure zvol is not suspended during last close
	* (hold zv_suspend_lock) and respect proper lock acquisition
	* ordering - zv_suspend_lock before zv_state_lock.
	*/
	if (zv->zv_open_count == 1) {
	if (!rw_tryenter(&zv->zv_suspend_lock, ZVOL_RW_READER)) {
	mutex_exit(&zv->zv_state_lock);
	rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);
	mutex_enter(&zv->zv_state_lock);
	/* Check to see if zv_suspend_lock is needed. */
	if (zv->zv_open_count != 1) {
	rw_exit(&zv->zv_suspend_lock);
	drop_suspend = B_FALSE;
	}
	}
	} else {
	drop_suspend = B_FALSE;
	}
	rw_exit(&zvol_state_lock);

	ASSERT(MUTEX_HELD(&zv->zv_state_lock));

	/*
	* You may get multiple opens, but only one close.
	*/
	zv->zv_open_count--;
	if (flags & O_SYNC) {
	zsd = &zv->zv_zso->zso_dev;
	zsd->zsd_sync_cnt--;
	}

	if (zv->zv_open_count == 0) {
	ASSERT(ZVOL_RW_READ_HELD(&zv->zv_suspend_lock));
	zvol_last_close(zv);
	wakeup(zv);
	}

	mutex_exit(&zv->zv_state_lock);

	if (drop_suspend)
	rw_exit(&zv->zv_suspend_lock);
	return (0);
	}

	static int
	zvol_cdev_ioctl(struct cdev *dev, ulong_t cmd, caddr_t data,
	int fflag, struct thread *td)
	{
	zvol_state_t *zv;
	zfs_locked_range_t *lr;
	off_t offset, length;
	int error;
	boolean_t sync;

	zv = dev->si_drv2;

	error = 0;
	KASSERT(zv->zv_open_count > 0,
	("Device with zero access count in %s", __func__));

	switch (cmd) {
	case DIOCGSECTORSIZE:
	(uint32_t )data = DEV_BSIZE;
	break;
	case DIOCGMEDIASIZE:
	(off_t )data = zv->zv_volsize;
	break;
	case DIOCGFLUSH:
	rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);
	if (zv->zv_zilog != NULL)
	zil_commit(zv->zv_zilog, ZVOL_OBJ);
	rw_exit(&zv->zv_suspend_lock);
	break;
	case DIOCGDELETE:
	if (!zvol_unmap_enabled)
	break;

	offset = ((off_t *)data)[0];
	length = ((off_t *)data)[1];
	if ((offset % DEV_BSIZE) != 0 \|\| (length % DEV_BSIZE) != 0 \|\|
	offset < 0 \|\| offset >= zv->zv_volsize \|\|
	length <= 0) {
	printf("%s: offset=%jd length=%jd\n", __func__, offset,
	length);
	error = SET_ERROR(EINVAL);
	break;
	}
	rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);
	zvol_ensure_zilog(zv);
	lr = zfs_rangelock_enter(&zv->zv_rangelock, offset, length,
	RL_WRITER);
	dmu_tx_t *tx = dmu_tx_create(zv->zv_objset);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error != 0) {
	sync = FALSE;
	dmu_tx_abort(tx);
	} else {
	sync = (zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS);
	zvol_log_truncate(zv, tx, offset, length, sync);
	dmu_tx_commit(tx);
	error = dmu_free_long_range(zv->zv_objset, ZVOL_OBJ,
	offset, length);
	}
	zfs_rangelock_exit(lr);
	if (sync)
	zil_commit(zv->zv_zilog, ZVOL_OBJ);
	rw_exit(&zv->zv_suspend_lock);
	break;
	case DIOCGSTRIPESIZE:
	(off_t )data = zv->zv_volblocksize;
	break;
	case DIOCGSTRIPEOFFSET:
	(off_t )data = 0;
	break;
	case DIOCGATTR: {
	spa_t *spa = dmu_objset_spa(zv->zv_objset);
	struct diocgattr_arg arg = (struct diocgattr_arg )data;
	uint64_t refd, avail, usedobjs, availobjs;

	if (strcmp(arg->name, "GEOM::candelete") == 0)
	arg->value.i = 1;
	else if (strcmp(arg->name, "blocksavail") == 0) {
	dmu_objset_space(zv->zv_objset, &refd, &avail,
	&usedobjs, &availobjs);
	arg->value.off = avail / DEV_BSIZE;
	} else if (strcmp(arg->name, "blocksused") == 0) {
	dmu_objset_space(zv->zv_objset, &refd, &avail,
	&usedobjs, &availobjs);
	arg->value.off = refd / DEV_BSIZE;
	} else if (strcmp(arg->name, "poolblocksavail") == 0) {
	avail = metaslab_class_get_space(spa_normal_class(spa));
	avail -= metaslab_class_get_alloc(
	spa_normal_class(spa));
	arg->value.off = avail / DEV_BSIZE;
	} else if (strcmp(arg->name, "poolblocksused") == 0) {
	refd = metaslab_class_get_alloc(spa_normal_class(spa));
	arg->value.off = refd / DEV_BSIZE;
	} else
	error = SET_ERROR(ENOIOCTL);
	break;
	}
	case FIOSEEKHOLE:
	case FIOSEEKDATA: {
	off_t off = (off_t )data;
	uint64_t noff;
	boolean_t hole;

	hole = (cmd == FIOSEEKHOLE);
	noff = *off;
	lr = zfs_rangelock_enter(&zv->zv_rangelock, 0, UINT64_MAX,
	RL_READER);
	error = dmu_offset_next(zv->zv_objset, ZVOL_OBJ, hole, &noff);
	zfs_rangelock_exit(lr);
	*off = noff;
	break;
	}
	default:
	error = SET_ERROR(ENOIOCTL);
	}

	return (error);
	}

	/*
	* Misc. helpers
	*/

	static void
	zvol_ensure_zilog(zvol_state_t *zv)
	{
	ASSERT(ZVOL_RW_READ_HELD(&zv->zv_suspend_lock));

	/*
	* Open a ZIL if this is the first time we have written to this
	* zvol. We protect zv->zv_zilog with zv_suspend_lock rather
	* than zv_state_lock so that we don't need to acquire an
	* additional lock in this path.
	*/
	if (zv->zv_zilog == NULL) {
	if (!rw_tryupgrade(&zv->zv_suspend_lock)) {
	rw_exit(&zv->zv_suspend_lock);
	rw_enter(&zv->zv_suspend_lock, RW_WRITER);
	}
	if (zv->zv_zilog == NULL) {
	zv->zv_zilog = zil_open(zv->zv_objset,
	zvol_get_data, &zv->zv_kstat.dk_zil_sums);
	zv->zv_flags \|= ZVOL_WRITTEN_TO;
	/* replay / destroy done in zvol_os_create_minor() */
	VERIFY0(zv->zv_zilog->zl_header->zh_flags &
	ZIL_REPLAY_NEEDED);
	}
	rw_downgrade(&zv->zv_suspend_lock);
	}
	}

	boolean_t
	zvol_os_is_zvol(const char *device)
	{
	return (device && strncmp(device, ZVOL_DIR, strlen(ZVOL_DIR)) == 0);
	}

	void
	zvol_os_rename_minor(zvol_state_t zv, const char newname)
	{
	ASSERT(RW_LOCK_HELD(&zvol_state_lock));
	ASSERT(MUTEX_HELD(&zv->zv_state_lock));

	/* Move to a new hashtable entry. */
	- zv->zv_hash = zvol_name_hash(zv->zv_name);
	+ zv->zv_hash = zvol_name_hash(newname);
	hlist_del(&zv->zv_hlink);
	hlist_add_head(&zv->zv_hlink, ZVOL_HT_HEAD(zv->zv_hash));

	if (zv->zv_volmode == ZFS_VOLMODE_GEOM) {
	struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom;
	struct g_provider *pp = zsg->zsg_provider;
	struct g_geom *gp;

	g_topology_lock();
	gp = pp->geom;
	ASSERT3P(gp, !=, NULL);

	zsg->zsg_provider = NULL;
	g_wither_provider(pp, ENXIO);

	pp = g_new_providerf(gp, "%s/%s", ZVOL_DRIVER, newname);
	pp->flags \|= G_PF_DIRECT_RECEIVE \| G_PF_DIRECT_SEND;
	pp->sectorsize = DEV_BSIZE;
	pp->mediasize = zv->zv_volsize;
	pp->private = zv;
	zsg->zsg_provider = pp;
	g_error_provider(pp, 0);
	g_topology_unlock();
	} else if (zv->zv_volmode == ZFS_VOLMODE_DEV) {
	struct zvol_state_dev *zsd = &zv->zv_zso->zso_dev;
	struct cdev *dev;
	struct make_dev_args args;

	dev = zsd->zsd_cdev;
	if (dev != NULL) {
	destroy_dev(dev);
	dev = zsd->zsd_cdev = NULL;
	if (zv->zv_open_count > 0) {
	zv->zv_flags &= ~ZVOL_EXCL;
	zv->zv_open_count = 0;
	/* XXX need suspend lock but lock order */
	zvol_last_close(zv);
	}
	}

	make_dev_args_init(&args);
	args.mda_flags = MAKEDEV_CHECKNAME \| MAKEDEV_WAITOK;
	args.mda_devsw = &zvol_cdevsw;
	args.mda_cr = NULL;
	args.mda_uid = UID_ROOT;
	args.mda_gid = GID_OPERATOR;
	args.mda_mode = 0640;
	args.mda_si_drv2 = zv;
	if (make_dev_s(&args, &dev, "%s/%s", ZVOL_DRIVER, newname)
	== 0) {
	#if __FreeBSD_version > 1300130
	dev->si_iosize_max = maxphys;
	#else
	dev->si_iosize_max = MAXPHYS;
	#endif
	zsd->zsd_cdev = dev;
	}
	}
	strlcpy(zv->zv_name, newname, sizeof (zv->zv_name));
	dataset_kstats_rename(&zv->zv_kstat, newname);
	}

	/*
	* Remove minor node for the specified volume.
	*/
	void
	zvol_os_free(zvol_state_t *zv)
	{
	ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock));
	ASSERT(!MUTEX_HELD(&zv->zv_state_lock));
	ASSERT0(zv->zv_open_count);

	ZFS_LOG(1, "ZVOL %s destroyed.", zv->zv_name);

	rw_destroy(&zv->zv_suspend_lock);
	zfs_rangelock_fini(&zv->zv_rangelock);

	if (zv->zv_volmode == ZFS_VOLMODE_GEOM) {
	struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom;
	struct g_provider *pp __maybe_unused = zsg->zsg_provider;

	ASSERT3P(pp->private, ==, NULL);

	g_topology_lock();
	zvol_geom_destroy(zv);
	g_topology_unlock();
	mtx_destroy(&zsg->zsg_queue_mtx);
	} else if (zv->zv_volmode == ZFS_VOLMODE_DEV) {
	struct zvol_state_dev *zsd = &zv->zv_zso->zso_dev;
	struct cdev *dev = zsd->zsd_cdev;

	if (dev != NULL) {
	ASSERT3P(dev->si_drv2, ==, NULL);
	destroy_dev(dev);
	knlist_clear(&zsd->zsd_selinfo.si_note, 0);
	knlist_destroy(&zsd->zsd_selinfo.si_note);
	}
	}

	mutex_destroy(&zv->zv_state_lock);
	dataset_kstats_destroy(&zv->zv_kstat);
	kmem_free(zv->zv_zso, sizeof (struct zvol_state_os));
	kmem_free(zv, sizeof (zvol_state_t));
	zvol_minors--;
	}

	/*
	* Create a minor node (plus a whole lot more) for the specified volume.
	*/
	int
	zvol_os_create_minor(const char *name)
	{
	zvol_state_t *zv;
	objset_t *os;
	dmu_object_info_t *doi;
	uint64_t volsize;
	uint64_t volmode, hash;
	int error;
	bool replayed_zil = B_FALSE;

	ZFS_LOG(1, "Creating ZVOL %s...", name);
	hash = zvol_name_hash(name);
	if ((zv = zvol_find_by_name_hash(name, hash, RW_NONE)) != NULL) {
	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
	mutex_exit(&zv->zv_state_lock);
	return (SET_ERROR(EEXIST));
	}

	DROP_GIANT();

	doi = kmem_alloc(sizeof (dmu_object_info_t), KM_SLEEP);

	/* Lie and say we're read-only. */
	error = dmu_objset_own(name, DMU_OST_ZVOL, B_TRUE, B_TRUE, FTAG, &os);
	if (error)
	goto out_doi;

	error = dmu_object_info(os, ZVOL_OBJ, doi);
	if (error)
	goto out_dmu_objset_disown;

	error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize);
	if (error)
	goto out_dmu_objset_disown;

	error = dsl_prop_get_integer(name,
	zfs_prop_to_name(ZFS_PROP_VOLMODE), &volmode, NULL);
	if (error \|\| volmode == ZFS_VOLMODE_DEFAULT)
	volmode = zvol_volmode;
	error = 0;

	/*
	* zvol_alloc equivalent ...
	*/
	zv = kmem_zalloc(sizeof (*zv), KM_SLEEP);
	zv->zv_hash = hash;
	mutex_init(&zv->zv_state_lock, NULL, MUTEX_DEFAULT, NULL);
	zv->zv_zso = kmem_zalloc(sizeof (struct zvol_state_os), KM_SLEEP);
	zv->zv_volmode = volmode;
	if (zv->zv_volmode == ZFS_VOLMODE_GEOM) {
	struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom;
	struct g_provider *pp;
	struct g_geom *gp;

	zsg->zsg_state = ZVOL_GEOM_UNINIT;
	mtx_init(&zsg->zsg_queue_mtx, "zvol", NULL, MTX_DEF);

	g_topology_lock();
	gp = g_new_geomf(&zfs_zvol_class, "zfs::zvol::%s", name);
	gp->start = zvol_geom_bio_start;
	gp->access = zvol_geom_access;
	pp = g_new_providerf(gp, "%s/%s", ZVOL_DRIVER, name);
	pp->flags \|= G_PF_DIRECT_RECEIVE \| G_PF_DIRECT_SEND;
	pp->sectorsize = DEV_BSIZE;
	pp->mediasize = 0;
	pp->private = zv;

	zsg->zsg_provider = pp;
	bioq_init(&zsg->zsg_queue);
	} else if (zv->zv_volmode == ZFS_VOLMODE_DEV) {
	struct zvol_state_dev *zsd = &zv->zv_zso->zso_dev;
	struct cdev *dev;
	struct make_dev_args args;

	make_dev_args_init(&args);
	args.mda_flags = MAKEDEV_CHECKNAME \| MAKEDEV_WAITOK;
	args.mda_devsw = &zvol_cdevsw;
	args.mda_cr = NULL;
	args.mda_uid = UID_ROOT;
	args.mda_gid = GID_OPERATOR;
	args.mda_mode = 0640;
	args.mda_si_drv2 = zv;
	if (make_dev_s(&args, &dev, "%s/%s", ZVOL_DRIVER, name)
	== 0) {
	#if __FreeBSD_version > 1300130
	dev->si_iosize_max = maxphys;
	#else
	dev->si_iosize_max = MAXPHYS;
	#endif
	zsd->zsd_cdev = dev;
	knlist_init_sx(&zsd->zsd_selinfo.si_note,
	&zv->zv_state_lock);
	}
	}
	(void) strlcpy(zv->zv_name, name, MAXPATHLEN);
	rw_init(&zv->zv_suspend_lock, NULL, RW_DEFAULT, NULL);
	zfs_rangelock_init(&zv->zv_rangelock, NULL, NULL);

	if (dmu_objset_is_snapshot(os) \|\| !spa_writeable(dmu_objset_spa(os)))
	zv->zv_flags \|= ZVOL_RDONLY;

	zv->zv_volblocksize = doi->doi_data_block_size;
	zv->zv_volsize = volsize;
	zv->zv_objset = os;

	ASSERT3P(zv->zv_kstat.dk_kstats, ==, NULL);
	error = dataset_kstats_create(&zv->zv_kstat, zv->zv_objset);
	if (error)
	goto out_dmu_objset_disown;
	ASSERT3P(zv->zv_zilog, ==, NULL);
	zv->zv_zilog = zil_open(os, zvol_get_data, &zv->zv_kstat.dk_zil_sums);
	if (spa_writeable(dmu_objset_spa(os))) {
	if (zil_replay_disable)
	replayed_zil = zil_destroy(zv->zv_zilog, B_FALSE);
	else
	replayed_zil = zil_replay(os, zv, zvol_replay_vector);
	}
	if (replayed_zil)
	zil_close(zv->zv_zilog);
	zv->zv_zilog = NULL;

	/* TODO: prefetch for geom tasting */

	zv->zv_objset = NULL;
	out_dmu_objset_disown:
	dmu_objset_disown(os, B_TRUE, FTAG);

	if (error == 0 && volmode == ZFS_VOLMODE_GEOM) {
	zvol_geom_run(zv);
	g_topology_unlock();
	}
	out_doi:
	kmem_free(doi, sizeof (dmu_object_info_t));
	if (error == 0) {
	rw_enter(&zvol_state_lock, RW_WRITER);
	zvol_insert(zv);
	zvol_minors++;
	rw_exit(&zvol_state_lock);
	ZFS_LOG(1, "ZVOL %s created.", name);
	}
	PICKUP_GIANT();
	return (error);
	}

	void
	zvol_os_clear_private(zvol_state_t *zv)
	{
	ASSERT(RW_LOCK_HELD(&zvol_state_lock));
	if (zv->zv_volmode == ZFS_VOLMODE_GEOM) {
	struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom;
	struct g_provider *pp = zsg->zsg_provider;

	if (pp->private == NULL) /* already cleared */
	return;

	mtx_lock(&zsg->zsg_queue_mtx);
	zsg->zsg_state = ZVOL_GEOM_STOPPED;
	pp->private = NULL;
	wakeup_one(&zsg->zsg_queue);
	while (zsg->zsg_state != ZVOL_GEOM_RUNNING)
	msleep(&zsg->zsg_state, &zsg->zsg_queue_mtx,
	0, "zvol:w", 0);
	mtx_unlock(&zsg->zsg_queue_mtx);
	ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock));
	} else if (zv->zv_volmode == ZFS_VOLMODE_DEV) {
	struct zvol_state_dev *zsd = &zv->zv_zso->zso_dev;
	struct cdev *dev = zsd->zsd_cdev;

	if (dev != NULL)
	dev->si_drv2 = NULL;
	}
	}

	int
	zvol_os_update_volsize(zvol_state_t *zv, uint64_t volsize)
	{
	zv->zv_volsize = volsize;
	if (zv->zv_volmode == ZFS_VOLMODE_GEOM) {
	struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom;
	struct g_provider *pp = zsg->zsg_provider;

	g_topology_lock();

	if (pp->private == NULL) {
	g_topology_unlock();
	return (SET_ERROR(ENXIO));
	}

	/*
	* Do not invoke resize event when initial size was zero.
	* ZVOL initializes the size on first open, this is not
	* real resizing.
	*/
	if (pp->mediasize == 0)
	pp->mediasize = zv->zv_volsize;
	else
	g_resize_provider(pp, zv->zv_volsize);

	g_topology_unlock();
	} else if (zv->zv_volmode == ZFS_VOLMODE_DEV) {
	struct zvol_state_dev *zsd = &zv->zv_zso->zso_dev;

	KNOTE_UNLOCKED(&zsd->zsd_selinfo.si_note, NOTE_ATTRIB);
	}
	return (0);
	}

	void
	zvol_os_set_disk_ro(zvol_state_t *zv, int flags)
	{
	// XXX? set_disk_ro(zv->zv_zso->zvo_disk, flags);
	}

	void
	zvol_os_set_capacity(zvol_state_t *zv, uint64_t capacity)
	{
	// XXX? set_capacity(zv->zv_zso->zvo_disk, capacity);
	}

	/*
	* Public interfaces
	*/

	int
	zvol_busy(void)
	{
	return (zvol_minors != 0);
	}

	int
	zvol_init(void)
	{
	zvol_init_impl();
	return (0);
	}

	void
	zvol_fini(void)
	{
	zvol_fini_impl();
	}
	diff --git a/sys/contrib/openzfs/module/os/linux/spl/spl-taskq.c b/sys/contrib/openzfs/module/os/linux/spl/spl-taskq.c
	index d18f935b167c..0e44aa1fcd38 100644
	--- a/sys/contrib/openzfs/module/os/linux/spl/spl-taskq.c
	+++ b/sys/contrib/openzfs/module/os/linux/spl/spl-taskq.c
	@@ -1,1466 +1,1439 @@
	/*
	* Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
	* Copyright (C) 2007 The Regents of the University of California.
	* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
	* Written by Brian Behlendorf <behlendorf1@llnl.gov>.
	* UCRL-CODE-235197
	*
	* This file is part of the SPL, Solaris Porting Layer.
	*
	* The SPL is free software; you can redistribute it and/or modify it
	* under the terms of the GNU General Public License as published by the
	* Free Software Foundation; either version 2 of the License, or (at your
	* option) any later version.
	*
	* The SPL is distributed in the hope that it will be useful, but WITHOUT
	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	* for more details.
	*
	* You should have received a copy of the GNU General Public License along
	* with the SPL. If not, see <http://www.gnu.org/licenses/>.
	*
	* Solaris Porting Layer (SPL) Task Queue Implementation.
	*/

	#include <sys/timer.h>
	#include <sys/taskq.h>
	#include <sys/kmem.h>
	#include <sys/tsd.h>
	#include <sys/trace_spl.h>
	#ifdef HAVE_CPU_HOTPLUG
	#include <linux/cpuhotplug.h>
	#endif

	static int spl_taskq_thread_bind = 0;
	module_param(spl_taskq_thread_bind, int, 0644);
	MODULE_PARM_DESC(spl_taskq_thread_bind, "Bind taskq thread to CPU by default");

	-static uint_t spl_taskq_thread_timeout_ms = 10000;
	+static uint_t spl_taskq_thread_timeout_ms = 5000;
	/* BEGIN CSTYLED */
	module_param(spl_taskq_thread_timeout_ms, uint, 0644);
	/* END CSTYLED */
	MODULE_PARM_DESC(spl_taskq_thread_timeout_ms,
	- "Time to require a dynamic thread be idle before it gets cleaned up");
	+ "Minimum idle threads exit interval for dynamic taskqs");

	static int spl_taskq_thread_dynamic = 1;
	module_param(spl_taskq_thread_dynamic, int, 0444);
	MODULE_PARM_DESC(spl_taskq_thread_dynamic, "Allow dynamic taskq threads");

	static int spl_taskq_thread_priority = 1;
	module_param(spl_taskq_thread_priority, int, 0644);
	MODULE_PARM_DESC(spl_taskq_thread_priority,
	"Allow non-default priority for taskq threads");

	static uint_t spl_taskq_thread_sequential = 4;
	/* BEGIN CSTYLED */
	module_param(spl_taskq_thread_sequential, uint, 0644);
	/* END CSTYLED */
	MODULE_PARM_DESC(spl_taskq_thread_sequential,
	"Create new taskq threads after N sequential tasks");

	/*
	* Global system-wide dynamic task queue available for all consumers. This
	* taskq is not intended for long-running tasks; instead, a dedicated taskq
	* should be created.
	*/
	taskq_t *system_taskq;
	EXPORT_SYMBOL(system_taskq);
	/* Global dynamic task queue for long delay */
	taskq_t *system_delay_taskq;
	EXPORT_SYMBOL(system_delay_taskq);

	/* Private dedicated taskq for creating new taskq threads on demand. */
	static taskq_t *dynamic_taskq;
	static taskq_thread_t taskq_thread_create(taskq_t );

	#ifdef HAVE_CPU_HOTPLUG
	/* Multi-callback id for cpu hotplugging. */
	static int spl_taskq_cpuhp_state;
	#endif

	/* List of all taskqs */
	LIST_HEAD(tq_list);
	struct rw_semaphore tq_list_sem;
	static uint_t taskq_tsd;

	static int
	task_km_flags(uint_t flags)
	{
	if (flags & TQ_NOSLEEP)
	return (KM_NOSLEEP);

	if (flags & TQ_PUSHPAGE)
	return (KM_PUSHPAGE);

	return (KM_SLEEP);
	}

	/*
	* taskq_find_by_name - Find the largest instance number of a named taskq.
	*/
	static int
	taskq_find_by_name(const char *name)
	{
	struct list_head *tql = NULL;
	taskq_t *tq;

	list_for_each_prev(tql, &tq_list) {
	tq = list_entry(tql, taskq_t, tq_taskqs);
	if (strcmp(name, tq->tq_name) == 0)
	return (tq->tq_instance);
	}
	return (-1);
	}

	/*
	* NOTE: Must be called with tq->tq_lock held, returns a list_t which
	* is not attached to the free, work, or pending taskq lists.
	*/
	static taskq_ent_t *
	task_alloc(taskq_t tq, uint_t flags, unsigned long irqflags)
	{
	taskq_ent_t *t;
	int count = 0;

	ASSERT(tq);
	retry:
	/* Acquire taskq_ent_t's from free list if available */
	if (!list_empty(&tq->tq_free_list) && !(flags & TQ_NEW)) {
	t = list_entry(tq->tq_free_list.next, taskq_ent_t, tqent_list);

	ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC));
	ASSERT(!(t->tqent_flags & TQENT_FLAG_CANCEL));
	ASSERT(!timer_pending(&t->tqent_timer));

	list_del_init(&t->tqent_list);
	return (t);
	}

	/* Free list is empty and memory allocations are prohibited */
	if (flags & TQ_NOALLOC)
	return (NULL);

	/* Hit maximum taskq_ent_t pool size */
	if (tq->tq_nalloc >= tq->tq_maxalloc) {
	if (flags & TQ_NOSLEEP)
	return (NULL);

	/*
	* Sleep periodically polling the free list for an available
	* taskq_ent_t. Dispatching with TQ_SLEEP should always succeed
	* but we cannot block forever waiting for an taskq_ent_t to
	* show up in the free list, otherwise a deadlock can happen.
	*
	* Therefore, we need to allocate a new task even if the number
	* of allocated tasks is above tq->tq_maxalloc, but we still
	* end up delaying the task allocation by one second, thereby
	* throttling the task dispatch rate.
	*/
	spin_unlock_irqrestore(&tq->tq_lock, *irqflags);
	schedule_timeout(HZ / 100);
	spin_lock_irqsave_nested(&tq->tq_lock, *irqflags,
	tq->tq_lock_class);
	if (count < 100) {
	count++;
	goto retry;
	}
	}

	spin_unlock_irqrestore(&tq->tq_lock, *irqflags);
	t = kmem_alloc(sizeof (taskq_ent_t), task_km_flags(flags));
	spin_lock_irqsave_nested(&tq->tq_lock, *irqflags, tq->tq_lock_class);

	if (t) {
	taskq_init_ent(t);
	tq->tq_nalloc++;
	}

	return (t);
	}

	/*
	* NOTE: Must be called with tq->tq_lock held, expects the taskq_ent_t
	* to already be removed from the free, work, or pending taskq lists.
	*/
	static void
	task_free(taskq_t tq, taskq_ent_t t)
	{
	ASSERT(tq);
	ASSERT(t);
	ASSERT(list_empty(&t->tqent_list));
	ASSERT(!timer_pending(&t->tqent_timer));

	kmem_free(t, sizeof (taskq_ent_t));
	tq->tq_nalloc--;
	}

	/*
	* NOTE: Must be called with tq->tq_lock held, either destroys the
	* taskq_ent_t if too many exist or moves it to the free list for later use.
	*/
	static void
	task_done(taskq_t tq, taskq_ent_t t)
	{
	ASSERT(tq);
	ASSERT(t);

	/* Wake tasks blocked in taskq_wait_id() */
	wake_up_all(&t->tqent_waitq);

	list_del_init(&t->tqent_list);

	if (tq->tq_nalloc <= tq->tq_minalloc) {
	t->tqent_id = TASKQID_INVALID;
	t->tqent_func = NULL;
	t->tqent_arg = NULL;
	t->tqent_flags = 0;

	list_add_tail(&t->tqent_list, &tq->tq_free_list);
	} else {
	task_free(tq, t);
	}
	}

	/*
	* When a delayed task timer expires remove it from the delay list and
	* add it to the priority list in order for immediate processing.
	*/
	static void
	task_expire_impl(taskq_ent_t *t)
	{
	taskq_ent_t *w;
	taskq_t *tq = t->tqent_taskq;
	struct list_head *l = NULL;
	unsigned long flags;

	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);

	if (t->tqent_flags & TQENT_FLAG_CANCEL) {
	ASSERT(list_empty(&t->tqent_list));
	spin_unlock_irqrestore(&tq->tq_lock, flags);
	return;
	}

	t->tqent_birth = jiffies;
	DTRACE_PROBE1(taskq_ent__birth, taskq_ent_t *, t);

	/*
	* The priority list must be maintained in strict task id order
	* from lowest to highest for lowest_id to be easily calculable.
	*/
	list_del(&t->tqent_list);
	list_for_each_prev(l, &tq->tq_prio_list) {
	w = list_entry(l, taskq_ent_t, tqent_list);
	if (w->tqent_id < t->tqent_id) {
	list_add(&t->tqent_list, l);
	break;
	}
	}
	if (l == &tq->tq_prio_list)
	list_add(&t->tqent_list, &tq->tq_prio_list);

	spin_unlock_irqrestore(&tq->tq_lock, flags);

	wake_up(&tq->tq_work_waitq);
	}

	static void
	task_expire(spl_timer_list_t tl)
	{
	struct timer_list tmr = (struct timer_list )tl;
	taskq_ent_t *t = from_timer(t, tmr, tqent_timer);
	task_expire_impl(t);
	}

	/*
	* Returns the lowest incomplete taskqid_t. The taskqid_t may
	* be queued on the pending list, on the priority list, on the
	* delay list, or on the work list currently being handled, but
	* it is not 100% complete yet.
	*/
	static taskqid_t
	taskq_lowest_id(taskq_t *tq)
	{
	taskqid_t lowest_id = tq->tq_next_id;
	taskq_ent_t *t;
	taskq_thread_t *tqt;

	if (!list_empty(&tq->tq_pend_list)) {
	t = list_entry(tq->tq_pend_list.next, taskq_ent_t, tqent_list);
	lowest_id = MIN(lowest_id, t->tqent_id);
	}

	if (!list_empty(&tq->tq_prio_list)) {
	t = list_entry(tq->tq_prio_list.next, taskq_ent_t, tqent_list);
	lowest_id = MIN(lowest_id, t->tqent_id);
	}

	if (!list_empty(&tq->tq_delay_list)) {
	t = list_entry(tq->tq_delay_list.next, taskq_ent_t, tqent_list);
	lowest_id = MIN(lowest_id, t->tqent_id);
	}

	if (!list_empty(&tq->tq_active_list)) {
	tqt = list_entry(tq->tq_active_list.next, taskq_thread_t,
	tqt_active_list);
	ASSERT(tqt->tqt_id != TASKQID_INVALID);
	lowest_id = MIN(lowest_id, tqt->tqt_id);
	}

	return (lowest_id);
	}

	/*
	* Insert a task into a list keeping the list sorted by increasing taskqid.
	*/
	static void
	taskq_insert_in_order(taskq_t tq, taskq_thread_t tqt)
	{
	taskq_thread_t *w;
	struct list_head *l = NULL;

	ASSERT(tq);
	ASSERT(tqt);

	list_for_each_prev(l, &tq->tq_active_list) {
	w = list_entry(l, taskq_thread_t, tqt_active_list);
	if (w->tqt_id < tqt->tqt_id) {
	list_add(&tqt->tqt_active_list, l);
	break;
	}
	}
	if (l == &tq->tq_active_list)
	list_add(&tqt->tqt_active_list, &tq->tq_active_list);
	}

	/*
	* Find and return a task from the given list if it exists. The list
	* must be in lowest to highest task id order.
	*/
	static taskq_ent_t *
	taskq_find_list(taskq_t tq, struct list_head lh, taskqid_t id)
	{
	struct list_head *l = NULL;
	taskq_ent_t *t;

	list_for_each(l, lh) {
	t = list_entry(l, taskq_ent_t, tqent_list);

	if (t->tqent_id == id)
	return (t);

	if (t->tqent_id > id)
	break;
	}

	return (NULL);
	}

	/*
	* Find an already dispatched task given the task id regardless of what
	* state it is in. If a task is still pending it will be returned.
	* If a task is executing, then -EBUSY will be returned instead.
	* If the task has already been run then NULL is returned.
	*/
	static taskq_ent_t *
	taskq_find(taskq_t *tq, taskqid_t id)
	{
	taskq_thread_t *tqt;
	struct list_head *l = NULL;
	taskq_ent_t *t;

	t = taskq_find_list(tq, &tq->tq_delay_list, id);
	if (t)
	return (t);

	t = taskq_find_list(tq, &tq->tq_prio_list, id);
	if (t)
	return (t);

	t = taskq_find_list(tq, &tq->tq_pend_list, id);
	if (t)
	return (t);

	list_for_each(l, &tq->tq_active_list) {
	tqt = list_entry(l, taskq_thread_t, tqt_active_list);
	if (tqt->tqt_id == id) {
	/*
	* Instead of returning tqt_task, we just return a non
	* NULL value to prevent misuse, since tqt_task only
	* has two valid fields.
	*/
	return (ERR_PTR(-EBUSY));
	}
	}

	return (NULL);
	}

	/*
	* Theory for the taskq_wait_id(), taskq_wait_outstanding(), and
	* taskq_wait() functions below.
	*
	* Taskq waiting is accomplished by tracking the lowest outstanding task
	* id and the next available task id. As tasks are dispatched they are
	* added to the tail of the pending, priority, or delay lists. As worker
	* threads become available the tasks are removed from the heads of these
	* lists and linked to the worker threads. This ensures the lists are
	* kept sorted by lowest to highest task id.
	*
	* Therefore the lowest outstanding task id can be quickly determined by
	* checking the head item from all of these lists. This value is stored
	* with the taskq as the lowest id. It only needs to be recalculated when
	* either the task with the current lowest id completes or is canceled.
	*
	* By blocking until the lowest task id exceeds the passed task id the
	* taskq_wait_outstanding() function can be easily implemented. Similarly,
	* by blocking until the lowest task id matches the next task id taskq_wait()
	* can be implemented.
	*
	* Callers should be aware that when there are multiple worked threads it
	* is possible for larger task ids to complete before smaller ones. Also
	* when the taskq contains delay tasks with small task ids callers may
	* block for a considerable length of time waiting for them to expire and
	* execute.
	*/
	static int
	taskq_wait_id_check(taskq_t *tq, taskqid_t id)
	{
	int rc;
	unsigned long flags;

	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
	rc = (taskq_find(tq, id) == NULL);
	spin_unlock_irqrestore(&tq->tq_lock, flags);

	return (rc);
	}

	/*
	* The taskq_wait_id() function blocks until the passed task id completes.
	* This does not guarantee that all lower task ids have completed.
	*/
	void
	taskq_wait_id(taskq_t *tq, taskqid_t id)
	{
	wait_event(tq->tq_wait_waitq, taskq_wait_id_check(tq, id));
	}
	EXPORT_SYMBOL(taskq_wait_id);

	static int
	taskq_wait_outstanding_check(taskq_t *tq, taskqid_t id)
	{
	int rc;
	unsigned long flags;

	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
	rc = (id < tq->tq_lowest_id);
	spin_unlock_irqrestore(&tq->tq_lock, flags);

	return (rc);
	}

	/*
	* The taskq_wait_outstanding() function will block until all tasks with a
	* lower taskqid than the passed 'id' have been completed. Note that all
	* task id's are assigned monotonically at dispatch time. Zero may be
	* passed for the id to indicate all tasks dispatch up to this point,
	* but not after, should be waited for.
	*/
	void
	taskq_wait_outstanding(taskq_t *tq, taskqid_t id)
	{
	id = id ? id : tq->tq_next_id - 1;
	wait_event(tq->tq_wait_waitq, taskq_wait_outstanding_check(tq, id));
	}
	EXPORT_SYMBOL(taskq_wait_outstanding);

	static int
	taskq_wait_check(taskq_t *tq)
	{
	int rc;
	unsigned long flags;

	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
	rc = (tq->tq_lowest_id == tq->tq_next_id);
	spin_unlock_irqrestore(&tq->tq_lock, flags);

	return (rc);
	}

	/*
	* The taskq_wait() function will block until the taskq is empty.
	* This means that if a taskq re-dispatches work to itself taskq_wait()
	* callers will block indefinitely.
	*/
	void
	taskq_wait(taskq_t *tq)
	{
	wait_event(tq->tq_wait_waitq, taskq_wait_check(tq));
	}
	EXPORT_SYMBOL(taskq_wait);

	int
	taskq_member(taskq_t tq, kthread_t t)
	{
	return (tq == (taskq_t *)tsd_get_by_thread(taskq_tsd, t));
	}
	EXPORT_SYMBOL(taskq_member);

	taskq_t *
	taskq_of_curthread(void)
	{
	return (tsd_get(taskq_tsd));
	}
	EXPORT_SYMBOL(taskq_of_curthread);

	/*
	* Cancel an already dispatched task given the task id. Still pending tasks
	* will be immediately canceled, and if the task is active the function will
	* block until it completes. Preallocated tasks which are canceled must be
	* freed by the caller.
	*/
	int
	taskq_cancel_id(taskq_t *tq, taskqid_t id)
	{
	taskq_ent_t *t;
	int rc = ENOENT;
	unsigned long flags;

	ASSERT(tq);

	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
	t = taskq_find(tq, id);
	if (t && t != ERR_PTR(-EBUSY)) {
	list_del_init(&t->tqent_list);
	t->tqent_flags \|= TQENT_FLAG_CANCEL;

	/*
	* When canceling the lowest outstanding task id we
	* must recalculate the new lowest outstanding id.
	*/
	if (tq->tq_lowest_id == t->tqent_id) {
	tq->tq_lowest_id = taskq_lowest_id(tq);
	ASSERT3S(tq->tq_lowest_id, >, t->tqent_id);
	}

	/*
	* The task_expire() function takes the tq->tq_lock so drop
	* drop the lock before synchronously cancelling the timer.
	*/
	if (timer_pending(&t->tqent_timer)) {
	spin_unlock_irqrestore(&tq->tq_lock, flags);
	del_timer_sync(&t->tqent_timer);
	spin_lock_irqsave_nested(&tq->tq_lock, flags,
	tq->tq_lock_class);
	}

	if (!(t->tqent_flags & TQENT_FLAG_PREALLOC))
	task_done(tq, t);

	rc = 0;
	}
	spin_unlock_irqrestore(&tq->tq_lock, flags);

	if (t == ERR_PTR(-EBUSY)) {
	taskq_wait_id(tq, id);
	rc = EBUSY;
	}

	return (rc);
	}
	EXPORT_SYMBOL(taskq_cancel_id);

	static int taskq_thread_spawn(taskq_t *tq);

	taskqid_t
	taskq_dispatch(taskq_t tq, task_func_t func, void arg, uint_t flags)
	{
	taskq_ent_t *t;
	taskqid_t rc = TASKQID_INVALID;
	unsigned long irqflags;

	ASSERT(tq);
	ASSERT(func);

	spin_lock_irqsave_nested(&tq->tq_lock, irqflags, tq->tq_lock_class);

	/* Taskq being destroyed and all tasks drained */
	if (!(tq->tq_flags & TASKQ_ACTIVE))
	goto out;

	/* Do not queue the task unless there is idle thread for it */
	ASSERT(tq->tq_nactive <= tq->tq_nthreads);
	if ((flags & TQ_NOQUEUE) && (tq->tq_nactive == tq->tq_nthreads)) {
	/* Dynamic taskq may be able to spawn another thread */
	- if (!(tq->tq_flags & TASKQ_DYNAMIC) \|\|
	- taskq_thread_spawn(tq) == 0)
	+ if (taskq_thread_spawn(tq) == 0)
	goto out;
	}

	if ((t = task_alloc(tq, flags, &irqflags)) == NULL)
	goto out;

	spin_lock(&t->tqent_lock);

	/* Queue to the front of the list to enforce TQ_NOQUEUE semantics */
	if (flags & TQ_NOQUEUE)
	list_add(&t->tqent_list, &tq->tq_prio_list);
	/* Queue to the priority list instead of the pending list */
	else if (flags & TQ_FRONT)
	list_add_tail(&t->tqent_list, &tq->tq_prio_list);
	else
	list_add_tail(&t->tqent_list, &tq->tq_pend_list);

	t->tqent_id = rc = tq->tq_next_id;
	tq->tq_next_id++;
	t->tqent_func = func;
	t->tqent_arg = arg;
	t->tqent_taskq = tq;
	t->tqent_timer.function = NULL;
	t->tqent_timer.expires = 0;

	t->tqent_birth = jiffies;
	DTRACE_PROBE1(taskq_ent__birth, taskq_ent_t *, t);

	ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC));

	spin_unlock(&t->tqent_lock);

	wake_up(&tq->tq_work_waitq);
	-out:
	+
	/* Spawn additional taskq threads if required. */
	if (!(flags & TQ_NOQUEUE) && tq->tq_nactive == tq->tq_nthreads)
	(void) taskq_thread_spawn(tq);
	-
	+out:
	spin_unlock_irqrestore(&tq->tq_lock, irqflags);
	return (rc);
	}
	EXPORT_SYMBOL(taskq_dispatch);

	taskqid_t
	taskq_dispatch_delay(taskq_t tq, task_func_t func, void arg,
	uint_t flags, clock_t expire_time)
	{
	taskqid_t rc = TASKQID_INVALID;
	taskq_ent_t *t;
	unsigned long irqflags;

	ASSERT(tq);
	ASSERT(func);

	spin_lock_irqsave_nested(&tq->tq_lock, irqflags, tq->tq_lock_class);

	/* Taskq being destroyed and all tasks drained */
	if (!(tq->tq_flags & TASKQ_ACTIVE))
	goto out;

	if ((t = task_alloc(tq, flags, &irqflags)) == NULL)
	goto out;

	spin_lock(&t->tqent_lock);

	/* Queue to the delay list for subsequent execution */
	list_add_tail(&t->tqent_list, &tq->tq_delay_list);

	t->tqent_id = rc = tq->tq_next_id;
	tq->tq_next_id++;
	t->tqent_func = func;
	t->tqent_arg = arg;
	t->tqent_taskq = tq;
	t->tqent_timer.function = task_expire;
	t->tqent_timer.expires = (unsigned long)expire_time;
	add_timer(&t->tqent_timer);

	ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC));

	spin_unlock(&t->tqent_lock);
	-out:
	+
	/* Spawn additional taskq threads if required. */
	if (tq->tq_nactive == tq->tq_nthreads)
	(void) taskq_thread_spawn(tq);
	+out:
	spin_unlock_irqrestore(&tq->tq_lock, irqflags);
	return (rc);
	}
	EXPORT_SYMBOL(taskq_dispatch_delay);

	void
	taskq_dispatch_ent(taskq_t tq, task_func_t func, void arg, uint_t flags,
	taskq_ent_t *t)
	{
	unsigned long irqflags;
	ASSERT(tq);
	ASSERT(func);

	spin_lock_irqsave_nested(&tq->tq_lock, irqflags,
	tq->tq_lock_class);

	/* Taskq being destroyed and all tasks drained */
	if (!(tq->tq_flags & TASKQ_ACTIVE)) {
	t->tqent_id = TASKQID_INVALID;
	goto out;
	}

	if ((flags & TQ_NOQUEUE) && (tq->tq_nactive == tq->tq_nthreads)) {
	/* Dynamic taskq may be able to spawn another thread */
	- if (!(tq->tq_flags & TASKQ_DYNAMIC) \|\|
	- taskq_thread_spawn(tq) == 0)
	- goto out2;
	+ if (taskq_thread_spawn(tq) == 0)
	+ goto out;
	flags \|= TQ_FRONT;
	}

	spin_lock(&t->tqent_lock);

	/*
	* Make sure the entry is not on some other taskq; it is important to
	* ASSERT() under lock
	*/
	ASSERT(taskq_empty_ent(t));

	/*
	* Mark it as a prealloc'd task. This is important
	* to ensure that we don't free it later.
	*/
	t->tqent_flags \|= TQENT_FLAG_PREALLOC;

	/* Queue to the priority list instead of the pending list */
	if (flags & TQ_FRONT)
	list_add_tail(&t->tqent_list, &tq->tq_prio_list);
	else
	list_add_tail(&t->tqent_list, &tq->tq_pend_list);

	t->tqent_id = tq->tq_next_id;
	tq->tq_next_id++;
	t->tqent_func = func;
	t->tqent_arg = arg;
	t->tqent_taskq = tq;

	t->tqent_birth = jiffies;
	DTRACE_PROBE1(taskq_ent__birth, taskq_ent_t *, t);

	spin_unlock(&t->tqent_lock);

	wake_up(&tq->tq_work_waitq);
	-out:
	+
	/* Spawn additional taskq threads if required. */
	if (tq->tq_nactive == tq->tq_nthreads)
	(void) taskq_thread_spawn(tq);
	-out2:
	+out:
	spin_unlock_irqrestore(&tq->tq_lock, irqflags);
	}
	EXPORT_SYMBOL(taskq_dispatch_ent);

	int
	taskq_empty_ent(taskq_ent_t *t)
	{
	return (list_empty(&t->tqent_list));
	}
	EXPORT_SYMBOL(taskq_empty_ent);

	void
	taskq_init_ent(taskq_ent_t *t)
	{
	spin_lock_init(&t->tqent_lock);
	init_waitqueue_head(&t->tqent_waitq);
	timer_setup(&t->tqent_timer, NULL, 0);
	INIT_LIST_HEAD(&t->tqent_list);
	t->tqent_id = 0;
	t->tqent_func = NULL;
	t->tqent_arg = NULL;
	t->tqent_flags = 0;
	t->tqent_taskq = NULL;
	}
	EXPORT_SYMBOL(taskq_init_ent);

	/*
	* Return the next pending task, preference is given to tasks on the
	* priority list which were dispatched with TQ_FRONT.
	*/
	static taskq_ent_t *
	taskq_next_ent(taskq_t *tq)
	{
	struct list_head *list;

	if (!list_empty(&tq->tq_prio_list))
	list = &tq->tq_prio_list;
	else if (!list_empty(&tq->tq_pend_list))
	list = &tq->tq_pend_list;
	else
	return (NULL);

	return (list_entry(list->next, taskq_ent_t, tqent_list));
	}

	/*
	* Spawns a new thread for the specified taskq.
	*/
	static void
	taskq_thread_spawn_task(void *arg)
	{
	taskq_t tq = (taskq_t )arg;
	unsigned long flags;

	if (taskq_thread_create(tq) == NULL) {
	/* restore spawning count if failed */
	spin_lock_irqsave_nested(&tq->tq_lock, flags,
	tq->tq_lock_class);
	tq->tq_nspawn--;
	spin_unlock_irqrestore(&tq->tq_lock, flags);
	}
	}

	/*
	* Spawn addition threads for dynamic taskqs (TASKQ_DYNAMIC) the current
	* number of threads is insufficient to handle the pending tasks. These
	* new threads must be created by the dedicated dynamic_taskq to avoid
	* deadlocks between thread creation and memory reclaim. The system_taskq
	* which is also a dynamic taskq cannot be safely used for this.
	*/
	static int
	taskq_thread_spawn(taskq_t *tq)
	{
	int spawning = 0;

	if (!(tq->tq_flags & TASKQ_DYNAMIC))
	return (0);

	+ tq->lastspawnstop = jiffies;
	if ((tq->tq_nthreads + tq->tq_nspawn < tq->tq_maxthreads) &&
	(tq->tq_flags & TASKQ_ACTIVE)) {
	spawning = (++tq->tq_nspawn);
	taskq_dispatch(dynamic_taskq, taskq_thread_spawn_task,
	tq, TQ_NOSLEEP);
	}

	return (spawning);
	}

	/*
	- * Threads in a dynamic taskq should only exit once it has been completely
	- * drained and no other threads are actively servicing tasks. This prevents
	- * threads from being created and destroyed more than is required.
	+ * Threads in a dynamic taskq may exit once there is no more work to do.
	+ * To prevent threads from being created and destroyed too often limit
	+ * the exit rate to one per spl_taskq_thread_timeout_ms.
	*
	* The first thread is the thread list is treated as the primary thread.
	* There is nothing special about the primary thread but in order to avoid
	* all the taskq pids from changing we opt to make it long running.
	*/
	static int
	taskq_thread_should_stop(taskq_t tq, taskq_thread_t tqt)
	{
	- if (!(tq->tq_flags & TASKQ_DYNAMIC))
	+ ASSERT(!taskq_next_ent(tq));
	+ if (!(tq->tq_flags & TASKQ_DYNAMIC) \|\| !spl_taskq_thread_dynamic)
	return (0);
	-
	+ if (!(tq->tq_flags & TASKQ_ACTIVE))
	+ return (1);
	if (list_first_entry(&(tq->tq_thread_list), taskq_thread_t,
	tqt_thread_list) == tqt)
	return (0);
	-
	- int no_work =
	- ((tq->tq_nspawn == 0) && /* No threads are being spawned */
	- (tq->tq_nactive == 0) && /* No threads are handling tasks */
	- (tq->tq_nthreads > 1) && /* More than 1 thread is running */
	- (!taskq_next_ent(tq)) && /* There are no pending tasks */
	- (spl_taskq_thread_dynamic)); /* Dynamic taskqs are allowed */
	-
	- /*
	- * If we would have said stop before, let's instead wait a bit, maybe
	- * we'll see more work come our way soon...
	- */
	- if (no_work) {
	- /* if it's 0, we want the old behavior. */
	- /* if the taskq is being torn down, we also want to go away. */
	- if (spl_taskq_thread_timeout_ms == 0 \|\|
	- !(tq->tq_flags & TASKQ_ACTIVE))
	- return (1);
	- unsigned long lasttime = tq->lastshouldstop;
	- if (lasttime > 0) {
	- if (time_after(jiffies, lasttime +
	- msecs_to_jiffies(spl_taskq_thread_timeout_ms)))
	- return (1);
	- else
	- return (0);
	- } else {
	- tq->lastshouldstop = jiffies;
	- }
	- } else {
	- tq->lastshouldstop = 0;
	- }
	- return (0);
	+ ASSERT3U(tq->tq_nthreads, >, 1);
	+ if (tq->tq_nspawn != 0)
	+ return (0);
	+ if (time_before(jiffies, tq->lastspawnstop +
	+ msecs_to_jiffies(spl_taskq_thread_timeout_ms)))
	+ return (0);
	+ tq->lastspawnstop = jiffies;
	+ return (1);
	}

	static int
	taskq_thread(void *args)
	{
	DECLARE_WAITQUEUE(wait, current);
	sigset_t blocked;
	taskq_thread_t *tqt = args;
	taskq_t *tq;
	taskq_ent_t *t;
	int seq_tasks = 0;
	unsigned long flags;
	taskq_ent_t dup_task = {};

	ASSERT(tqt);
	ASSERT(tqt->tqt_tq);
	tq = tqt->tqt_tq;
	current->flags \|= PF_NOFREEZE;

	(void) spl_fstrans_mark();

	sigfillset(&blocked);
	sigprocmask(SIG_BLOCK, &blocked, NULL);
	flush_signals(current);

	tsd_set(taskq_tsd, tq);
	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
	/*
	* If we are dynamically spawned, decrease spawning count. Note that
	* we could be created during taskq_create, in which case we shouldn't
	* do the decrement. But it's fine because taskq_create will reset
	* tq_nspawn later.
	*/
	if (tq->tq_flags & TASKQ_DYNAMIC)
	tq->tq_nspawn--;

	/* Immediately exit if more threads than allowed were created. */
	if (tq->tq_nthreads >= tq->tq_maxthreads)
	goto error;

	tq->tq_nthreads++;
	list_add_tail(&tqt->tqt_thread_list, &tq->tq_thread_list);
	wake_up(&tq->tq_wait_waitq);
	set_current_state(TASK_INTERRUPTIBLE);

	while (!kthread_should_stop()) {

	if (list_empty(&tq->tq_pend_list) &&
	list_empty(&tq->tq_prio_list)) {

	- if (taskq_thread_should_stop(tq, tqt)) {
	- wake_up_all(&tq->tq_wait_waitq);
	+ if (taskq_thread_should_stop(tq, tqt))
	break;
	- }

	add_wait_queue_exclusive(&tq->tq_work_waitq, &wait);
	spin_unlock_irqrestore(&tq->tq_lock, flags);

	schedule();
	seq_tasks = 0;

	spin_lock_irqsave_nested(&tq->tq_lock, flags,
	tq->tq_lock_class);
	remove_wait_queue(&tq->tq_work_waitq, &wait);
	} else {
	__set_current_state(TASK_RUNNING);
	}

	if ((t = taskq_next_ent(tq)) != NULL) {
	list_del_init(&t->tqent_list);

	/*
	* A TQENT_FLAG_PREALLOC task may be reused or freed
	* during the task function call. Store tqent_id and
	* tqent_flags here.
	*
	* Also use an on stack taskq_ent_t for tqt_task
	* assignment in this case; we want to make sure
	* to duplicate all fields, so the values are
	* correct when it's accessed via DTRACE_PROBE*.
	*/
	tqt->tqt_id = t->tqent_id;
	tqt->tqt_flags = t->tqent_flags;

	if (t->tqent_flags & TQENT_FLAG_PREALLOC) {
	dup_task = *t;
	t = &dup_task;
	}
	tqt->tqt_task = t;

	taskq_insert_in_order(tq, tqt);
	tq->tq_nactive++;
	spin_unlock_irqrestore(&tq->tq_lock, flags);

	DTRACE_PROBE1(taskq_ent__start, taskq_ent_t *, t);

	/* Perform the requested task */
	t->tqent_func(t->tqent_arg);

	DTRACE_PROBE1(taskq_ent__finish, taskq_ent_t *, t);

	spin_lock_irqsave_nested(&tq->tq_lock, flags,
	tq->tq_lock_class);
	tq->tq_nactive--;
	list_del_init(&tqt->tqt_active_list);
	tqt->tqt_task = NULL;

	/* For prealloc'd tasks, we don't free anything. */
	if (!(tqt->tqt_flags & TQENT_FLAG_PREALLOC))
	task_done(tq, t);

	/*
	* When the current lowest outstanding taskqid is
	* done calculate the new lowest outstanding id
	*/
	if (tq->tq_lowest_id == tqt->tqt_id) {
	tq->tq_lowest_id = taskq_lowest_id(tq);
	ASSERT3S(tq->tq_lowest_id, >, tqt->tqt_id);
	}

	/* Spawn additional taskq threads if required. */
	if ((++seq_tasks) > spl_taskq_thread_sequential &&
	taskq_thread_spawn(tq))
	seq_tasks = 0;

	tqt->tqt_id = TASKQID_INVALID;
	tqt->tqt_flags = 0;
	wake_up_all(&tq->tq_wait_waitq);
	- } else {
	- if (taskq_thread_should_stop(tq, tqt))
	- break;
	}

	set_current_state(TASK_INTERRUPTIBLE);

	}

	__set_current_state(TASK_RUNNING);
	tq->tq_nthreads--;
	list_del_init(&tqt->tqt_thread_list);
	error:
	kmem_free(tqt, sizeof (taskq_thread_t));
	spin_unlock_irqrestore(&tq->tq_lock, flags);

	tsd_set(taskq_tsd, NULL);
	thread_exit();

	return (0);
	}

	static taskq_thread_t *
	taskq_thread_create(taskq_t *tq)
	{
	static int last_used_cpu = 0;
	taskq_thread_t *tqt;

	tqt = kmem_alloc(sizeof (*tqt), KM_PUSHPAGE);
	INIT_LIST_HEAD(&tqt->tqt_thread_list);
	INIT_LIST_HEAD(&tqt->tqt_active_list);
	tqt->tqt_tq = tq;
	tqt->tqt_id = TASKQID_INVALID;

	tqt->tqt_thread = spl_kthread_create(taskq_thread, tqt,
	"%s", tq->tq_name);
	if (tqt->tqt_thread == NULL) {
	kmem_free(tqt, sizeof (taskq_thread_t));
	return (NULL);
	}

	if (spl_taskq_thread_bind) {
	last_used_cpu = (last_used_cpu + 1) % num_online_cpus();
	kthread_bind(tqt->tqt_thread, last_used_cpu);
	}

	if (spl_taskq_thread_priority)
	set_user_nice(tqt->tqt_thread, PRIO_TO_NICE(tq->tq_pri));

	wake_up_process(tqt->tqt_thread);

	return (tqt);
	}

	taskq_t *
	taskq_create(const char *name, int threads_arg, pri_t pri,
	int minalloc, int maxalloc, uint_t flags)
	{
	taskq_t *tq;
	taskq_thread_t *tqt;
	int count = 0, rc = 0, i;
	unsigned long irqflags;
	int nthreads = threads_arg;

	ASSERT(name != NULL);
	ASSERT(minalloc >= 0);
	ASSERT(!(flags & (TASKQ_CPR_SAFE))); /* Unsupported */

	/* Scale the number of threads using nthreads as a percentage */
	if (flags & TASKQ_THREADS_CPU_PCT) {
	ASSERT(nthreads <= 100);
	ASSERT(nthreads >= 0);
	nthreads = MIN(threads_arg, 100);
	nthreads = MAX(nthreads, 0);
	nthreads = MAX((num_online_cpus() * nthreads) /100, 1);
	}

	tq = kmem_alloc(sizeof (*tq), KM_PUSHPAGE);
	if (tq == NULL)
	return (NULL);

	tq->tq_hp_support = B_FALSE;
	#ifdef HAVE_CPU_HOTPLUG
	if (flags & TASKQ_THREADS_CPU_PCT) {
	tq->tq_hp_support = B_TRUE;
	if (cpuhp_state_add_instance_nocalls(spl_taskq_cpuhp_state,
	&tq->tq_hp_cb_node) != 0) {
	kmem_free(tq, sizeof (*tq));
	return (NULL);
	}
	}
	#endif

	spin_lock_init(&tq->tq_lock);
	INIT_LIST_HEAD(&tq->tq_thread_list);
	INIT_LIST_HEAD(&tq->tq_active_list);
	tq->tq_name = kmem_strdup(name);
	tq->tq_nactive = 0;
	tq->tq_nthreads = 0;
	tq->tq_nspawn = 0;
	tq->tq_maxthreads = nthreads;
	tq->tq_cpu_pct = threads_arg;
	tq->tq_pri = pri;
	tq->tq_minalloc = minalloc;
	tq->tq_maxalloc = maxalloc;
	tq->tq_nalloc = 0;
	tq->tq_flags = (flags \| TASKQ_ACTIVE);
	tq->tq_next_id = TASKQID_INITIAL;
	tq->tq_lowest_id = TASKQID_INITIAL;
	- tq->lastshouldstop = 0;
	+ tq->lastspawnstop = jiffies;
	INIT_LIST_HEAD(&tq->tq_free_list);
	INIT_LIST_HEAD(&tq->tq_pend_list);
	INIT_LIST_HEAD(&tq->tq_prio_list);
	INIT_LIST_HEAD(&tq->tq_delay_list);
	init_waitqueue_head(&tq->tq_work_waitq);
	init_waitqueue_head(&tq->tq_wait_waitq);
	tq->tq_lock_class = TQ_LOCK_GENERAL;
	INIT_LIST_HEAD(&tq->tq_taskqs);

	if (flags & TASKQ_PREPOPULATE) {
	spin_lock_irqsave_nested(&tq->tq_lock, irqflags,
	tq->tq_lock_class);

	for (i = 0; i < minalloc; i++)
	task_done(tq, task_alloc(tq, TQ_PUSHPAGE \| TQ_NEW,
	&irqflags));

	spin_unlock_irqrestore(&tq->tq_lock, irqflags);
	}

	if ((flags & TASKQ_DYNAMIC) && spl_taskq_thread_dynamic)
	nthreads = 1;

	for (i = 0; i < nthreads; i++) {
	tqt = taskq_thread_create(tq);
	if (tqt == NULL)
	rc = 1;
	else
	count++;
	}

	/* Wait for all threads to be started before potential destroy */
	wait_event(tq->tq_wait_waitq, tq->tq_nthreads == count);
	/*
	* taskq_thread might have touched nspawn, but we don't want them to
	* because they're not dynamically spawned. So we reset it to 0
	*/
	tq->tq_nspawn = 0;

	if (rc) {
	taskq_destroy(tq);
	tq = NULL;
	} else {
	down_write(&tq_list_sem);
	tq->tq_instance = taskq_find_by_name(name) + 1;
	list_add_tail(&tq->tq_taskqs, &tq_list);
	up_write(&tq_list_sem);
	}

	return (tq);
	}
	EXPORT_SYMBOL(taskq_create);

	void
	taskq_destroy(taskq_t *tq)
	{
	struct task_struct *thread;
	taskq_thread_t *tqt;
	taskq_ent_t *t;
	unsigned long flags;

	ASSERT(tq);
	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
	tq->tq_flags &= ~TASKQ_ACTIVE;
	spin_unlock_irqrestore(&tq->tq_lock, flags);

	#ifdef HAVE_CPU_HOTPLUG
	if (tq->tq_hp_support) {
	VERIFY0(cpuhp_state_remove_instance_nocalls(
	spl_taskq_cpuhp_state, &tq->tq_hp_cb_node));
	}
	#endif
	/*
	* When TASKQ_ACTIVE is clear new tasks may not be added nor may
	* new worker threads be spawned for dynamic taskq.
	*/
	if (dynamic_taskq != NULL)
	taskq_wait_outstanding(dynamic_taskq, 0);

	taskq_wait(tq);

	/* remove taskq from global list used by the kstats */
	down_write(&tq_list_sem);
	list_del(&tq->tq_taskqs);
	up_write(&tq_list_sem);

	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
	/* wait for spawning threads to insert themselves to the list */
	while (tq->tq_nspawn) {
	spin_unlock_irqrestore(&tq->tq_lock, flags);
	schedule_timeout_interruptible(1);
	spin_lock_irqsave_nested(&tq->tq_lock, flags,
	tq->tq_lock_class);
	}

	/*
	* Signal each thread to exit and block until it does. Each thread
	* is responsible for removing itself from the list and freeing its
	* taskq_thread_t. This allows for idle threads to opt to remove
	* themselves from the taskq. They can be recreated as needed.
	*/
	while (!list_empty(&tq->tq_thread_list)) {
	tqt = list_entry(tq->tq_thread_list.next,
	taskq_thread_t, tqt_thread_list);
	thread = tqt->tqt_thread;
	spin_unlock_irqrestore(&tq->tq_lock, flags);

	kthread_stop(thread);

	spin_lock_irqsave_nested(&tq->tq_lock, flags,
	tq->tq_lock_class);
	}

	while (!list_empty(&tq->tq_free_list)) {
	t = list_entry(tq->tq_free_list.next, taskq_ent_t, tqent_list);

	ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC));

	list_del_init(&t->tqent_list);
	task_free(tq, t);
	}

	ASSERT0(tq->tq_nthreads);
	ASSERT0(tq->tq_nalloc);
	ASSERT0(tq->tq_nspawn);
	ASSERT(list_empty(&tq->tq_thread_list));
	ASSERT(list_empty(&tq->tq_active_list));
	ASSERT(list_empty(&tq->tq_free_list));
	ASSERT(list_empty(&tq->tq_pend_list));
	ASSERT(list_empty(&tq->tq_prio_list));
	ASSERT(list_empty(&tq->tq_delay_list));

	spin_unlock_irqrestore(&tq->tq_lock, flags);

	kmem_strfree(tq->tq_name);
	kmem_free(tq, sizeof (taskq_t));
	}
	EXPORT_SYMBOL(taskq_destroy);

	static unsigned int spl_taskq_kick = 0;

	/*
	* 2.6.36 API Change
	* module_param_cb is introduced to take kernel_param_ops and
	* module_param_call is marked as obsolete. Also set and get operations
	* were changed to take a 'const struct kernel_param *'.
	*/
	static int
	#ifdef module_param_cb
	param_set_taskq_kick(const char val, const struct kernel_param kp)
	#else
	param_set_taskq_kick(const char val, struct kernel_param kp)
	#endif
	{
	int ret;
	taskq_t *tq = NULL;
	taskq_ent_t *t;
	unsigned long flags;

	ret = param_set_uint(val, kp);
	if (ret < 0 \|\| !spl_taskq_kick)
	return (ret);
	/* reset value */
	spl_taskq_kick = 0;

	down_read(&tq_list_sem);
	list_for_each_entry(tq, &tq_list, tq_taskqs) {
	spin_lock_irqsave_nested(&tq->tq_lock, flags,
	tq->tq_lock_class);
	/* Check if the first pending is older than 5 seconds */
	t = taskq_next_ent(tq);
	if (t && time_after(jiffies, t->tqent_birth + 5*HZ)) {
	(void) taskq_thread_spawn(tq);
	printk(KERN_INFO "spl: Kicked taskq %s/%d\n",
	tq->tq_name, tq->tq_instance);
	}
	spin_unlock_irqrestore(&tq->tq_lock, flags);
	}
	up_read(&tq_list_sem);
	return (ret);
	}

	#ifdef module_param_cb
	static const struct kernel_param_ops param_ops_taskq_kick = {
	.set = param_set_taskq_kick,
	.get = param_get_uint,
	};
	module_param_cb(spl_taskq_kick, &param_ops_taskq_kick, &spl_taskq_kick, 0644);
	#else
	module_param_call(spl_taskq_kick, param_set_taskq_kick, param_get_uint,
	&spl_taskq_kick, 0644);
	#endif
	MODULE_PARM_DESC(spl_taskq_kick,
	"Write nonzero to kick stuck taskqs to spawn more threads");

	#ifdef HAVE_CPU_HOTPLUG
	/*
	* This callback will be called exactly once for each core that comes online,
	* for each dynamic taskq. We attempt to expand taskqs that have
	* TASKQ_THREADS_CPU_PCT set. We need to redo the percentage calculation every
	* time, to correctly determine whether or not to add a thread.
	*/
	static int
	spl_taskq_expand(unsigned int cpu, struct hlist_node *node)
	{
	taskq_t *tq = list_entry(node, taskq_t, tq_hp_cb_node);
	unsigned long flags;
	int err = 0;

	ASSERT(tq);
	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);

	if (!(tq->tq_flags & TASKQ_ACTIVE)) {
	spin_unlock_irqrestore(&tq->tq_lock, flags);
	return (err);
	}

	ASSERT(tq->tq_flags & TASKQ_THREADS_CPU_PCT);
	int nthreads = MIN(tq->tq_cpu_pct, 100);
	nthreads = MAX(((num_online_cpus() + 1) * nthreads) / 100, 1);
	tq->tq_maxthreads = nthreads;

	if (!((tq->tq_flags & TASKQ_DYNAMIC) && spl_taskq_thread_dynamic) &&
	tq->tq_maxthreads > tq->tq_nthreads) {
	spin_unlock_irqrestore(&tq->tq_lock, flags);
	taskq_thread_t *tqt = taskq_thread_create(tq);
	if (tqt == NULL)
	err = -1;
	return (err);
	}
	spin_unlock_irqrestore(&tq->tq_lock, flags);
	return (err);
	}

	/*
	* While we don't support offlining CPUs, it is possible that CPUs will fail
	* to online successfully. We do need to be able to handle this case
	* gracefully.
	*/
	static int
	spl_taskq_prepare_down(unsigned int cpu, struct hlist_node *node)
	{
	taskq_t *tq = list_entry(node, taskq_t, tq_hp_cb_node);
	unsigned long flags;

	ASSERT(tq);
	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);

	if (!(tq->tq_flags & TASKQ_ACTIVE))
	goto out;

	ASSERT(tq->tq_flags & TASKQ_THREADS_CPU_PCT);
	int nthreads = MIN(tq->tq_cpu_pct, 100);
	nthreads = MAX(((num_online_cpus()) * nthreads) / 100, 1);
	tq->tq_maxthreads = nthreads;

	if (!((tq->tq_flags & TASKQ_DYNAMIC) && spl_taskq_thread_dynamic) &&
	tq->tq_maxthreads < tq->tq_nthreads) {
	ASSERT3U(tq->tq_maxthreads, ==, tq->tq_nthreads - 1);
	taskq_thread_t *tqt = list_entry(tq->tq_thread_list.next,
	taskq_thread_t, tqt_thread_list);
	struct task_struct *thread = tqt->tqt_thread;
	spin_unlock_irqrestore(&tq->tq_lock, flags);

	kthread_stop(thread);

	return (0);
	}

	out:
	spin_unlock_irqrestore(&tq->tq_lock, flags);
	return (0);
	}
	#endif

	int
	spl_taskq_init(void)
	{
	init_rwsem(&tq_list_sem);
	tsd_create(&taskq_tsd, NULL);

	#ifdef HAVE_CPU_HOTPLUG
	spl_taskq_cpuhp_state = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
	"fs/spl_taskq:online", spl_taskq_expand, spl_taskq_prepare_down);
	#endif

	system_taskq = taskq_create("spl_system_taskq", MAX(boot_ncpus, 64),
	maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE\|TASKQ_DYNAMIC);
	if (system_taskq == NULL)
	return (-ENOMEM);

	system_delay_taskq = taskq_create("spl_delay_taskq", MAX(boot_ncpus, 4),
	maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE\|TASKQ_DYNAMIC);
	if (system_delay_taskq == NULL) {
	#ifdef HAVE_CPU_HOTPLUG
	cpuhp_remove_multi_state(spl_taskq_cpuhp_state);
	#endif
	taskq_destroy(system_taskq);
	return (-ENOMEM);
	}

	dynamic_taskq = taskq_create("spl_dynamic_taskq", 1,
	maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE);
	if (dynamic_taskq == NULL) {
	#ifdef HAVE_CPU_HOTPLUG
	cpuhp_remove_multi_state(spl_taskq_cpuhp_state);
	#endif
	taskq_destroy(system_taskq);
	taskq_destroy(system_delay_taskq);
	return (-ENOMEM);
	}

	/*
	* This is used to annotate tq_lock, so
	* taskq_dispatch -> taskq_thread_spawn -> taskq_dispatch
	* does not trigger a lockdep warning re: possible recursive locking
	*/
	dynamic_taskq->tq_lock_class = TQ_LOCK_DYNAMIC;

	return (0);
	}

	void
	spl_taskq_fini(void)
	{
	taskq_destroy(dynamic_taskq);
	dynamic_taskq = NULL;

	taskq_destroy(system_delay_taskq);
	system_delay_taskq = NULL;

	taskq_destroy(system_taskq);
	system_taskq = NULL;

	tsd_destroy(&taskq_tsd);

	#ifdef HAVE_CPU_HOTPLUG
	cpuhp_remove_multi_state(spl_taskq_cpuhp_state);
	spl_taskq_cpuhp_state = 0;
	#endif
	}
	diff --git a/sys/contrib/openzfs/module/os/linux/spl/spl-xdr.c b/sys/contrib/openzfs/module/os/linux/spl/spl-xdr.c
	index 6b77524181db..e1773da5d173 100644
	--- a/sys/contrib/openzfs/module/os/linux/spl/spl-xdr.c
	+++ b/sys/contrib/openzfs/module/os/linux/spl/spl-xdr.c
	@@ -1,512 +1,513 @@
	/*
	* Copyright (c) 2008-2010 Sun Microsystems, Inc.
	* Written by Ricardo Correia <Ricardo.M.Correia@Sun.COM>
	*
	* This file is part of the SPL, Solaris Porting Layer.
	*
	* The SPL is free software; you can redistribute it and/or modify it
	* under the terms of the GNU General Public License as published by the
	* Free Software Foundation; either version 2 of the License, or (at your
	* option) any later version.
	*
	* The SPL is distributed in the hope that it will be useful, but WITHOUT
	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	* for more details.
	*
	* You should have received a copy of the GNU General Public License along
	* with the SPL. If not, see <http://www.gnu.org/licenses/>.
	*
	* Solaris Porting Layer (SPL) XDR Implementation.
	*/

	#include <linux/string.h>
	#include <sys/kmem.h>
	#include <sys/debug.h>
	#include <sys/types.h>
	#include <sys/sysmacros.h>
	+#include <rpc/types.h>
	#include <rpc/xdr.h>

	/*
	* SPL's XDR mem implementation.
	*
	* This is used by libnvpair to serialize/deserialize the name-value pair data
	* structures into byte arrays in a well-defined and portable manner.
	*
	* These data structures are used by the DMU/ZFS to flexibly manipulate various
	* information in memory and later serialize it/deserialize it to disk.
	* Examples of usages include the pool configuration, lists of pool and dataset
	* properties, etc.
	*
	* Reference documentation for the XDR representation and XDR operations can be
	* found in RFC 1832 and xdr(3), respectively.
	*
	* === Implementation shortcomings ===
	*
	* It is assumed that the following C types have the following sizes:
	*
	* char/unsigned char: 1 byte
	* short/unsigned short: 2 bytes
	* int/unsigned int: 4 bytes
	* longlong_t/u_longlong_t: 8 bytes
	*
	* The C standard allows these types to be larger (and in the case of ints,
	* shorter), so if that is the case on some compiler/architecture, the build
	* will fail (on purpose).
	*
	* If someone wants to fix the code to work properly on such environments, then:
	*
	* 1) Preconditions should be added to xdrmem_enc functions to make sure the
	* caller doesn't pass arguments which exceed the expected range.
	* 2) Functions which take signed integers should be changed to properly do
	* sign extension.
	* 3) For ints with less than 32 bits, well.. I suspect you'll have bigger
	* problems than this implementation.
	*
	* It is also assumed that:
	*
	* 1) Chars have 8 bits.
	* 2) We can always do 32-bit-aligned int memory accesses and byte-aligned
	* memcpy, memset and memcmp.
	* 3) Arrays passed to xdr_array() are packed and the compiler/architecture
	* supports element-sized-aligned memory accesses.
	* 4) Negative integers are natively stored in two's complement binary
	* representation.
	*
	* No checks are done for the 4 assumptions above, though.
	*
	* === Caller expectations ===
	*
	* Existing documentation does not describe the semantics of XDR operations very
	* well. Therefore, some assumptions about failure semantics will be made and
	* will be described below:
	*
	* 1) If any encoding operation fails (e.g., due to lack of buffer space), the
	* the stream should be considered valid only up to the encoding operation
	* previous to the one that first failed. However, the stream size as returned
	* by xdr_control() cannot be considered to be strictly correct (it may be
	* bigger).
	*
	* Putting it another way, if there is an encoding failure it's undefined
	* whether anything is added to the stream in that operation and therefore
	* neither xdr_control() nor future encoding operations on the same stream can
	* be relied upon to produce correct results.
	*
	* 2) If a decoding operation fails, it's undefined whether anything will be
	* decoded into passed buffers/pointers during that operation, or what the
	* values on those buffers will look like.
	*
	* Future decoding operations on the same stream will also have similar
	* undefined behavior.
	*
	* 3) When the first decoding operation fails it is OK to trust the results of
	* previous decoding operations on the same stream, as long as the caller
	* expects a failure to be possible (e.g. due to end-of-stream).
	*
	* However, this is highly discouraged because the caller should know the
	* stream size and should be coded to expect any decoding failure to be data
	* corruption due to hardware, accidental or even malicious causes, which should
	* be handled gracefully in all cases.
	*
	* In very rare situations where there are strong reasons to believe the data
	* can be trusted to be valid and non-tampered with, then the caller may assume
	* a decoding failure to be a bug (e.g. due to mismatched data types) and may
	* fail non-gracefully.
	*
	* 4) Non-zero padding bytes will cause the decoding operation to fail.
	*
	* 5) Zero bytes on string types will also cause the decoding operation to fail.
	*
	* 6) It is assumed that either the pointer to the stream buffer given by the
	* caller is 32-bit aligned or the architecture supports non-32-bit-aligned int
	* memory accesses.
	*
	* 7) The stream buffer and encoding/decoding buffers/ptrs should not overlap.
	*
	* 8) If a caller passes pointers to non-kernel memory (e.g., pointers to user
	* space or MMIO space), the computer may explode.
	*/

	static const struct xdr_ops xdrmem_encode_ops;
	static const struct xdr_ops xdrmem_decode_ops;

	void
	xdrmem_create(XDR *xdrs, const caddr_t addr, const uint_t size,
	const enum xdr_op op)
	{
	switch (op) {
	case XDR_ENCODE:
	xdrs->x_ops = &xdrmem_encode_ops;
	break;
	case XDR_DECODE:
	xdrs->x_ops = &xdrmem_decode_ops;
	break;
	default:
	xdrs->x_ops = NULL; /* Let the caller know we failed */
	return;
	}

	xdrs->x_op = op;
	xdrs->x_addr = addr;
	xdrs->x_addr_end = addr + size;

	if (xdrs->x_addr_end < xdrs->x_addr) {
	xdrs->x_ops = NULL;
	}
	}
	EXPORT_SYMBOL(xdrmem_create);

	static bool_t
	xdrmem_control(XDR xdrs, int req, void info)
	{
	struct xdr_bytesrec rec = (struct xdr_bytesrec )info;

	if (req != XDR_GET_BYTES_AVAIL)
	return (FALSE);

	rec->xc_is_last_record = TRUE; /* always TRUE in xdrmem streams */
	rec->xc_num_avail = xdrs->x_addr_end - xdrs->x_addr;

	return (TRUE);
	}

	static bool_t
	xdrmem_enc_bytes(XDR *xdrs, caddr_t cp, const uint_t cnt)
	{
	uint_t size = roundup(cnt, 4);
	uint_t pad;

	if (size < cnt)
	return (FALSE); /* Integer overflow */

	if (xdrs->x_addr > xdrs->x_addr_end)
	return (FALSE);

	if (xdrs->x_addr_end - xdrs->x_addr < size)
	return (FALSE);

	memcpy(xdrs->x_addr, cp, cnt);

	xdrs->x_addr += cnt;

	pad = size - cnt;
	if (pad > 0) {
	memset(xdrs->x_addr, 0, pad);
	xdrs->x_addr += pad;
	}

	return (TRUE);
	}

	static bool_t
	xdrmem_dec_bytes(XDR *xdrs, caddr_t cp, const uint_t cnt)
	{
	static uint32_t zero = 0;
	uint_t size = roundup(cnt, 4);
	uint_t pad;

	if (size < cnt)
	return (FALSE); /* Integer overflow */

	if (xdrs->x_addr > xdrs->x_addr_end)
	return (FALSE);

	if (xdrs->x_addr_end - xdrs->x_addr < size)
	return (FALSE);

	memcpy(cp, xdrs->x_addr, cnt);
	xdrs->x_addr += cnt;

	pad = size - cnt;
	if (pad > 0) {
	/* An inverted memchr() would be useful here... */
	if (memcmp(&zero, xdrs->x_addr, pad) != 0)
	return (FALSE);

	xdrs->x_addr += pad;
	}

	return (TRUE);
	}

	static bool_t
	xdrmem_enc_uint32(XDR *xdrs, uint32_t val)
	{
	if (xdrs->x_addr + sizeof (uint32_t) > xdrs->x_addr_end)
	return (FALSE);

	((uint32_t )xdrs->x_addr) = cpu_to_be32(val);

	xdrs->x_addr += sizeof (uint32_t);

	return (TRUE);
	}

	static bool_t
	xdrmem_dec_uint32(XDR xdrs, uint32_t val)
	{
	if (xdrs->x_addr + sizeof (uint32_t) > xdrs->x_addr_end)
	return (FALSE);

	val = be32_to_cpu(((uint32_t *)xdrs->x_addr));

	xdrs->x_addr += sizeof (uint32_t);

	return (TRUE);
	}

	static bool_t
	xdrmem_enc_char(XDR xdrs, char cp)
	{
	uint32_t val;

	BUILD_BUG_ON(sizeof (char) != 1);
	val = ((unsigned char ) cp);

	return (xdrmem_enc_uint32(xdrs, val));
	}

	static bool_t
	xdrmem_dec_char(XDR xdrs, char cp)
	{
	uint32_t val;

	BUILD_BUG_ON(sizeof (char) != 1);

	if (!xdrmem_dec_uint32(xdrs, &val))
	return (FALSE);

	/*
	* If any of the 3 other bytes are non-zero then val will be greater
	* than 0xff and we fail because according to the RFC, this block does
	* not have a char encoded in it.
	*/
	if (val > 0xff)
	return (FALSE);

	((unsigned char ) cp) = val;

	return (TRUE);
	}

	static bool_t
	xdrmem_enc_ushort(XDR xdrs, unsigned short usp)
	{
	BUILD_BUG_ON(sizeof (unsigned short) != 2);

	return (xdrmem_enc_uint32(xdrs, *usp));
	}

	static bool_t
	xdrmem_dec_ushort(XDR xdrs, unsigned short usp)
	{
	uint32_t val;

	BUILD_BUG_ON(sizeof (unsigned short) != 2);

	if (!xdrmem_dec_uint32(xdrs, &val))
	return (FALSE);

	/*
	* Short ints are not in the RFC, but we assume similar logic as in
	* xdrmem_dec_char().
	*/
	if (val > 0xffff)
	return (FALSE);

	*usp = val;

	return (TRUE);
	}

	static bool_t
	xdrmem_enc_uint(XDR xdrs, unsigned up)
	{
	BUILD_BUG_ON(sizeof (unsigned) != 4);

	return (xdrmem_enc_uint32(xdrs, *up));
	}

	static bool_t
	xdrmem_dec_uint(XDR xdrs, unsigned up)
	{
	BUILD_BUG_ON(sizeof (unsigned) != 4);

	return (xdrmem_dec_uint32(xdrs, (uint32_t *)up));
	}

	static bool_t
	xdrmem_enc_ulonglong(XDR xdrs, u_longlong_t ullp)
	{
	BUILD_BUG_ON(sizeof (u_longlong_t) != 8);

	if (!xdrmem_enc_uint32(xdrs, *ullp >> 32))
	return (FALSE);

	return (xdrmem_enc_uint32(xdrs, *ullp & 0xffffffff));
	}

	static bool_t
	xdrmem_dec_ulonglong(XDR xdrs, u_longlong_t ullp)
	{
	uint32_t low, high;

	BUILD_BUG_ON(sizeof (u_longlong_t) != 8);

	if (!xdrmem_dec_uint32(xdrs, &high))
	return (FALSE);
	if (!xdrmem_dec_uint32(xdrs, &low))
	return (FALSE);

	*ullp = ((u_longlong_t)high << 32) \| low;

	return (TRUE);
	}

	static bool_t
	xdr_enc_array(XDR xdrs, caddr_t arrp, uint_t *sizep, const uint_t maxsize,
	const uint_t elsize, const xdrproc_t elproc)
	{
	uint_t i;
	caddr_t addr = *arrp;

	if (sizep > maxsize \|\| sizep > UINT_MAX / elsize)
	return (FALSE);

	if (!xdrmem_enc_uint(xdrs, sizep))
	return (FALSE);

	for (i = 0; i < *sizep; i++) {
	if (!elproc(xdrs, addr))
	return (FALSE);
	addr += elsize;
	}

	return (TRUE);
	}

	static bool_t
	xdr_dec_array(XDR xdrs, caddr_t arrp, uint_t *sizep, const uint_t maxsize,
	const uint_t elsize, const xdrproc_t elproc)
	{
	uint_t i, size;
	bool_t alloc = FALSE;
	caddr_t addr;

	if (!xdrmem_dec_uint(xdrs, sizep))
	return (FALSE);

	size = *sizep;

	if (size > maxsize \|\| size > UINT_MAX / elsize)
	return (FALSE);

	/*
	* The Solaris man page says: "If *arrp is NULL when decoding,
	* xdr_array() allocates memory and *arrp points to it".
	*/
	if (*arrp == NULL) {
	BUILD_BUG_ON(sizeof (uint_t) > sizeof (size_t));

	arrp = kmem_alloc(size elsize, KM_NOSLEEP);
	if (*arrp == NULL)
	return (FALSE);

	alloc = TRUE;
	}

	addr = *arrp;

	for (i = 0; i < size; i++) {
	if (!elproc(xdrs, addr)) {
	if (alloc)
	kmem_free(arrp, size elsize);
	return (FALSE);
	}
	addr += elsize;
	}

	return (TRUE);
	}

	static bool_t
	xdr_enc_string(XDR xdrs, char *sp, const uint_t maxsize)
	{
	size_t slen = strlen(*sp);
	uint_t len;

	if (slen > maxsize)
	return (FALSE);

	len = slen;

	if (!xdrmem_enc_uint(xdrs, &len))
	return (FALSE);

	return (xdrmem_enc_bytes(xdrs, *sp, len));
	}

	static bool_t
	xdr_dec_string(XDR xdrs, char *sp, const uint_t maxsize)
	{
	uint_t size;
	bool_t alloc = FALSE;

	if (!xdrmem_dec_uint(xdrs, &size))
	return (FALSE);

	if (size > maxsize \|\| size > UINT_MAX - 1)
	return (FALSE);

	/*
	* Solaris man page: "If *sp is NULL when decoding, xdr_string()
	* allocates memory and *sp points to it".
	*/
	if (*sp == NULL) {
	BUILD_BUG_ON(sizeof (uint_t) > sizeof (size_t));

	*sp = kmem_alloc(size + 1, KM_NOSLEEP);
	if (*sp == NULL)
	return (FALSE);

	alloc = TRUE;
	}

	if (!xdrmem_dec_bytes(xdrs, *sp, size))
	goto fail;

	if (memchr(*sp, 0, size) != NULL)
	goto fail;

	(*sp)[size] = '\0';

	return (TRUE);

	fail:
	if (alloc)
	kmem_free(*sp, size + 1);

	return (FALSE);
	}

	static const struct xdr_ops xdrmem_encode_ops = {
	.xdr_control = xdrmem_control,
	.xdr_char = xdrmem_enc_char,
	.xdr_u_short = xdrmem_enc_ushort,
	.xdr_u_int = xdrmem_enc_uint,
	.xdr_u_longlong_t = xdrmem_enc_ulonglong,
	.xdr_opaque = xdrmem_enc_bytes,
	.xdr_string = xdr_enc_string,
	.xdr_array = xdr_enc_array
	};

	static const struct xdr_ops xdrmem_decode_ops = {
	.xdr_control = xdrmem_control,
	.xdr_char = xdrmem_dec_char,
	.xdr_u_short = xdrmem_dec_ushort,
	.xdr_u_int = xdrmem_dec_uint,
	.xdr_u_longlong_t = xdrmem_dec_ulonglong,
	.xdr_opaque = xdrmem_dec_bytes,
	.xdr_string = xdr_dec_string,
	.xdr_array = xdr_dec_array
	};
	diff --git a/sys/contrib/openzfs/module/os/linux/zfs/abd_os.c b/sys/contrib/openzfs/module/os/linux/zfs/abd_os.c
	index 24390fbbf125..d3255dcbc0f7 100644
	--- a/sys/contrib/openzfs/module/os/linux/zfs/abd_os.c
	+++ b/sys/contrib/openzfs/module/os/linux/zfs/abd_os.c
	@@ -1,1166 +1,1273 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2014 by Chunwei Chen. All rights reserved.
	* Copyright (c) 2019 by Delphix. All rights reserved.
	+ * Copyright (c) 2023, 2024, Klara Inc.
	*/

	/*
	* See abd.c for a general overview of the arc buffered data (ABD).
	*
	* Linear buffers act exactly like normal buffers and are always mapped into the
	* kernel's virtual memory space, while scattered ABD data chunks are allocated
	* as physical pages and then mapped in only while they are actually being
	* accessed through one of the abd_* library functions. Using scattered ABDs
	* provides several benefits:
	*
	* (1) They avoid use of kmem_*, preventing performance problems where running
	* kmem_reap on very large memory systems never finishes and causes
	* constant TLB shootdowns.
	*
	* (2) Fragmentation is less of an issue since when we are at the limit of
	* allocatable space, we won't have to search around for a long free
	* hole in the VA space for large ARC allocations. Each chunk is mapped in
	* individually, so even if we are using HIGHMEM (see next point) we
	* wouldn't need to worry about finding a contiguous address range.
	*
	* (3) If we are not using HIGHMEM, then all physical memory is always
	* mapped into the kernel's address space, so we also avoid the map /
	* unmap costs on each ABD access.
	*
	* If we are not using HIGHMEM, scattered buffers which have only one chunk
	* can be treated as linear buffers, because they are contiguous in the
	* kernel's virtual address space. See abd_alloc_chunks() for details.
	*/

	#include <sys/abd_impl.h>
	#include <sys/param.h>
	#include <sys/zio.h>
	#include <sys/arc.h>
	#include <sys/zfs_context.h>
	#include <sys/zfs_znode.h>
	#ifdef _KERNEL
	#include <linux/kmap_compat.h>
	+#include <linux/mm_compat.h>
	#include <linux/scatterlist.h>
	+#include <linux/version.h>
	#endif

	#ifdef _KERNEL
	#if defined(MAX_ORDER)
	#define ABD_MAX_ORDER (MAX_ORDER)
	#elif defined(MAX_PAGE_ORDER)
	#define ABD_MAX_ORDER (MAX_PAGE_ORDER)
	#endif
	#else
	#define ABD_MAX_ORDER (1)
	#endif

	typedef struct abd_stats {
	kstat_named_t abdstat_struct_size;
	kstat_named_t abdstat_linear_cnt;
	kstat_named_t abdstat_linear_data_size;
	kstat_named_t abdstat_scatter_cnt;
	kstat_named_t abdstat_scatter_data_size;
	kstat_named_t abdstat_scatter_chunk_waste;
	kstat_named_t abdstat_scatter_orders[ABD_MAX_ORDER];
	kstat_named_t abdstat_scatter_page_multi_chunk;
	kstat_named_t abdstat_scatter_page_multi_zone;
	kstat_named_t abdstat_scatter_page_alloc_retry;
	kstat_named_t abdstat_scatter_sg_table_retry;
	} abd_stats_t;

	static abd_stats_t abd_stats = {
	/* Amount of memory occupied by all of the abd_t struct allocations */
	{ "struct_size", KSTAT_DATA_UINT64 },
	/*
	* The number of linear ABDs which are currently allocated, excluding
	* ABDs which don't own their data (for instance the ones which were
	* allocated through abd_get_offset() and abd_get_from_buf()). If an
	* ABD takes ownership of its buf then it will become tracked.
	*/
	{ "linear_cnt", KSTAT_DATA_UINT64 },
	/* Amount of data stored in all linear ABDs tracked by linear_cnt */
	{ "linear_data_size", KSTAT_DATA_UINT64 },
	/*
	* The number of scatter ABDs which are currently allocated, excluding
	* ABDs which don't own their data (for instance the ones which were
	* allocated through abd_get_offset()).
	*/
	{ "scatter_cnt", KSTAT_DATA_UINT64 },
	/* Amount of data stored in all scatter ABDs tracked by scatter_cnt */
	{ "scatter_data_size", KSTAT_DATA_UINT64 },
	/*
	* The amount of space wasted at the end of the last chunk across all
	* scatter ABDs tracked by scatter_cnt.
	*/
	{ "scatter_chunk_waste", KSTAT_DATA_UINT64 },
	/*
	* The number of compound allocations of a given order. These
	* allocations are spread over all currently allocated ABDs, and
	* act as a measure of memory fragmentation.
	*/
	{ { "scatter_order_N", KSTAT_DATA_UINT64 } },
	/*
	* The number of scatter ABDs which contain multiple chunks.
	* ABDs are preferentially allocated from the minimum number of
	* contiguous multi-page chunks, a single chunk is optimal.
	*/
	{ "scatter_page_multi_chunk", KSTAT_DATA_UINT64 },
	/*
	* The number of scatter ABDs which are split across memory zones.
	* ABDs are preferentially allocated using pages from a single zone.
	*/
	{ "scatter_page_multi_zone", KSTAT_DATA_UINT64 },
	/*
	* The total number of retries encountered when attempting to
	* allocate the pages to populate the scatter ABD.
	*/
	{ "scatter_page_alloc_retry", KSTAT_DATA_UINT64 },
	/*
	* The total number of retries encountered when attempting to
	* allocate the sg table for an ABD.
	*/
	{ "scatter_sg_table_retry", KSTAT_DATA_UINT64 },
	};

	static struct {
	wmsum_t abdstat_struct_size;
	wmsum_t abdstat_linear_cnt;
	wmsum_t abdstat_linear_data_size;
	wmsum_t abdstat_scatter_cnt;
	wmsum_t abdstat_scatter_data_size;
	wmsum_t abdstat_scatter_chunk_waste;
	wmsum_t abdstat_scatter_orders[ABD_MAX_ORDER];
	wmsum_t abdstat_scatter_page_multi_chunk;
	wmsum_t abdstat_scatter_page_multi_zone;
	wmsum_t abdstat_scatter_page_alloc_retry;
	wmsum_t abdstat_scatter_sg_table_retry;
	} abd_sums;

	#define abd_for_each_sg(abd, sg, n, i) \
	for_each_sg(ABD_SCATTER(abd).abd_sgl, sg, n, i)

	/*
	* zfs_abd_scatter_min_size is the minimum allocation size to use scatter
	* ABD's. Smaller allocations will use linear ABD's which uses
	* zio_[data_]buf_alloc().
	*
	* Scatter ABD's use at least one page each, so sub-page allocations waste
	* some space when allocated as scatter (e.g. 2KB scatter allocation wastes
	* half of each page). Using linear ABD's for small allocations means that
	* they will be put on slabs which contain many allocations. This can
	* improve memory efficiency, but it also makes it much harder for ARC
	* evictions to actually free pages, because all the buffers on one slab need
	* to be freed in order for the slab (and underlying pages) to be freed.
	* Typically, 512B and 1KB kmem caches have 16 buffers per slab, so it's
	* possible for them to actually waste more memory than scatter (one page per
	* buf = wasting 3/4 or 7/8th; one buf per slab = wasting 15/16th).
	*
	* Spill blocks are typically 512B and are heavily used on systems running
	* selinux with the default dnode size and the `xattr=sa` property set.
	*
	* By default we use linear allocations for 512B and 1KB, and scatter
	* allocations for larger (1.5KB and up).
	*/
	static int zfs_abd_scatter_min_size = 512 * 3;

	/*
	* We use a scattered SPA_MAXBLOCKSIZE sized ABD whose pages are
	* just a single zero'd page. This allows us to conserve memory by
	* only using a single zero page for the scatterlist.
	*/
	abd_t *abd_zero_scatter = NULL;

	struct page;
	/*
	* _KERNEL - Will point to ZERO_PAGE if it is available or it will be
	* an allocated zero'd PAGESIZE buffer.
	* Userspace - Will be an allocated zero'ed PAGESIZE buffer.
	*
	* abd_zero_page is assigned to each of the pages of abd_zero_scatter.
	*/
	static struct page *abd_zero_page = NULL;

	static kmem_cache_t *abd_cache = NULL;
	static kstat_t *abd_ksp;

	static uint_t
	abd_chunkcnt_for_bytes(size_t size)
	{
	return (P2ROUNDUP(size, PAGESIZE) / PAGESIZE);
	}

	abd_t *
	abd_alloc_struct_impl(size_t size)
	{
	/*
	* In Linux we do not use the size passed in during ABD
	* allocation, so we just ignore it.
	*/
	(void) size;
	abd_t *abd = kmem_cache_alloc(abd_cache, KM_PUSHPAGE);
	ASSERT3P(abd, !=, NULL);
	ABDSTAT_INCR(abdstat_struct_size, sizeof (abd_t));

	return (abd);
	}

	void
	abd_free_struct_impl(abd_t *abd)
	{
	kmem_cache_free(abd_cache, abd);
	ABDSTAT_INCR(abdstat_struct_size, -(int)sizeof (abd_t));
	}

	#ifdef _KERNEL
	static unsigned zfs_abd_scatter_max_order = ABD_MAX_ORDER - 1;

	/*
	* Mark zfs data pages so they can be excluded from kernel crash dumps
	*/
	#ifdef _LP64
	#define ABD_FILE_CACHE_PAGE 0x2F5ABDF11ECAC4E

	static inline void
	abd_mark_zfs_page(struct page *page)
	{
	get_page(page);
	SetPagePrivate(page);
	set_page_private(page, ABD_FILE_CACHE_PAGE);
	}

	static inline void
	abd_unmark_zfs_page(struct page *page)
	{
	set_page_private(page, 0UL);
	ClearPagePrivate(page);
	put_page(page);
	}
	#else
	#define abd_mark_zfs_page(page)
	#define abd_unmark_zfs_page(page)
	#endif /* _LP64 */

	#ifndef CONFIG_HIGHMEM

	#ifndef __GFP_RECLAIM
	#define __GFP_RECLAIM __GFP_WAIT
	#endif

	/*
	* The goal is to minimize fragmentation by preferentially populating ABDs
	* with higher order compound pages from a single zone. Allocation size is
	* progressively decreased until it can be satisfied without performing
	* reclaim or compaction. When necessary this function will degenerate to
	* allocating individual pages and allowing reclaim to satisfy allocations.
	*/
	void
	abd_alloc_chunks(abd_t *abd, size_t size)
	{
	struct list_head pages;
	struct sg_table table;
	struct scatterlist *sg;
	struct page page, tmp_page = NULL;
	gfp_t gfp = __GFP_NOWARN \| GFP_NOIO;
	gfp_t gfp_comp = (gfp \| __GFP_NORETRY \| __GFP_COMP) & ~__GFP_RECLAIM;
	unsigned int max_order = MIN(zfs_abd_scatter_max_order,
	ABD_MAX_ORDER - 1);
	unsigned int nr_pages = abd_chunkcnt_for_bytes(size);
	unsigned int chunks = 0, zones = 0;
	size_t remaining_size;
	int nid = NUMA_NO_NODE;
	unsigned int alloc_pages = 0;

	INIT_LIST_HEAD(&pages);

	ASSERT3U(alloc_pages, <, nr_pages);

	while (alloc_pages < nr_pages) {
	unsigned int chunk_pages;
	unsigned int order;

	order = MIN(highbit64(nr_pages - alloc_pages) - 1, max_order);
	chunk_pages = (1U << order);

	page = alloc_pages_node(nid, order ? gfp_comp : gfp, order);
	if (page == NULL) {
	if (order == 0) {
	ABDSTAT_BUMP(abdstat_scatter_page_alloc_retry);
	schedule_timeout_interruptible(1);
	} else {
	max_order = MAX(0, order - 1);
	}
	continue;
	}

	list_add_tail(&page->lru, &pages);

	if ((nid != NUMA_NO_NODE) && (page_to_nid(page) != nid))
	zones++;

	nid = page_to_nid(page);
	ABDSTAT_BUMP(abdstat_scatter_orders[order]);
	chunks++;
	alloc_pages += chunk_pages;
	}

	ASSERT3S(alloc_pages, ==, nr_pages);

	while (sg_alloc_table(&table, chunks, gfp)) {
	ABDSTAT_BUMP(abdstat_scatter_sg_table_retry);
	schedule_timeout_interruptible(1);
	}

	sg = table.sgl;
	remaining_size = size;
	list_for_each_entry_safe(page, tmp_page, &pages, lru) {
	size_t sg_size = MIN(PAGESIZE << compound_order(page),
	remaining_size);
	sg_set_page(sg, page, sg_size, 0);
	abd_mark_zfs_page(page);
	remaining_size -= sg_size;

	sg = sg_next(sg);
	list_del(&page->lru);
	}

	/*
	* These conditions ensure that a possible transformation to a linear
	* ABD would be valid.
	*/
	ASSERT(!PageHighMem(sg_page(table.sgl)));
	ASSERT0(ABD_SCATTER(abd).abd_offset);

	if (table.nents == 1) {
	/*
	* Since there is only one entry, this ABD can be represented
	* as a linear buffer. All single-page (4K) ABD's can be
	* represented this way. Some multi-page ABD's can also be
	* represented this way, if we were able to allocate a single
	* "chunk" (higher-order "page" which represents a power-of-2
	* series of physically-contiguous pages). This is often the
	* case for 2-page (8K) ABD's.
	*
	* Representing a single-entry scatter ABD as a linear ABD
	* has the performance advantage of avoiding the copy (and
	* allocation) in abd_borrow_buf_copy / abd_return_buf_copy.
	* A performance increase of around 5% has been observed for
	* ARC-cached reads (of small blocks which can take advantage
	* of this).
	*
	* Note that this optimization is only possible because the
	* pages are always mapped into the kernel's address space.
	* This is not the case for highmem pages, so the
	* optimization can not be made there.
	*/
	abd->abd_flags \|= ABD_FLAG_LINEAR;
	abd->abd_flags \|= ABD_FLAG_LINEAR_PAGE;
	abd->abd_u.abd_linear.abd_sgl = table.sgl;
	ABD_LINEAR_BUF(abd) = page_address(sg_page(table.sgl));
	} else if (table.nents > 1) {
	ABDSTAT_BUMP(abdstat_scatter_page_multi_chunk);
	abd->abd_flags \|= ABD_FLAG_MULTI_CHUNK;

	if (zones) {
	ABDSTAT_BUMP(abdstat_scatter_page_multi_zone);
	abd->abd_flags \|= ABD_FLAG_MULTI_ZONE;
	}

	ABD_SCATTER(abd).abd_sgl = table.sgl;
	ABD_SCATTER(abd).abd_nents = table.nents;
	}
	}
	#else

	/*
	* Allocate N individual pages to construct a scatter ABD. This function
	* makes no attempt to request contiguous pages and requires the minimal
	* number of kernel interfaces. It's designed for maximum compatibility.
	*/
	void
	abd_alloc_chunks(abd_t *abd, size_t size)
	{
	struct scatterlist *sg = NULL;
	struct sg_table table;
	struct page *page;
	gfp_t gfp = __GFP_NOWARN \| GFP_NOIO;
	int nr_pages = abd_chunkcnt_for_bytes(size);
	int i = 0;

	while (sg_alloc_table(&table, nr_pages, gfp)) {
	ABDSTAT_BUMP(abdstat_scatter_sg_table_retry);
	schedule_timeout_interruptible(1);
	}

	ASSERT3U(table.nents, ==, nr_pages);
	ABD_SCATTER(abd).abd_sgl = table.sgl;
	ABD_SCATTER(abd).abd_nents = nr_pages;

	abd_for_each_sg(abd, sg, nr_pages, i) {
	while ((page = __page_cache_alloc(gfp)) == NULL) {
	ABDSTAT_BUMP(abdstat_scatter_page_alloc_retry);
	schedule_timeout_interruptible(1);
	}

	ABDSTAT_BUMP(abdstat_scatter_orders[0]);
	sg_set_page(sg, page, PAGESIZE, 0);
	abd_mark_zfs_page(page);
	}

	if (nr_pages > 1) {
	ABDSTAT_BUMP(abdstat_scatter_page_multi_chunk);
	abd->abd_flags \|= ABD_FLAG_MULTI_CHUNK;
	}
	}
	#endif /* !CONFIG_HIGHMEM */

	/*
	* This must be called if any of the sg_table allocation functions
	* are called.
	*/
	static void
	abd_free_sg_table(abd_t *abd)
	{
	struct sg_table table;

	table.sgl = ABD_SCATTER(abd).abd_sgl;
	table.nents = table.orig_nents = ABD_SCATTER(abd).abd_nents;
	sg_free_table(&table);
	}

	void
	abd_free_chunks(abd_t *abd)
	{
	struct scatterlist *sg = NULL;
	struct page *page;
	int nr_pages = ABD_SCATTER(abd).abd_nents;
	int order, i = 0;

	if (abd->abd_flags & ABD_FLAG_MULTI_ZONE)
	ABDSTAT_BUMPDOWN(abdstat_scatter_page_multi_zone);

	if (abd->abd_flags & ABD_FLAG_MULTI_CHUNK)
	ABDSTAT_BUMPDOWN(abdstat_scatter_page_multi_chunk);

	abd_for_each_sg(abd, sg, nr_pages, i) {
	page = sg_page(sg);
	abd_unmark_zfs_page(page);
	order = compound_order(page);
	__free_pages(page, order);
	ASSERT3U(sg->length, <=, PAGE_SIZE << order);
	ABDSTAT_BUMPDOWN(abdstat_scatter_orders[order]);
	}
	abd_free_sg_table(abd);
	}

	/*
	* Allocate scatter ABD of size SPA_MAXBLOCKSIZE, where each page in
	* the scatterlist will be set to the zero'd out buffer abd_zero_page.
	*/
	static void
	abd_alloc_zero_scatter(void)
	{
	struct scatterlist *sg = NULL;
	struct sg_table table;
	gfp_t gfp = __GFP_NOWARN \| GFP_NOIO;
	int nr_pages = abd_chunkcnt_for_bytes(SPA_MAXBLOCKSIZE);
	int i = 0;

	#if defined(HAVE_ZERO_PAGE_GPL_ONLY)
	gfp_t gfp_zero_page = gfp \| __GFP_ZERO;
	while ((abd_zero_page = __page_cache_alloc(gfp_zero_page)) == NULL) {
	ABDSTAT_BUMP(abdstat_scatter_page_alloc_retry);
	schedule_timeout_interruptible(1);
	}
	abd_mark_zfs_page(abd_zero_page);
	#else
	abd_zero_page = ZERO_PAGE(0);
	#endif /* HAVE_ZERO_PAGE_GPL_ONLY */

	while (sg_alloc_table(&table, nr_pages, gfp)) {
	ABDSTAT_BUMP(abdstat_scatter_sg_table_retry);
	schedule_timeout_interruptible(1);
	}
	ASSERT3U(table.nents, ==, nr_pages);

	abd_zero_scatter = abd_alloc_struct(SPA_MAXBLOCKSIZE);
	abd_zero_scatter->abd_flags \|= ABD_FLAG_OWNER;
	ABD_SCATTER(abd_zero_scatter).abd_offset = 0;
	ABD_SCATTER(abd_zero_scatter).abd_sgl = table.sgl;
	ABD_SCATTER(abd_zero_scatter).abd_nents = nr_pages;
	abd_zero_scatter->abd_size = SPA_MAXBLOCKSIZE;
	abd_zero_scatter->abd_flags \|= ABD_FLAG_MULTI_CHUNK \| ABD_FLAG_ZEROS;

	abd_for_each_sg(abd_zero_scatter, sg, nr_pages, i) {
	sg_set_page(sg, abd_zero_page, PAGESIZE, 0);
	}

	ABDSTAT_BUMP(abdstat_scatter_cnt);
	ABDSTAT_INCR(abdstat_scatter_data_size, PAGESIZE);
	ABDSTAT_BUMP(abdstat_scatter_page_multi_chunk);
	}

	#else /* _KERNEL */

	#ifndef PAGE_SHIFT
	#define PAGE_SHIFT (highbit64(PAGESIZE)-1)
	#endif

	#define zfs_kmap_atomic(chunk) ((void *)chunk)
	#define zfs_kunmap_atomic(addr) do { (void)(addr); } while (0)
	#define local_irq_save(flags) do { (void)(flags); } while (0)
	#define local_irq_restore(flags) do { (void)(flags); } while (0)
	#define nth_page(pg, i) \
	((struct page )((void )(pg) + (i) * PAGESIZE))

	struct scatterlist {
	struct page *page;
	int length;
	int end;
	};

	static void
	sg_init_table(struct scatterlist *sg, int nr)
	{
	memset(sg, 0, nr * sizeof (struct scatterlist));
	sg[nr - 1].end = 1;
	}

	/*
	* This must be called if any of the sg_table allocation functions
	* are called.
	*/
	static void
	abd_free_sg_table(abd_t *abd)
	{
	int nents = ABD_SCATTER(abd).abd_nents;
	vmem_free(ABD_SCATTER(abd).abd_sgl,
	nents * sizeof (struct scatterlist));
	}

	#define for_each_sg(sgl, sg, nr, i) \
	for ((i) = 0, (sg) = (sgl); (i) < (nr); (i)++, (sg) = sg_next(sg))

	static inline void
	sg_set_page(struct scatterlist sg, struct page page, unsigned int len,
	unsigned int offset)
	{
	/* currently we don't use offset */
	ASSERT(offset == 0);
	sg->page = page;
	sg->length = len;
	}

	static inline struct page *
	sg_page(struct scatterlist *sg)
	{
	return (sg->page);
	}

	static inline struct scatterlist *
	sg_next(struct scatterlist *sg)
	{
	if (sg->end)
	return (NULL);

	return (sg + 1);
	}

	void
	abd_alloc_chunks(abd_t *abd, size_t size)
	{
	unsigned nr_pages = abd_chunkcnt_for_bytes(size);
	struct scatterlist *sg;
	int i;

	ABD_SCATTER(abd).abd_sgl = vmem_alloc(nr_pages *
	sizeof (struct scatterlist), KM_SLEEP);
	sg_init_table(ABD_SCATTER(abd).abd_sgl, nr_pages);

	abd_for_each_sg(abd, sg, nr_pages, i) {
	struct page *p = umem_alloc_aligned(PAGESIZE, 64, KM_SLEEP);
	sg_set_page(sg, p, PAGESIZE, 0);
	}
	ABD_SCATTER(abd).abd_nents = nr_pages;
	}

	void
	abd_free_chunks(abd_t *abd)
	{
	int i, n = ABD_SCATTER(abd).abd_nents;
	struct scatterlist *sg;

	abd_for_each_sg(abd, sg, n, i) {
	struct page *p = nth_page(sg_page(sg), 0);
	umem_free_aligned(p, PAGESIZE);
	}
	abd_free_sg_table(abd);
	}

	static void
	abd_alloc_zero_scatter(void)
	{
	unsigned nr_pages = abd_chunkcnt_for_bytes(SPA_MAXBLOCKSIZE);
	struct scatterlist *sg;
	int i;

	abd_zero_page = umem_alloc_aligned(PAGESIZE, 64, KM_SLEEP);
	memset(abd_zero_page, 0, PAGESIZE);
	abd_zero_scatter = abd_alloc_struct(SPA_MAXBLOCKSIZE);
	abd_zero_scatter->abd_flags \|= ABD_FLAG_OWNER;
	abd_zero_scatter->abd_flags \|= ABD_FLAG_MULTI_CHUNK \| ABD_FLAG_ZEROS;
	ABD_SCATTER(abd_zero_scatter).abd_offset = 0;
	ABD_SCATTER(abd_zero_scatter).abd_nents = nr_pages;
	abd_zero_scatter->abd_size = SPA_MAXBLOCKSIZE;
	ABD_SCATTER(abd_zero_scatter).abd_sgl = vmem_alloc(nr_pages *
	sizeof (struct scatterlist), KM_SLEEP);

	sg_init_table(ABD_SCATTER(abd_zero_scatter).abd_sgl, nr_pages);

	abd_for_each_sg(abd_zero_scatter, sg, nr_pages, i) {
	sg_set_page(sg, abd_zero_page, PAGESIZE, 0);
	}

	ABDSTAT_BUMP(abdstat_scatter_cnt);
	ABDSTAT_INCR(abdstat_scatter_data_size, PAGESIZE);
	ABDSTAT_BUMP(abdstat_scatter_page_multi_chunk);
	}

	#endif /* _KERNEL */

	boolean_t
	abd_size_alloc_linear(size_t size)
	{
	return (!zfs_abd_scatter_enabled \|\| size < zfs_abd_scatter_min_size);
	}

	void
	abd_update_scatter_stats(abd_t *abd, abd_stats_op_t op)
	{
	ASSERT(op == ABDSTAT_INCR \|\| op == ABDSTAT_DECR);
	int waste = P2ROUNDUP(abd->abd_size, PAGESIZE) - abd->abd_size;
	if (op == ABDSTAT_INCR) {
	ABDSTAT_BUMP(abdstat_scatter_cnt);
	ABDSTAT_INCR(abdstat_scatter_data_size, abd->abd_size);
	ABDSTAT_INCR(abdstat_scatter_chunk_waste, waste);
	arc_space_consume(waste, ARC_SPACE_ABD_CHUNK_WASTE);
	} else {
	ABDSTAT_BUMPDOWN(abdstat_scatter_cnt);
	ABDSTAT_INCR(abdstat_scatter_data_size, -(int)abd->abd_size);
	ABDSTAT_INCR(abdstat_scatter_chunk_waste, -waste);
	arc_space_return(waste, ARC_SPACE_ABD_CHUNK_WASTE);
	}
	}

	void
	abd_update_linear_stats(abd_t *abd, abd_stats_op_t op)
	{
	ASSERT(op == ABDSTAT_INCR \|\| op == ABDSTAT_DECR);
	if (op == ABDSTAT_INCR) {
	ABDSTAT_BUMP(abdstat_linear_cnt);
	ABDSTAT_INCR(abdstat_linear_data_size, abd->abd_size);
	} else {
	ABDSTAT_BUMPDOWN(abdstat_linear_cnt);
	ABDSTAT_INCR(abdstat_linear_data_size, -(int)abd->abd_size);
	}
	}

	void
	abd_verify_scatter(abd_t *abd)
	{
	size_t n;
	int i = 0;
	struct scatterlist *sg = NULL;

	ASSERT3U(ABD_SCATTER(abd).abd_nents, >, 0);
	ASSERT3U(ABD_SCATTER(abd).abd_offset, <,
	ABD_SCATTER(abd).abd_sgl->length);
	n = ABD_SCATTER(abd).abd_nents;
	abd_for_each_sg(abd, sg, n, i) {
	ASSERT3P(sg_page(sg), !=, NULL);
	}
	}

	static void
	abd_free_zero_scatter(void)
	{
	ABDSTAT_BUMPDOWN(abdstat_scatter_cnt);
	ABDSTAT_INCR(abdstat_scatter_data_size, -(int)PAGESIZE);
	ABDSTAT_BUMPDOWN(abdstat_scatter_page_multi_chunk);

	abd_free_sg_table(abd_zero_scatter);
	abd_free_struct(abd_zero_scatter);
	abd_zero_scatter = NULL;
	ASSERT3P(abd_zero_page, !=, NULL);
	#if defined(_KERNEL)
	#if defined(HAVE_ZERO_PAGE_GPL_ONLY)
	abd_unmark_zfs_page(abd_zero_page);
	__free_page(abd_zero_page);
	#endif /* HAVE_ZERO_PAGE_GPL_ONLY */
	#else
	umem_free_aligned(abd_zero_page, PAGESIZE);
	#endif /* _KERNEL */
	}

	static int
	abd_kstats_update(kstat_t *ksp, int rw)
	{
	abd_stats_t *as = ksp->ks_data;

	if (rw == KSTAT_WRITE)
	return (EACCES);
	as->abdstat_struct_size.value.ui64 =
	wmsum_value(&abd_sums.abdstat_struct_size);
	as->abdstat_linear_cnt.value.ui64 =
	wmsum_value(&abd_sums.abdstat_linear_cnt);
	as->abdstat_linear_data_size.value.ui64 =
	wmsum_value(&abd_sums.abdstat_linear_data_size);
	as->abdstat_scatter_cnt.value.ui64 =
	wmsum_value(&abd_sums.abdstat_scatter_cnt);
	as->abdstat_scatter_data_size.value.ui64 =
	wmsum_value(&abd_sums.abdstat_scatter_data_size);
	as->abdstat_scatter_chunk_waste.value.ui64 =
	wmsum_value(&abd_sums.abdstat_scatter_chunk_waste);
	for (int i = 0; i < ABD_MAX_ORDER; i++) {
	as->abdstat_scatter_orders[i].value.ui64 =
	wmsum_value(&abd_sums.abdstat_scatter_orders[i]);
	}
	as->abdstat_scatter_page_multi_chunk.value.ui64 =
	wmsum_value(&abd_sums.abdstat_scatter_page_multi_chunk);
	as->abdstat_scatter_page_multi_zone.value.ui64 =
	wmsum_value(&abd_sums.abdstat_scatter_page_multi_zone);
	as->abdstat_scatter_page_alloc_retry.value.ui64 =
	wmsum_value(&abd_sums.abdstat_scatter_page_alloc_retry);
	as->abdstat_scatter_sg_table_retry.value.ui64 =
	wmsum_value(&abd_sums.abdstat_scatter_sg_table_retry);
	return (0);
	}

	void
	abd_init(void)
	{
	int i;

	abd_cache = kmem_cache_create("abd_t", sizeof (abd_t),
	0, NULL, NULL, NULL, NULL, NULL, 0);

	wmsum_init(&abd_sums.abdstat_struct_size, 0);
	wmsum_init(&abd_sums.abdstat_linear_cnt, 0);
	wmsum_init(&abd_sums.abdstat_linear_data_size, 0);
	wmsum_init(&abd_sums.abdstat_scatter_cnt, 0);
	wmsum_init(&abd_sums.abdstat_scatter_data_size, 0);
	wmsum_init(&abd_sums.abdstat_scatter_chunk_waste, 0);
	for (i = 0; i < ABD_MAX_ORDER; i++)
	wmsum_init(&abd_sums.abdstat_scatter_orders[i], 0);
	wmsum_init(&abd_sums.abdstat_scatter_page_multi_chunk, 0);
	wmsum_init(&abd_sums.abdstat_scatter_page_multi_zone, 0);
	wmsum_init(&abd_sums.abdstat_scatter_page_alloc_retry, 0);
	wmsum_init(&abd_sums.abdstat_scatter_sg_table_retry, 0);

	abd_ksp = kstat_create("zfs", 0, "abdstats", "misc", KSTAT_TYPE_NAMED,
	sizeof (abd_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
	if (abd_ksp != NULL) {
	for (i = 0; i < ABD_MAX_ORDER; i++) {
	snprintf(abd_stats.abdstat_scatter_orders[i].name,
	KSTAT_STRLEN, "scatter_order_%d", i);
	abd_stats.abdstat_scatter_orders[i].data_type =
	KSTAT_DATA_UINT64;
	}
	abd_ksp->ks_data = &abd_stats;
	abd_ksp->ks_update = abd_kstats_update;
	kstat_install(abd_ksp);
	}

	abd_alloc_zero_scatter();
	}

	void
	abd_fini(void)
	{
	abd_free_zero_scatter();

	if (abd_ksp != NULL) {
	kstat_delete(abd_ksp);
	abd_ksp = NULL;
	}

	wmsum_fini(&abd_sums.abdstat_struct_size);
	wmsum_fini(&abd_sums.abdstat_linear_cnt);
	wmsum_fini(&abd_sums.abdstat_linear_data_size);
	wmsum_fini(&abd_sums.abdstat_scatter_cnt);
	wmsum_fini(&abd_sums.abdstat_scatter_data_size);
	wmsum_fini(&abd_sums.abdstat_scatter_chunk_waste);
	for (int i = 0; i < ABD_MAX_ORDER; i++)
	wmsum_fini(&abd_sums.abdstat_scatter_orders[i]);
	wmsum_fini(&abd_sums.abdstat_scatter_page_multi_chunk);
	wmsum_fini(&abd_sums.abdstat_scatter_page_multi_zone);
	wmsum_fini(&abd_sums.abdstat_scatter_page_alloc_retry);
	wmsum_fini(&abd_sums.abdstat_scatter_sg_table_retry);

	if (abd_cache) {
	kmem_cache_destroy(abd_cache);
	abd_cache = NULL;
	}
	}

	void
	abd_free_linear_page(abd_t *abd)
	{
	/* Transform it back into a scatter ABD for freeing */
	struct scatterlist *sg = abd->abd_u.abd_linear.abd_sgl;
	abd->abd_flags &= ~ABD_FLAG_LINEAR;
	abd->abd_flags &= ~ABD_FLAG_LINEAR_PAGE;
	ABD_SCATTER(abd).abd_nents = 1;
	ABD_SCATTER(abd).abd_offset = 0;
	ABD_SCATTER(abd).abd_sgl = sg;
	abd_free_chunks(abd);

	abd_update_scatter_stats(abd, ABDSTAT_DECR);
	}

	/*
	* If we're going to use this ABD for doing I/O using the block layer, the
	* consumer of the ABD data doesn't care if it's scattered or not, and we don't
	* plan to store this ABD in memory for a long period of time, we should
	* allocate the ABD type that requires the least data copying to do the I/O.
	*
	* On Linux the optimal thing to do would be to use abd_get_offset() and
	* construct a new ABD which shares the original pages thereby eliminating
	* the copy. But for the moment a new linear ABD is allocated until this
	* performance optimization can be implemented.
	*/
	abd_t *
	abd_alloc_for_io(size_t size, boolean_t is_metadata)
	{
	return (abd_alloc(size, is_metadata));
	}

	abd_t *
	abd_get_offset_scatter(abd_t abd, abd_t sabd, size_t off,
	size_t size)
	{
	(void) size;
	int i = 0;
	struct scatterlist *sg = NULL;

	abd_verify(sabd);
	ASSERT3U(off, <=, sabd->abd_size);

	size_t new_offset = ABD_SCATTER(sabd).abd_offset + off;

	if (abd == NULL)
	abd = abd_alloc_struct(0);

	/*
	* Even if this buf is filesystem metadata, we only track that
	* if we own the underlying data buffer, which is not true in
	* this case. Therefore, we don't ever use ABD_FLAG_META here.
	*/

	abd_for_each_sg(sabd, sg, ABD_SCATTER(sabd).abd_nents, i) {
	if (new_offset < sg->length)
	break;
	new_offset -= sg->length;
	}

	ABD_SCATTER(abd).abd_sgl = sg;
	ABD_SCATTER(abd).abd_offset = new_offset;
	ABD_SCATTER(abd).abd_nents = ABD_SCATTER(sabd).abd_nents - i;

	return (abd);
	}

	/*
	* Initialize the abd_iter.
	*/
	void
	abd_iter_init(struct abd_iter aiter, abd_t abd)
	{
	ASSERT(!abd_is_gang(abd));
	abd_verify(abd);
	+ memset(aiter, 0, sizeof (struct abd_iter));
	aiter->iter_abd = abd;
	- aiter->iter_mapaddr = NULL;
	- aiter->iter_mapsize = 0;
	- aiter->iter_pos = 0;
	- if (abd_is_linear(abd)) {
	- aiter->iter_offset = 0;
	- aiter->iter_sg = NULL;
	- } else {
	+ if (!abd_is_linear(abd)) {
	aiter->iter_offset = ABD_SCATTER(abd).abd_offset;
	aiter->iter_sg = ABD_SCATTER(abd).abd_sgl;
	}
	}

	/*
	* This is just a helper function to see if we have exhausted the
	* abd_iter and reached the end.
	*/
	boolean_t
	abd_iter_at_end(struct abd_iter *aiter)
	{
	+ ASSERT3U(aiter->iter_pos, <=, aiter->iter_abd->abd_size);
	return (aiter->iter_pos == aiter->iter_abd->abd_size);
	}

	/*
	* Advance the iterator by a certain amount. Cannot be called when a chunk is
	* in use. This can be safely called when the aiter has already exhausted, in
	* which case this does nothing.
	*/
	void
	abd_iter_advance(struct abd_iter *aiter, size_t amount)
	{
	+ /*
	+ * Ensure that last chunk is not in use. abd_iterate_*() must clear
	+ * this state (directly or abd_iter_unmap()) before advancing.
	+ */
	ASSERT3P(aiter->iter_mapaddr, ==, NULL);
	ASSERT0(aiter->iter_mapsize);
	+ ASSERT3P(aiter->iter_page, ==, NULL);
	+ ASSERT0(aiter->iter_page_doff);
	+ ASSERT0(aiter->iter_page_dsize);

	/* There's nothing left to advance to, so do nothing */
	if (abd_iter_at_end(aiter))
	return;

	aiter->iter_pos += amount;
	aiter->iter_offset += amount;
	if (!abd_is_linear(aiter->iter_abd)) {
	while (aiter->iter_offset >= aiter->iter_sg->length) {
	aiter->iter_offset -= aiter->iter_sg->length;
	aiter->iter_sg = sg_next(aiter->iter_sg);
	if (aiter->iter_sg == NULL) {
	ASSERT0(aiter->iter_offset);
	break;
	}
	}
	}
	}

	/*
	* Map the current chunk into aiter. This can be safely called when the aiter
	* has already exhausted, in which case this does nothing.
	*/
	void
	abd_iter_map(struct abd_iter *aiter)
	{
	void *paddr;
	size_t offset = 0;

	ASSERT3P(aiter->iter_mapaddr, ==, NULL);
	ASSERT0(aiter->iter_mapsize);

	/* There's nothing left to iterate over, so do nothing */
	if (abd_iter_at_end(aiter))
	return;

	if (abd_is_linear(aiter->iter_abd)) {
	ASSERT3U(aiter->iter_pos, ==, aiter->iter_offset);
	offset = aiter->iter_offset;
	aiter->iter_mapsize = aiter->iter_abd->abd_size - offset;
	paddr = ABD_LINEAR_BUF(aiter->iter_abd);
	} else {
	offset = aiter->iter_offset;
	aiter->iter_mapsize = MIN(aiter->iter_sg->length - offset,
	aiter->iter_abd->abd_size - aiter->iter_pos);

	paddr = zfs_kmap_atomic(sg_page(aiter->iter_sg));
	}

	aiter->iter_mapaddr = (char *)paddr + offset;
	}

	/*
	* Unmap the current chunk from aiter. This can be safely called when the aiter
	* has already exhausted, in which case this does nothing.
	*/
	void
	abd_iter_unmap(struct abd_iter *aiter)
	{
	/* There's nothing left to unmap, so do nothing */
	if (abd_iter_at_end(aiter))
	return;

	if (!abd_is_linear(aiter->iter_abd)) {
	/* LINTED E_FUNC_SET_NOT_USED */
	zfs_kunmap_atomic(aiter->iter_mapaddr - aiter->iter_offset);
	}

	ASSERT3P(aiter->iter_mapaddr, !=, NULL);
	ASSERT3U(aiter->iter_mapsize, >, 0);

	aiter->iter_mapaddr = NULL;
	aiter->iter_mapsize = 0;
	}

	void
	abd_cache_reap_now(void)
	{
	}

	#if defined(_KERNEL)
	+/*
	+ * Yield the next page struct and data offset and size within it, without
	+ * mapping it into the address space.
	+ */
	+void
	+abd_iter_page(struct abd_iter *aiter)
	+{
	+ if (abd_iter_at_end(aiter)) {
	+ aiter->iter_page = NULL;
	+ aiter->iter_page_doff = 0;
	+ aiter->iter_page_dsize = 0;
	+ return;
	+ }
	+
	+ struct page *page;
	+ size_t doff, dsize;
	+
	+ if (abd_is_linear(aiter->iter_abd)) {
	+ ASSERT3U(aiter->iter_pos, ==, aiter->iter_offset);
	+
	+ /* memory address at iter_pos */
	+ void *paddr = ABD_LINEAR_BUF(aiter->iter_abd) + aiter->iter_pos;
	+
	+ /* struct page for address */
	+ page = is_vmalloc_addr(paddr) ?
	+ vmalloc_to_page(paddr) : virt_to_page(paddr);
	+
	+ /* offset of address within the page */
	+ doff = offset_in_page(paddr);
	+
	+ /* total data remaining in abd from this position */
	+ dsize = aiter->iter_abd->abd_size - aiter->iter_offset;
	+ } else {
	+ ASSERT(!abd_is_gang(aiter->iter_abd));
	+
	+ /* current scatter page */
	+ page = sg_page(aiter->iter_sg);
	+
	+ /* position within page */
	+ doff = aiter->iter_offset;
	+
	+ /* remaining data in scatterlist */
	+ dsize = MIN(aiter->iter_sg->length - aiter->iter_offset,
	+ aiter->iter_abd->abd_size - aiter->iter_pos);
	+ }
	+ ASSERT(page);
	+
	+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0)
	+ if (PageTail(page)) {
	+ /*
	+ * This page is part of a "compound page", which is a group of
	+ * pages that can be referenced from a single struct page *.
	+ * Its organised as a "head" page, followed by a series of
	+ * "tail" pages.
	+ *
	+ * In OpenZFS, compound pages are allocated using the
	+ * __GFP_COMP flag, which we get from scatter ABDs and SPL
	+ * vmalloc slabs (ie >16K allocations). So a great many of the
	+ * IO buffers we get are going to be of this type.
	+ *
	+ * The tail pages are just regular PAGE_SIZE pages, and can be
	+ * safely used as-is. However, the head page has length
	+ * covering itself and all the tail pages. If this ABD chunk
	+ * spans multiple pages, then we can use the head page and a
	+ * >PAGE_SIZE length, which is far more efficient.
	+ *
	+ * To do this, we need to adjust the offset to be counted from
	+ * the head page. struct page for compound pages are stored
	+ * contiguously, so we can just adjust by a simple offset.
	+ *
	+ * Before kernel 4.5, compound page heads were refcounted
	+ * separately, such that moving back to the head page would
	+ * require us to take a reference to it and releasing it once
	+ * we're completely finished with it. In practice, that means
	+ * when our caller is done with the ABD, which we have no
	+ * insight into from here. Rather than contort this API to
	+ * track head page references on such ancient kernels, we just
	+ * compile this block out and use the tail pages directly. This
	+ * is slightly less efficient, but makes everything far
	+ * simpler.
	+ */
	+ struct page *head = compound_head(page);
	+ doff += ((page - head) * PAGESIZE);
	+ page = head;
	+ }
	+#endif
	+
	+ /* final page and position within it */
	+ aiter->iter_page = page;
	+ aiter->iter_page_doff = doff;
	+
	+ /* amount of data in the chunk, up to the end of the page */
	+ aiter->iter_page_dsize = MIN(dsize, page_size(page) - doff);
	+}
	+
	+/*
	+ * Note: ABD BIO functions only needed to support vdev_classic. See comments in
	+ * vdev_disk.c.
	+ */
	+
	/*
	* bio_nr_pages for ABD.
	* @off is the offset in @abd
	*/
	unsigned long
	abd_nr_pages_off(abd_t *abd, unsigned int size, size_t off)
	{
	unsigned long pos;

	if (abd_is_gang(abd)) {
	unsigned long count = 0;

	for (abd_t *cabd = abd_gang_get_offset(abd, &off);
	cabd != NULL && size != 0;
	cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
	ASSERT3U(off, <, cabd->abd_size);
	int mysize = MIN(size, cabd->abd_size - off);
	count += abd_nr_pages_off(cabd, mysize, off);
	size -= mysize;
	off = 0;
	}
	return (count);
	}

	if (abd_is_linear(abd))
	pos = (unsigned long)abd_to_buf(abd) + off;
	else
	pos = ABD_SCATTER(abd).abd_offset + off;

	return (((pos + size + PAGESIZE - 1) >> PAGE_SHIFT) -
	(pos >> PAGE_SHIFT));
	}

	static unsigned int
	bio_map(struct bio bio, void buf_ptr, unsigned int bio_size)
	{
	unsigned int offset, size, i;
	struct page *page;

	offset = offset_in_page(buf_ptr);
	for (i = 0; i < bio->bi_max_vecs; i++) {
	size = PAGE_SIZE - offset;

	if (bio_size <= 0)
	break;

	if (size > bio_size)
	size = bio_size;

	if (is_vmalloc_addr(buf_ptr))
	page = vmalloc_to_page(buf_ptr);
	else
	page = virt_to_page(buf_ptr);

	/*
	* Some network related block device uses tcp_sendpage, which
	* doesn't behave well when using 0-count page, this is a
	* safety net to catch them.
	*/
	ASSERT3S(page_count(page), >, 0);

	if (bio_add_page(bio, page, size, offset) != size)
	break;

	buf_ptr += size;
	bio_size -= size;
	offset = 0;
	}

	return (bio_size);
	}

	/*
	* bio_map for gang ABD.
	*/
	static unsigned int
	abd_gang_bio_map_off(struct bio bio, abd_t abd,
	unsigned int io_size, size_t off)
	{
	ASSERT(abd_is_gang(abd));

	for (abd_t *cabd = abd_gang_get_offset(abd, &off);
	cabd != NULL;
	cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
	ASSERT3U(off, <, cabd->abd_size);
	int size = MIN(io_size, cabd->abd_size - off);
	int remainder = abd_bio_map_off(bio, cabd, size, off);
	io_size -= (size - remainder);
	if (io_size == 0 \|\| remainder > 0)
	return (io_size);
	off = 0;
	}
	ASSERT0(io_size);
	return (io_size);
	}

	/*
	* bio_map for ABD.
	* @off is the offset in @abd
	* Remaining IO size is returned
	*/
	unsigned int
	abd_bio_map_off(struct bio bio, abd_t abd,
	unsigned int io_size, size_t off)
	{
	struct abd_iter aiter;

	ASSERT3U(io_size, <=, abd->abd_size - off);
	if (abd_is_linear(abd))
	return (bio_map(bio, ((char *)abd_to_buf(abd)) + off, io_size));

	ASSERT(!abd_is_linear(abd));
	if (abd_is_gang(abd))
	return (abd_gang_bio_map_off(bio, abd, io_size, off));

	abd_iter_init(&aiter, abd);
	abd_iter_advance(&aiter, off);

	for (int i = 0; i < bio->bi_max_vecs; i++) {
	struct page *pg;
	size_t len, sgoff, pgoff;
	struct scatterlist *sg;

	if (io_size <= 0)
	break;

	sg = aiter.iter_sg;
	sgoff = aiter.iter_offset;
	pgoff = sgoff & (PAGESIZE - 1);
	len = MIN(io_size, PAGESIZE - pgoff);
	ASSERT(len > 0);

	pg = nth_page(sg_page(sg), sgoff >> PAGE_SHIFT);
	if (bio_add_page(bio, pg, len, pgoff) != len)
	break;

	io_size -= len;
	abd_iter_advance(&aiter, len);
	}

	return (io_size);
	}

	/* Tunable Parameters */
	module_param(zfs_abd_scatter_enabled, int, 0644);
	MODULE_PARM_DESC(zfs_abd_scatter_enabled,
	"Toggle whether ABD allocations must be linear.");
	module_param(zfs_abd_scatter_min_size, int, 0644);
	MODULE_PARM_DESC(zfs_abd_scatter_min_size,
	"Minimum size of scatter allocations.");
	/* CSTYLED */
	module_param(zfs_abd_scatter_max_order, uint, 0644);
	MODULE_PARM_DESC(zfs_abd_scatter_max_order,
	"Maximum order allocation used for a scatter ABD.");
	-#endif
	+
	+#endif /* _KERNEL */
	diff --git a/sys/contrib/openzfs/module/os/linux/zfs/vdev_disk.c b/sys/contrib/openzfs/module/os/linux/zfs/vdev_disk.c
	index b0bda5fa2012..943e534ef5b0 100644
	--- a/sys/contrib/openzfs/module/os/linux/zfs/vdev_disk.c
	+++ b/sys/contrib/openzfs/module/os/linux/zfs/vdev_disk.c
	@@ -1,1176 +1,1663 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC.
	* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
	* Rewritten for Linux by Brian Behlendorf <behlendorf1@llnl.gov>.
	* LLNL-CODE-403049.
	* Copyright (c) 2012, 2019 by Delphix. All rights reserved.
	+ * Copyright (c) 2023, 2024, Klara Inc.
	*/

	#include <sys/zfs_context.h>
	#include <sys/spa_impl.h>
	#include <sys/vdev_disk.h>
	#include <sys/vdev_impl.h>
	#include <sys/vdev_trim.h>
	#include <sys/abd.h>
	#include <sys/fs/zfs.h>
	#include <sys/zio.h>
	#include <linux/blkpg.h>
	#include <linux/msdos_fs.h>
	#include <linux/vfs_compat.h>
	#ifdef HAVE_LINUX_BLK_CGROUP_HEADER
	#include <linux/blk-cgroup.h>
	#endif

	/*
	* Linux 6.8.x uses a bdev_handle as an instance/refcount for an underlying
	* block_device. Since it carries the block_device inside, its convenient to
	- * just use the handle as a proxy. For pre-6.8, we just emulate this with
	- * a cast, since we don't need any of the other fields inside the handle.
	+ * just use the handle as a proxy.
	+ *
	+ * Linux 6.9.x uses a file for the same purpose.
	+ *
	+ * For pre-6.8, we just emulate this with a cast, since we don't need any of
	+ * the other fields inside the handle.
	*/
	-#ifdef HAVE_BDEV_OPEN_BY_PATH
	+#if defined(HAVE_BDEV_OPEN_BY_PATH)
	typedef struct bdev_handle zfs_bdev_handle_t;
	#define BDH_BDEV(bdh) ((bdh)->bdev)
	#define BDH_IS_ERR(bdh) (IS_ERR(bdh))
	#define BDH_PTR_ERR(bdh) (PTR_ERR(bdh))
	#define BDH_ERR_PTR(err) (ERR_PTR(err))
	+#elif defined(HAVE_BDEV_FILE_OPEN_BY_PATH)
	+typedef struct file zfs_bdev_handle_t;
	+#define BDH_BDEV(bdh) (file_bdev(bdh))
	+#define BDH_IS_ERR(bdh) (IS_ERR(bdh))
	+#define BDH_PTR_ERR(bdh) (PTR_ERR(bdh))
	+#define BDH_ERR_PTR(err) (ERR_PTR(err))
	#else
	typedef void zfs_bdev_handle_t;
	#define BDH_BDEV(bdh) ((struct block_device *)bdh)
	#define BDH_IS_ERR(bdh) (IS_ERR(BDH_BDEV(bdh)))
	#define BDH_PTR_ERR(bdh) (PTR_ERR(BDH_BDEV(bdh)))
	#define BDH_ERR_PTR(err) (ERR_PTR(err))
	#endif

	typedef struct vdev_disk {
	zfs_bdev_handle_t *vd_bdh;
	krwlock_t vd_lock;
	} vdev_disk_t;

	+/*
	+ * Maximum number of segments to add to a bio (min 4). If this is higher than
	+ * the maximum allowed by the device queue or the kernel itself, it will be
	+ * clamped. Setting it to zero will cause the kernel's ideal size to be used.
	+ */
	+uint_t zfs_vdev_disk_max_segs = 0;
	+
	/*
	* Unique identifier for the exclusive vdev holder.
	*/
	static void *zfs_vdev_holder = VDEV_HOLDER;

	/*
	* Wait up to zfs_vdev_open_timeout_ms milliseconds before determining the
	* device is missing. The missing path may be transient since the links
	* can be briefly removed and recreated in response to udev events.
	*/
	static uint_t zfs_vdev_open_timeout_ms = 1000;

	/*
	* Size of the "reserved" partition, in blocks.
	*/
	#define EFI_MIN_RESV_SIZE (16 * 1024)

	-/*
	- * Virtual device vector for disks.
	- */
	-typedef struct dio_request {
	- zio_t dr_zio; / Parent ZIO */
	- atomic_t dr_ref; /* References */
	- int dr_error; /* Bio error */
	- int dr_bio_count; /* Count of bio's */
	- struct bio dr_bio[]; / Attached bio's */
	-} dio_request_t;
	-
	/*
	* BIO request failfast mask.
	*/

	static unsigned int zfs_vdev_failfast_mask = 1;

	+/*
	+ * Convert SPA mode flags into bdev open mode flags.
	+ */
	#ifdef HAVE_BLK_MODE_T
	-static blk_mode_t
	+typedef blk_mode_t vdev_bdev_mode_t;
	+#define VDEV_BDEV_MODE_READ BLK_OPEN_READ
	+#define VDEV_BDEV_MODE_WRITE BLK_OPEN_WRITE
	+#define VDEV_BDEV_MODE_EXCL BLK_OPEN_EXCL
	+#define VDEV_BDEV_MODE_MASK (BLK_OPEN_READ\|BLK_OPEN_WRITE\|BLK_OPEN_EXCL)
	#else
	-static fmode_t
	+typedef fmode_t vdev_bdev_mode_t;
	+#define VDEV_BDEV_MODE_READ FMODE_READ
	+#define VDEV_BDEV_MODE_WRITE FMODE_WRITE
	+#define VDEV_BDEV_MODE_EXCL FMODE_EXCL
	+#define VDEV_BDEV_MODE_MASK (FMODE_READ\|FMODE_WRITE\|FMODE_EXCL)
	#endif
	-vdev_bdev_mode(spa_mode_t spa_mode, boolean_t exclusive)
	-{
	-#ifdef HAVE_BLK_MODE_T
	- blk_mode_t mode = 0;

	- if (spa_mode & SPA_MODE_READ)
	- mode \|= BLK_OPEN_READ;
	-
	- if (spa_mode & SPA_MODE_WRITE)
	- mode \|= BLK_OPEN_WRITE;
	+static vdev_bdev_mode_t
	+vdev_bdev_mode(spa_mode_t smode)
	+{
	+ ASSERT3U(smode, !=, SPA_MODE_UNINIT);
	+ ASSERT0(smode & ~(SPA_MODE_READ\|SPA_MODE_WRITE));

	- if (exclusive)
	- mode \|= BLK_OPEN_EXCL;
	-#else
	- fmode_t mode = 0;
	+ vdev_bdev_mode_t bmode = VDEV_BDEV_MODE_EXCL;

	- if (spa_mode & SPA_MODE_READ)
	- mode \|= FMODE_READ;
	+ if (smode & SPA_MODE_READ)
	+ bmode \|= VDEV_BDEV_MODE_READ;

	- if (spa_mode & SPA_MODE_WRITE)
	- mode \|= FMODE_WRITE;
	+ if (smode & SPA_MODE_WRITE)
	+ bmode \|= VDEV_BDEV_MODE_WRITE;

	- if (exclusive)
	- mode \|= FMODE_EXCL;
	-#endif
	+ ASSERT(bmode & VDEV_BDEV_MODE_MASK);
	+ ASSERT0(bmode & ~VDEV_BDEV_MODE_MASK);

	- return (mode);
	+ return (bmode);
	}

	/*
	* Returns the usable capacity (in bytes) for the partition or disk.
	*/
	static uint64_t
	bdev_capacity(struct block_device *bdev)
	{
	return (i_size_read(bdev->bd_inode));
	}

	#if !defined(HAVE_BDEV_WHOLE)
	static inline struct block_device *
	bdev_whole(struct block_device *bdev)
	{
	return (bdev->bd_contains);
	}
	#endif

	#if defined(HAVE_BDEVNAME)
	#define vdev_bdevname(bdev, name) bdevname(bdev, name)
	#else
	static inline void
	vdev_bdevname(struct block_device bdev, char name)
	{
	snprintf(name, BDEVNAME_SIZE, "%pg", bdev);
	}
	#endif

	/*
	* Returns the maximum expansion capacity of the block device (in bytes).
	*
	* It is possible to expand a vdev when it has been created as a wholedisk
	* and the containing block device has increased in capacity. Or when the
	* partition containing the pool has been manually increased in size.
	*
	* This function is only responsible for calculating the potential expansion
	* size so it can be reported by 'zpool list'. The efi_use_whole_disk() is
	* responsible for verifying the expected partition layout in the wholedisk
	* case, and updating the partition table if appropriate. Once the partition
	* size has been increased the additional capacity will be visible using
	* bdev_capacity().
	*
	* The returned maximum expansion capacity is always expected to be larger, or
	* at the very least equal, to its usable capacity to prevent overestimating
	* the pool expandsize.
	*/
	static uint64_t
	bdev_max_capacity(struct block_device *bdev, uint64_t wholedisk)
	{
	uint64_t psize;
	int64_t available;

	if (wholedisk && bdev != bdev_whole(bdev)) {
	/*
	* When reporting maximum expansion capacity for a wholedisk
	* deduct any capacity which is expected to be lost due to
	* alignment restrictions. Over reporting this value isn't
	* harmful and would only result in slightly less capacity
	* than expected post expansion.
	* The estimated available space may be slightly smaller than
	* bdev_capacity() for devices where the number of sectors is
	* not a multiple of the alignment size and the partition layout
	* is keeping less than PARTITION_END_ALIGNMENT bytes after the
	* "reserved" EFI partition: in such cases return the device
	* usable capacity.
	*/
	available = i_size_read(bdev_whole(bdev)->bd_inode) -
	((EFI_MIN_RESV_SIZE + NEW_START_BLOCK +
	PARTITION_END_ALIGNMENT) << SECTOR_BITS);
	psize = MAX(available, bdev_capacity(bdev));
	} else {
	psize = bdev_capacity(bdev);
	}

	return (psize);
	}

	static void
	vdev_disk_error(zio_t *zio)
	{
	/*
	* This function can be called in interrupt context, for instance while
	* handling IRQs coming from a misbehaving disk device; use printk()
	* which is safe from any context.
	*/
	printk(KERN_WARNING "zio pool=%s vdev=%s error=%d type=%d "
	"offset=%llu size=%llu flags=%llu\n", spa_name(zio->io_spa),
	zio->io_vd->vdev_path, zio->io_error, zio->io_type,
	(u_longlong_t)zio->io_offset, (u_longlong_t)zio->io_size,
	zio->io_flags);
	}

	static void
	vdev_disk_kobj_evt_post(vdev_t *v)
	{
	vdev_disk_t *vd = v->vdev_tsd;
	if (vd && vd->vd_bdh) {
	spl_signal_kobj_evt(BDH_BDEV(vd->vd_bdh));
	} else {
	vdev_dbgmsg(v, "vdev_disk_t is NULL for VDEV:%s\n",
	v->vdev_path);
	}
	}

	static zfs_bdev_handle_t *
	-vdev_blkdev_get_by_path(const char path, spa_mode_t mode, void holder)
	+vdev_blkdev_get_by_path(const char path, spa_mode_t smode, void holder)
	{
	-#if defined(HAVE_BDEV_OPEN_BY_PATH)
	- return (bdev_open_by_path(path,
	- vdev_bdev_mode(mode, B_TRUE), holder, NULL));
	+ vdev_bdev_mode_t bmode = vdev_bdev_mode(smode);
	+
	+#if defined(HAVE_BDEV_FILE_OPEN_BY_PATH)
	+ return (bdev_file_open_by_path(path, bmode, holder, NULL));
	+#elif defined(HAVE_BDEV_OPEN_BY_PATH)
	+ return (bdev_open_by_path(path, bmode, holder, NULL));
	#elif defined(HAVE_BLKDEV_GET_BY_PATH_4ARG)
	- return (blkdev_get_by_path(path,
	- vdev_bdev_mode(mode, B_TRUE), holder, NULL));
	+ return (blkdev_get_by_path(path, bmode, holder, NULL));
	#else
	- return (blkdev_get_by_path(path,
	- vdev_bdev_mode(mode, B_TRUE), holder));
	+ return (blkdev_get_by_path(path, bmode, holder));
	#endif
	}

	static void
	-vdev_blkdev_put(zfs_bdev_handle_t bdh, spa_mode_t mode, void holder)
	+vdev_blkdev_put(zfs_bdev_handle_t bdh, spa_mode_t smode, void holder)
	{
	#if defined(HAVE_BDEV_RELEASE)
	return (bdev_release(bdh));
	#elif defined(HAVE_BLKDEV_PUT_HOLDER)
	return (blkdev_put(BDH_BDEV(bdh), holder));
	+#elif defined(HAVE_BLKDEV_PUT)
	+ return (blkdev_put(BDH_BDEV(bdh), vdev_bdev_mode(smode)));
	#else
	- return (blkdev_put(BDH_BDEV(bdh),
	- vdev_bdev_mode(mode, B_TRUE)));
	+ fput(bdh);
	#endif
	}

	static int
	vdev_disk_open(vdev_t v, uint64_t psize, uint64_t *max_psize,
	uint64_t logical_ashift, uint64_t physical_ashift)
	{
	zfs_bdev_handle_t *bdh;
	-#ifdef HAVE_BLK_MODE_T
	- blk_mode_t mode = vdev_bdev_mode(spa_mode(v->vdev_spa), B_FALSE);
	-#else
	- fmode_t mode = vdev_bdev_mode(spa_mode(v->vdev_spa), B_FALSE);
	-#endif
	+ spa_mode_t smode = spa_mode(v->vdev_spa);
	hrtime_t timeout = MSEC2NSEC(zfs_vdev_open_timeout_ms);
	vdev_disk_t *vd;

	/* Must have a pathname and it must be absolute. */
	if (v->vdev_path == NULL \|\| v->vdev_path[0] != '/') {
	v->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
	vdev_dbgmsg(v, "invalid vdev_path");
	return (SET_ERROR(EINVAL));
	}

	/*
	* Reopen the device if it is currently open. When expanding a
	* partition force re-scanning the partition table if userland
	* did not take care of this already. We need to do this while closed
	* in order to get an accurate updated block device size. Then
	* since udev may need to recreate the device links increase the
	* open retry timeout before reporting the device as unavailable.
	*/
	vd = v->vdev_tsd;
	if (vd) {
	char disk_name[BDEVNAME_SIZE + 6] = "/dev/";
	boolean_t reread_part = B_FALSE;

	rw_enter(&vd->vd_lock, RW_WRITER);
	bdh = vd->vd_bdh;
	vd->vd_bdh = NULL;

	if (bdh) {
	struct block_device *bdev = BDH_BDEV(bdh);
	if (v->vdev_expanding && bdev != bdev_whole(bdev)) {
	vdev_bdevname(bdev_whole(bdev), disk_name + 5);
	/*
	* If userland has BLKPG_RESIZE_PARTITION,
	* then it should have updated the partition
	* table already. We can detect this by
	* comparing our current physical size
	* with that of the device. If they are
	* the same, then we must not have
	* BLKPG_RESIZE_PARTITION or it failed to
	* update the partition table online. We
	* fallback to rescanning the partition
	* table from the kernel below. However,
	* if the capacity already reflects the
	* updated partition, then we skip
	* rescanning the partition table here.
	*/
	if (v->vdev_psize == bdev_capacity(bdev))
	reread_part = B_TRUE;
	}

	- vdev_blkdev_put(bdh, mode, zfs_vdev_holder);
	+ vdev_blkdev_put(bdh, smode, zfs_vdev_holder);
	}

	if (reread_part) {
	- bdh = vdev_blkdev_get_by_path(disk_name, mode,
	+ bdh = vdev_blkdev_get_by_path(disk_name, smode,
	zfs_vdev_holder);
	if (!BDH_IS_ERR(bdh)) {
	int error =
	vdev_bdev_reread_part(BDH_BDEV(bdh));
	- vdev_blkdev_put(bdh, mode, zfs_vdev_holder);
	+ vdev_blkdev_put(bdh, smode, zfs_vdev_holder);
	if (error == 0) {
	timeout = MSEC2NSEC(
	zfs_vdev_open_timeout_ms * 2);
	}
	}
	}
	} else {
	vd = kmem_zalloc(sizeof (vdev_disk_t), KM_SLEEP);

	rw_init(&vd->vd_lock, NULL, RW_DEFAULT, NULL);
	rw_enter(&vd->vd_lock, RW_WRITER);
	}

	/*
	* Devices are always opened by the path provided at configuration
	* time. This means that if the provided path is a udev by-id path
	* then drives may be re-cabled without an issue. If the provided
	* path is a udev by-path path, then the physical location information
	* will be preserved. This can be critical for more complicated
	* configurations where drives are located in specific physical
	* locations to maximize the systems tolerance to component failure.
	*
	* Alternatively, you can provide your own udev rule to flexibly map
	* the drives as you see fit. It is not advised that you use the
	* /dev/[hd]d devices which may be reordered due to probing order.
	* Devices in the wrong locations will be detected by the higher
	* level vdev validation.
	*
	* The specified paths may be briefly removed and recreated in
	* response to udev events. This should be exceptionally unlikely
	* because the zpool command makes every effort to verify these paths
	* have already settled prior to reaching this point. Therefore,
	* a ENOENT failure at this point is highly likely to be transient
	* and it is reasonable to sleep and retry before giving up. In
	* practice delays have been observed to be on the order of 100ms.
	*
	* When ERESTARTSYS is returned it indicates the block device is
	* a zvol which could not be opened due to the deadlock detection
	* logic in zvol_open(). Extend the timeout and retry the open
	* subsequent attempts are expected to eventually succeed.
	*/
	hrtime_t start = gethrtime();
	bdh = BDH_ERR_PTR(-ENXIO);
	while (BDH_IS_ERR(bdh) && ((gethrtime() - start) < timeout)) {
	- bdh = vdev_blkdev_get_by_path(v->vdev_path, mode,
	+ bdh = vdev_blkdev_get_by_path(v->vdev_path, smode,
	zfs_vdev_holder);
	if (unlikely(BDH_PTR_ERR(bdh) == -ENOENT)) {
	/*
	* There is no point of waiting since device is removed
	* explicitly
	*/
	if (v->vdev_removed)
	break;

	schedule_timeout(MSEC_TO_TICK(10));
	} else if (unlikely(BDH_PTR_ERR(bdh) == -ERESTARTSYS)) {
	timeout = MSEC2NSEC(zfs_vdev_open_timeout_ms * 10);
	continue;
	} else if (BDH_IS_ERR(bdh)) {
	break;
	}
	}

	if (BDH_IS_ERR(bdh)) {
	int error = -BDH_PTR_ERR(bdh);
	vdev_dbgmsg(v, "open error=%d timeout=%llu/%llu", error,
	(u_longlong_t)(gethrtime() - start),
	(u_longlong_t)timeout);
	vd->vd_bdh = NULL;
	v->vdev_tsd = vd;
	rw_exit(&vd->vd_lock);
	return (SET_ERROR(error));
	} else {
	vd->vd_bdh = bdh;
	v->vdev_tsd = vd;
	rw_exit(&vd->vd_lock);
	}

	struct block_device *bdev = BDH_BDEV(vd->vd_bdh);

	/* Determine the physical block size */
	int physical_block_size = bdev_physical_block_size(bdev);

	/* Determine the logical block size */
	int logical_block_size = bdev_logical_block_size(bdev);

	/* Clear the nowritecache bit, causes vdev_reopen() to try again. */
	v->vdev_nowritecache = B_FALSE;

	/* Set when device reports it supports TRIM. */
	v->vdev_has_trim = bdev_discard_supported(bdev);

	/* Set when device reports it supports secure TRIM. */
	v->vdev_has_securetrim = bdev_secure_discard_supported(bdev);

	/* Inform the ZIO pipeline that we are non-rotational */
	v->vdev_nonrot = blk_queue_nonrot(bdev_get_queue(bdev));

	/* Physical volume size in bytes for the partition */
	*psize = bdev_capacity(bdev);

	/* Physical volume size in bytes including possible expansion space */
	*max_psize = bdev_max_capacity(bdev, v->vdev_wholedisk);

	/* Based on the minimum sector size set the block size */
	*physical_ashift = highbit64(MAX(physical_block_size,
	SPA_MINBLOCKSIZE)) - 1;

	*logical_ashift = highbit64(MAX(logical_block_size,
	SPA_MINBLOCKSIZE)) - 1;

	return (0);
	}

	static void
	vdev_disk_close(vdev_t *v)
	{
	vdev_disk_t *vd = v->vdev_tsd;

	if (v->vdev_reopening \|\| vd == NULL)
	return;

	- if (vd->vd_bdh != NULL) {
	+ if (vd->vd_bdh != NULL)
	vdev_blkdev_put(vd->vd_bdh, spa_mode(v->vdev_spa),
	zfs_vdev_holder);
	- }

	rw_destroy(&vd->vd_lock);
	kmem_free(vd, sizeof (vdev_disk_t));
	v->vdev_tsd = NULL;
	}

	-static dio_request_t *
	-vdev_disk_dio_alloc(int bio_count)
	-{
	- dio_request_t *dr = kmem_zalloc(sizeof (dio_request_t) +
	- sizeof (struct bio ) bio_count, KM_SLEEP);
	- atomic_set(&dr->dr_ref, 0);
	- dr->dr_bio_count = bio_count;
	- dr->dr_error = 0;
	-
	- for (int i = 0; i < dr->dr_bio_count; i++)
	- dr->dr_bio[i] = NULL;
	-
	- return (dr);
	-}
	-
	-static void
	-vdev_disk_dio_free(dio_request_t *dr)
	-{
	- int i;
	-
	- for (i = 0; i < dr->dr_bio_count; i++)
	- if (dr->dr_bio[i])
	- bio_put(dr->dr_bio[i]);
	-
	- kmem_free(dr, sizeof (dio_request_t) +
	- sizeof (struct bio ) dr->dr_bio_count);
	-}
	-
	-static void
	-vdev_disk_dio_get(dio_request_t *dr)
	-{
	- atomic_inc(&dr->dr_ref);
	-}
	-
	-static void
	-vdev_disk_dio_put(dio_request_t *dr)
	-{
	- int rc = atomic_dec_return(&dr->dr_ref);
	-
	- /*
	- * Free the dio_request when the last reference is dropped and
	- * ensure zio_interpret is called only once with the correct zio
	- */
	- if (rc == 0) {
	- zio_t *zio = dr->dr_zio;
	- int error = dr->dr_error;
	-
	- vdev_disk_dio_free(dr);
	-
	- if (zio) {
	- zio->io_error = error;
	- ASSERT3S(zio->io_error, >=, 0);
	- if (zio->io_error)
	- vdev_disk_error(zio);
	-
	- zio_delay_interrupt(zio);
	- }
	- }
	-}
	-
	-BIO_END_IO_PROTO(vdev_disk_physio_completion, bio, error)
	-{
	- dio_request_t *dr = bio->bi_private;
	-
	- if (dr->dr_error == 0) {
	-#ifdef HAVE_1ARG_BIO_END_IO_T
	- dr->dr_error = BIO_END_IO_ERROR(bio);
	-#else
	- if (error)
	- dr->dr_error = -(error);
	- else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
	- dr->dr_error = EIO;
	-#endif
	- }
	-
	- /* Drop reference acquired by __vdev_disk_physio */
	- vdev_disk_dio_put(dr);
	-}
	-
	static inline void
	vdev_submit_bio_impl(struct bio *bio)
	{
	#ifdef HAVE_1ARG_SUBMIT_BIO
	(void) submit_bio(bio);
	#else
	(void) submit_bio(bio_data_dir(bio), bio);
	#endif
	}

	/*
	* preempt_schedule_notrace is GPL-only which breaks the ZFS build, so
	* replace it with preempt_schedule under the following condition:
	*/
	#if defined(CONFIG_ARM64) && \
	defined(CONFIG_PREEMPTION) && \
	defined(CONFIG_BLK_CGROUP)
	#define preempt_schedule_notrace(x) preempt_schedule(x)
	#endif

	/*
	* As for the Linux 5.18 kernel bio_alloc() expects a block_device struct
	* as an argument removing the need to set it with bio_set_dev(). This
	* removes the need for all of the following compatibility code.
	*/
	#if !defined(HAVE_BIO_ALLOC_4ARG)

	#ifdef HAVE_BIO_SET_DEV
	#if defined(CONFIG_BLK_CGROUP) && defined(HAVE_BIO_SET_DEV_GPL_ONLY)
	/*
	* The Linux 5.5 kernel updated percpu_ref_tryget() which is inlined by
	* blkg_tryget() to use rcu_read_lock() instead of rcu_read_lock_sched().
	* As a side effect the function was converted to GPL-only. Define our
	* own version when needed which uses rcu_read_lock_sched().
	*
	* The Linux 5.17 kernel split linux/blk-cgroup.h into a private and a public
	* part, moving blkg_tryget into the private one. Define our own version.
	*/
	#if defined(HAVE_BLKG_TRYGET_GPL_ONLY) \|\| !defined(HAVE_BLKG_TRYGET)
	static inline bool
	vdev_blkg_tryget(struct blkcg_gq *blkg)
	{
	struct percpu_ref *ref = &blkg->refcnt;
	unsigned long __percpu *count;
	bool rc;

	rcu_read_lock_sched();

	if (__ref_is_percpu(ref, &count)) {
	this_cpu_inc(*count);
	rc = true;
	} else {
	#ifdef ZFS_PERCPU_REF_COUNT_IN_DATA
	rc = atomic_long_inc_not_zero(&ref->data->count);
	#else
	rc = atomic_long_inc_not_zero(&ref->count);
	#endif
	}

	rcu_read_unlock_sched();

	return (rc);
	}
	#else
	#define vdev_blkg_tryget(bg) blkg_tryget(bg)
	#endif
	#ifdef HAVE_BIO_SET_DEV_MACRO
	/*
	* The Linux 5.0 kernel updated the bio_set_dev() macro so it calls the
	* GPL-only bio_associate_blkg() symbol thus inadvertently converting
	* the entire macro. Provide a minimal version which always assigns the
	* request queue's root_blkg to the bio.
	*/
	static inline void
	vdev_bio_associate_blkg(struct bio *bio)
	{
	#if defined(HAVE_BIO_BDEV_DISK)
	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
	#else
	struct request_queue *q = bio->bi_disk->queue;
	#endif

	ASSERT3P(q, !=, NULL);
	ASSERT3P(bio->bi_blkg, ==, NULL);

	if (q->root_blkg && vdev_blkg_tryget(q->root_blkg))
	bio->bi_blkg = q->root_blkg;
	}

	#define bio_associate_blkg vdev_bio_associate_blkg
	#else
	static inline void
	vdev_bio_set_dev(struct bio bio, struct block_device bdev)
	{
	#if defined(HAVE_BIO_BDEV_DISK)
	struct request_queue *q = bdev->bd_disk->queue;
	#else
	struct request_queue *q = bio->bi_disk->queue;
	#endif
	bio_clear_flag(bio, BIO_REMAPPED);
	if (bio->bi_bdev != bdev)
	bio_clear_flag(bio, BIO_THROTTLED);
	bio->bi_bdev = bdev;

	ASSERT3P(q, !=, NULL);
	ASSERT3P(bio->bi_blkg, ==, NULL);

	if (q->root_blkg && vdev_blkg_tryget(q->root_blkg))
	bio->bi_blkg = q->root_blkg;
	}
	#define bio_set_dev vdev_bio_set_dev
	#endif
	#endif
	#else
	/*
	* Provide a bio_set_dev() helper macro for pre-Linux 4.14 kernels.
	*/
	static inline void
	bio_set_dev(struct bio bio, struct block_device bdev)
	{
	bio->bi_bdev = bdev;
	}
	#endif /* HAVE_BIO_SET_DEV */
	#endif /* !HAVE_BIO_ALLOC_4ARG */

	static inline void
	vdev_submit_bio(struct bio *bio)
	{
	struct bio_list *bio_list = current->bio_list;
	current->bio_list = NULL;
	vdev_submit_bio_impl(bio);
	current->bio_list = bio_list;
	}

	static inline struct bio *
	vdev_bio_alloc(struct block_device *bdev, gfp_t gfp_mask,
	unsigned short nr_vecs)
	{
	struct bio *bio;

	#ifdef HAVE_BIO_ALLOC_4ARG
	bio = bio_alloc(bdev, nr_vecs, 0, gfp_mask);
	#else
	bio = bio_alloc(gfp_mask, nr_vecs);
	if (likely(bio != NULL))
	bio_set_dev(bio, bdev);
	#endif

	return (bio);
	}

	+static inline uint_t
	+vdev_bio_max_segs(struct block_device *bdev)
	+{
	+ /*
	+ * Smallest of the device max segs and the tuneable max segs. Minimum
	+ * 4, so there's room to finish split pages if they come up.
	+ */
	+ const uint_t dev_max_segs = queue_max_segments(bdev_get_queue(bdev));
	+ const uint_t tune_max_segs = (zfs_vdev_disk_max_segs > 0) ?
	+ MAX(4, zfs_vdev_disk_max_segs) : dev_max_segs;
	+ const uint_t max_segs = MIN(tune_max_segs, dev_max_segs);
	+
	+#ifdef HAVE_BIO_MAX_SEGS
	+ return (bio_max_segs(max_segs));
	+#else
	+ return (MIN(max_segs, BIO_MAX_PAGES));
	+#endif
	+}
	+
	+static inline uint_t
	+vdev_bio_max_bytes(struct block_device *bdev)
	+{
	+ return (queue_max_sectors(bdev_get_queue(bdev)) << 9);
	+}
	+
	+
	+/*
	+ * Virtual block IO object (VBIO)
	+ *
	+ * Linux block IO (BIO) objects have a limit on how many data segments (pages)
	+ * they can hold. Depending on how they're allocated and structured, a large
	+ * ZIO can require more than one BIO to be submitted to the kernel, which then
	+ * all have to complete before we can return the completed ZIO back to ZFS.
	+ *
	+ * A VBIO is a wrapper around multiple BIOs, carrying everything needed to
	+ * translate a ZIO down into the kernel block layer and back again.
	+ *
	+ * Note that these are only used for data ZIOs (read/write). Meta-operations
	+ * (flush/trim) don't need multiple BIOs and so can just make the call
	+ * directly.
	+ */
	+typedef struct {
	+ zio_t vbio_zio; / parent zio */
	+
	+ struct block_device vbio_bdev; / blockdev to submit bios to */
	+
	+ abd_t vbio_abd; / abd carrying borrowed linear buf */
	+
	+ uint_t vbio_max_segs; /* max segs per bio */
	+
	+ uint_t vbio_max_bytes; /* max bytes per bio */
	+ uint_t vbio_lbs_mask; /* logical block size mask */
	+
	+ uint64_t vbio_offset; /* start offset of next bio */
	+
	+ struct bio vbio_bio; / pointer to the current bio */
	+ int vbio_flags; /* bio flags */
	+} vbio_t;
	+
	+static vbio_t *
	+vbio_alloc(zio_t zio, struct block_device bdev, int flags)
	+{
	+ vbio_t *vbio = kmem_zalloc(sizeof (vbio_t), KM_SLEEP);
	+
	+ vbio->vbio_zio = zio;
	+ vbio->vbio_bdev = bdev;
	+ vbio->vbio_abd = NULL;
	+ vbio->vbio_max_segs = vdev_bio_max_segs(bdev);
	+ vbio->vbio_max_bytes = vdev_bio_max_bytes(bdev);
	+ vbio->vbio_lbs_mask = ~(bdev_logical_block_size(bdev)-1);
	+ vbio->vbio_offset = zio->io_offset;
	+ vbio->vbio_bio = NULL;
	+ vbio->vbio_flags = flags;
	+
	+ return (vbio);
	+}
	+
	+BIO_END_IO_PROTO(vbio_completion, bio, error);
	+
	+static int
	+vbio_add_page(vbio_t vbio, struct page page, uint_t size, uint_t offset)
	+{
	+ struct bio *bio = vbio->vbio_bio;
	+ uint_t ssize;
	+
	+ while (size > 0) {
	+ if (bio == NULL) {
	+ /* New BIO, allocate and set up */
	+ bio = vdev_bio_alloc(vbio->vbio_bdev, GFP_NOIO,
	+ vbio->vbio_max_segs);
	+ VERIFY(bio);
	+
	+ BIO_BI_SECTOR(bio) = vbio->vbio_offset >> 9;
	+ bio_set_op_attrs(bio,
	+ vbio->vbio_zio->io_type == ZIO_TYPE_WRITE ?
	+ WRITE : READ, vbio->vbio_flags);
	+
	+ if (vbio->vbio_bio) {
	+ bio_chain(vbio->vbio_bio, bio);
	+ vdev_submit_bio(vbio->vbio_bio);
	+ }
	+ vbio->vbio_bio = bio;
	+ }
	+
	+ /*
	+ * Only load as much of the current page data as will fit in
	+ * the space left in the BIO, respecting lbs alignment. Older
	+ * kernels will error if we try to overfill the BIO, while
	+ * newer ones will accept it and split the BIO. This ensures
	+ * everything works on older kernels, and avoids an additional
	+ * overhead on the new.
	+ */
	+ ssize = MIN(size, (vbio->vbio_max_bytes - BIO_BI_SIZE(bio)) &
	+ vbio->vbio_lbs_mask);
	+ if (ssize > 0 &&
	+ bio_add_page(bio, page, ssize, offset) == ssize) {
	+ /* Accepted, adjust and load any remaining. */
	+ size -= ssize;
	+ offset += ssize;
	+ continue;
	+ }
	+
	+ /* No room, set up for a new BIO and loop */
	+ vbio->vbio_offset += BIO_BI_SIZE(bio);
	+
	+ /* Signal new BIO allocation wanted */
	+ bio = NULL;
	+ }
	+
	+ return (0);
	+}
	+
	+/* Iterator callback to submit ABD pages to the vbio. */
	+static int
	+vbio_fill_cb(struct page page, size_t off, size_t len, void priv)
	+{
	+ vbio_t *vbio = priv;
	+ return (vbio_add_page(vbio, page, len, off));
	+}
	+
	+/* Create some BIOs, fill them with data and submit them */
	+static void
	+vbio_submit(vbio_t vbio, abd_t abd, uint64_t size)
	+{
	+ /*
	+ * We plug so we can submit the BIOs as we go and only unplug them when
	+ * they are fully created and submitted. This is important; if we don't
	+ * plug, then the kernel may start executing earlier BIOs while we're
	+ * still creating and executing later ones, and if the device goes
	+ * away while that's happening, older kernels can get confused and
	+ * trample memory.
	+ */
	+ struct blk_plug plug;
	+ blk_start_plug(&plug);
	+
	+ (void) abd_iterate_page_func(abd, 0, size, vbio_fill_cb, vbio);
	+ ASSERT(vbio->vbio_bio);
	+
	+ vbio->vbio_bio->bi_end_io = vbio_completion;
	+ vbio->vbio_bio->bi_private = vbio;
	+
	+ /*
	+ * Once submitted, vbio_bio now owns vbio (through bi_private) and we
	+ * can't touch it again. The bio may complete and vbio_completion() be
	+ * called and free the vbio before this task is run again, so we must
	+ * consider it invalid from this point.
	+ */
	+ vdev_submit_bio(vbio->vbio_bio);
	+
	+ blk_finish_plug(&plug);
	+}
	+
	+/* IO completion callback */
	+BIO_END_IO_PROTO(vbio_completion, bio, error)
	+{
	+ vbio_t *vbio = bio->bi_private;
	+ zio_t *zio = vbio->vbio_zio;
	+
	+ ASSERT(zio);
	+
	+ /* Capture and log any errors */
	+#ifdef HAVE_1ARG_BIO_END_IO_T
	+ zio->io_error = BIO_END_IO_ERROR(bio);
	+#else
	+ zio->io_error = 0;
	+ if (error)
	+ zio->io_error = -(error);
	+ else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
	+ zio->io_error = EIO;
	+#endif
	+ ASSERT3U(zio->io_error, >=, 0);
	+
	+ if (zio->io_error)
	+ vdev_disk_error(zio);
	+
	+ /* Return the BIO to the kernel */
	+ bio_put(bio);
	+
	+ /*
	+ * If we copied the ABD before issuing it, clean up and return the copy
	+ * to the ADB, with changes if appropriate.
	+ */
	+ if (vbio->vbio_abd != NULL) {
	+ void *buf = abd_to_buf(vbio->vbio_abd);
	+ abd_free(vbio->vbio_abd);
	+ vbio->vbio_abd = NULL;
	+
	+ if (zio->io_type == ZIO_TYPE_READ)
	+ abd_return_buf_copy(zio->io_abd, buf, zio->io_size);
	+ else
	+ abd_return_buf(zio->io_abd, buf, zio->io_size);
	+ }
	+
	+ /* Final cleanup */
	+ kmem_free(vbio, sizeof (vbio_t));
	+
	+ /* All done, submit for processing */
	+ zio_delay_interrupt(zio);
	+}
	+
	+/*
	+ * Iterator callback to count ABD pages and check their size & alignment.
	+ *
	+ * On Linux, each BIO segment can take a page pointer, and an offset+length of
	+ * the data within that page. A page can be arbitrarily large ("compound"
	+ * pages) but we still have to ensure the data portion is correctly sized and
	+ * aligned to the logical block size, to ensure that if the kernel wants to
	+ * split the BIO, the two halves will still be properly aligned.
	+ */
	+typedef struct {
	+ uint_t bmask;
	+ uint_t npages;
	+ uint_t end;
	+} vdev_disk_check_pages_t;
	+
	+static int
	+vdev_disk_check_pages_cb(struct page page, size_t off, size_t len, void priv)
	+{
	+ vdev_disk_check_pages_t *s = priv;
	+
	+ /*
	+ * If we didn't finish on a block size boundary last time, then there
	+ * would be a gap if we tried to use this ABD as-is, so abort.
	+ */
	+ if (s->end != 0)
	+ return (1);
	+
	+ /*
	+ * Note if we're taking less than a full block, so we can check it
	+ * above on the next call.
	+ */
	+ s->end = (off+len) & s->bmask;
	+
	+ /* All blocks after the first must start on a block size boundary. */
	+ if (s->npages != 0 && (off & s->bmask) != 0)
	+ return (1);
	+
	+ s->npages++;
	+ return (0);
	+}
	+
	+/*
	+ * Check if we can submit the pages in this ABD to the kernel as-is. Returns
	+ * the number of pages, or 0 if it can't be submitted like this.
	+ */
	+static boolean_t
	+vdev_disk_check_pages(abd_t abd, uint64_t size, struct block_device bdev)
	+{
	+ vdev_disk_check_pages_t s = {
	+ .bmask = bdev_logical_block_size(bdev)-1,
	+ .npages = 0,
	+ .end = 0,
	+ };
	+
	+ if (abd_iterate_page_func(abd, 0, size, vdev_disk_check_pages_cb, &s))
	+ return (B_FALSE);
	+
	+ return (B_TRUE);
	+}
	+
	+static int
	+vdev_disk_io_rw(zio_t *zio)
	+{
	+ vdev_t *v = zio->io_vd;
	+ vdev_disk_t *vd = v->vdev_tsd;
	+ struct block_device *bdev = BDH_BDEV(vd->vd_bdh);
	+ int flags = 0;
	+
	+ /*
	+ * Accessing outside the block device is never allowed.
	+ */
	+ if (zio->io_offset + zio->io_size > bdev->bd_inode->i_size) {
	+ vdev_dbgmsg(zio->io_vd,
	+ "Illegal access %llu size %llu, device size %llu",
	+ (u_longlong_t)zio->io_offset,
	+ (u_longlong_t)zio->io_size,
	+ (u_longlong_t)i_size_read(bdev->bd_inode));
	+ return (SET_ERROR(EIO));
	+ }
	+
	+ if (!(zio->io_flags & (ZIO_FLAG_IO_RETRY \| ZIO_FLAG_TRYHARD)) &&
	+ v->vdev_failfast == B_TRUE) {
	+ bio_set_flags_failfast(bdev, &flags, zfs_vdev_failfast_mask & 1,
	+ zfs_vdev_failfast_mask & 2, zfs_vdev_failfast_mask & 4);
	+ }
	+
	+ /*
	+ * Check alignment of the incoming ABD. If any part of it would require
	+ * submitting a page that is not aligned to the logical block size,
	+ * then we take a copy into a linear buffer and submit that instead.
	+ * This should be impossible on a 512b LBS, and fairly rare on 4K,
	+ * usually requiring abnormally-small data blocks (eg gang blocks)
	+ * mixed into the same ABD as larger ones (eg aggregated).
	+ */
	+ abd_t *abd = zio->io_abd;
	+ if (!vdev_disk_check_pages(abd, zio->io_size, bdev)) {
	+ void *buf;
	+ if (zio->io_type == ZIO_TYPE_READ)
	+ buf = abd_borrow_buf(zio->io_abd, zio->io_size);
	+ else
	+ buf = abd_borrow_buf_copy(zio->io_abd, zio->io_size);
	+
	+ /*
	+ * Wrap the copy in an abd_t, so we can use the same iterators
	+ * to count and fill the vbio later.
	+ */
	+ abd = abd_get_from_buf(buf, zio->io_size);
	+
	+ /*
	+ * False here would mean the borrowed copy has an invalid
	+ * alignment too, which would mean we've somehow been passed a
	+ * linear ABD with an interior page that has a non-zero offset
	+ * or a size not a multiple of PAGE_SIZE. This is not possible.
	+ * It would mean either zio_buf_alloc() or its underlying
	+ * allocators have done something extremely strange, or our
	+ * math in vdev_disk_check_pages() is wrong. In either case,
	+ * something in seriously wrong and its not safe to continue.
	+ */
	+ VERIFY(vdev_disk_check_pages(abd, zio->io_size, bdev));
	+ }
	+
	+ /* Allocate vbio, with a pointer to the borrowed ABD if necessary */
	+ vbio_t *vbio = vbio_alloc(zio, bdev, flags);
	+ if (abd != zio->io_abd)
	+ vbio->vbio_abd = abd;
	+
	+ /* Fill it with data pages and submit it to the kernel */
	+ vbio_submit(vbio, abd, zio->io_size);
	+ return (0);
	+}
	+
	+/* ========== */
	+
	+/*
	+ * This is the classic, battle-tested BIO submission code. Until we're totally
	+ * sure that the new code is safe and correct in all cases, this will remain
	+ * available.
	+ *
	+ * It is enabled by setting zfs_vdev_disk_classic=1 at module load time. It is
	+ * enabled (=1) by default since 2.2.4, and disabled by default (=0) on master.
	+ *
	+ * These functions have been renamed to vdev_classic_* to make it clear what
	+ * they belong to, but their implementations are unchanged.
	+ */
	+
	+/*
	+ * Virtual device vector for disks.
	+ */
	+typedef struct dio_request {
	+ zio_t dr_zio; / Parent ZIO */
	+ atomic_t dr_ref; /* References */
	+ int dr_error; /* Bio error */
	+ int dr_bio_count; /* Count of bio's */
	+ struct bio dr_bio[]; / Attached bio's */
	+} dio_request_t;
	+
	+static dio_request_t *
	+vdev_classic_dio_alloc(int bio_count)
	+{
	+ dio_request_t *dr = kmem_zalloc(sizeof (dio_request_t) +
	+ sizeof (struct bio ) bio_count, KM_SLEEP);
	+ atomic_set(&dr->dr_ref, 0);
	+ dr->dr_bio_count = bio_count;
	+ dr->dr_error = 0;
	+
	+ for (int i = 0; i < dr->dr_bio_count; i++)
	+ dr->dr_bio[i] = NULL;
	+
	+ return (dr);
	+}
	+
	+static void
	+vdev_classic_dio_free(dio_request_t *dr)
	+{
	+ int i;
	+
	+ for (i = 0; i < dr->dr_bio_count; i++)
	+ if (dr->dr_bio[i])
	+ bio_put(dr->dr_bio[i]);
	+
	+ kmem_free(dr, sizeof (dio_request_t) +
	+ sizeof (struct bio ) dr->dr_bio_count);
	+}
	+
	+static void
	+vdev_classic_dio_get(dio_request_t *dr)
	+{
	+ atomic_inc(&dr->dr_ref);
	+}
	+
	+static void
	+vdev_classic_dio_put(dio_request_t *dr)
	+{
	+ int rc = atomic_dec_return(&dr->dr_ref);
	+
	+ /*
	+ * Free the dio_request when the last reference is dropped and
	+ * ensure zio_interpret is called only once with the correct zio
	+ */
	+ if (rc == 0) {
	+ zio_t *zio = dr->dr_zio;
	+ int error = dr->dr_error;
	+
	+ vdev_classic_dio_free(dr);
	+
	+ if (zio) {
	+ zio->io_error = error;
	+ ASSERT3S(zio->io_error, >=, 0);
	+ if (zio->io_error)
	+ vdev_disk_error(zio);
	+
	+ zio_delay_interrupt(zio);
	+ }
	+ }
	+}
	+
	+BIO_END_IO_PROTO(vdev_classic_physio_completion, bio, error)
	+{
	+ dio_request_t *dr = bio->bi_private;
	+
	+ if (dr->dr_error == 0) {
	+#ifdef HAVE_1ARG_BIO_END_IO_T
	+ dr->dr_error = BIO_END_IO_ERROR(bio);
	+#else
	+ if (error)
	+ dr->dr_error = -(error);
	+ else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
	+ dr->dr_error = EIO;
	+#endif
	+ }
	+
	+ /* Drop reference acquired by vdev_classic_physio */
	+ vdev_classic_dio_put(dr);
	+}
	+
	static inline unsigned int
	-vdev_bio_max_segs(zio_t *zio, int bio_size, uint64_t abd_offset)
	+vdev_classic_bio_max_segs(zio_t *zio, int bio_size, uint64_t abd_offset)
	{
	unsigned long nr_segs = abd_nr_pages_off(zio->io_abd,
	bio_size, abd_offset);

	#ifdef HAVE_BIO_MAX_SEGS
	return (bio_max_segs(nr_segs));
	#else
	return (MIN(nr_segs, BIO_MAX_PAGES));
	#endif
	}

	static int
	-__vdev_disk_physio(struct block_device bdev, zio_t zio,
	- size_t io_size, uint64_t io_offset, int rw, int flags)
	+vdev_classic_physio(zio_t *zio)
	{
	+ vdev_t *v = zio->io_vd;
	+ vdev_disk_t *vd = v->vdev_tsd;
	+ struct block_device *bdev = BDH_BDEV(vd->vd_bdh);
	+ size_t io_size = zio->io_size;
	+ uint64_t io_offset = zio->io_offset;
	+ int rw = zio->io_type == ZIO_TYPE_READ ? READ : WRITE;
	+ int flags = 0;
	+
	dio_request_t *dr;
	uint64_t abd_offset;
	uint64_t bio_offset;
	int bio_size;
	int bio_count = 16;
	int error = 0;
	struct blk_plug plug;
	unsigned short nr_vecs;

	/*
	* Accessing outside the block device is never allowed.
	*/
	if (io_offset + io_size > bdev->bd_inode->i_size) {
	vdev_dbgmsg(zio->io_vd,
	"Illegal access %llu size %llu, device size %llu",
	(u_longlong_t)io_offset,
	(u_longlong_t)io_size,
	(u_longlong_t)i_size_read(bdev->bd_inode));
	return (SET_ERROR(EIO));
	}

	retry:
	- dr = vdev_disk_dio_alloc(bio_count);
	+ dr = vdev_classic_dio_alloc(bio_count);

	if (!(zio->io_flags & (ZIO_FLAG_IO_RETRY \| ZIO_FLAG_TRYHARD)) &&
	zio->io_vd->vdev_failfast == B_TRUE) {
	bio_set_flags_failfast(bdev, &flags, zfs_vdev_failfast_mask & 1,
	zfs_vdev_failfast_mask & 2, zfs_vdev_failfast_mask & 4);
	}

	dr->dr_zio = zio;

	/*
	* Since bio's can have up to BIO_MAX_PAGES=256 iovec's, each of which
	* is at least 512 bytes and at most PAGESIZE (typically 4K), one bio
	* can cover at least 128KB and at most 1MB. When the required number
	* of iovec's exceeds this, we are forced to break the IO in multiple
	* bio's and wait for them all to complete. This is likely if the
	* recordsize property is increased beyond 1MB. The default
	* bio_count=16 should typically accommodate the maximum-size zio of
	* 16MB.
	*/

	abd_offset = 0;
	bio_offset = io_offset;
	bio_size = io_size;
	for (int i = 0; i <= dr->dr_bio_count; i++) {

	/* Finished constructing bio's for given buffer */
	if (bio_size <= 0)
	break;

	/*
	* If additional bio's are required, we have to retry, but
	* this should be rare - see the comment above.
	*/
	if (dr->dr_bio_count == i) {
	- vdev_disk_dio_free(dr);
	+ vdev_classic_dio_free(dr);
	bio_count *= 2;
	goto retry;
	}

	- nr_vecs = vdev_bio_max_segs(zio, bio_size, abd_offset);
	+ nr_vecs = vdev_classic_bio_max_segs(zio, bio_size, abd_offset);
	dr->dr_bio[i] = vdev_bio_alloc(bdev, GFP_NOIO, nr_vecs);
	if (unlikely(dr->dr_bio[i] == NULL)) {
	- vdev_disk_dio_free(dr);
	+ vdev_classic_dio_free(dr);
	return (SET_ERROR(ENOMEM));
	}

	- /* Matching put called by vdev_disk_physio_completion */
	- vdev_disk_dio_get(dr);
	+ /* Matching put called by vdev_classic_physio_completion */
	+ vdev_classic_dio_get(dr);

	BIO_BI_SECTOR(dr->dr_bio[i]) = bio_offset >> 9;
	- dr->dr_bio[i]->bi_end_io = vdev_disk_physio_completion;
	+ dr->dr_bio[i]->bi_end_io = vdev_classic_physio_completion;
	dr->dr_bio[i]->bi_private = dr;
	bio_set_op_attrs(dr->dr_bio[i], rw, flags);

	/* Remaining size is returned to become the new size */
	bio_size = abd_bio_map_off(dr->dr_bio[i], zio->io_abd,
	bio_size, abd_offset);

	/* Advance in buffer and construct another bio if needed */
	abd_offset += BIO_BI_SIZE(dr->dr_bio[i]);
	bio_offset += BIO_BI_SIZE(dr->dr_bio[i]);
	}

	/* Extra reference to protect dio_request during vdev_submit_bio */
	- vdev_disk_dio_get(dr);
	+ vdev_classic_dio_get(dr);

	if (dr->dr_bio_count > 1)
	blk_start_plug(&plug);

	/* Submit all bio's associated with this dio */
	for (int i = 0; i < dr->dr_bio_count; i++) {
	if (dr->dr_bio[i])
	vdev_submit_bio(dr->dr_bio[i]);
	}

	if (dr->dr_bio_count > 1)
	blk_finish_plug(&plug);

	- vdev_disk_dio_put(dr);
	+ vdev_classic_dio_put(dr);

	return (error);
	}

	+/* ========== */
	+
	BIO_END_IO_PROTO(vdev_disk_io_flush_completion, bio, error)
	{
	zio_t *zio = bio->bi_private;
	#ifdef HAVE_1ARG_BIO_END_IO_T
	zio->io_error = BIO_END_IO_ERROR(bio);
	#else
	zio->io_error = -error;
	#endif

	if (zio->io_error && (zio->io_error == EOPNOTSUPP))
	zio->io_vd->vdev_nowritecache = B_TRUE;

	bio_put(bio);
	ASSERT3S(zio->io_error, >=, 0);
	if (zio->io_error)
	vdev_disk_error(zio);
	zio_interrupt(zio);
	}

	static int
	vdev_disk_io_flush(struct block_device bdev, zio_t zio)
	{
	struct request_queue *q;
	struct bio *bio;

	q = bdev_get_queue(bdev);
	if (!q)
	return (SET_ERROR(ENXIO));

	bio = vdev_bio_alloc(bdev, GFP_NOIO, 0);
	if (unlikely(bio == NULL))
	return (SET_ERROR(ENOMEM));

	bio->bi_end_io = vdev_disk_io_flush_completion;
	bio->bi_private = zio;
	bio_set_flush(bio);
	vdev_submit_bio(bio);
	invalidate_bdev(bdev);

	return (0);
	}

	-#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE) \|\| \
	- defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC)
	BIO_END_IO_PROTO(vdev_disk_discard_end_io, bio, error)
	{
	zio_t *zio = bio->bi_private;
	#ifdef HAVE_1ARG_BIO_END_IO_T
	zio->io_error = BIO_END_IO_ERROR(bio);
	#else
	zio->io_error = -error;
	#endif
	bio_put(bio);
	if (zio->io_error)
	vdev_disk_error(zio);
	zio_interrupt(zio);
	}

	+/*
	+ * Wrappers for the different secure erase and discard APIs. We use async
	+ * when available; in this case, *biop is set to the last bio in the chain.
	+ */
	static int
	-vdev_issue_discard_trim(zio_t *zio, unsigned long flags)
	+vdev_bdev_issue_secure_erase(zfs_bdev_handle_t *bdh, sector_t sector,
	+ sector_t nsect, struct bio **biop)
	{
	- int ret;
	- struct bio *bio = NULL;
	+ *biop = NULL;
	+ int error;

	-#if defined(BLKDEV_DISCARD_SECURE)
	- ret = - __blkdev_issue_discard(
	- BDH_BDEV(((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh),
	- zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS, flags, &bio);
	+#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE)
	+ error = blkdev_issue_secure_erase(BDH_BDEV(bdh),
	+ sector, nsect, GFP_NOFS);
	+#elif defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS)
	+ error = __blkdev_issue_discard(BDH_BDEV(bdh),
	+ sector, nsect, GFP_NOFS, BLKDEV_DISCARD_SECURE, biop);
	+#elif defined(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS)
	+ error = blkdev_issue_discard(BDH_BDEV(bdh),
	+ sector, nsect, GFP_NOFS, BLKDEV_DISCARD_SECURE);
	#else
	- (void) flags;
	- ret = - __blkdev_issue_discard(
	- BDH_BDEV(((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh),
	- zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS, &bio);
	+#error "unsupported kernel"
	#endif
	- if (!ret && bio) {
	- bio->bi_private = zio;
	- bio->bi_end_io = vdev_disk_discard_end_io;
	- vdev_submit_bio(bio);
	- }
	- return (ret);
	+
	+ return (error);
	}
	+
	+static int
	+vdev_bdev_issue_discard(zfs_bdev_handle_t *bdh, sector_t sector,
	+ sector_t nsect, struct bio **biop)
	+{
	+ *biop = NULL;
	+ int error;
	+
	+#if defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS)
	+ error = __blkdev_issue_discard(BDH_BDEV(bdh),
	+ sector, nsect, GFP_NOFS, 0, biop);
	+#elif defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_NOFLAGS)
	+ error = __blkdev_issue_discard(BDH_BDEV(bdh),
	+ sector, nsect, GFP_NOFS, biop);
	+#elif defined(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS)
	+ error = blkdev_issue_discard(BDH_BDEV(bdh),
	+ sector, nsect, GFP_NOFS, 0);
	+#elif defined(HAVE_BLKDEV_ISSUE_DISCARD_NOFLAGS)
	+ error = blkdev_issue_discard(BDH_BDEV(bdh),
	+ sector, nsect, GFP_NOFS);
	+#else
	+#error "unsupported kernel"
	#endif

	+ return (error);
	+}
	+
	+/*
	+ * Entry point for TRIM ops. This calls the right wrapper for secure erase or
	+ * discard, and then does the appropriate finishing work for error vs success
	+ * and async vs sync.
	+ */
	static int
	vdev_disk_io_trim(zio_t *zio)
	{
	- unsigned long trim_flags = 0;
	- if (zio->io_trim_flags & ZIO_TRIM_SECURE) {
	-#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE)
	- return (-blkdev_issue_secure_erase(
	- BDH_BDEV(((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh),
	- zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS));
	-#elif defined(BLKDEV_DISCARD_SECURE)
	- trim_flags \|= BLKDEV_DISCARD_SECURE;
	-#endif
	+ int error;
	+ struct bio *bio;
	+
	+ zfs_bdev_handle_t bdh = ((vdev_disk_t )zio->io_vd->vdev_tsd)->vd_bdh;
	+ sector_t sector = zio->io_offset >> 9;
	+ sector_t nsects = zio->io_size >> 9;
	+
	+ if (zio->io_trim_flags & ZIO_TRIM_SECURE)
	+ error = vdev_bdev_issue_secure_erase(bdh, sector, nsects, &bio);
	+ else
	+ error = vdev_bdev_issue_discard(bdh, sector, nsects, &bio);
	+
	+ if (error != 0)
	+ return (SET_ERROR(-error));
	+
	+ if (bio == NULL) {
	+ /*
	+ * This was a synchronous op that completed successfully, so
	+ * return it to ZFS immediately.
	+ */
	+ zio_interrupt(zio);
	+ } else {
	+ /*
	+ * This was an asynchronous op; set up completion callback and
	+ * issue it.
	+ */
	+ bio->bi_private = zio;
	+ bio->bi_end_io = vdev_disk_discard_end_io;
	+ vdev_submit_bio(bio);
	}
	-#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE) \|\| \
	- defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC)
	- return (vdev_issue_discard_trim(zio, trim_flags));
	-#elif defined(HAVE_BLKDEV_ISSUE_DISCARD)
	- return (-blkdev_issue_discard(
	- BDH_BDEV(((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh),
	- zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS, trim_flags));
	-#else
	-#error "Unsupported kernel"
	-#endif
	+
	+ return (0);
	}

	+int (vdev_disk_io_rw_fn)(zio_t zio) = NULL;
	+
	static void
	vdev_disk_io_start(zio_t *zio)
	{
	vdev_t *v = zio->io_vd;
	vdev_disk_t *vd = v->vdev_tsd;
	- int rw, error;
	+ int error;

	/*
	* If the vdev is closed, it's likely in the REMOVED or FAULTED state.
	* Nothing to be done here but return failure.
	*/
	if (vd == NULL) {
	zio->io_error = ENXIO;
	zio_interrupt(zio);
	return;
	}

	rw_enter(&vd->vd_lock, RW_READER);

	/*
	* If the vdev is closed, it's likely due to a failed reopen and is
	* in the UNAVAIL state. Nothing to be done here but return failure.
	*/
	if (vd->vd_bdh == NULL) {
	rw_exit(&vd->vd_lock);
	zio->io_error = ENXIO;
	zio_interrupt(zio);
	return;
	}

	switch (zio->io_type) {
	case ZIO_TYPE_IOCTL:

	if (!vdev_readable(v)) {
	rw_exit(&vd->vd_lock);
	zio->io_error = SET_ERROR(ENXIO);
	zio_interrupt(zio);
	return;
	}

	switch (zio->io_cmd) {
	case DKIOCFLUSHWRITECACHE:

	if (zfs_nocacheflush)
	break;

	if (v->vdev_nowritecache) {
	zio->io_error = SET_ERROR(ENOTSUP);
	break;
	}

	error = vdev_disk_io_flush(BDH_BDEV(vd->vd_bdh), zio);
	if (error == 0) {
	rw_exit(&vd->vd_lock);
	return;
	}

	zio->io_error = error;

	break;

	default:
	zio->io_error = SET_ERROR(ENOTSUP);
	}

	rw_exit(&vd->vd_lock);
	zio_execute(zio);
	return;
	- case ZIO_TYPE_WRITE:
	- rw = WRITE;
	- break;
	-
	- case ZIO_TYPE_READ:
	- rw = READ;
	- break;

	case ZIO_TYPE_TRIM:
	- zio->io_error = vdev_disk_io_trim(zio);
	+ error = vdev_disk_io_trim(zio);
	rw_exit(&vd->vd_lock);
	-#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE)
	- if (zio->io_trim_flags & ZIO_TRIM_SECURE)
	+ if (error) {
	+ zio->io_error = error;
	+ zio_execute(zio);
	+ }
	+ return;
	+
	+ case ZIO_TYPE_READ:
	+ case ZIO_TYPE_WRITE:
	+ zio->io_target_timestamp = zio_handle_io_delay(zio);
	+ error = vdev_disk_io_rw_fn(zio);
	+ rw_exit(&vd->vd_lock);
	+ if (error) {
	+ zio->io_error = error;
	zio_interrupt(zio);
	-#elif defined(HAVE_BLKDEV_ISSUE_DISCARD)
	- zio_interrupt(zio);
	-#endif
	+ }
	return;

	default:
	+ /*
	+ * Getting here means our parent vdev has made a very strange
	+ * request of us, and shouldn't happen. Assert here to force a
	+ * crash in dev builds, but in production return the IO
	+ * unhandled. The pool will likely suspend anyway but that's
	+ * nicer than crashing the kernel.
	+ */
	+ ASSERT3S(zio->io_type, ==, -1);
	+
	rw_exit(&vd->vd_lock);
	zio->io_error = SET_ERROR(ENOTSUP);
	zio_interrupt(zio);
	return;
	}

	- zio->io_target_timestamp = zio_handle_io_delay(zio);
	- error = __vdev_disk_physio(BDH_BDEV(vd->vd_bdh), zio,
	- zio->io_size, zio->io_offset, rw, 0);
	- rw_exit(&vd->vd_lock);
	-
	- if (error) {
	- zio->io_error = error;
	- zio_interrupt(zio);
	- return;
	- }
	+ __builtin_unreachable();
	}

	static void
	vdev_disk_io_done(zio_t *zio)
	{
	/*
	* If the device returned EIO, we revalidate the media. If it is
	* determined the media has changed this triggers the asynchronous
	* removal of the device from the configuration.
	*/
	if (zio->io_error == EIO) {
	vdev_t *v = zio->io_vd;
	vdev_disk_t *vd = v->vdev_tsd;

	if (!zfs_check_disk_status(BDH_BDEV(vd->vd_bdh))) {
	invalidate_bdev(BDH_BDEV(vd->vd_bdh));
	v->vdev_remove_wanted = B_TRUE;
	spa_async_request(zio->io_spa, SPA_ASYNC_REMOVE);
	}
	}
	}

	static void
	vdev_disk_hold(vdev_t *vd)
	{
	ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_WRITER));

	/* We must have a pathname, and it must be absolute. */
	if (vd->vdev_path == NULL \|\| vd->vdev_path[0] != '/')
	return;

	/*
	* Only prefetch path and devid info if the device has
	* never been opened.
	*/
	if (vd->vdev_tsd != NULL)
	return;

	}

	static void
	vdev_disk_rele(vdev_t *vd)
	{
	ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_WRITER));

	/* XXX: Implement me as a vnode rele for the device */
	}

	+/*
	+ * BIO submission method. See comment above about vdev_classic.
	+ * Set zfs_vdev_disk_classic=0 for new, =1 for classic
	+ */
	+static uint_t zfs_vdev_disk_classic = 1; /* default classic */
	+
	+/* Set submission function from module parameter */
	+static int
	+vdev_disk_param_set_classic(const char buf, zfs_kernel_param_t kp)
	+{
	+ int err = param_set_uint(buf, kp);
	+ if (err < 0)
	+ return (SET_ERROR(err));
	+
	+ vdev_disk_io_rw_fn =
	+ zfs_vdev_disk_classic ? vdev_classic_physio : vdev_disk_io_rw;
	+
	+ printk(KERN_INFO "ZFS: forcing %s BIO submission\n",
	+ zfs_vdev_disk_classic ? "classic" : "new");
	+
	+ return (0);
	+}
	+
	+/*
	+ * At first use vdev use, set the submission function from the default value if
	+ * it hasn't been set already.
	+ */
	+static int
	+vdev_disk_init(spa_t spa, nvlist_t nv, void **tsd)
	+{
	+ (void) spa;
	+ (void) nv;
	+ (void) tsd;
	+
	+ if (vdev_disk_io_rw_fn == NULL)
	+ vdev_disk_io_rw_fn = zfs_vdev_disk_classic ?
	+ vdev_classic_physio : vdev_disk_io_rw;
	+
	+ return (0);
	+}
	+
	vdev_ops_t vdev_disk_ops = {
	- .vdev_op_init = NULL,
	+ .vdev_op_init = vdev_disk_init,
	.vdev_op_fini = NULL,
	.vdev_op_open = vdev_disk_open,
	.vdev_op_close = vdev_disk_close,
	.vdev_op_asize = vdev_default_asize,
	.vdev_op_min_asize = vdev_default_min_asize,
	.vdev_op_min_alloc = NULL,
	.vdev_op_io_start = vdev_disk_io_start,
	.vdev_op_io_done = vdev_disk_io_done,
	.vdev_op_state_change = NULL,
	.vdev_op_need_resilver = NULL,
	.vdev_op_hold = vdev_disk_hold,
	.vdev_op_rele = vdev_disk_rele,
	.vdev_op_remap = NULL,
	.vdev_op_xlate = vdev_default_xlate,
	.vdev_op_rebuild_asize = NULL,
	.vdev_op_metaslab_init = NULL,
	.vdev_op_config_generate = NULL,
	.vdev_op_nparity = NULL,
	.vdev_op_ndisks = NULL,
	.vdev_op_type = VDEV_TYPE_DISK, /* name of this vdev type */
	.vdev_op_leaf = B_TRUE, /* leaf vdev */
	.vdev_op_kobj_evt_post = vdev_disk_kobj_evt_post
	};

	/*
	* The zfs_vdev_scheduler module option has been deprecated. Setting this
	* value no longer has any effect. It has not yet been entirely removed
	* to allow the module to be loaded if this option is specified in the
	* /etc/modprobe.d/zfs.conf file. The following warning will be logged.
	*/
	static int
	param_set_vdev_scheduler(const char val, zfs_kernel_param_t kp)
	{
	int error = param_set_charp(val, kp);
	if (error == 0) {
	printk(KERN_INFO "The 'zfs_vdev_scheduler' module option "
	"is not supported.\n");
	}

	return (error);
	}

	static const char *zfs_vdev_scheduler = "unused";
	module_param_call(zfs_vdev_scheduler, param_set_vdev_scheduler,
	param_get_charp, &zfs_vdev_scheduler, 0644);
	MODULE_PARM_DESC(zfs_vdev_scheduler, "I/O scheduler");

	int
	param_set_min_auto_ashift(const char buf, zfs_kernel_param_t kp)
	{
	uint_t val;
	int error;

	error = kstrtouint(buf, 0, &val);
	if (error < 0)
	return (SET_ERROR(error));

	if (val < ASHIFT_MIN \|\| val > zfs_vdev_max_auto_ashift)
	return (SET_ERROR(-EINVAL));

	error = param_set_uint(buf, kp);
	if (error < 0)
	return (SET_ERROR(error));

	return (0);
	}

	int
	param_set_max_auto_ashift(const char buf, zfs_kernel_param_t kp)
	{
	uint_t val;
	int error;

	error = kstrtouint(buf, 0, &val);
	if (error < 0)
	return (SET_ERROR(error));

	if (val > ASHIFT_MAX \|\| val < zfs_vdev_min_auto_ashift)
	return (SET_ERROR(-EINVAL));

	error = param_set_uint(buf, kp);
	if (error < 0)
	return (SET_ERROR(error));

	return (0);
	}

	ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, open_timeout_ms, UINT, ZMOD_RW,
	"Timeout before determining that a device is missing");

	ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, failfast_mask, UINT, ZMOD_RW,
	"Defines failfast mask: 1 - device, 2 - transport, 4 - driver");
	+
	+ZFS_MODULE_PARAM(zfs_vdev_disk, zfs_vdev_disk_, max_segs, UINT, ZMOD_RW,
	+ "Maximum number of data segments to add to an IO request (min 4)");
	+
	+ZFS_MODULE_PARAM_CALL(zfs_vdev_disk, zfs_vdev_disk_, classic,
	+ vdev_disk_param_set_classic, param_get_uint, ZMOD_RD,
	+ "Use classic BIO submission method");
	diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_vnops_os.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_vnops_os.c
	index c06a75662bf7..be528f6e8176 100644
	--- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_vnops_os.c
	+++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_vnops_os.c
	@@ -1,4251 +1,4245 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2012, 2018 by Delphix. All rights reserved.
	* Copyright (c) 2015 by Chunwei Chen. All rights reserved.
	* Copyright 2017 Nexenta Systems, Inc.
	*/

	/* Portions Copyright 2007 Jeremy Teo */
	/* Portions Copyright 2010 Robert Milkowski */


	#include <sys/types.h>
	#include <sys/param.h>
	#include <sys/time.h>
	#include <sys/sysmacros.h>
	#include <sys/vfs.h>
	#include <sys/file.h>
	#include <sys/stat.h>
	#include <sys/kmem.h>
	#include <sys/taskq.h>
	#include <sys/uio.h>
	#include <sys/vmsystm.h>
	#include <sys/atomic.h>
	#include <sys/pathname.h>
	#include <sys/cmn_err.h>
	#include <sys/errno.h>
	#include <sys/zfs_dir.h>
	#include <sys/zfs_acl.h>
	#include <sys/zfs_ioctl.h>
	#include <sys/fs/zfs.h>
	#include <sys/dmu.h>
	#include <sys/dmu_objset.h>
	#include <sys/spa.h>
	#include <sys/txg.h>
	#include <sys/dbuf.h>
	#include <sys/zap.h>
	#include <sys/sa.h>
	#include <sys/policy.h>
	#include <sys/sunddi.h>
	#include <sys/sid.h>
	#include <sys/zfs_ctldir.h>
	#include <sys/zfs_fuid.h>
	#include <sys/zfs_quota.h>
	#include <sys/zfs_sa.h>
	#include <sys/zfs_vnops.h>
	#include <sys/zfs_rlock.h>
	#include <sys/cred.h>
	#include <sys/zpl.h>
	#include <sys/zil.h>
	#include <sys/sa_impl.h>

	/*
	* Programming rules.
	*
	* Each vnode op performs some logical unit of work. To do this, the ZPL must
	* properly lock its in-core state, create a DMU transaction, do the work,
	* record this work in the intent log (ZIL), commit the DMU transaction,
	* and wait for the intent log to commit if it is a synchronous operation.
	* Moreover, the vnode ops must work in both normal and log replay context.
	* The ordering of events is important to avoid deadlocks and references
	* to freed memory. The example below illustrates the following Big Rules:
	*
	* (1) A check must be made in each zfs thread for a mounted file system.
	* This is done avoiding races using zfs_enter(zfsvfs).
	* A zfs_exit(zfsvfs) is needed before all returns. Any znodes
	* must be checked with zfs_verify_zp(zp). Both of these macros
	* can return EIO from the calling function.
	*
	* (2) zrele() should always be the last thing except for zil_commit() (if
	* necessary) and zfs_exit(). This is for 3 reasons: First, if it's the
	* last reference, the vnode/znode can be freed, so the zp may point to
	* freed memory. Second, the last reference will call zfs_zinactive(),
	* which may induce a lot of work -- pushing cached pages (which acquires
	* range locks) and syncing out cached atime changes. Third,
	* zfs_zinactive() may require a new tx, which could deadlock the system
	* if you were already holding one. This deadlock occurs because the tx
	* currently being operated on prevents a txg from syncing, which
	* prevents the new tx from progressing, resulting in a deadlock. If you
	* must call zrele() within a tx, use zfs_zrele_async(). Note that iput()
	* is a synonym for zrele().
	*
	* (3) All range locks must be grabbed before calling dmu_tx_assign(),
	* as they can span dmu_tx_assign() calls.
	*
	* (4) If ZPL locks are held, pass TXG_NOWAIT as the second argument to
	* dmu_tx_assign(). This is critical because we don't want to block
	* while holding locks.
	*
	* If no ZPL locks are held (aside from zfs_enter()), use TXG_WAIT. This
	* reduces lock contention and CPU usage when we must wait (note that if
	* throughput is constrained by the storage, nearly every transaction
	* must wait).
	*
	* Note, in particular, that if a lock is sometimes acquired before
	* the tx assigns, and sometimes after (e.g. z_lock), then failing
	* to use a non-blocking assign can deadlock the system. The scenario:
	*
	* Thread A has grabbed a lock before calling dmu_tx_assign().
	* Thread B is in an already-assigned tx, and blocks for this lock.
	* Thread A calls dmu_tx_assign(TXG_WAIT) and blocks in txg_wait_open()
	* forever, because the previous txg can't quiesce until B's tx commits.
	*
	* If dmu_tx_assign() returns ERESTART and zfsvfs->z_assign is TXG_NOWAIT,
	* then drop all locks, call dmu_tx_wait(), and try again. On subsequent
	* calls to dmu_tx_assign(), pass TXG_NOTHROTTLE in addition to TXG_NOWAIT,
	* to indicate that this operation has already called dmu_tx_wait().
	* This will ensure that we don't retry forever, waiting a short bit
	* each time.
	*
	* (5) If the operation succeeded, generate the intent log entry for it
	* before dropping locks. This ensures that the ordering of events
	* in the intent log matches the order in which they actually occurred.
	* During ZIL replay the zfs_log_* functions will update the sequence
	* number to indicate the zil transaction has replayed.
	*
	* (6) At the end of each vnode op, the DMU tx must always commit,
	* regardless of whether there were any errors.
	*
	* (7) After dropping all locks, invoke zil_commit(zilog, foid)
	* to ensure that synchronous semantics are provided when necessary.
	*
	* In general, this is how things should be ordered in each vnode op:
	*
	* zfs_enter(zfsvfs); // exit if unmounted
	* top:
	* zfs_dirent_lock(&dl, ...) // lock directory entry (may igrab())
	* rw_enter(...); // grab any other locks you need
	* tx = dmu_tx_create(...); // get DMU tx
	* dmu_tx_hold_*(); // hold each object you might modify
	* error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) \| TXG_NOWAIT);
	* if (error) {
	* rw_exit(...); // drop locks
	* zfs_dirent_unlock(dl); // unlock directory entry
	* zrele(...); // release held znodes
	* if (error == ERESTART) {
	* waited = B_TRUE;
	* dmu_tx_wait(tx);
	* dmu_tx_abort(tx);
	* goto top;
	* }
	* dmu_tx_abort(tx); // abort DMU tx
	* zfs_exit(zfsvfs); // finished in zfs
	* return (error); // really out of space
	* }
	* error = do_real_work(); // do whatever this VOP does
	* if (error == 0)
	* zfs_log_*(...); // on success, make ZIL entry
	* dmu_tx_commit(tx); // commit DMU tx -- error or not
	* rw_exit(...); // drop locks
	* zfs_dirent_unlock(dl); // unlock directory entry
	* zrele(...); // release held znodes
	* zil_commit(zilog, foid); // synchronous when necessary
	* zfs_exit(zfsvfs); // finished in zfs
	* return (error); // done, report error
	*/
	int
	zfs_open(struct inode ip, int mode, int flag, cred_t cr)
	{
	(void) cr;
	znode_t *zp = ITOZ(ip);
	zfsvfs_t *zfsvfs = ITOZSB(ip);
	int error;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	/* Honor ZFS_APPENDONLY file attribute */
	if (blk_mode_is_open_write(mode) && (zp->z_pflags & ZFS_APPENDONLY) &&
	((flag & O_APPEND) == 0)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EPERM));
	}

	/* Keep a count of the synchronous opens in the znode */
	if (flag & O_SYNC)
	atomic_inc_32(&zp->z_sync_cnt);

	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	int
	zfs_close(struct inode ip, int flag, cred_t cr)
	{
	(void) cr;
	znode_t *zp = ITOZ(ip);
	zfsvfs_t *zfsvfs = ITOZSB(ip);
	int error;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	/* Decrement the synchronous opens in the znode */
	if (flag & O_SYNC)
	atomic_dec_32(&zp->z_sync_cnt);

	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	#if defined(_KERNEL)

	static int zfs_fillpage(struct inode ip, struct page pp);

	/*
	* When a file is memory mapped, we must keep the IO data synchronized
	* between the DMU cache and the memory mapped pages. Update all mapped
	* pages with the contents of the coresponding dmu buffer.
	*/
	void
	update_pages(znode_t zp, int64_t start, int len, objset_t os)
	{
	struct address_space *mp = ZTOI(zp)->i_mapping;
	int64_t off = start & (PAGE_SIZE - 1);

	for (start &= PAGE_MASK; len > 0; start += PAGE_SIZE) {
	uint64_t nbytes = MIN(PAGE_SIZE - off, len);

	struct page *pp = find_lock_page(mp, start >> PAGE_SHIFT);
	if (pp) {
	if (mapping_writably_mapped(mp))
	flush_dcache_page(pp);

	void *pb = kmap(pp);
	int error = dmu_read(os, zp->z_id, start + off,
	nbytes, pb + off, DMU_READ_PREFETCH);
	kunmap(pp);

	if (error) {
	SetPageError(pp);
	ClearPageUptodate(pp);
	} else {
	ClearPageError(pp);
	SetPageUptodate(pp);

	if (mapping_writably_mapped(mp))
	flush_dcache_page(pp);

	mark_page_accessed(pp);
	}

	unlock_page(pp);
	put_page(pp);
	}

	len -= nbytes;
	off = 0;
	}
	}

	/*
	* When a file is memory mapped, we must keep the I/O data synchronized
	* between the DMU cache and the memory mapped pages. Preferentially read
	* from memory mapped pages, otherwise fallback to reading through the dmu.
	*/
	int
	mappedread(znode_t zp, int nbytes, zfs_uio_t uio)
	{
	struct inode *ip = ZTOI(zp);
	struct address_space *mp = ip->i_mapping;
	int64_t start = uio->uio_loffset;
	int64_t off = start & (PAGE_SIZE - 1);
	int len = nbytes;
	int error = 0;

	for (start &= PAGE_MASK; len > 0; start += PAGE_SIZE) {
	uint64_t bytes = MIN(PAGE_SIZE - off, len);

	struct page *pp = find_lock_page(mp, start >> PAGE_SHIFT);
	if (pp) {
	/*
	* If filemap_fault() retries there exists a window
	* where the page will be unlocked and not up to date.
	* In this case we must try and fill the page.
	*/
	if (unlikely(!PageUptodate(pp))) {
	error = zfs_fillpage(ip, pp);
	if (error) {
	unlock_page(pp);
	put_page(pp);
	return (error);
	}
	}

	ASSERT(PageUptodate(pp) \|\| PageDirty(pp));

	unlock_page(pp);

	void *pb = kmap(pp);
	error = zfs_uiomove(pb + off, bytes, UIO_READ, uio);
	kunmap(pp);

	if (mapping_writably_mapped(mp))
	flush_dcache_page(pp);

	mark_page_accessed(pp);
	put_page(pp);
	} else {
	error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl),
	uio, bytes);
	}

	len -= bytes;
	off = 0;

	if (error)
	break;
	}

	return (error);
	}
	#endif /* _KERNEL */

	static unsigned long zfs_delete_blocks = DMU_MAX_DELETEBLKCNT;

	/*
	* Write the bytes to a file.
	*
	* IN: zp - znode of file to be written to
	* data - bytes to write
	* len - number of bytes to write
	* pos - offset to start writing at
	*
	* OUT: resid - remaining bytes to write
	*
	* RETURN: 0 if success
	* positive error code if failure. EIO is returned
	* for a short write when residp isn't provided.
	*
	* Timestamps:
	* zp - ctime\|mtime updated if byte count > 0
	*/
	int
	zfs_write_simple(znode_t zp, const void data, size_t len,
	loff_t pos, size_t *residp)
	{
	fstrans_cookie_t cookie;
	int error;

	struct iovec iov;
	iov.iov_base = (void *)data;
	iov.iov_len = len;

	zfs_uio_t uio;
	zfs_uio_iovec_init(&uio, &iov, 1, pos, UIO_SYSSPACE, len, 0);

	cookie = spl_fstrans_mark();
	error = zfs_write(zp, &uio, 0, kcred);
	spl_fstrans_unmark(cookie);

	if (error == 0) {
	if (residp != NULL)
	*residp = zfs_uio_resid(&uio);
	else if (zfs_uio_resid(&uio) != 0)
	error = SET_ERROR(EIO);
	}

	return (error);
	}

	static void
	zfs_rele_async_task(void *arg)
	{
	iput(arg);
	}

	void
	zfs_zrele_async(znode_t *zp)
	{
	struct inode *ip = ZTOI(zp);
	objset_t *os = ITOZSB(ip)->z_os;

	ASSERT(atomic_read(&ip->i_count) > 0);
	ASSERT(os != NULL);

	/*
	* If decrementing the count would put us at 0, we can't do it inline
	* here, because that would be synchronous. Instead, dispatch an iput
	* to run later.
	*
	* For more information on the dangers of a synchronous iput, see the
	* header comment of this file.
	*/
	if (!atomic_add_unless(&ip->i_count, -1, 1)) {
	VERIFY(taskq_dispatch(dsl_pool_zrele_taskq(dmu_objset_pool(os)),
	zfs_rele_async_task, ip, TQ_SLEEP) != TASKQID_INVALID);
	}
	}


	/*
	* Lookup an entry in a directory, or an extended attribute directory.
	* If it exists, return a held inode reference for it.
	*
	* IN: zdp - znode of directory to search.
	* nm - name of entry to lookup.
	* flags - LOOKUP_XATTR set if looking for an attribute.
	* cr - credentials of caller.
	* direntflags - directory lookup flags
	* realpnp - returned pathname.
	*
	* OUT: zpp - znode of located entry, NULL if not found.
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* NA
	*/
	int
	zfs_lookup(znode_t zdp, char nm, znode_t *zpp, int flags, cred_t cr,
	int direntflags, pathname_t realpnp)
	{
	zfsvfs_t *zfsvfs = ZTOZSB(zdp);
	int error = 0;

	/*
	* Fast path lookup, however we must skip DNLC lookup
	* for case folding or normalizing lookups because the
	* DNLC code only stores the passed in name. This means
	* creating 'a' and removing 'A' on a case insensitive
	* file system would work, but DNLC still thinks 'a'
	* exists and won't let you create it again on the next
	* pass through fast path.
	*/
	if (!(flags & (LOOKUP_XATTR \| FIGNORECASE))) {

	if (!S_ISDIR(ZTOI(zdp)->i_mode)) {
	return (SET_ERROR(ENOTDIR));
	} else if (zdp->z_sa_hdl == NULL) {
	return (SET_ERROR(EIO));
	}

	if (nm[0] == 0 \|\| (nm[0] == '.' && nm[1] == '\0')) {
	error = zfs_fastaccesschk_execute(zdp, cr);
	if (!error) {
	*zpp = zdp;
	zhold(*zpp);
	return (0);
	}
	return (error);
	}
	}

	if ((error = zfs_enter_verify_zp(zfsvfs, zdp, FTAG)) != 0)
	return (error);

	*zpp = NULL;

	if (flags & LOOKUP_XATTR) {
	/*
	* We don't allow recursive attributes..
	* Maybe someday we will.
	*/
	if (zdp->z_pflags & ZFS_XATTR) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	if ((error = zfs_get_xattrdir(zdp, zpp, cr, flags))) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Do we have permission to get into attribute directory?
	*/

	if ((error = zfs_zaccess(*zpp, ACE_EXECUTE, 0,
	B_TRUE, cr, zfs_init_idmap))) {
	zrele(*zpp);
	*zpp = NULL;
	}

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	if (!S_ISDIR(ZTOI(zdp)->i_mode)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(ENOTDIR));
	}

	/*
	* Check accessibility of directory.
	*/

	if ((error = zfs_zaccess(zdp, ACE_EXECUTE, 0, B_FALSE, cr,
	zfs_init_idmap))) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	if (zfsvfs->z_utf8 && u8_validate(nm, strlen(nm),
	NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EILSEQ));
	}

	error = zfs_dirlook(zdp, nm, zpp, flags, direntflags, realpnp);
	if ((error == 0) && (*zpp))
	zfs_znode_update_vfs(*zpp);

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Attempt to create a new entry in a directory. If the entry
	* already exists, truncate the file if permissible, else return
	* an error. Return the ip of the created or trunc'd file.
	*
	* IN: dzp - znode of directory to put new file entry in.
	* name - name of new file entry.
	* vap - attributes of new file.
	* excl - flag indicating exclusive or non-exclusive mode.
	* mode - mode to open file with.
	* cr - credentials of caller.
	* flag - file flag.
	* vsecp - ACL to be set
	* mnt_ns - user namespace of the mount
	*
	* OUT: zpp - znode of created or trunc'd entry.
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* dzp - ctime\|mtime updated if new entry created
	* zp - ctime\|mtime always, atime if new
	*/
	int
	zfs_create(znode_t dzp, char name, vattr_t *vap, int excl,
	int mode, znode_t *zpp, cred_t cr, int flag, vsecattr_t *vsecp,
	zidmap_t *mnt_ns)
	{
	znode_t *zp;
	zfsvfs_t *zfsvfs = ZTOZSB(dzp);
	zilog_t *zilog;
	objset_t *os;
	zfs_dirlock_t *dl;
	dmu_tx_t *tx;
	int error;
	uid_t uid;
	gid_t gid;
	zfs_acl_ids_t acl_ids;
	boolean_t fuid_dirtied;
	boolean_t have_acl = B_FALSE;
	boolean_t waited = B_FALSE;
	boolean_t skip_acl = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;

	/*
	* If we have an ephemeral id, ACL, or XVATTR then
	* make sure file system is at proper version
	*/

	gid = crgetgid(cr);
	uid = crgetuid(cr);

	if (zfsvfs->z_use_fuids == B_FALSE &&
	(vsecp \|\| IS_EPHEMERAL(uid) \|\| IS_EPHEMERAL(gid)))
	return (SET_ERROR(EINVAL));

	if (name == NULL)
	return (SET_ERROR(EINVAL));

	if ((error = zfs_enter_verify_zp(zfsvfs, dzp, FTAG)) != 0)
	return (error);
	os = zfsvfs->z_os;
	zilog = zfsvfs->z_log;

	if (zfsvfs->z_utf8 && u8_validate(name, strlen(name),
	NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EILSEQ));
	}

	if (vap->va_mask & ATTR_XVATTR) {
	if ((error = secpolicy_xvattr((xvattr_t *)vap,
	crgetuid(cr), cr, vap->va_mode)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	}

	top:
	*zpp = NULL;
	if (*name == '\0') {
	/*
	* Null component name refers to the directory itself.
	*/
	zhold(dzp);
	zp = dzp;
	dl = NULL;
	error = 0;
	} else {
	/* possible igrab(zp) */
	int zflg = 0;

	if (flag & FIGNORECASE)
	zflg \|= ZCILOOK;

	error = zfs_dirent_lock(&dl, dzp, name, &zp, zflg,
	NULL, NULL);
	if (error) {
	if (have_acl)
	zfs_acl_ids_free(&acl_ids);
	if (strcmp(name, "..") == 0)
	error = SET_ERROR(EISDIR);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	}

	if (zp == NULL) {
	uint64_t txtype;
	uint64_t projid = ZFS_DEFAULT_PROJID;

	/*
	* Create a new file object and update the directory
	* to reference it.
	*/
	if ((error = zfs_zaccess(dzp, ACE_ADD_FILE, 0, skip_acl, cr,
	mnt_ns))) {
	if (have_acl)
	zfs_acl_ids_free(&acl_ids);
	goto out;
	}

	/*
	* We only support the creation of regular files in
	* extended attribute directories.
	*/

	if ((dzp->z_pflags & ZFS_XATTR) && !S_ISREG(vap->va_mode)) {
	if (have_acl)
	zfs_acl_ids_free(&acl_ids);
	error = SET_ERROR(EINVAL);
	goto out;
	}

	if (!have_acl && (error = zfs_acl_ids_create(dzp, 0, vap,
	cr, vsecp, &acl_ids, mnt_ns)) != 0)
	goto out;
	have_acl = B_TRUE;

	if (S_ISREG(vap->va_mode) \|\| S_ISDIR(vap->va_mode))
	projid = zfs_inherit_projid(dzp);
	if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, projid)) {
	zfs_acl_ids_free(&acl_ids);
	error = SET_ERROR(EDQUOT);
	goto out;
	}

	tx = dmu_tx_create(os);

	dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes +
	ZFS_SA_BASE_ATTR_SIZE);

	fuid_dirtied = zfsvfs->z_fuid_dirty;
	if (fuid_dirtied)
	zfs_fuid_txhold(zfsvfs, tx);
	dmu_tx_hold_zap(tx, dzp->z_id, TRUE, name);
	dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE);
	if (!zfsvfs->z_use_sa &&
	acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) {
	dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
	0, acl_ids.z_aclp->z_acl_bytes);
	}

	error = dmu_tx_assign(tx,
	(waited ? TXG_NOTHROTTLE : 0) \| TXG_NOWAIT);
	if (error) {
	zfs_dirent_unlock(dl);
	if (error == ERESTART) {
	waited = B_TRUE;
	dmu_tx_wait(tx);
	dmu_tx_abort(tx);
	goto top;
	}
	zfs_acl_ids_free(&acl_ids);
	dmu_tx_abort(tx);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	zfs_mknode(dzp, vap, tx, cr, 0, &zp, &acl_ids);

	error = zfs_link_create(dl, zp, tx, ZNEW);
	if (error != 0) {
	/*
	* Since, we failed to add the directory entry for it,
	* delete the newly created dnode.
	*/
	zfs_znode_delete(zp, tx);
	remove_inode_hash(ZTOI(zp));
	zfs_acl_ids_free(&acl_ids);
	dmu_tx_commit(tx);
	goto out;
	}

	if (fuid_dirtied)
	zfs_fuid_sync(zfsvfs, tx);

	txtype = zfs_log_create_txtype(Z_FILE, vsecp, vap);
	if (flag & FIGNORECASE)
	txtype \|= TX_CI;
	zfs_log_create(zilog, tx, txtype, dzp, zp, name,
	vsecp, acl_ids.z_fuidp, vap);
	zfs_acl_ids_free(&acl_ids);
	dmu_tx_commit(tx);
	} else {
	int aflags = (flag & O_APPEND) ? V_APPEND : 0;

	if (have_acl)
	zfs_acl_ids_free(&acl_ids);

	/*
	* A directory entry already exists for this name.
	*/
	/*
	* Can't truncate an existing file if in exclusive mode.
	*/
	if (excl) {
	error = SET_ERROR(EEXIST);
	goto out;
	}
	/*
	* Can't open a directory for writing.
	*/
	if (S_ISDIR(ZTOI(zp)->i_mode)) {
	error = SET_ERROR(EISDIR);
	goto out;
	}
	/*
	* Verify requested access to file.
	*/
	if (mode && (error = zfs_zaccess_rwx(zp, mode, aflags, cr,
	mnt_ns))) {
	goto out;
	}

	mutex_enter(&dzp->z_lock);
	dzp->z_seq++;
	mutex_exit(&dzp->z_lock);

	/*
	* Truncate regular files if requested.
	*/
	if (S_ISREG(ZTOI(zp)->i_mode) &&
	(vap->va_mask & ATTR_SIZE) && (vap->va_size == 0)) {
	/* we can't hold any locks when calling zfs_freesp() */
	if (dl) {
	zfs_dirent_unlock(dl);
	dl = NULL;
	}
	error = zfs_freesp(zp, 0, 0, mode, TRUE);
	}
	}
	out:

	if (dl)
	zfs_dirent_unlock(dl);

	if (error) {
	if (zp)
	zrele(zp);
	} else {
	zfs_znode_update_vfs(dzp);
	zfs_znode_update_vfs(zp);
	*zpp = zp;
	}

	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	int
	zfs_tmpfile(struct inode dip, vattr_t vap, int excl,
	int mode, struct inode *ipp, cred_t cr, int flag, vsecattr_t *vsecp,
	zidmap_t *mnt_ns)
	{
	(void) excl, (void) mode, (void) flag;
	znode_t zp = NULL, dzp = ITOZ(dip);
	zfsvfs_t *zfsvfs = ITOZSB(dip);
	objset_t *os;
	dmu_tx_t *tx;
	int error;
	uid_t uid;
	gid_t gid;
	zfs_acl_ids_t acl_ids;
	uint64_t projid = ZFS_DEFAULT_PROJID;
	boolean_t fuid_dirtied;
	boolean_t have_acl = B_FALSE;
	boolean_t waited = B_FALSE;

	/*
	* If we have an ephemeral id, ACL, or XVATTR then
	* make sure file system is at proper version
	*/

	gid = crgetgid(cr);
	uid = crgetuid(cr);

	if (zfsvfs->z_use_fuids == B_FALSE &&
	(vsecp \|\| IS_EPHEMERAL(uid) \|\| IS_EPHEMERAL(gid)))
	return (SET_ERROR(EINVAL));

	if ((error = zfs_enter_verify_zp(zfsvfs, dzp, FTAG)) != 0)
	return (error);
	os = zfsvfs->z_os;

	if (vap->va_mask & ATTR_XVATTR) {
	if ((error = secpolicy_xvattr((xvattr_t *)vap,
	crgetuid(cr), cr, vap->va_mode)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	}

	top:
	*ipp = NULL;

	/*
	* Create a new file object and update the directory
	* to reference it.
	*/
	if ((error = zfs_zaccess(dzp, ACE_ADD_FILE, 0, B_FALSE, cr, mnt_ns))) {
	if (have_acl)
	zfs_acl_ids_free(&acl_ids);
	goto out;
	}

	if (!have_acl && (error = zfs_acl_ids_create(dzp, 0, vap,
	cr, vsecp, &acl_ids, mnt_ns)) != 0)
	goto out;
	have_acl = B_TRUE;

	if (S_ISREG(vap->va_mode) \|\| S_ISDIR(vap->va_mode))
	projid = zfs_inherit_projid(dzp);
	if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, projid)) {
	zfs_acl_ids_free(&acl_ids);
	error = SET_ERROR(EDQUOT);
	goto out;
	}

	tx = dmu_tx_create(os);

	dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes +
	ZFS_SA_BASE_ATTR_SIZE);
	dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);

	fuid_dirtied = zfsvfs->z_fuid_dirty;
	if (fuid_dirtied)
	zfs_fuid_txhold(zfsvfs, tx);
	if (!zfsvfs->z_use_sa &&
	acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) {
	dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
	0, acl_ids.z_aclp->z_acl_bytes);
	}
	error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) \| TXG_NOWAIT);
	if (error) {
	if (error == ERESTART) {
	waited = B_TRUE;
	dmu_tx_wait(tx);
	dmu_tx_abort(tx);
	goto top;
	}
	zfs_acl_ids_free(&acl_ids);
	dmu_tx_abort(tx);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	zfs_mknode(dzp, vap, tx, cr, IS_TMPFILE, &zp, &acl_ids);

	if (fuid_dirtied)
	zfs_fuid_sync(zfsvfs, tx);

	/* Add to unlinked set */
	zp->z_unlinked = B_TRUE;
	zfs_unlinked_add(zp, tx);
	zfs_acl_ids_free(&acl_ids);
	dmu_tx_commit(tx);
	out:

	if (error) {
	if (zp)
	zrele(zp);
	} else {
	zfs_znode_update_vfs(dzp);
	zfs_znode_update_vfs(zp);
	*ipp = ZTOI(zp);
	}

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Remove an entry from a directory.
	*
	* IN: dzp - znode of directory to remove entry from.
	* name - name of entry to remove.
	* cr - credentials of caller.
	* flags - case flags.
	*
	* RETURN: 0 if success
	* error code if failure
	*
	* Timestamps:
	* dzp - ctime\|mtime
	* ip - ctime (if nlink > 0)
	*/

	static uint64_t null_xattr = 0;

	int
	zfs_remove(znode_t dzp, char name, cred_t *cr, int flags)
	{
	znode_t *zp;
	znode_t *xzp;
	zfsvfs_t *zfsvfs = ZTOZSB(dzp);
	zilog_t *zilog;
	uint64_t acl_obj, xattr_obj;
	uint64_t xattr_obj_unlinked = 0;
	uint64_t obj = 0;
	uint64_t links;
	zfs_dirlock_t *dl;
	dmu_tx_t *tx;
	boolean_t may_delete_now, delete_now = FALSE;
	boolean_t unlinked, toobig = FALSE;
	uint64_t txtype;
	pathname_t *realnmp = NULL;
	pathname_t realnm;
	int error;
	int zflg = ZEXISTS;
	boolean_t waited = B_FALSE;

	if (name == NULL)
	return (SET_ERROR(EINVAL));

	if ((error = zfs_enter_verify_zp(zfsvfs, dzp, FTAG)) != 0)
	return (error);
	zilog = zfsvfs->z_log;

	if (flags & FIGNORECASE) {
	zflg \|= ZCILOOK;
	pn_alloc(&realnm);
	realnmp = &realnm;
	}

	top:
	xattr_obj = 0;
	xzp = NULL;
	/*
	* Attempt to lock directory; fail if entry doesn't exist.
	*/
	if ((error = zfs_dirent_lock(&dl, dzp, name, &zp, zflg,
	NULL, realnmp))) {
	if (realnmp)
	pn_free(realnmp);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	if ((error = zfs_zaccess_delete(dzp, zp, cr, zfs_init_idmap))) {
	goto out;
	}

	/*
	* Need to use rmdir for removing directories.
	*/
	if (S_ISDIR(ZTOI(zp)->i_mode)) {
	error = SET_ERROR(EPERM);
	goto out;
	}

	mutex_enter(&zp->z_lock);
	may_delete_now = atomic_read(&ZTOI(zp)->i_count) == 1 &&
	!zn_has_cached_data(zp, 0, LLONG_MAX);
	mutex_exit(&zp->z_lock);

	/*
	* We may delete the znode now, or we may put it in the unlinked set;
	* it depends on whether we're the last link, and on whether there are
	* other holds on the inode. So we dmu_tx_hold() the right things to
	* allow for either case.
	*/
	obj = zp->z_id;
	tx = dmu_tx_create(zfsvfs->z_os);
	dmu_tx_hold_zap(tx, dzp->z_id, FALSE, name);
	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
	zfs_sa_upgrade_txholds(tx, zp);
	zfs_sa_upgrade_txholds(tx, dzp);
	if (may_delete_now) {
	toobig = zp->z_size > zp->z_blksz * zfs_delete_blocks;
	/* if the file is too big, only hold_free a token amount */
	dmu_tx_hold_free(tx, zp->z_id, 0,
	(toobig ? DMU_MAX_ACCESS : DMU_OBJECT_END));
	}

	/* are there any extended attributes? */
	error = sa_lookup(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs),
	&xattr_obj, sizeof (xattr_obj));
	if (error == 0 && xattr_obj) {
	error = zfs_zget(zfsvfs, xattr_obj, &xzp);
	ASSERT0(error);
	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE);
	dmu_tx_hold_sa(tx, xzp->z_sa_hdl, B_FALSE);
	}

	mutex_enter(&zp->z_lock);
	if ((acl_obj = zfs_external_acl(zp)) != 0 && may_delete_now)
	dmu_tx_hold_free(tx, acl_obj, 0, DMU_OBJECT_END);
	mutex_exit(&zp->z_lock);

	/* charge as an update -- would be nice not to charge at all */
	dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);

	/*
	* Mark this transaction as typically resulting in a net free of space
	*/
	dmu_tx_mark_netfree(tx);

	error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) \| TXG_NOWAIT);
	if (error) {
	zfs_dirent_unlock(dl);
	if (error == ERESTART) {
	waited = B_TRUE;
	dmu_tx_wait(tx);
	dmu_tx_abort(tx);
	zrele(zp);
	if (xzp)
	zrele(xzp);
	goto top;
	}
	if (realnmp)
	pn_free(realnmp);
	dmu_tx_abort(tx);
	zrele(zp);
	if (xzp)
	zrele(xzp);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Remove the directory entry.
	*/
	error = zfs_link_destroy(dl, zp, tx, zflg, &unlinked);

	if (error) {
	dmu_tx_commit(tx);
	goto out;
	}

	if (unlinked) {
	/*
	* Hold z_lock so that we can make sure that the ACL obj
	* hasn't changed. Could have been deleted due to
	* zfs_sa_upgrade().
	*/
	mutex_enter(&zp->z_lock);
	(void) sa_lookup(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs),
	&xattr_obj_unlinked, sizeof (xattr_obj_unlinked));
	delete_now = may_delete_now && !toobig &&
	atomic_read(&ZTOI(zp)->i_count) == 1 &&
	!zn_has_cached_data(zp, 0, LLONG_MAX) &&
	xattr_obj == xattr_obj_unlinked &&
	zfs_external_acl(zp) == acl_obj;
	VERIFY_IMPLY(xattr_obj_unlinked, xzp);
	}

	if (delete_now) {
	if (xattr_obj_unlinked) {
	ASSERT3U(ZTOI(xzp)->i_nlink, ==, 2);
	mutex_enter(&xzp->z_lock);
	xzp->z_unlinked = B_TRUE;
	clear_nlink(ZTOI(xzp));
	links = 0;
	error = sa_update(xzp->z_sa_hdl, SA_ZPL_LINKS(zfsvfs),
	&links, sizeof (links), tx);
	ASSERT3U(error, ==, 0);
	mutex_exit(&xzp->z_lock);
	zfs_unlinked_add(xzp, tx);

	if (zp->z_is_sa)
	error = sa_remove(zp->z_sa_hdl,
	SA_ZPL_XATTR(zfsvfs), tx);
	else
	error = sa_update(zp->z_sa_hdl,
	SA_ZPL_XATTR(zfsvfs), &null_xattr,
	sizeof (uint64_t), tx);
	ASSERT0(error);
	}
	/*
	* Add to the unlinked set because a new reference could be
	* taken concurrently resulting in a deferred destruction.
	*/
	zfs_unlinked_add(zp, tx);
	mutex_exit(&zp->z_lock);
	} else if (unlinked) {
	mutex_exit(&zp->z_lock);
	zfs_unlinked_add(zp, tx);
	}

	txtype = TX_REMOVE;
	if (flags & FIGNORECASE)
	txtype \|= TX_CI;
	zfs_log_remove(zilog, tx, txtype, dzp, name, obj, unlinked);

	dmu_tx_commit(tx);
	out:
	if (realnmp)
	pn_free(realnmp);

	zfs_dirent_unlock(dl);
	zfs_znode_update_vfs(dzp);
	zfs_znode_update_vfs(zp);

	if (delete_now)
	zrele(zp);
	else
	zfs_zrele_async(zp);

	if (xzp) {
	zfs_znode_update_vfs(xzp);
	zfs_zrele_async(xzp);
	}

	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Create a new directory and insert it into dzp using the name
	* provided. Return a pointer to the inserted directory.
	*
	* IN: dzp - znode of directory to add subdir to.
	* dirname - name of new directory.
	* vap - attributes of new directory.
	* cr - credentials of caller.
	* flags - case flags.
	* vsecp - ACL to be set
	* mnt_ns - user namespace of the mount
	*
	* OUT: zpp - znode of created directory.
	*
	* RETURN: 0 if success
	* error code if failure
	*
	* Timestamps:
	* dzp - ctime\|mtime updated
	* zpp - ctime\|mtime\|atime updated
	*/
	int
	zfs_mkdir(znode_t dzp, char dirname, vattr_t vap, znode_t *zpp,
	cred_t cr, int flags, vsecattr_t vsecp, zidmap_t *mnt_ns)
	{
	znode_t *zp;
	zfsvfs_t *zfsvfs = ZTOZSB(dzp);
	zilog_t *zilog;
	zfs_dirlock_t *dl;
	uint64_t txtype;
	dmu_tx_t *tx;
	int error;
	int zf = ZNEW;
	uid_t uid;
	gid_t gid = crgetgid(cr);
	zfs_acl_ids_t acl_ids;
	boolean_t fuid_dirtied;
	boolean_t waited = B_FALSE;

	ASSERT(S_ISDIR(vap->va_mode));

	/*
	* If we have an ephemeral id, ACL, or XVATTR then
	* make sure file system is at proper version
	*/

	uid = crgetuid(cr);
	if (zfsvfs->z_use_fuids == B_FALSE &&
	(vsecp \|\| IS_EPHEMERAL(uid) \|\| IS_EPHEMERAL(gid)))
	return (SET_ERROR(EINVAL));

	if (dirname == NULL)
	return (SET_ERROR(EINVAL));

	if ((error = zfs_enter_verify_zp(zfsvfs, dzp, FTAG)) != 0)
	return (error);
	zilog = zfsvfs->z_log;

	if (dzp->z_pflags & ZFS_XATTR) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	if (zfsvfs->z_utf8 && u8_validate(dirname,
	strlen(dirname), NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EILSEQ));
	}
	if (flags & FIGNORECASE)
	zf \|= ZCILOOK;

	if (vap->va_mask & ATTR_XVATTR) {
	if ((error = secpolicy_xvattr((xvattr_t *)vap,
	crgetuid(cr), cr, vap->va_mode)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	}

	if ((error = zfs_acl_ids_create(dzp, 0, vap, cr,
	vsecp, &acl_ids, mnt_ns)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	/*
	* First make sure the new directory doesn't exist.
	*
	* Existence is checked first to make sure we don't return
	* EACCES instead of EEXIST which can cause some applications
	* to fail.
	*/
	top:
	*zpp = NULL;

	if ((error = zfs_dirent_lock(&dl, dzp, dirname, &zp, zf,
	NULL, NULL))) {
	zfs_acl_ids_free(&acl_ids);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	if ((error = zfs_zaccess(dzp, ACE_ADD_SUBDIRECTORY, 0, B_FALSE, cr,
	mnt_ns))) {
	zfs_acl_ids_free(&acl_ids);
	zfs_dirent_unlock(dl);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, zfs_inherit_projid(dzp))) {
	zfs_acl_ids_free(&acl_ids);
	zfs_dirent_unlock(dl);
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EDQUOT));
	}

	/*
	* Add a new entry to the directory.
	*/
	tx = dmu_tx_create(zfsvfs->z_os);
	dmu_tx_hold_zap(tx, dzp->z_id, TRUE, dirname);
	dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, FALSE, NULL);
	fuid_dirtied = zfsvfs->z_fuid_dirty;
	if (fuid_dirtied)
	zfs_fuid_txhold(zfsvfs, tx);
	if (!zfsvfs->z_use_sa && acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) {
	dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0,
	acl_ids.z_aclp->z_acl_bytes);
	}

	dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes +
	ZFS_SA_BASE_ATTR_SIZE);

	error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) \| TXG_NOWAIT);
	if (error) {
	zfs_dirent_unlock(dl);
	if (error == ERESTART) {
	waited = B_TRUE;
	dmu_tx_wait(tx);
	dmu_tx_abort(tx);
	goto top;
	}
	zfs_acl_ids_free(&acl_ids);
	dmu_tx_abort(tx);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Create new node.
	*/
	zfs_mknode(dzp, vap, tx, cr, 0, &zp, &acl_ids);

	/*
	* Now put new name in parent dir.
	*/
	error = zfs_link_create(dl, zp, tx, ZNEW);
	if (error != 0) {
	zfs_znode_delete(zp, tx);
	remove_inode_hash(ZTOI(zp));
	goto out;
	}

	if (fuid_dirtied)
	zfs_fuid_sync(zfsvfs, tx);

	*zpp = zp;

	txtype = zfs_log_create_txtype(Z_DIR, vsecp, vap);
	if (flags & FIGNORECASE)
	txtype \|= TX_CI;
	zfs_log_create(zilog, tx, txtype, dzp, zp, dirname, vsecp,
	acl_ids.z_fuidp, vap);

	out:
	zfs_acl_ids_free(&acl_ids);

	dmu_tx_commit(tx);

	zfs_dirent_unlock(dl);

	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);

	if (error != 0) {
	zrele(zp);
	} else {
	zfs_znode_update_vfs(dzp);
	zfs_znode_update_vfs(zp);
	}
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Remove a directory subdir entry. If the current working
	* directory is the same as the subdir to be removed, the
	* remove will fail.
	*
	* IN: dzp - znode of directory to remove from.
	* name - name of directory to be removed.
	* cwd - inode of current working directory.
	* cr - credentials of caller.
	* flags - case flags
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* dzp - ctime\|mtime updated
	*/
	int
	zfs_rmdir(znode_t dzp, char name, znode_t cwd, cred_t cr,
	int flags)
	{
	znode_t *zp;
	zfsvfs_t *zfsvfs = ZTOZSB(dzp);
	zilog_t *zilog;
	zfs_dirlock_t *dl;
	dmu_tx_t *tx;
	int error;
	int zflg = ZEXISTS;
	boolean_t waited = B_FALSE;

	if (name == NULL)
	return (SET_ERROR(EINVAL));

	if ((error = zfs_enter_verify_zp(zfsvfs, dzp, FTAG)) != 0)
	return (error);
	zilog = zfsvfs->z_log;

	if (flags & FIGNORECASE)
	zflg \|= ZCILOOK;
	top:
	zp = NULL;

	/*
	* Attempt to lock directory; fail if entry doesn't exist.
	*/
	if ((error = zfs_dirent_lock(&dl, dzp, name, &zp, zflg,
	NULL, NULL))) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	if ((error = zfs_zaccess_delete(dzp, zp, cr, zfs_init_idmap))) {
	goto out;
	}

	if (!S_ISDIR(ZTOI(zp)->i_mode)) {
	error = SET_ERROR(ENOTDIR);
	goto out;
	}

	if (zp == cwd) {
	error = SET_ERROR(EINVAL);
	goto out;
	}

	/*
	* Grab a lock on the directory to make sure that no one is
	* trying to add (or lookup) entries while we are removing it.
	*/
	rw_enter(&zp->z_name_lock, RW_WRITER);

	/*
	* Grab a lock on the parent pointer to make sure we play well
	* with the treewalk and directory rename code.
	*/
	rw_enter(&zp->z_parent_lock, RW_WRITER);

	tx = dmu_tx_create(zfsvfs->z_os);
	dmu_tx_hold_zap(tx, dzp->z_id, FALSE, name);
	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
	dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);
	zfs_sa_upgrade_txholds(tx, zp);
	zfs_sa_upgrade_txholds(tx, dzp);
	dmu_tx_mark_netfree(tx);
	error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) \| TXG_NOWAIT);
	if (error) {
	rw_exit(&zp->z_parent_lock);
	rw_exit(&zp->z_name_lock);
	zfs_dirent_unlock(dl);
	if (error == ERESTART) {
	waited = B_TRUE;
	dmu_tx_wait(tx);
	dmu_tx_abort(tx);
	zrele(zp);
	goto top;
	}
	dmu_tx_abort(tx);
	zrele(zp);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	error = zfs_link_destroy(dl, zp, tx, zflg, NULL);

	if (error == 0) {
	uint64_t txtype = TX_RMDIR;
	if (flags & FIGNORECASE)
	txtype \|= TX_CI;
	zfs_log_remove(zilog, tx, txtype, dzp, name, ZFS_NO_OBJECT,
	B_FALSE);
	}

	dmu_tx_commit(tx);

	rw_exit(&zp->z_parent_lock);
	rw_exit(&zp->z_name_lock);
	out:
	zfs_dirent_unlock(dl);

	zfs_znode_update_vfs(dzp);
	zfs_znode_update_vfs(zp);
	zrele(zp);

	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Read directory entries from the given directory cursor position and emit
	* name and position for each entry.
	*
	* IN: ip - inode of directory to read.
	* ctx - directory entry context.
	* cr - credentials of caller.
	*
	* RETURN: 0 if success
	* error code if failure
	*
	* Timestamps:
	* ip - atime updated
	*
	* Note that the low 4 bits of the cookie returned by zap is always zero.
	* This allows us to use the low range for "special" directory entries:
	* We use 0 for '.', and 1 for '..'. If this is the root of the filesystem,
	* we use the offset 2 for the '.zfs' directory.
	*/
	int
	zfs_readdir(struct inode ip, zpl_dir_context_t ctx, cred_t *cr)
	{
	(void) cr;
	znode_t *zp = ITOZ(ip);
	zfsvfs_t *zfsvfs = ITOZSB(ip);
	objset_t *os;
	zap_cursor_t zc;
	zap_attribute_t zap;
	int error;
	uint8_t prefetch;
	uint8_t type;
	int done = 0;
	uint64_t parent;
	uint64_t offset; /* must be unsigned; checks for < 1 */

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_PARENT(zfsvfs),
	&parent, sizeof (parent))) != 0)
	goto out;

	/*
	* Quit if directory has been removed (posix)
	*/
	if (zp->z_unlinked)
	goto out;

	error = 0;
	os = zfsvfs->z_os;
	offset = ctx->pos;
	prefetch = zp->z_zn_prefetch;

	/*
	* Initialize the iterator cursor.
	*/
	if (offset <= 3) {
	/*
	* Start iteration from the beginning of the directory.
	*/
	zap_cursor_init(&zc, os, zp->z_id);
	} else {
	/*
	* The offset is a serialized cursor.
	*/
	zap_cursor_init_serialized(&zc, os, zp->z_id, offset);
	}

	/*
	* Transform to file-system independent format
	*/
	while (!done) {
	uint64_t objnum;
	/*
	* Special case `.', `..', and `.zfs'.
	*/
	if (offset == 0) {
	(void) strcpy(zap.za_name, ".");
	zap.za_normalization_conflict = 0;
	objnum = zp->z_id;
	type = DT_DIR;
	} else if (offset == 1) {
	(void) strcpy(zap.za_name, "..");
	zap.za_normalization_conflict = 0;
	objnum = parent;
	type = DT_DIR;
	} else if (offset == 2 && zfs_show_ctldir(zp)) {
	(void) strcpy(zap.za_name, ZFS_CTLDIR_NAME);
	zap.za_normalization_conflict = 0;
	objnum = ZFSCTL_INO_ROOT;
	type = DT_DIR;
	} else {
	/*
	* Grab next entry.
	*/
	if ((error = zap_cursor_retrieve(&zc, &zap))) {
	if (error == ENOENT)
	break;
	else
	goto update;
	}

	/*
	* Allow multiple entries provided the first entry is
	* the object id. Non-zpl consumers may safely make
	* use of the additional space.
	*
	* XXX: This should be a feature flag for compatibility
	*/
	if (zap.za_integer_length != 8 \|\|
	zap.za_num_integers == 0) {
	cmn_err(CE_WARN, "zap_readdir: bad directory "
	"entry, obj = %lld, offset = %lld, "
	"length = %d, num = %lld\n",
	(u_longlong_t)zp->z_id,
	(u_longlong_t)offset,
	zap.za_integer_length,
	(u_longlong_t)zap.za_num_integers);
	error = SET_ERROR(ENXIO);
	goto update;
	}

	objnum = ZFS_DIRENT_OBJ(zap.za_first_integer);
	type = ZFS_DIRENT_TYPE(zap.za_first_integer);
	}

	done = !zpl_dir_emit(ctx, zap.za_name, strlen(zap.za_name),
	objnum, type);
	if (done)
	break;

	- /* Prefetch znode */
	- if (prefetch) {
	- dmu_prefetch(os, objnum, 0, 0, 0,
	- ZIO_PRIORITY_SYNC_READ);
	- }
	+ if (prefetch)
	+ dmu_prefetch_dnode(os, objnum, ZIO_PRIORITY_SYNC_READ);

	/*
	* Move to the next entry, fill in the previous offset.
	*/
	if (offset > 2 \|\| (offset == 2 && !zfs_show_ctldir(zp))) {
	zap_cursor_advance(&zc);
	offset = zap_cursor_serialize(&zc);
	} else {
	offset += 1;
	}
	ctx->pos = offset;
	}
	zp->z_zn_prefetch = B_FALSE; /* a lookup will re-enable pre-fetching */

	update:
	zap_cursor_fini(&zc);
	if (error == ENOENT)
	error = 0;
	out:
	zfs_exit(zfsvfs, FTAG);

	return (error);
	}

	/*
	* Get the basic file attributes and place them in the provided kstat
	* structure. The inode is assumed to be the authoritative source
	* for most of the attributes. However, the znode currently has the
	* authoritative atime, blksize, and block count.
	*
	* IN: ip - inode of file.
	*
	* OUT: sp - kstat values.
	*
	* RETURN: 0 (always succeeds)
	*/
	int
	#ifdef HAVE_GENERIC_FILLATTR_IDMAP_REQMASK
	zfs_getattr_fast(zidmap_t user_ns, u32 request_mask, struct inode ip,
	struct kstat *sp)
	#else
	zfs_getattr_fast(zidmap_t user_ns, struct inode ip, struct kstat *sp)
	#endif
	{
	znode_t *zp = ITOZ(ip);
	zfsvfs_t *zfsvfs = ITOZSB(ip);
	uint32_t blksize;
	u_longlong_t nblocks;
	int error;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	mutex_enter(&zp->z_lock);

	#ifdef HAVE_GENERIC_FILLATTR_IDMAP_REQMASK
	zpl_generic_fillattr(user_ns, request_mask, ip, sp);
	#else
	zpl_generic_fillattr(user_ns, ip, sp);
	#endif
	/*
	* +1 link count for root inode with visible '.zfs' directory.
	*/
	if ((zp->z_id == zfsvfs->z_root) && zfs_show_ctldir(zp))
	if (sp->nlink < ZFS_LINK_MAX)
	sp->nlink++;

	sa_object_size(zp->z_sa_hdl, &blksize, &nblocks);
	sp->blksize = blksize;
	sp->blocks = nblocks;

	if (unlikely(zp->z_blksz == 0)) {
	/*
	* Block size hasn't been set; suggest maximal I/O transfers.
	*/
	sp->blksize = zfsvfs->z_max_blksz;
	}

	mutex_exit(&zp->z_lock);

	/*
	* Required to prevent NFS client from detecting different inode
	* numbers of snapshot root dentry before and after snapshot mount.
	*/
	if (zfsvfs->z_issnap) {
	if (ip->i_sb->s_root->d_inode == ip)
	sp->ino = ZFSCTL_INO_SNAPDIRS -
	dmu_objset_id(zfsvfs->z_os);
	}

	zfs_exit(zfsvfs, FTAG);

	return (0);
	}

	/*
	* For the operation of changing file's user/group/project, we need to
	* handle not only the main object that is assigned to the file directly,
	* but also the ones that are used by the file via hidden xattr directory.
	*
	* Because the xattr directory may contains many EA entries, as to it may
	* be impossible to change all of them via the transaction of changing the
	* main object's user/group/project attributes. Then we have to change them
	* via other multiple independent transactions one by one. It may be not good
	* solution, but we have no better idea yet.
	*/
	static int
	zfs_setattr_dir(znode_t *dzp)
	{
	struct inode *dxip = ZTOI(dzp);
	struct inode *xip = NULL;
	zfsvfs_t *zfsvfs = ZTOZSB(dzp);
	objset_t *os = zfsvfs->z_os;
	zap_cursor_t zc;
	zap_attribute_t zap;
	zfs_dirlock_t *dl;
	znode_t *zp = NULL;
	dmu_tx_t *tx = NULL;
	uint64_t uid, gid;
	sa_bulk_attr_t bulk[4];
	int count;
	int err;

	zap_cursor_init(&zc, os, dzp->z_id);
	while ((err = zap_cursor_retrieve(&zc, &zap)) == 0) {
	count = 0;
	if (zap.za_integer_length != 8 \|\| zap.za_num_integers != 1) {
	err = ENXIO;
	break;
	}

	err = zfs_dirent_lock(&dl, dzp, (char *)zap.za_name, &zp,
	ZEXISTS, NULL, NULL);
	if (err == ENOENT)
	goto next;
	if (err)
	break;

	xip = ZTOI(zp);
	if (KUID_TO_SUID(xip->i_uid) == KUID_TO_SUID(dxip->i_uid) &&
	KGID_TO_SGID(xip->i_gid) == KGID_TO_SGID(dxip->i_gid) &&
	zp->z_projid == dzp->z_projid)
	goto next;

	tx = dmu_tx_create(os);
	if (!(zp->z_pflags & ZFS_PROJID))
	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE);
	else
	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);

	err = dmu_tx_assign(tx, TXG_WAIT);
	if (err)
	break;

	mutex_enter(&dzp->z_lock);

	if (KUID_TO_SUID(xip->i_uid) != KUID_TO_SUID(dxip->i_uid)) {
	xip->i_uid = dxip->i_uid;
	uid = zfs_uid_read(dxip);
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zfsvfs), NULL,
	&uid, sizeof (uid));
	}

	if (KGID_TO_SGID(xip->i_gid) != KGID_TO_SGID(dxip->i_gid)) {
	xip->i_gid = dxip->i_gid;
	gid = zfs_gid_read(dxip);
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zfsvfs), NULL,
	&gid, sizeof (gid));
	}

	if (zp->z_projid != dzp->z_projid) {
	if (!(zp->z_pflags & ZFS_PROJID)) {
	zp->z_pflags \|= ZFS_PROJID;
	SA_ADD_BULK_ATTR(bulk, count,
	SA_ZPL_FLAGS(zfsvfs), NULL, &zp->z_pflags,
	sizeof (zp->z_pflags));
	}

	zp->z_projid = dzp->z_projid;
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_PROJID(zfsvfs),
	NULL, &zp->z_projid, sizeof (zp->z_projid));
	}

	mutex_exit(&dzp->z_lock);

	if (likely(count > 0)) {
	err = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
	dmu_tx_commit(tx);
	} else {
	dmu_tx_abort(tx);
	}
	tx = NULL;
	if (err != 0 && err != ENOENT)
	break;

	next:
	if (zp) {
	zrele(zp);
	zp = NULL;
	zfs_dirent_unlock(dl);
	}
	zap_cursor_advance(&zc);
	}

	if (tx)
	dmu_tx_abort(tx);
	if (zp) {
	zrele(zp);
	zfs_dirent_unlock(dl);
	}
	zap_cursor_fini(&zc);

	return (err == ENOENT ? 0 : err);
	}

	/*
	* Set the file attributes to the values contained in the
	* vattr structure.
	*
	* IN: zp - znode of file to be modified.
	* vap - new attribute values.
	* If ATTR_XVATTR set, then optional attrs are being set
	* flags - ATTR_UTIME set if non-default time values provided.
	* - ATTR_NOACLCHECK (CIFS context only).
	* cr - credentials of caller.
	* mnt_ns - user namespace of the mount
	*
	* RETURN: 0 if success
	* error code if failure
	*
	* Timestamps:
	* ip - ctime updated, mtime updated if size changed.
	*/
	int
	zfs_setattr(znode_t zp, vattr_t vap, int flags, cred_t cr, zidmap_t mnt_ns)
	{
	struct inode *ip;
	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	objset_t *os;
	zilog_t *zilog;
	dmu_tx_t *tx;
	vattr_t oldva;
	xvattr_t *tmpxvattr;
	uint_t mask = vap->va_mask;
	uint_t saved_mask = 0;
	int trim_mask = 0;
	uint64_t new_mode;
	uint64_t new_kuid = 0, new_kgid = 0, new_uid, new_gid;
	uint64_t xattr_obj;
	uint64_t mtime[2], ctime[2], atime[2];
	uint64_t projid = ZFS_INVALID_PROJID;
	znode_t *attrzp;
	int need_policy = FALSE;
	int err, err2 = 0;
	zfs_fuid_info_t *fuidp = NULL;
	xvattr_t xvap = (xvattr_t )vap; /* vap may be an xvattr_t * */
	xoptattr_t *xoap;
	zfs_acl_t *aclp;
	boolean_t skipaclchk = (flags & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
	boolean_t fuid_dirtied = B_FALSE;
	boolean_t handle_eadir = B_FALSE;
	sa_bulk_attr_t bulk, xattr_bulk;
	int count = 0, xattr_count = 0, bulks = 8;

	if (mask == 0)
	return (0);

	if ((err = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (err);
	ip = ZTOI(zp);
	os = zfsvfs->z_os;

	/*
	* If this is a xvattr_t, then get a pointer to the structure of
	* optional attributes. If this is NULL, then we have a vattr_t.
	*/
	xoap = xva_getxoptattr(xvap);
	if (xoap != NULL && (mask & ATTR_XVATTR)) {
	if (XVA_ISSET_REQ(xvap, XAT_PROJID)) {
	if (!dmu_objset_projectquota_enabled(os) \|\|
	(!S_ISREG(ip->i_mode) && !S_ISDIR(ip->i_mode))) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(ENOTSUP));
	}

	projid = xoap->xoa_projid;
	if (unlikely(projid == ZFS_INVALID_PROJID)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	if (projid == zp->z_projid && zp->z_pflags & ZFS_PROJID)
	projid = ZFS_INVALID_PROJID;
	else
	need_policy = TRUE;
	}

	if (XVA_ISSET_REQ(xvap, XAT_PROJINHERIT) &&
	(xoap->xoa_projinherit !=
	((zp->z_pflags & ZFS_PROJINHERIT) != 0)) &&
	(!dmu_objset_projectquota_enabled(os) \|\|
	(!S_ISREG(ip->i_mode) && !S_ISDIR(ip->i_mode)))) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(ENOTSUP));
	}
	}

	zilog = zfsvfs->z_log;

	/*
	* Make sure that if we have ephemeral uid/gid or xvattr specified
	* that file system is at proper version level
	*/

	if (zfsvfs->z_use_fuids == B_FALSE &&
	(((mask & ATTR_UID) && IS_EPHEMERAL(vap->va_uid)) \|\|
	((mask & ATTR_GID) && IS_EPHEMERAL(vap->va_gid)) \|\|
	(mask & ATTR_XVATTR))) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	if (mask & ATTR_SIZE && S_ISDIR(ip->i_mode)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EISDIR));
	}

	if (mask & ATTR_SIZE && !S_ISREG(ip->i_mode) && !S_ISFIFO(ip->i_mode)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	tmpxvattr = kmem_alloc(sizeof (xvattr_t), KM_SLEEP);
	xva_init(tmpxvattr);

	bulk = kmem_alloc(sizeof (sa_bulk_attr_t) * bulks, KM_SLEEP);
	xattr_bulk = kmem_alloc(sizeof (sa_bulk_attr_t) * bulks, KM_SLEEP);

	/*
	* Immutable files can only alter immutable bit and atime
	*/
	if ((zp->z_pflags & ZFS_IMMUTABLE) &&
	((mask & (ATTR_SIZE\|ATTR_UID\|ATTR_GID\|ATTR_MTIME\|ATTR_MODE)) \|\|
	((mask & ATTR_XVATTR) && XVA_ISSET_REQ(xvap, XAT_CREATETIME)))) {
	err = SET_ERROR(EPERM);
	goto out3;
	}

	if ((mask & ATTR_SIZE) && (zp->z_pflags & ZFS_READONLY)) {
	err = SET_ERROR(EPERM);
	goto out3;
	}

	/*
	* Verify timestamps doesn't overflow 32 bits.
	* ZFS can handle large timestamps, but 32bit syscalls can't
	* handle times greater than 2039. This check should be removed
	* once large timestamps are fully supported.
	*/
	if (mask & (ATTR_ATIME \| ATTR_MTIME)) {
	if (((mask & ATTR_ATIME) &&
	TIMESPEC_OVERFLOW(&vap->va_atime)) \|\|
	((mask & ATTR_MTIME) &&
	TIMESPEC_OVERFLOW(&vap->va_mtime))) {
	err = SET_ERROR(EOVERFLOW);
	goto out3;
	}
	}

	top:
	attrzp = NULL;
	aclp = NULL;

	/* Can this be moved to before the top label? */
	if (zfs_is_readonly(zfsvfs)) {
	err = SET_ERROR(EROFS);
	goto out3;
	}

	/*
	* First validate permissions
	*/

	if (mask & ATTR_SIZE) {
	err = zfs_zaccess(zp, ACE_WRITE_DATA, 0, skipaclchk, cr,
	mnt_ns);
	if (err)
	goto out3;

	/*
	* XXX - Note, we are not providing any open
	* mode flags here (like FNDELAY), so we may
	* block if there are locks present... this
	* should be addressed in openat().
	*/
	/* XXX - would it be OK to generate a log record here? */
	err = zfs_freesp(zp, vap->va_size, 0, 0, FALSE);
	if (err)
	goto out3;
	}

	if (mask & (ATTR_ATIME\|ATTR_MTIME) \|\|
	((mask & ATTR_XVATTR) && (XVA_ISSET_REQ(xvap, XAT_HIDDEN) \|\|
	XVA_ISSET_REQ(xvap, XAT_READONLY) \|\|
	XVA_ISSET_REQ(xvap, XAT_ARCHIVE) \|\|
	XVA_ISSET_REQ(xvap, XAT_OFFLINE) \|\|
	XVA_ISSET_REQ(xvap, XAT_SPARSE) \|\|
	XVA_ISSET_REQ(xvap, XAT_CREATETIME) \|\|
	XVA_ISSET_REQ(xvap, XAT_SYSTEM)))) {
	need_policy = zfs_zaccess(zp, ACE_WRITE_ATTRIBUTES, 0,
	skipaclchk, cr, mnt_ns);
	}

	if (mask & (ATTR_UID\|ATTR_GID)) {
	int idmask = (mask & (ATTR_UID\|ATTR_GID));
	int take_owner;
	int take_group;
	uid_t uid;
	gid_t gid;

	/*
	* NOTE: even if a new mode is being set,
	* we may clear S_ISUID/S_ISGID bits.
	*/

	if (!(mask & ATTR_MODE))
	vap->va_mode = zp->z_mode;

	/*
	* Take ownership or chgrp to group we are a member of
	*/

	uid = zfs_uid_to_vfsuid(mnt_ns, zfs_i_user_ns(ip),
	vap->va_uid);
	gid = zfs_gid_to_vfsgid(mnt_ns, zfs_i_user_ns(ip),
	vap->va_gid);
	take_owner = (mask & ATTR_UID) && (uid == crgetuid(cr));
	take_group = (mask & ATTR_GID) &&
	zfs_groupmember(zfsvfs, gid, cr);

	/*
	* If both ATTR_UID and ATTR_GID are set then take_owner and
	* take_group must both be set in order to allow taking
	* ownership.
	*
	* Otherwise, send the check through secpolicy_vnode_setattr()
	*
	*/

	if (((idmask == (ATTR_UID\|ATTR_GID)) &&
	take_owner && take_group) \|\|
	((idmask == ATTR_UID) && take_owner) \|\|
	((idmask == ATTR_GID) && take_group)) {
	if (zfs_zaccess(zp, ACE_WRITE_OWNER, 0,
	skipaclchk, cr, mnt_ns) == 0) {
	/*
	* Remove setuid/setgid for non-privileged users
	*/
	(void) secpolicy_setid_clear(vap, cr);
	trim_mask = (mask & (ATTR_UID\|ATTR_GID));
	} else {
	need_policy = TRUE;
	}
	} else {
	need_policy = TRUE;
	}
	}

	mutex_enter(&zp->z_lock);
	oldva.va_mode = zp->z_mode;
	zfs_fuid_map_ids(zp, cr, &oldva.va_uid, &oldva.va_gid);
	if (mask & ATTR_XVATTR) {
	/*
	* Update xvattr mask to include only those attributes
	* that are actually changing.
	*
	* the bits will be restored prior to actually setting
	* the attributes so the caller thinks they were set.
	*/
	if (XVA_ISSET_REQ(xvap, XAT_APPENDONLY)) {
	if (xoap->xoa_appendonly !=
	((zp->z_pflags & ZFS_APPENDONLY) != 0)) {
	need_policy = TRUE;
	} else {
	XVA_CLR_REQ(xvap, XAT_APPENDONLY);
	XVA_SET_REQ(tmpxvattr, XAT_APPENDONLY);
	}
	}

	if (XVA_ISSET_REQ(xvap, XAT_PROJINHERIT)) {
	if (xoap->xoa_projinherit !=
	((zp->z_pflags & ZFS_PROJINHERIT) != 0)) {
	need_policy = TRUE;
	} else {
	XVA_CLR_REQ(xvap, XAT_PROJINHERIT);
	XVA_SET_REQ(tmpxvattr, XAT_PROJINHERIT);
	}
	}

	if (XVA_ISSET_REQ(xvap, XAT_NOUNLINK)) {
	if (xoap->xoa_nounlink !=
	((zp->z_pflags & ZFS_NOUNLINK) != 0)) {
	need_policy = TRUE;
	} else {
	XVA_CLR_REQ(xvap, XAT_NOUNLINK);
	XVA_SET_REQ(tmpxvattr, XAT_NOUNLINK);
	}
	}

	if (XVA_ISSET_REQ(xvap, XAT_IMMUTABLE)) {
	if (xoap->xoa_immutable !=
	((zp->z_pflags & ZFS_IMMUTABLE) != 0)) {
	need_policy = TRUE;
	} else {
	XVA_CLR_REQ(xvap, XAT_IMMUTABLE);
	XVA_SET_REQ(tmpxvattr, XAT_IMMUTABLE);
	}
	}

	if (XVA_ISSET_REQ(xvap, XAT_NODUMP)) {
	if (xoap->xoa_nodump !=
	((zp->z_pflags & ZFS_NODUMP) != 0)) {
	need_policy = TRUE;
	} else {
	XVA_CLR_REQ(xvap, XAT_NODUMP);
	XVA_SET_REQ(tmpxvattr, XAT_NODUMP);
	}
	}

	if (XVA_ISSET_REQ(xvap, XAT_AV_MODIFIED)) {
	if (xoap->xoa_av_modified !=
	((zp->z_pflags & ZFS_AV_MODIFIED) != 0)) {
	need_policy = TRUE;
	} else {
	XVA_CLR_REQ(xvap, XAT_AV_MODIFIED);
	XVA_SET_REQ(tmpxvattr, XAT_AV_MODIFIED);
	}
	}

	if (XVA_ISSET_REQ(xvap, XAT_AV_QUARANTINED)) {
	if ((!S_ISREG(ip->i_mode) &&
	xoap->xoa_av_quarantined) \|\|
	xoap->xoa_av_quarantined !=
	((zp->z_pflags & ZFS_AV_QUARANTINED) != 0)) {
	need_policy = TRUE;
	} else {
	XVA_CLR_REQ(xvap, XAT_AV_QUARANTINED);
	XVA_SET_REQ(tmpxvattr, XAT_AV_QUARANTINED);
	}
	}

	if (XVA_ISSET_REQ(xvap, XAT_REPARSE)) {
	mutex_exit(&zp->z_lock);
	err = SET_ERROR(EPERM);
	goto out3;
	}

	if (need_policy == FALSE &&
	(XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP) \|\|
	XVA_ISSET_REQ(xvap, XAT_OPAQUE))) {
	need_policy = TRUE;
	}
	}

	mutex_exit(&zp->z_lock);

	if (mask & ATTR_MODE) {
	if (zfs_zaccess(zp, ACE_WRITE_ACL, 0, skipaclchk, cr,
	mnt_ns) == 0) {
	err = secpolicy_setid_setsticky_clear(ip, vap,
	&oldva, cr, mnt_ns, zfs_i_user_ns(ip));
	if (err)
	goto out3;
	trim_mask \|= ATTR_MODE;
	} else {
	need_policy = TRUE;
	}
	}

	if (need_policy) {
	/*
	* If trim_mask is set then take ownership
	* has been granted or write_acl is present and user
	* has the ability to modify mode. In that case remove
	* UID\|GID and or MODE from mask so that
	* secpolicy_vnode_setattr() doesn't revoke it.
	*/

	if (trim_mask) {
	saved_mask = vap->va_mask;
	vap->va_mask &= ~trim_mask;
	}
	err = secpolicy_vnode_setattr(cr, ip, vap, &oldva, flags,
	zfs_zaccess_unix, zp);
	if (err)
	goto out3;

	if (trim_mask)
	vap->va_mask \|= saved_mask;
	}

	/*
	* secpolicy_vnode_setattr, or take ownership may have
	* changed va_mask
	*/
	mask = vap->va_mask;

	if ((mask & (ATTR_UID \| ATTR_GID)) \|\| projid != ZFS_INVALID_PROJID) {
	handle_eadir = B_TRUE;
	err = sa_lookup(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs),
	&xattr_obj, sizeof (xattr_obj));

	if (err == 0 && xattr_obj) {
	err = zfs_zget(ZTOZSB(zp), xattr_obj, &attrzp);
	if (err)
	goto out2;
	}
	if (mask & ATTR_UID) {
	new_kuid = zfs_fuid_create(zfsvfs,
	(uint64_t)vap->va_uid, cr, ZFS_OWNER, &fuidp);
	if (new_kuid != KUID_TO_SUID(ZTOI(zp)->i_uid) &&
	zfs_id_overquota(zfsvfs, DMU_USERUSED_OBJECT,
	new_kuid)) {
	if (attrzp)
	zrele(attrzp);
	err = SET_ERROR(EDQUOT);
	goto out2;
	}
	}

	if (mask & ATTR_GID) {
	new_kgid = zfs_fuid_create(zfsvfs,
	(uint64_t)vap->va_gid, cr, ZFS_GROUP, &fuidp);
	if (new_kgid != KGID_TO_SGID(ZTOI(zp)->i_gid) &&
	zfs_id_overquota(zfsvfs, DMU_GROUPUSED_OBJECT,
	new_kgid)) {
	if (attrzp)
	zrele(attrzp);
	err = SET_ERROR(EDQUOT);
	goto out2;
	}
	}

	if (projid != ZFS_INVALID_PROJID &&
	zfs_id_overquota(zfsvfs, DMU_PROJECTUSED_OBJECT, projid)) {
	if (attrzp)
	zrele(attrzp);
	err = EDQUOT;
	goto out2;
	}
	}
	tx = dmu_tx_create(os);

	if (mask & ATTR_MODE) {
	uint64_t pmode = zp->z_mode;
	uint64_t acl_obj;
	new_mode = (pmode & S_IFMT) \| (vap->va_mode & ~S_IFMT);

	if (ZTOZSB(zp)->z_acl_mode == ZFS_ACL_RESTRICTED &&
	!(zp->z_pflags & ZFS_ACL_TRIVIAL)) {
	err = EPERM;
	goto out;
	}

	if ((err = zfs_acl_chmod_setattr(zp, &aclp, new_mode)))
	goto out;

	mutex_enter(&zp->z_lock);
	if (!zp->z_is_sa && ((acl_obj = zfs_external_acl(zp)) != 0)) {
	/*
	* Are we upgrading ACL from old V0 format
	* to V1 format?
	*/
	if (zfsvfs->z_version >= ZPL_VERSION_FUID &&
	zfs_znode_acl_version(zp) ==
	ZFS_ACL_VERSION_INITIAL) {
	dmu_tx_hold_free(tx, acl_obj, 0,
	DMU_OBJECT_END);
	dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
	0, aclp->z_acl_bytes);
	} else {
	dmu_tx_hold_write(tx, acl_obj, 0,
	aclp->z_acl_bytes);
	}
	} else if (!zp->z_is_sa && aclp->z_acl_bytes > ZFS_ACE_SPACE) {
	dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
	0, aclp->z_acl_bytes);
	}
	mutex_exit(&zp->z_lock);
	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE);
	} else {
	if (((mask & ATTR_XVATTR) &&
	XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP)) \|\|
	(projid != ZFS_INVALID_PROJID &&
	!(zp->z_pflags & ZFS_PROJID)))
	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE);
	else
	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
	}

	if (attrzp) {
	dmu_tx_hold_sa(tx, attrzp->z_sa_hdl, B_FALSE);
	}

	fuid_dirtied = zfsvfs->z_fuid_dirty;
	if (fuid_dirtied)
	zfs_fuid_txhold(zfsvfs, tx);

	zfs_sa_upgrade_txholds(tx, zp);

	err = dmu_tx_assign(tx, TXG_WAIT);
	if (err)
	goto out;

	count = 0;
	/*
	* Set each attribute requested.
	* We group settings according to the locks they need to acquire.
	*
	* Note: you cannot set ctime directly, although it will be
	* updated as a side-effect of calling this function.
	*/

	if (projid != ZFS_INVALID_PROJID && !(zp->z_pflags & ZFS_PROJID)) {
	/*
	* For the existed object that is upgraded from old system,
	* its on-disk layout has no slot for the project ID attribute.
	* But quota accounting logic needs to access related slots by
	* offset directly. So we need to adjust old objects' layout
	* to make the project ID to some unified and fixed offset.
	*/
	if (attrzp)
	err = sa_add_projid(attrzp->z_sa_hdl, tx, projid);
	if (err == 0)
	err = sa_add_projid(zp->z_sa_hdl, tx, projid);

	if (unlikely(err == EEXIST))
	err = 0;
	else if (err != 0)
	goto out;
	else
	projid = ZFS_INVALID_PROJID;
	}

	if (mask & (ATTR_UID\|ATTR_GID\|ATTR_MODE))
	mutex_enter(&zp->z_acl_lock);
	mutex_enter(&zp->z_lock);

	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
	&zp->z_pflags, sizeof (zp->z_pflags));

	if (attrzp) {
	if (mask & (ATTR_UID\|ATTR_GID\|ATTR_MODE))
	mutex_enter(&attrzp->z_acl_lock);
	mutex_enter(&attrzp->z_lock);
	SA_ADD_BULK_ATTR(xattr_bulk, xattr_count,
	SA_ZPL_FLAGS(zfsvfs), NULL, &attrzp->z_pflags,
	sizeof (attrzp->z_pflags));
	if (projid != ZFS_INVALID_PROJID) {
	attrzp->z_projid = projid;
	SA_ADD_BULK_ATTR(xattr_bulk, xattr_count,
	SA_ZPL_PROJID(zfsvfs), NULL, &attrzp->z_projid,
	sizeof (attrzp->z_projid));
	}
	}

	if (mask & (ATTR_UID\|ATTR_GID)) {

	if (mask & ATTR_UID) {
	ZTOI(zp)->i_uid = SUID_TO_KUID(new_kuid);
	new_uid = zfs_uid_read(ZTOI(zp));
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zfsvfs), NULL,
	&new_uid, sizeof (new_uid));
	if (attrzp) {
	SA_ADD_BULK_ATTR(xattr_bulk, xattr_count,
	SA_ZPL_UID(zfsvfs), NULL, &new_uid,
	sizeof (new_uid));
	ZTOI(attrzp)->i_uid = SUID_TO_KUID(new_uid);
	}
	}

	if (mask & ATTR_GID) {
	ZTOI(zp)->i_gid = SGID_TO_KGID(new_kgid);
	new_gid = zfs_gid_read(ZTOI(zp));
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zfsvfs),
	NULL, &new_gid, sizeof (new_gid));
	if (attrzp) {
	SA_ADD_BULK_ATTR(xattr_bulk, xattr_count,
	SA_ZPL_GID(zfsvfs), NULL, &new_gid,
	sizeof (new_gid));
	ZTOI(attrzp)->i_gid = SGID_TO_KGID(new_kgid);
	}
	}
	if (!(mask & ATTR_MODE)) {
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs),
	NULL, &new_mode, sizeof (new_mode));
	new_mode = zp->z_mode;
	}
	err = zfs_acl_chown_setattr(zp);
	ASSERT(err == 0);
	if (attrzp) {
	err = zfs_acl_chown_setattr(attrzp);
	ASSERT(err == 0);
	}
	}

	if (mask & ATTR_MODE) {
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs), NULL,
	&new_mode, sizeof (new_mode));
	zp->z_mode = ZTOI(zp)->i_mode = new_mode;
	ASSERT3P(aclp, !=, NULL);
	err = zfs_aclset_common(zp, aclp, cr, tx);
	ASSERT0(err);
	if (zp->z_acl_cached)
	zfs_acl_free(zp->z_acl_cached);
	zp->z_acl_cached = aclp;
	aclp = NULL;
	}

	if ((mask & ATTR_ATIME) \|\| zp->z_atime_dirty) {
	zp->z_atime_dirty = B_FALSE;
	inode_timespec_t tmp_atime = zpl_inode_get_atime(ip);
	ZFS_TIME_ENCODE(&tmp_atime, atime);
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ATIME(zfsvfs), NULL,
	&atime, sizeof (atime));
	}

	if (mask & (ATTR_MTIME \| ATTR_SIZE)) {
	ZFS_TIME_ENCODE(&vap->va_mtime, mtime);
	zpl_inode_set_mtime_to_ts(ZTOI(zp),
	zpl_inode_timestamp_truncate(vap->va_mtime, ZTOI(zp)));

	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL,
	mtime, sizeof (mtime));
	}

	if (mask & (ATTR_CTIME \| ATTR_SIZE)) {
	ZFS_TIME_ENCODE(&vap->va_ctime, ctime);
	zpl_inode_set_ctime_to_ts(ZTOI(zp),
	zpl_inode_timestamp_truncate(vap->va_ctime, ZTOI(zp)));
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL,
	ctime, sizeof (ctime));
	}

	if (projid != ZFS_INVALID_PROJID) {
	zp->z_projid = projid;
	SA_ADD_BULK_ATTR(bulk, count,
	SA_ZPL_PROJID(zfsvfs), NULL, &zp->z_projid,
	sizeof (zp->z_projid));
	}

	if (attrzp && mask) {
	SA_ADD_BULK_ATTR(xattr_bulk, xattr_count,
	SA_ZPL_CTIME(zfsvfs), NULL, &ctime,
	sizeof (ctime));
	}

	/*
	* Do this after setting timestamps to prevent timestamp
	* update from toggling bit
	*/

	if (xoap && (mask & ATTR_XVATTR)) {

	/*
	* restore trimmed off masks
	* so that return masks can be set for caller.
	*/

	if (XVA_ISSET_REQ(tmpxvattr, XAT_APPENDONLY)) {
	XVA_SET_REQ(xvap, XAT_APPENDONLY);
	}
	if (XVA_ISSET_REQ(tmpxvattr, XAT_NOUNLINK)) {
	XVA_SET_REQ(xvap, XAT_NOUNLINK);
	}
	if (XVA_ISSET_REQ(tmpxvattr, XAT_IMMUTABLE)) {
	XVA_SET_REQ(xvap, XAT_IMMUTABLE);
	}
	if (XVA_ISSET_REQ(tmpxvattr, XAT_NODUMP)) {
	XVA_SET_REQ(xvap, XAT_NODUMP);
	}
	if (XVA_ISSET_REQ(tmpxvattr, XAT_AV_MODIFIED)) {
	XVA_SET_REQ(xvap, XAT_AV_MODIFIED);
	}
	if (XVA_ISSET_REQ(tmpxvattr, XAT_AV_QUARANTINED)) {
	XVA_SET_REQ(xvap, XAT_AV_QUARANTINED);
	}
	if (XVA_ISSET_REQ(tmpxvattr, XAT_PROJINHERIT)) {
	XVA_SET_REQ(xvap, XAT_PROJINHERIT);
	}

	if (XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP))
	ASSERT(S_ISREG(ip->i_mode));

	zfs_xvattr_set(zp, xvap, tx);
	}

	if (fuid_dirtied)
	zfs_fuid_sync(zfsvfs, tx);

	if (mask != 0)
	zfs_log_setattr(zilog, tx, TX_SETATTR, zp, vap, mask, fuidp);

	mutex_exit(&zp->z_lock);
	if (mask & (ATTR_UID\|ATTR_GID\|ATTR_MODE))
	mutex_exit(&zp->z_acl_lock);

	if (attrzp) {
	if (mask & (ATTR_UID\|ATTR_GID\|ATTR_MODE))
	mutex_exit(&attrzp->z_acl_lock);
	mutex_exit(&attrzp->z_lock);
	}
	out:
	if (err == 0 && xattr_count > 0) {
	err2 = sa_bulk_update(attrzp->z_sa_hdl, xattr_bulk,
	xattr_count, tx);
	ASSERT(err2 == 0);
	}

	if (aclp)
	zfs_acl_free(aclp);

	if (fuidp) {
	zfs_fuid_info_free(fuidp);
	fuidp = NULL;
	}

	if (err) {
	dmu_tx_abort(tx);
	if (attrzp)
	zrele(attrzp);
	if (err == ERESTART)
	goto top;
	} else {
	if (count > 0)
	err2 = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
	dmu_tx_commit(tx);
	if (attrzp) {
	if (err2 == 0 && handle_eadir)
	err = zfs_setattr_dir(attrzp);
	zrele(attrzp);
	}
	zfs_znode_update_vfs(zp);
	}

	out2:
	if (os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);

	out3:
	kmem_free(xattr_bulk, sizeof (sa_bulk_attr_t) * bulks);
	kmem_free(bulk, sizeof (sa_bulk_attr_t) * bulks);
	kmem_free(tmpxvattr, sizeof (xvattr_t));
	zfs_exit(zfsvfs, FTAG);
	return (err);
	}

	typedef struct zfs_zlock {
	krwlock_t zl_rwlock; / lock we acquired */
	znode_t zl_znode; / znode we held */
	struct zfs_zlock zl_next; / next in list */
	} zfs_zlock_t;

	/*
	* Drop locks and release vnodes that were held by zfs_rename_lock().
	*/
	static void
	zfs_rename_unlock(zfs_zlock_t **zlpp)
	{
	zfs_zlock_t *zl;

	while ((zl = *zlpp) != NULL) {
	if (zl->zl_znode != NULL)
	zfs_zrele_async(zl->zl_znode);
	rw_exit(zl->zl_rwlock);
	*zlpp = zl->zl_next;
	kmem_free(zl, sizeof (*zl));
	}
	}

	/*
	* Search back through the directory tree, using the ".." entries.
	* Lock each directory in the chain to prevent concurrent renames.
	* Fail any attempt to move a directory into one of its own descendants.
	* XXX - z_parent_lock can overlap with map or grow locks
	*/
	static int
	zfs_rename_lock(znode_t szp, znode_t tdzp, znode_t sdzp, zfs_zlock_t *zlpp)
	{
	zfs_zlock_t *zl;
	znode_t *zp = tdzp;
	uint64_t rootid = ZTOZSB(zp)->z_root;
	uint64_t oidp = zp->z_id;
	krwlock_t *rwlp = &szp->z_parent_lock;
	krw_t rw = RW_WRITER;

	/*
	* First pass write-locks szp and compares to zp->z_id.
	* Later passes read-lock zp and compare to zp->z_parent.
	*/
	do {
	if (!rw_tryenter(rwlp, rw)) {
	/*
	* Another thread is renaming in this path.
	* Note that if we are a WRITER, we don't have any
	* parent_locks held yet.
	*/
	if (rw == RW_READER && zp->z_id > szp->z_id) {
	/*
	* Drop our locks and restart
	*/
	zfs_rename_unlock(&zl);
	*zlpp = NULL;
	zp = tdzp;
	oidp = zp->z_id;
	rwlp = &szp->z_parent_lock;
	rw = RW_WRITER;
	continue;
	} else {
	/*
	* Wait for other thread to drop its locks
	*/
	rw_enter(rwlp, rw);
	}
	}

	zl = kmem_alloc(sizeof (*zl), KM_SLEEP);
	zl->zl_rwlock = rwlp;
	zl->zl_znode = NULL;
	zl->zl_next = *zlpp;
	*zlpp = zl;

	if (oidp == szp->z_id) /* We're a descendant of szp */
	return (SET_ERROR(EINVAL));

	if (oidp == rootid) /* We've hit the top */
	return (0);

	if (rw == RW_READER) { /* i.e. not the first pass */
	int error = zfs_zget(ZTOZSB(zp), oidp, &zp);
	if (error)
	return (error);
	zl->zl_znode = zp;
	}
	(void) sa_lookup(zp->z_sa_hdl, SA_ZPL_PARENT(ZTOZSB(zp)),
	&oidp, sizeof (oidp));
	rwlp = &zp->z_parent_lock;
	rw = RW_READER;

	} while (zp->z_id != sdzp->z_id);

	return (0);
	}

	/*
	* Move an entry from the provided source directory to the target
	* directory. Change the entry name as indicated.
	*
	* IN: sdzp - Source directory containing the "old entry".
	* snm - Old entry name.
	* tdzp - Target directory to contain the "new entry".
	* tnm - New entry name.
	* cr - credentials of caller.
	* flags - case flags
	* rflags - RENAME_* flags
	* wa_vap - attributes for RENAME_WHITEOUT (must be a char 0:0).
	* mnt_ns - user namespace of the mount
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* sdzp,tdzp - ctime\|mtime updated
	*/
	int
	zfs_rename(znode_t sdzp, char snm, znode_t tdzp, char tnm,
	cred_t cr, int flags, uint64_t rflags, vattr_t wo_vap, zidmap_t *mnt_ns)
	{
	znode_t szp, tzp;
	zfsvfs_t *zfsvfs = ZTOZSB(sdzp);
	zilog_t *zilog;
	zfs_dirlock_t sdl, tdl;
	dmu_tx_t *tx;
	zfs_zlock_t *zl;
	int cmp, serr, terr;
	int error = 0;
	int zflg = 0;
	boolean_t waited = B_FALSE;
	/* Needed for whiteout inode creation. */
	boolean_t fuid_dirtied;
	zfs_acl_ids_t acl_ids;
	boolean_t have_acl = B_FALSE;
	znode_t *wzp = NULL;


	if (snm == NULL \|\| tnm == NULL)
	return (SET_ERROR(EINVAL));

	if (rflags & ~(RENAME_NOREPLACE \| RENAME_EXCHANGE \| RENAME_WHITEOUT))
	return (SET_ERROR(EINVAL));

	/* Already checked by Linux VFS, but just to make sure. */
	if (rflags & RENAME_EXCHANGE &&
	(rflags & (RENAME_NOREPLACE \| RENAME_WHITEOUT)))
	return (SET_ERROR(EINVAL));

	/*
	* Make sure we only get wo_vap iff. RENAME_WHITEOUT and that it's the
	* right kind of vattr_t for the whiteout file. These are set
	* internally by ZFS so should never be incorrect.
	*/
	VERIFY_EQUIV(rflags & RENAME_WHITEOUT, wo_vap != NULL);
	VERIFY_IMPLY(wo_vap, wo_vap->va_mode == S_IFCHR);
	VERIFY_IMPLY(wo_vap, wo_vap->va_rdev == makedevice(0, 0));

	if ((error = zfs_enter_verify_zp(zfsvfs, sdzp, FTAG)) != 0)
	return (error);
	zilog = zfsvfs->z_log;

	if ((error = zfs_verify_zp(tdzp)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* We check i_sb because snapshots and the ctldir must have different
	* super blocks.
	*/
	if (ZTOI(tdzp)->i_sb != ZTOI(sdzp)->i_sb \|\|
	zfsctl_is_node(ZTOI(tdzp))) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EXDEV));
	}

	if (zfsvfs->z_utf8 && u8_validate(tnm,
	strlen(tnm), NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EILSEQ));
	}

	if (flags & FIGNORECASE)
	zflg \|= ZCILOOK;

	top:
	szp = NULL;
	tzp = NULL;
	zl = NULL;

	/*
	* This is to prevent the creation of links into attribute space
	* by renaming a linked file into/outof an attribute directory.
	* See the comment in zfs_link() for why this is considered bad.
	*/
	if ((tdzp->z_pflags & ZFS_XATTR) != (sdzp->z_pflags & ZFS_XATTR)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	/*
	* Lock source and target directory entries. To prevent deadlock,
	* a lock ordering must be defined. We lock the directory with
	* the smallest object id first, or if it's a tie, the one with
	* the lexically first name.
	*/
	if (sdzp->z_id < tdzp->z_id) {
	cmp = -1;
	} else if (sdzp->z_id > tdzp->z_id) {
	cmp = 1;
	} else {
	/*
	* First compare the two name arguments without
	* considering any case folding.
	*/
	int nofold = (zfsvfs->z_norm & ~U8_TEXTPREP_TOUPPER);

	cmp = u8_strcmp(snm, tnm, 0, nofold, U8_UNICODE_LATEST, &error);
	ASSERT(error == 0 \|\| !zfsvfs->z_utf8);
	if (cmp == 0) {
	/*
	* POSIX: "If the old argument and the new argument
	* both refer to links to the same existing file,
	* the rename() function shall return successfully
	* and perform no other action."
	*/
	zfs_exit(zfsvfs, FTAG);
	return (0);
	}
	/*
	* If the file system is case-folding, then we may
	* have some more checking to do. A case-folding file
	* system is either supporting mixed case sensitivity
	* access or is completely case-insensitive. Note
	* that the file system is always case preserving.
	*
	* In mixed sensitivity mode case sensitive behavior
	* is the default. FIGNORECASE must be used to
	* explicitly request case insensitive behavior.
	*
	* If the source and target names provided differ only
	* by case (e.g., a request to rename 'tim' to 'Tim'),
	* we will treat this as a special case in the
	* case-insensitive mode: as long as the source name
	* is an exact match, we will allow this to proceed as
	* a name-change request.
	*/
	if ((zfsvfs->z_case == ZFS_CASE_INSENSITIVE \|\|
	(zfsvfs->z_case == ZFS_CASE_MIXED &&
	flags & FIGNORECASE)) &&
	u8_strcmp(snm, tnm, 0, zfsvfs->z_norm, U8_UNICODE_LATEST,
	&error) == 0) {
	/*
	* case preserving rename request, require exact
	* name matches
	*/
	zflg \|= ZCIEXACT;
	zflg &= ~ZCILOOK;
	}
	}

	/*
	* If the source and destination directories are the same, we should
	* grab the z_name_lock of that directory only once.
	*/
	if (sdzp == tdzp) {
	zflg \|= ZHAVELOCK;
	rw_enter(&sdzp->z_name_lock, RW_READER);
	}

	if (cmp < 0) {
	serr = zfs_dirent_lock(&sdl, sdzp, snm, &szp,
	ZEXISTS \| zflg, NULL, NULL);
	terr = zfs_dirent_lock(&tdl,
	tdzp, tnm, &tzp, ZRENAMING \| zflg, NULL, NULL);
	} else {
	terr = zfs_dirent_lock(&tdl,
	tdzp, tnm, &tzp, zflg, NULL, NULL);
	serr = zfs_dirent_lock(&sdl,
	sdzp, snm, &szp, ZEXISTS \| ZRENAMING \| zflg,
	NULL, NULL);
	}

	if (serr) {
	/*
	* Source entry invalid or not there.
	*/
	if (!terr) {
	zfs_dirent_unlock(tdl);
	if (tzp)
	zrele(tzp);
	}

	if (sdzp == tdzp)
	rw_exit(&sdzp->z_name_lock);

	if (strcmp(snm, "..") == 0)
	serr = EINVAL;
	zfs_exit(zfsvfs, FTAG);
	return (serr);
	}
	if (terr) {
	zfs_dirent_unlock(sdl);
	zrele(szp);

	if (sdzp == tdzp)
	rw_exit(&sdzp->z_name_lock);

	if (strcmp(tnm, "..") == 0)
	terr = EINVAL;
	zfs_exit(zfsvfs, FTAG);
	return (terr);
	}

	/*
	* If we are using project inheritance, means if the directory has
	* ZFS_PROJINHERIT set, then its descendant directories will inherit
	* not only the project ID, but also the ZFS_PROJINHERIT flag. Under
	* such case, we only allow renames into our tree when the project
	* IDs are the same.
	*/
	if (tdzp->z_pflags & ZFS_PROJINHERIT &&
	tdzp->z_projid != szp->z_projid) {
	error = SET_ERROR(EXDEV);
	goto out;
	}

	/*
	* Must have write access at the source to remove the old entry
	* and write access at the target to create the new entry.
	* Note that if target and source are the same, this can be
	* done in a single check.
	*/
	if ((error = zfs_zaccess_rename(sdzp, szp, tdzp, tzp, cr, mnt_ns)))
	goto out;

	if (S_ISDIR(ZTOI(szp)->i_mode)) {
	/*
	* Check to make sure rename is valid.
	* Can't do a move like this: /usr/a/b to /usr/a/b/c/d
	*/
	if ((error = zfs_rename_lock(szp, tdzp, sdzp, &zl)))
	goto out;
	}

	/*
	* Does target exist?
	*/
	if (tzp) {
	if (rflags & RENAME_NOREPLACE) {
	error = SET_ERROR(EEXIST);
	goto out;
	}
	/*
	* Source and target must be the same type (unless exchanging).
	*/
	if (!(rflags & RENAME_EXCHANGE)) {
	boolean_t s_is_dir = S_ISDIR(ZTOI(szp)->i_mode) != 0;
	boolean_t t_is_dir = S_ISDIR(ZTOI(tzp)->i_mode) != 0;

	if (s_is_dir != t_is_dir) {
	error = SET_ERROR(s_is_dir ? ENOTDIR : EISDIR);
	goto out;
	}
	}
	/*
	* POSIX dictates that when the source and target
	* entries refer to the same file object, rename
	* must do nothing and exit without error.
	*/
	if (szp->z_id == tzp->z_id) {
	error = 0;
	goto out;
	}
	} else if (rflags & RENAME_EXCHANGE) {
	/* Target must exist for RENAME_EXCHANGE. */
	error = SET_ERROR(ENOENT);
	goto out;
	}

	/* Set up inode creation for RENAME_WHITEOUT. */
	if (rflags & RENAME_WHITEOUT) {
	/*
	* Whiteout files are not regular files or directories, so to
	* match zfs_create() we do not inherit the project id.
	*/
	uint64_t wo_projid = ZFS_DEFAULT_PROJID;

	error = zfs_zaccess(sdzp, ACE_ADD_FILE, 0, B_FALSE, cr, mnt_ns);
	if (error)
	goto out;

	if (!have_acl) {
	error = zfs_acl_ids_create(sdzp, 0, wo_vap, cr, NULL,
	&acl_ids, mnt_ns);
	if (error)
	goto out;
	have_acl = B_TRUE;
	}

	if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, wo_projid)) {
	error = SET_ERROR(EDQUOT);
	goto out;
	}
	}

	tx = dmu_tx_create(zfsvfs->z_os);
	dmu_tx_hold_sa(tx, szp->z_sa_hdl, B_FALSE);
	dmu_tx_hold_sa(tx, sdzp->z_sa_hdl, B_FALSE);
	dmu_tx_hold_zap(tx, sdzp->z_id,
	(rflags & RENAME_EXCHANGE) ? TRUE : FALSE, snm);
	dmu_tx_hold_zap(tx, tdzp->z_id, TRUE, tnm);
	if (sdzp != tdzp) {
	dmu_tx_hold_sa(tx, tdzp->z_sa_hdl, B_FALSE);
	zfs_sa_upgrade_txholds(tx, tdzp);
	}
	if (tzp) {
	dmu_tx_hold_sa(tx, tzp->z_sa_hdl, B_FALSE);
	zfs_sa_upgrade_txholds(tx, tzp);
	}
	if (rflags & RENAME_WHITEOUT) {
	dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes +
	ZFS_SA_BASE_ATTR_SIZE);

	dmu_tx_hold_zap(tx, sdzp->z_id, TRUE, snm);
	dmu_tx_hold_sa(tx, sdzp->z_sa_hdl, B_FALSE);
	if (!zfsvfs->z_use_sa &&
	acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) {
	dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
	0, acl_ids.z_aclp->z_acl_bytes);
	}
	}
	fuid_dirtied = zfsvfs->z_fuid_dirty;
	if (fuid_dirtied)
	zfs_fuid_txhold(zfsvfs, tx);
	zfs_sa_upgrade_txholds(tx, szp);
	dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);
	error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) \| TXG_NOWAIT);
	if (error) {
	if (zl != NULL)
	zfs_rename_unlock(&zl);
	zfs_dirent_unlock(sdl);
	zfs_dirent_unlock(tdl);

	if (sdzp == tdzp)
	rw_exit(&sdzp->z_name_lock);

	if (error == ERESTART) {
	waited = B_TRUE;
	dmu_tx_wait(tx);
	dmu_tx_abort(tx);
	zrele(szp);
	if (tzp)
	zrele(tzp);
	goto top;
	}
	dmu_tx_abort(tx);
	zrele(szp);
	if (tzp)
	zrele(tzp);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Unlink the source.
	*/
	szp->z_pflags \|= ZFS_AV_MODIFIED;
	if (tdzp->z_pflags & ZFS_PROJINHERIT)
	szp->z_pflags \|= ZFS_PROJINHERIT;

	error = sa_update(szp->z_sa_hdl, SA_ZPL_FLAGS(zfsvfs),
	(void *)&szp->z_pflags, sizeof (uint64_t), tx);
	VERIFY0(error);

	error = zfs_link_destroy(sdl, szp, tx, ZRENAMING, NULL);
	if (error)
	goto commit;

	/*
	* Unlink the target.
	*/
	if (tzp) {
	int tzflg = zflg;

	if (rflags & RENAME_EXCHANGE) {
	/* This inode will be re-linked soon. */
	tzflg \|= ZRENAMING;

	tzp->z_pflags \|= ZFS_AV_MODIFIED;
	if (sdzp->z_pflags & ZFS_PROJINHERIT)
	tzp->z_pflags \|= ZFS_PROJINHERIT;

	error = sa_update(tzp->z_sa_hdl, SA_ZPL_FLAGS(zfsvfs),
	(void *)&tzp->z_pflags, sizeof (uint64_t), tx);
	ASSERT0(error);
	}
	error = zfs_link_destroy(tdl, tzp, tx, tzflg, NULL);
	if (error)
	goto commit_link_szp;
	}

	/*
	* Create the new target links:
	* * We always link the target.
	* * RENAME_EXCHANGE: Link the old target to the source.
	* * RENAME_WHITEOUT: Create a whiteout inode in-place of the source.
	*/
	error = zfs_link_create(tdl, szp, tx, ZRENAMING);
	if (error) {
	/*
	* If we have removed the existing target, a subsequent call to
	* zfs_link_create() to add back the same entry, but with a new
	* dnode (szp), should not fail.
	*/
	ASSERT3P(tzp, ==, NULL);
	goto commit_link_tzp;
	}

	switch (rflags & (RENAME_EXCHANGE \| RENAME_WHITEOUT)) {
	case RENAME_EXCHANGE:
	error = zfs_link_create(sdl, tzp, tx, ZRENAMING);
	/*
	* The same argument as zfs_link_create() failing for
	* szp applies here, since the source directory must
	* have had an entry we are replacing.
	*/
	ASSERT0(error);
	if (error)
	goto commit_unlink_td_szp;
	break;
	case RENAME_WHITEOUT:
	zfs_mknode(sdzp, wo_vap, tx, cr, 0, &wzp, &acl_ids);
	error = zfs_link_create(sdl, wzp, tx, ZNEW);
	if (error) {
	zfs_znode_delete(wzp, tx);
	remove_inode_hash(ZTOI(wzp));
	goto commit_unlink_td_szp;
	}
	break;
	}

	if (fuid_dirtied)
	zfs_fuid_sync(zfsvfs, tx);

	switch (rflags & (RENAME_EXCHANGE \| RENAME_WHITEOUT)) {
	case RENAME_EXCHANGE:
	zfs_log_rename_exchange(zilog, tx,
	(flags & FIGNORECASE ? TX_CI : 0), sdzp, sdl->dl_name,
	tdzp, tdl->dl_name, szp);
	break;
	case RENAME_WHITEOUT:
	zfs_log_rename_whiteout(zilog, tx,
	(flags & FIGNORECASE ? TX_CI : 0), sdzp, sdl->dl_name,
	tdzp, tdl->dl_name, szp, wzp);
	break;
	default:
	ASSERT0(rflags & ~RENAME_NOREPLACE);
	zfs_log_rename(zilog, tx, (flags & FIGNORECASE ? TX_CI : 0),
	sdzp, sdl->dl_name, tdzp, tdl->dl_name, szp);
	break;
	}

	commit:
	dmu_tx_commit(tx);
	out:
	if (have_acl)
	zfs_acl_ids_free(&acl_ids);

	zfs_znode_update_vfs(sdzp);
	if (sdzp == tdzp)
	rw_exit(&sdzp->z_name_lock);

	if (sdzp != tdzp)
	zfs_znode_update_vfs(tdzp);

	zfs_znode_update_vfs(szp);
	zrele(szp);
	if (wzp) {
	zfs_znode_update_vfs(wzp);
	zrele(wzp);
	}
	if (tzp) {
	zfs_znode_update_vfs(tzp);
	zrele(tzp);
	}

	if (zl != NULL)
	zfs_rename_unlock(&zl);

	zfs_dirent_unlock(sdl);
	zfs_dirent_unlock(tdl);

	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);

	zfs_exit(zfsvfs, FTAG);
	return (error);

	/*
	* Clean-up path for broken link state.
	*
	* At this point we are in a (very) bad state, so we need to do our
	* best to correct the state. In particular, all of the nlinks are
	* wrong because we were destroying and creating links with ZRENAMING.
	*
	* In some form, all of these operations have to resolve the state:
	*
	* * link_destroy() must succeed. Fortunately, this is very likely
	* since we only just created it.
	*
	* * link_create()s are allowed to fail (though they shouldn't because
	* we only just unlinked them and are putting the entries back
	* during clean-up). But if they fail, we can just forcefully drop
	* the nlink value to (at the very least) avoid broken nlink values
	* -- though in the case of non-empty directories we will have to
	* panic (otherwise we'd have a leaked directory with a broken ..).
	*/
	commit_unlink_td_szp:
	VERIFY0(zfs_link_destroy(tdl, szp, tx, ZRENAMING, NULL));
	commit_link_tzp:
	if (tzp) {
	if (zfs_link_create(tdl, tzp, tx, ZRENAMING))
	VERIFY0(zfs_drop_nlink(tzp, tx, NULL));
	}
	commit_link_szp:
	if (zfs_link_create(sdl, szp, tx, ZRENAMING))
	VERIFY0(zfs_drop_nlink(szp, tx, NULL));
	goto commit;
	}

	/*
	* Insert the indicated symbolic reference entry into the directory.
	*
	* IN: dzp - Directory to contain new symbolic link.
	* name - Name of directory entry in dip.
	* vap - Attributes of new entry.
	* link - Name for new symlink entry.
	* cr - credentials of caller.
	* flags - case flags
	* mnt_ns - user namespace of the mount
	*
	* OUT: zpp - Znode for new symbolic link.
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* dip - ctime\|mtime updated
	*/
	int
	zfs_symlink(znode_t dzp, char name, vattr_t vap, char link,
	znode_t *zpp, cred_t cr, int flags, zidmap_t *mnt_ns)
	{
	znode_t *zp;
	zfs_dirlock_t *dl;
	dmu_tx_t *tx;
	zfsvfs_t *zfsvfs = ZTOZSB(dzp);
	zilog_t *zilog;
	uint64_t len = strlen(link);
	int error;
	int zflg = ZNEW;
	zfs_acl_ids_t acl_ids;
	boolean_t fuid_dirtied;
	uint64_t txtype = TX_SYMLINK;
	boolean_t waited = B_FALSE;

	ASSERT(S_ISLNK(vap->va_mode));

	if (name == NULL)
	return (SET_ERROR(EINVAL));

	if ((error = zfs_enter_verify_zp(zfsvfs, dzp, FTAG)) != 0)
	return (error);
	zilog = zfsvfs->z_log;

	if (zfsvfs->z_utf8 && u8_validate(name, strlen(name),
	NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EILSEQ));
	}
	if (flags & FIGNORECASE)
	zflg \|= ZCILOOK;

	if (len > MAXPATHLEN) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(ENAMETOOLONG));
	}

	if ((error = zfs_acl_ids_create(dzp, 0,
	vap, cr, NULL, &acl_ids, mnt_ns)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	top:
	*zpp = NULL;

	/*
	* Attempt to lock directory; fail if entry already exists.
	*/
	error = zfs_dirent_lock(&dl, dzp, name, &zp, zflg, NULL, NULL);
	if (error) {
	zfs_acl_ids_free(&acl_ids);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	if ((error = zfs_zaccess(dzp, ACE_ADD_FILE, 0, B_FALSE, cr, mnt_ns))) {
	zfs_acl_ids_free(&acl_ids);
	zfs_dirent_unlock(dl);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, ZFS_DEFAULT_PROJID)) {
	zfs_acl_ids_free(&acl_ids);
	zfs_dirent_unlock(dl);
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EDQUOT));
	}
	tx = dmu_tx_create(zfsvfs->z_os);
	fuid_dirtied = zfsvfs->z_fuid_dirty;
	dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0, MAX(1, len));
	dmu_tx_hold_zap(tx, dzp->z_id, TRUE, name);
	dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes +
	ZFS_SA_BASE_ATTR_SIZE + len);
	dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE);
	if (!zfsvfs->z_use_sa && acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) {
	dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0,
	acl_ids.z_aclp->z_acl_bytes);
	}
	if (fuid_dirtied)
	zfs_fuid_txhold(zfsvfs, tx);
	error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) \| TXG_NOWAIT);
	if (error) {
	zfs_dirent_unlock(dl);
	if (error == ERESTART) {
	waited = B_TRUE;
	dmu_tx_wait(tx);
	dmu_tx_abort(tx);
	goto top;
	}
	zfs_acl_ids_free(&acl_ids);
	dmu_tx_abort(tx);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Create a new object for the symlink.
	* for version 4 ZPL datasets the symlink will be an SA attribute
	*/
	zfs_mknode(dzp, vap, tx, cr, 0, &zp, &acl_ids);

	if (fuid_dirtied)
	zfs_fuid_sync(zfsvfs, tx);

	mutex_enter(&zp->z_lock);
	if (zp->z_is_sa)
	error = sa_update(zp->z_sa_hdl, SA_ZPL_SYMLINK(zfsvfs),
	link, len, tx);
	else
	zfs_sa_symlink(zp, link, len, tx);
	mutex_exit(&zp->z_lock);

	zp->z_size = len;
	(void) sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(zfsvfs),
	&zp->z_size, sizeof (zp->z_size), tx);
	/*
	* Insert the new object into the directory.
	*/
	error = zfs_link_create(dl, zp, tx, ZNEW);
	if (error != 0) {
	zfs_znode_delete(zp, tx);
	remove_inode_hash(ZTOI(zp));
	} else {
	if (flags & FIGNORECASE)
	txtype \|= TX_CI;
	zfs_log_symlink(zilog, tx, txtype, dzp, zp, name, link);

	zfs_znode_update_vfs(dzp);
	zfs_znode_update_vfs(zp);
	}

	zfs_acl_ids_free(&acl_ids);

	dmu_tx_commit(tx);

	zfs_dirent_unlock(dl);

	if (error == 0) {
	*zpp = zp;

	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);
	} else {
	zrele(zp);
	}

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Return, in the buffer contained in the provided uio structure,
	* the symbolic path referred to by ip.
	*
	* IN: ip - inode of symbolic link
	* uio - structure to contain the link path.
	* cr - credentials of caller.
	*
	* RETURN: 0 if success
	* error code if failure
	*
	* Timestamps:
	* ip - atime updated
	*/
	int
	zfs_readlink(struct inode ip, zfs_uio_t uio, cred_t *cr)
	{
	(void) cr;
	znode_t *zp = ITOZ(ip);
	zfsvfs_t *zfsvfs = ITOZSB(ip);
	int error;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	mutex_enter(&zp->z_lock);
	if (zp->z_is_sa)
	error = sa_lookup_uio(zp->z_sa_hdl,
	SA_ZPL_SYMLINK(zfsvfs), uio);
	else
	error = zfs_sa_readlink(zp, uio);
	mutex_exit(&zp->z_lock);

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Insert a new entry into directory tdzp referencing szp.
	*
	* IN: tdzp - Directory to contain new entry.
	* szp - znode of new entry.
	* name - name of new entry.
	* cr - credentials of caller.
	* flags - case flags.
	*
	* RETURN: 0 if success
	* error code if failure
	*
	* Timestamps:
	* tdzp - ctime\|mtime updated
	* szp - ctime updated
	*/
	int
	zfs_link(znode_t tdzp, znode_t szp, char name, cred_t cr,
	int flags)
	{
	struct inode *sip = ZTOI(szp);
	znode_t *tzp;
	zfsvfs_t *zfsvfs = ZTOZSB(tdzp);
	zilog_t *zilog;
	zfs_dirlock_t *dl;
	dmu_tx_t *tx;
	int error;
	int zf = ZNEW;
	uint64_t parent;
	uid_t owner;
	boolean_t waited = B_FALSE;
	boolean_t is_tmpfile = 0;
	uint64_t txg;
	#ifdef HAVE_TMPFILE
	is_tmpfile = (sip->i_nlink == 0 && (sip->i_state & I_LINKABLE));
	#endif
	ASSERT(S_ISDIR(ZTOI(tdzp)->i_mode));

	if (name == NULL)
	return (SET_ERROR(EINVAL));

	if ((error = zfs_enter_verify_zp(zfsvfs, tdzp, FTAG)) != 0)
	return (error);
	zilog = zfsvfs->z_log;

	/*
	* POSIX dictates that we return EPERM here.
	* Better choices include ENOTSUP or EISDIR.
	*/
	if (S_ISDIR(sip->i_mode)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EPERM));
	}

	if ((error = zfs_verify_zp(szp)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* If we are using project inheritance, means if the directory has
	* ZFS_PROJINHERIT set, then its descendant directories will inherit
	* not only the project ID, but also the ZFS_PROJINHERIT flag. Under
	* such case, we only allow hard link creation in our tree when the
	* project IDs are the same.
	*/
	if (tdzp->z_pflags & ZFS_PROJINHERIT &&
	tdzp->z_projid != szp->z_projid) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EXDEV));
	}

	/*
	* We check i_sb because snapshots and the ctldir must have different
	* super blocks.
	*/
	if (sip->i_sb != ZTOI(tdzp)->i_sb \|\| zfsctl_is_node(sip)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EXDEV));
	}

	/* Prevent links to .zfs/shares files */

	if ((error = sa_lookup(szp->z_sa_hdl, SA_ZPL_PARENT(zfsvfs),
	&parent, sizeof (uint64_t))) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	if (parent == zfsvfs->z_shares_dir) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EPERM));
	}

	if (zfsvfs->z_utf8 && u8_validate(name,
	strlen(name), NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EILSEQ));
	}
	if (flags & FIGNORECASE)
	zf \|= ZCILOOK;

	/*
	* We do not support links between attributes and non-attributes
	* because of the potential security risk of creating links
	* into "normal" file space in order to circumvent restrictions
	* imposed in attribute space.
	*/
	if ((szp->z_pflags & ZFS_XATTR) != (tdzp->z_pflags & ZFS_XATTR)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	owner = zfs_fuid_map_id(zfsvfs, KUID_TO_SUID(sip->i_uid),
	cr, ZFS_OWNER);
	if (owner != crgetuid(cr) && secpolicy_basic_link(cr) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EPERM));
	}

	if ((error = zfs_zaccess(tdzp, ACE_ADD_FILE, 0, B_FALSE, cr,
	zfs_init_idmap))) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	top:
	/*
	* Attempt to lock directory; fail if entry already exists.
	*/
	error = zfs_dirent_lock(&dl, tdzp, name, &tzp, zf, NULL, NULL);
	if (error) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	tx = dmu_tx_create(zfsvfs->z_os);
	dmu_tx_hold_sa(tx, szp->z_sa_hdl, B_FALSE);
	dmu_tx_hold_zap(tx, tdzp->z_id, TRUE, name);
	if (is_tmpfile)
	dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);

	zfs_sa_upgrade_txholds(tx, szp);
	zfs_sa_upgrade_txholds(tx, tdzp);
	error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) \| TXG_NOWAIT);
	if (error) {
	zfs_dirent_unlock(dl);
	if (error == ERESTART) {
	waited = B_TRUE;
	dmu_tx_wait(tx);
	dmu_tx_abort(tx);
	goto top;
	}
	dmu_tx_abort(tx);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	/* unmark z_unlinked so zfs_link_create will not reject */
	if (is_tmpfile)
	szp->z_unlinked = B_FALSE;
	error = zfs_link_create(dl, szp, tx, 0);

	if (error == 0) {
	uint64_t txtype = TX_LINK;
	/*
	* tmpfile is created to be in z_unlinkedobj, so remove it.
	* Also, we don't log in ZIL, because all previous file
	* operation on the tmpfile are ignored by ZIL. Instead we
	* always wait for txg to sync to make sure all previous
	* operation are sync safe.
	*/
	if (is_tmpfile) {
	VERIFY(zap_remove_int(zfsvfs->z_os,
	zfsvfs->z_unlinkedobj, szp->z_id, tx) == 0);
	} else {
	if (flags & FIGNORECASE)
	txtype \|= TX_CI;
	zfs_log_link(zilog, tx, txtype, tdzp, szp, name);
	}
	} else if (is_tmpfile) {
	/* restore z_unlinked since when linking failed */
	szp->z_unlinked = B_TRUE;
	}
	txg = dmu_tx_get_txg(tx);
	dmu_tx_commit(tx);

	zfs_dirent_unlock(dl);

	if (!is_tmpfile && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);

	if (is_tmpfile && zfsvfs->z_os->os_sync != ZFS_SYNC_DISABLED)
	txg_wait_synced(dmu_objset_pool(zfsvfs->z_os), txg);

	zfs_znode_update_vfs(tdzp);
	zfs_znode_update_vfs(szp);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	static void
	zfs_putpage_sync_commit_cb(void *arg)
	{
	struct page *pp = arg;

	ClearPageError(pp);
	end_page_writeback(pp);
	}

	static void
	zfs_putpage_async_commit_cb(void *arg)
	{
	struct page *pp = arg;
	znode_t *zp = ITOZ(pp->mapping->host);

	ClearPageError(pp);
	end_page_writeback(pp);
	atomic_dec_32(&zp->z_async_writes_cnt);
	}

	/*
	* Push a page out to disk, once the page is on stable storage the
	* registered commit callback will be run as notification of completion.
	*
	* IN: ip - page mapped for inode.
	* pp - page to push (page is locked)
	* wbc - writeback control data
	* for_sync - does the caller intend to wait synchronously for the
	* page writeback to complete?
	*
	* RETURN: 0 if success
	* error code if failure
	*
	* Timestamps:
	* ip - ctime\|mtime updated
	*/
	int
	zfs_putpage(struct inode ip, struct page pp, struct writeback_control *wbc,
	boolean_t for_sync)
	{
	znode_t *zp = ITOZ(ip);
	zfsvfs_t *zfsvfs = ITOZSB(ip);
	loff_t offset;
	loff_t pgoff;
	unsigned int pglen;
	dmu_tx_t *tx;
	caddr_t va;
	int err = 0;
	uint64_t mtime[2], ctime[2];
	inode_timespec_t tmp_ts;
	sa_bulk_attr_t bulk[3];
	int cnt = 0;
	struct address_space *mapping;

	if ((err = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (err);

	ASSERT(PageLocked(pp));

	pgoff = page_offset(pp); /* Page byte-offset in file */
	offset = i_size_read(ip); /* File length in bytes */
	pglen = MIN(PAGE_SIZE, /* Page length in bytes */
	P2ROUNDUP(offset, PAGE_SIZE)-pgoff);

	/* Page is beyond end of file */
	if (pgoff >= offset) {
	unlock_page(pp);
	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	/* Truncate page length to end of file */
	if (pgoff + pglen > offset)
	pglen = offset - pgoff;

	#if 0
	/*
	* FIXME: Allow mmap writes past its quota. The correct fix
	* is to register a page_mkwrite() handler to count the page
	* against its quota when it is about to be dirtied.
	*/
	if (zfs_id_overblockquota(zfsvfs, DMU_USERUSED_OBJECT,
	KUID_TO_SUID(ip->i_uid)) \|\|
	zfs_id_overblockquota(zfsvfs, DMU_GROUPUSED_OBJECT,
	KGID_TO_SGID(ip->i_gid)) \|\|
	(zp->z_projid != ZFS_DEFAULT_PROJID &&
	zfs_id_overblockquota(zfsvfs, DMU_PROJECTUSED_OBJECT,
	zp->z_projid))) {
	err = EDQUOT;
	}
	#endif

	/*
	* The ordering here is critical and must adhere to the following
	* rules in order to avoid deadlocking in either zfs_read() or
	* zfs_free_range() due to a lock inversion.
	*
	* 1) The page must be unlocked prior to acquiring the range lock.
	* This is critical because zfs_read() calls find_lock_page()
	* which may block on the page lock while holding the range lock.
	*
	* 2) Before setting or clearing write back on a page the range lock
	* must be held in order to prevent a lock inversion with the
	* zfs_free_range() function.
	*
	* This presents a problem because upon entering this function the
	* page lock is already held. To safely acquire the range lock the
	* page lock must be dropped. This creates a window where another
	* process could truncate, invalidate, dirty, or write out the page.
	*
	* Therefore, after successfully reacquiring the range and page locks
	* the current page state is checked. In the common case everything
	* will be as is expected and it can be written out. However, if
	* the page state has changed it must be handled accordingly.
	*/
	mapping = pp->mapping;
	redirty_page_for_writepage(wbc, pp);
	unlock_page(pp);

	zfs_locked_range_t *lr = zfs_rangelock_enter(&zp->z_rangelock,
	pgoff, pglen, RL_WRITER);
	lock_page(pp);

	/* Page mapping changed or it was no longer dirty, we're done */
	if (unlikely((mapping != pp->mapping) \|\| !PageDirty(pp))) {
	unlock_page(pp);
	zfs_rangelock_exit(lr);
	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	/* Another process started write block if required */
	if (PageWriteback(pp)) {
	unlock_page(pp);
	zfs_rangelock_exit(lr);

	if (wbc->sync_mode != WB_SYNC_NONE) {
	/*
	* Speed up any non-sync page writebacks since
	* they may take several seconds to complete.
	* Refer to the comment in zpl_fsync() (when
	* HAVE_FSYNC_RANGE is defined) for details.
	*/
	if (atomic_load_32(&zp->z_async_writes_cnt) > 0) {
	zil_commit(zfsvfs->z_log, zp->z_id);
	}

	if (PageWriteback(pp))
	#ifdef HAVE_PAGEMAP_FOLIO_WAIT_BIT
	folio_wait_bit(page_folio(pp), PG_writeback);
	#else
	wait_on_page_bit(pp, PG_writeback);
	#endif
	}

	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	/* Clear the dirty flag the required locks are held */
	if (!clear_page_dirty_for_io(pp)) {
	unlock_page(pp);
	zfs_rangelock_exit(lr);
	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	/*
	* Counterpart for redirty_page_for_writepage() above. This page
	* was in fact not skipped and should not be counted as if it were.
	*/
	wbc->pages_skipped--;
	if (!for_sync)
	atomic_inc_32(&zp->z_async_writes_cnt);
	set_page_writeback(pp);
	unlock_page(pp);

	tx = dmu_tx_create(zfsvfs->z_os);
	dmu_tx_hold_write(tx, zp->z_id, pgoff, pglen);
	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
	zfs_sa_upgrade_txholds(tx, zp);

	- err = dmu_tx_assign(tx, TXG_NOWAIT);
	+ err = dmu_tx_assign(tx, TXG_WAIT);
	if (err != 0) {
	- if (err == ERESTART)
	- dmu_tx_wait(tx);
	-
	dmu_tx_abort(tx);
	#ifdef HAVE_VFS_FILEMAP_DIRTY_FOLIO
	filemap_dirty_folio(page_mapping(pp), page_folio(pp));
	#else
	__set_page_dirty_nobuffers(pp);
	#endif
	ClearPageError(pp);
	end_page_writeback(pp);
	if (!for_sync)
	atomic_dec_32(&zp->z_async_writes_cnt);
	zfs_rangelock_exit(lr);
	zfs_exit(zfsvfs, FTAG);
	return (err);
	}

	va = kmap(pp);
	ASSERT3U(pglen, <=, PAGE_SIZE);
	dmu_write(zfsvfs->z_os, zp->z_id, pgoff, pglen, va, tx);
	kunmap(pp);

	SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
	SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
	SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_FLAGS(zfsvfs), NULL,
	&zp->z_pflags, 8);

	/* Preserve the mtime and ctime provided by the inode */
	tmp_ts = zpl_inode_get_mtime(ip);
	ZFS_TIME_ENCODE(&tmp_ts, mtime);
	tmp_ts = zpl_inode_get_ctime(ip);
	ZFS_TIME_ENCODE(&tmp_ts, ctime);
	zp->z_atime_dirty = B_FALSE;
	zp->z_seq++;

	err = sa_bulk_update(zp->z_sa_hdl, bulk, cnt, tx);

	zfs_log_write(zfsvfs->z_log, tx, TX_WRITE, zp, pgoff, pglen, 0,
	for_sync ? zfs_putpage_sync_commit_cb :
	zfs_putpage_async_commit_cb, pp);

	dmu_tx_commit(tx);

	zfs_rangelock_exit(lr);

	if (wbc->sync_mode != WB_SYNC_NONE) {
	/*
	* Note that this is rarely called under writepages(), because
	* writepages() normally handles the entire commit for
	* performance reasons.
	*/
	zil_commit(zfsvfs->z_log, zp->z_id);
	} else if (!for_sync && atomic_load_32(&zp->z_sync_writes_cnt) > 0) {
	/*
	* If the caller does not intend to wait synchronously
	* for this page writeback to complete and there are active
	* synchronous calls on this file, do a commit so that
	* the latter don't accidentally end up waiting for
	* our writeback to complete. Refer to the comment in
	* zpl_fsync() (when HAVE_FSYNC_RANGE is defined) for details.
	*/
	zil_commit(zfsvfs->z_log, zp->z_id);
	}

	dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, pglen);

	zfs_exit(zfsvfs, FTAG);
	return (err);
	}

	/*
	* Update the system attributes when the inode has been dirtied. For the
	* moment we only update the mode, atime, mtime, and ctime.
	*/
	int
	zfs_dirty_inode(struct inode *ip, int flags)
	{
	znode_t *zp = ITOZ(ip);
	zfsvfs_t *zfsvfs = ITOZSB(ip);
	dmu_tx_t *tx;
	uint64_t mode, atime[2], mtime[2], ctime[2];
	inode_timespec_t tmp_ts;
	sa_bulk_attr_t bulk[4];
	int error = 0;
	int cnt = 0;

	if (zfs_is_readonly(zfsvfs) \|\| dmu_objset_is_snapshot(zfsvfs->z_os))
	return (0);

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	#ifdef I_DIRTY_TIME
	/*
	* This is the lazytime semantic introduced in Linux 4.0
	* This flag will only be called from update_time when lazytime is set.
	* (Note, I_DIRTY_SYNC will also set if not lazytime)
	* Fortunately mtime and ctime are managed within ZFS itself, so we
	* only need to dirty atime.
	*/
	if (flags == I_DIRTY_TIME) {
	zp->z_atime_dirty = B_TRUE;
	goto out;
	}
	#endif

	tx = dmu_tx_create(zfsvfs->z_os);

	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
	zfs_sa_upgrade_txholds(tx, zp);

	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error) {
	dmu_tx_abort(tx);
	goto out;
	}

	mutex_enter(&zp->z_lock);
	zp->z_atime_dirty = B_FALSE;

	SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_MODE(zfsvfs), NULL, &mode, 8);
	SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_ATIME(zfsvfs), NULL, &atime, 16);
	SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
	SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);

	/* Preserve the mode, mtime and ctime provided by the inode */
	tmp_ts = zpl_inode_get_atime(ip);
	ZFS_TIME_ENCODE(&tmp_ts, atime);
	tmp_ts = zpl_inode_get_mtime(ip);
	ZFS_TIME_ENCODE(&tmp_ts, mtime);
	tmp_ts = zpl_inode_get_ctime(ip);
	ZFS_TIME_ENCODE(&tmp_ts, ctime);
	mode = ip->i_mode;

	zp->z_mode = mode;

	error = sa_bulk_update(zp->z_sa_hdl, bulk, cnt, tx);
	mutex_exit(&zp->z_lock);

	dmu_tx_commit(tx);
	out:
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	void
	zfs_inactive(struct inode *ip)
	{
	znode_t *zp = ITOZ(ip);
	zfsvfs_t *zfsvfs = ITOZSB(ip);
	uint64_t atime[2];
	int error;
	int need_unlock = 0;

	/* Only read lock if we haven't already write locked, e.g. rollback */
	if (!RW_WRITE_HELD(&zfsvfs->z_teardown_inactive_lock)) {
	need_unlock = 1;
	rw_enter(&zfsvfs->z_teardown_inactive_lock, RW_READER);
	}
	if (zp->z_sa_hdl == NULL) {
	if (need_unlock)
	rw_exit(&zfsvfs->z_teardown_inactive_lock);
	return;
	}

	if (zp->z_atime_dirty && zp->z_unlinked == B_FALSE) {
	dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os);

	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
	zfs_sa_upgrade_txholds(tx, zp);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error) {
	dmu_tx_abort(tx);
	} else {
	inode_timespec_t tmp_atime;
	tmp_atime = zpl_inode_get_atime(ip);
	ZFS_TIME_ENCODE(&tmp_atime, atime);
	mutex_enter(&zp->z_lock);
	(void) sa_update(zp->z_sa_hdl, SA_ZPL_ATIME(zfsvfs),
	(void *)&atime, sizeof (atime), tx);
	zp->z_atime_dirty = B_FALSE;
	mutex_exit(&zp->z_lock);
	dmu_tx_commit(tx);
	}
	}

	zfs_zinactive(zp);
	if (need_unlock)
	rw_exit(&zfsvfs->z_teardown_inactive_lock);
	}

	/*
	* Fill pages with data from the disk.
	*/
	static int
	zfs_fillpage(struct inode ip, struct page pp)
	{
	zfsvfs_t *zfsvfs = ITOZSB(ip);
	loff_t i_size = i_size_read(ip);
	u_offset_t io_off = page_offset(pp);
	size_t io_len = PAGE_SIZE;

	ASSERT3U(io_off, <, i_size);

	if (io_off + io_len > i_size)
	io_len = i_size - io_off;

	void *va = kmap(pp);
	int error = dmu_read(zfsvfs->z_os, ITOZ(ip)->z_id, io_off,
	io_len, va, DMU_READ_PREFETCH);
	if (io_len != PAGE_SIZE)
	memset((char *)va + io_len, 0, PAGE_SIZE - io_len);
	kunmap(pp);

	if (error) {
	/* convert checksum errors into IO errors */
	if (error == ECKSUM)
	error = SET_ERROR(EIO);

	SetPageError(pp);
	ClearPageUptodate(pp);
	} else {
	ClearPageError(pp);
	SetPageUptodate(pp);
	}

	return (error);
	}

	/*
	* Uses zfs_fillpage to read data from the file and fill the page.
	*
	* IN: ip - inode of file to get data from.
	* pp - page to read
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* vp - atime updated
	*/
	int
	zfs_getpage(struct inode ip, struct page pp)
	{
	zfsvfs_t *zfsvfs = ITOZSB(ip);
	znode_t *zp = ITOZ(ip);
	int error;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	error = zfs_fillpage(ip, pp);
	if (error == 0)
	dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, PAGE_SIZE);

	zfs_exit(zfsvfs, FTAG);

	return (error);
	}

	/*
	* Check ZFS specific permissions to memory map a section of a file.
	*
	* IN: ip - inode of the file to mmap
	* off - file offset
	* addrp - start address in memory region
	* len - length of memory region
	* vm_flags- address flags
	*
	* RETURN: 0 if success
	* error code if failure
	*/
	int
	zfs_map(struct inode ip, offset_t off, caddr_t addrp, size_t len,
	unsigned long vm_flags)
	{
	(void) addrp;
	znode_t *zp = ITOZ(ip);
	zfsvfs_t *zfsvfs = ITOZSB(ip);
	int error;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	if ((vm_flags & VM_WRITE) && (vm_flags & VM_SHARED) &&
	(zp->z_pflags & (ZFS_IMMUTABLE \| ZFS_READONLY \| ZFS_APPENDONLY))) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EPERM));
	}

	if ((vm_flags & (VM_READ \| VM_EXEC)) &&
	(zp->z_pflags & ZFS_AV_QUARANTINED)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EACCES));
	}

	if (off < 0 \|\| len > MAXOFFSET_T - off) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(ENXIO));
	}

	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	/*
	* Free or allocate space in a file. Currently, this function only
	* supports the `F_FREESP' command. However, this command is somewhat
	* misnamed, as its functionality includes the ability to allocate as
	* well as free space.
	*
	* IN: zp - znode of file to free data in.
	* cmd - action to take (only F_FREESP supported).
	* bfp - section of file to free/alloc.
	* flag - current file open mode flags.
	* offset - current file offset.
	* cr - credentials of caller.
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Timestamps:
	* zp - ctime\|mtime updated
	*/
	int
	zfs_space(znode_t zp, int cmd, flock64_t bfp, int flag,
	offset_t offset, cred_t *cr)
	{
	(void) offset;
	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	uint64_t off, len;
	int error;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	if (cmd != F_FREESP) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	/*
	* Callers might not be able to detect properly that we are read-only,
	* so check it explicitly here.
	*/
	if (zfs_is_readonly(zfsvfs)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EROFS));
	}

	if (bfp->l_len < 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	/*
	* Permissions aren't checked on Solaris because on this OS
	* zfs_space() can only be called with an opened file handle.
	* On Linux we can get here through truncate_range() which
	* operates directly on inodes, so we need to check access rights.
	*/
	if ((error = zfs_zaccess(zp, ACE_WRITE_DATA, 0, B_FALSE, cr,
	zfs_init_idmap))) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	off = bfp->l_start;
	len = bfp->l_len; /* 0 means from off to end of file */

	error = zfs_freesp(zp, off, len, flag, TRUE);

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	int
	zfs_fid(struct inode ip, fid_t fidp)
	{
	znode_t *zp = ITOZ(ip);
	zfsvfs_t *zfsvfs = ITOZSB(ip);
	uint32_t gen;
	uint64_t gen64;
	uint64_t object = zp->z_id;
	zfid_short_t *zfid;
	int size, i, error;

	if ((error = zfs_enter(zfsvfs, FTAG)) != 0)
	return (error);

	if (fidp->fid_len < SHORT_FID_LEN) {
	fidp->fid_len = SHORT_FID_LEN;
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(ENOSPC));
	}

	if ((error = zfs_verify_zp(zp)) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_GEN(zfsvfs),
	&gen64, sizeof (uint64_t))) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	gen = (uint32_t)gen64;

	size = SHORT_FID_LEN;

	zfid = (zfid_short_t *)fidp;

	zfid->zf_len = size;

	for (i = 0; i < sizeof (zfid->zf_object); i++)
	zfid->zf_object[i] = (uint8_t)(object >> (8 * i));

	/* Must have a non-zero generation number to distinguish from .zfs */
	if (gen == 0)
	gen = 1;
	for (i = 0; i < sizeof (zfid->zf_gen); i++)
	zfid->zf_gen[i] = (uint8_t)(gen >> (8 * i));

	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	#if defined(_KERNEL)
	EXPORT_SYMBOL(zfs_open);
	EXPORT_SYMBOL(zfs_close);
	EXPORT_SYMBOL(zfs_lookup);
	EXPORT_SYMBOL(zfs_create);
	EXPORT_SYMBOL(zfs_tmpfile);
	EXPORT_SYMBOL(zfs_remove);
	EXPORT_SYMBOL(zfs_mkdir);
	EXPORT_SYMBOL(zfs_rmdir);
	EXPORT_SYMBOL(zfs_readdir);
	EXPORT_SYMBOL(zfs_getattr_fast);
	EXPORT_SYMBOL(zfs_setattr);
	EXPORT_SYMBOL(zfs_rename);
	EXPORT_SYMBOL(zfs_symlink);
	EXPORT_SYMBOL(zfs_readlink);
	EXPORT_SYMBOL(zfs_link);
	EXPORT_SYMBOL(zfs_inactive);
	EXPORT_SYMBOL(zfs_space);
	EXPORT_SYMBOL(zfs_fid);
	EXPORT_SYMBOL(zfs_getpage);
	EXPORT_SYMBOL(zfs_putpage);
	EXPORT_SYMBOL(zfs_dirty_inode);
	EXPORT_SYMBOL(zfs_map);

	/* CSTYLED */
	module_param(zfs_delete_blocks, ulong, 0644);
	MODULE_PARM_DESC(zfs_delete_blocks, "Delete files larger than N blocks async");
	#endif
	diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c b/sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c
	index 3caa0fc6c214..9dec52215c7c 100644
	--- a/sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c
	+++ b/sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c
	@@ -1,1391 +1,1391 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2011, Lawrence Livermore National Security, LLC.
	* Copyright (c) 2015 by Chunwei Chen. All rights reserved.
	*/


	#ifdef CONFIG_COMPAT
	#include <linux/compat.h>
	#endif
	#include <linux/fs.h>
	#include <sys/file.h>
	#include <sys/dmu_objset.h>
	#include <sys/zfs_znode.h>
	#include <sys/zfs_vfsops.h>
	#include <sys/zfs_vnops.h>
	#include <sys/zfs_project.h>
	#if defined(HAVE_VFS_SET_PAGE_DIRTY_NOBUFFERS) \|\| \
	defined(HAVE_VFS_FILEMAP_DIRTY_FOLIO)
	#include <linux/pagemap.h>
	#endif
	#ifdef HAVE_FILE_FADVISE
	#include <linux/fadvise.h>
	#endif
	#ifdef HAVE_VFS_FILEMAP_DIRTY_FOLIO
	#include <linux/writeback.h>
	#endif

	/*
	* When using fallocate(2) to preallocate space, inflate the requested
	* capacity check by 10% to account for the required metadata blocks.
	*/
	static unsigned int zfs_fallocate_reserve_percent = 110;

	static int
	zpl_open(struct inode ip, struct file filp)
	{
	cred_t *cr = CRED();
	int error;
	fstrans_cookie_t cookie;

	error = generic_file_open(ip, filp);
	if (error)
	return (error);

	crhold(cr);
	cookie = spl_fstrans_mark();
	error = -zfs_open(ip, filp->f_mode, filp->f_flags, cr);
	spl_fstrans_unmark(cookie);
	crfree(cr);
	ASSERT3S(error, <=, 0);

	return (error);
	}

	static int
	zpl_release(struct inode ip, struct file filp)
	{
	cred_t *cr = CRED();
	int error;
	fstrans_cookie_t cookie;

	cookie = spl_fstrans_mark();
	if (ITOZ(ip)->z_atime_dirty)
	zfs_mark_inode_dirty(ip);

	crhold(cr);
	error = -zfs_close(ip, filp->f_flags, cr);
	spl_fstrans_unmark(cookie);
	crfree(cr);
	ASSERT3S(error, <=, 0);

	return (error);
	}

	static int
	zpl_iterate(struct file filp, zpl_dir_context_t ctx)
	{
	cred_t *cr = CRED();
	int error;
	fstrans_cookie_t cookie;

	crhold(cr);
	cookie = spl_fstrans_mark();
	error = -zfs_readdir(file_inode(filp), ctx, cr);
	spl_fstrans_unmark(cookie);
	crfree(cr);
	ASSERT3S(error, <=, 0);

	return (error);
	}

	#if !defined(HAVE_VFS_ITERATE) && !defined(HAVE_VFS_ITERATE_SHARED)
	static int
	zpl_readdir(struct file filp, void dirent, filldir_t filldir)
	{
	zpl_dir_context_t ctx =
	ZPL_DIR_CONTEXT_INIT(dirent, filldir, filp->f_pos);
	int error;

	error = zpl_iterate(filp, &ctx);
	filp->f_pos = ctx.pos;

	return (error);
	}
	#endif /* !HAVE_VFS_ITERATE && !HAVE_VFS_ITERATE_SHARED */

	#if defined(HAVE_FSYNC_WITHOUT_DENTRY)
	/*
	* Linux 2.6.35 - 3.0 API,
	* As of 2.6.35 the dentry argument to the fops->fsync() hook was deemed
	* redundant. The dentry is still accessible via filp->f_path.dentry,
	* and we are guaranteed that filp will never be NULL.
	*/
	static int
	zpl_fsync(struct file *filp, int datasync)
	{
	struct inode *inode = filp->f_mapping->host;
	cred_t *cr = CRED();
	int error;
	fstrans_cookie_t cookie;

	crhold(cr);
	cookie = spl_fstrans_mark();
	error = -zfs_fsync(ITOZ(inode), datasync, cr);
	spl_fstrans_unmark(cookie);
	crfree(cr);
	ASSERT3S(error, <=, 0);

	return (error);
	}

	#ifdef HAVE_FILE_AIO_FSYNC
	static int
	zpl_aio_fsync(struct kiocb *kiocb, int datasync)
	{
	return (zpl_fsync(kiocb->ki_filp, datasync));
	}
	#endif

	#elif defined(HAVE_FSYNC_RANGE)
	/*
	* Linux 3.1 API,
	* As of 3.1 the responsibility to call filemap_write_and_wait_range() has
	* been pushed down in to the .fsync() vfs hook. Additionally, the i_mutex
	* lock is no longer held by the caller, for zfs we don't require the lock
	* to be held so we don't acquire it.
	*/
	static int
	zpl_fsync(struct file *filp, loff_t start, loff_t end, int datasync)
	{
	struct inode *inode = filp->f_mapping->host;
	znode_t *zp = ITOZ(inode);
	zfsvfs_t *zfsvfs = ITOZSB(inode);
	cred_t *cr = CRED();
	int error;
	fstrans_cookie_t cookie;

	/*
	* The variables z_sync_writes_cnt and z_async_writes_cnt work in
	* tandem so that sync writes can detect if there are any non-sync
	* writes going on and vice-versa. The "vice-versa" part to this logic
	* is located in zfs_putpage() where non-sync writes check if there are
	* any ongoing sync writes. If any sync and non-sync writes overlap,
	* we do a commit to complete the non-sync writes since the latter can
	* potentially take several seconds to complete and thus block sync
	* writes in the upcoming call to filemap_write_and_wait_range().
	*/
	atomic_inc_32(&zp->z_sync_writes_cnt);
	/*
	* If the following check does not detect an overlapping non-sync write
	* (say because it's just about to start), then it is guaranteed that
	* the non-sync write will detect this sync write. This is because we
	* always increment z_sync_writes_cnt / z_async_writes_cnt before doing
	* the check on z_async_writes_cnt / z_sync_writes_cnt here and in
	* zfs_putpage() respectively.
	*/
	if (atomic_load_32(&zp->z_async_writes_cnt) > 0) {
	if ((error = zpl_enter(zfsvfs, FTAG)) != 0) {
	atomic_dec_32(&zp->z_sync_writes_cnt);
	return (error);
	}
	zil_commit(zfsvfs->z_log, zp->z_id);
	zpl_exit(zfsvfs, FTAG);
	}

	error = filemap_write_and_wait_range(inode->i_mapping, start, end);

	/*
	* The sync write is not complete yet but we decrement
	* z_sync_writes_cnt since zfs_fsync() increments and decrements
	* it internally. If a non-sync write starts just after the decrement
	* operation but before we call zfs_fsync(), it may not detect this
	* overlapping sync write but it does not matter since we have already
	* gone past filemap_write_and_wait_range() and we won't block due to
	* the non-sync write.
	*/
	atomic_dec_32(&zp->z_sync_writes_cnt);

	if (error)
	return (error);

	crhold(cr);
	cookie = spl_fstrans_mark();
	error = -zfs_fsync(zp, datasync, cr);
	spl_fstrans_unmark(cookie);
	crfree(cr);
	ASSERT3S(error, <=, 0);

	return (error);
	}

	#ifdef HAVE_FILE_AIO_FSYNC
	static int
	zpl_aio_fsync(struct kiocb *kiocb, int datasync)
	{
	return (zpl_fsync(kiocb->ki_filp, kiocb->ki_pos, -1, datasync));
	}
	#endif

	#else
	#error "Unsupported fops->fsync() implementation"
	#endif

	static inline int
	zfs_io_flags(struct kiocb *kiocb)
	{
	int flags = 0;

	#if defined(IOCB_DSYNC)
	if (kiocb->ki_flags & IOCB_DSYNC)
	flags \|= O_DSYNC;
	#endif
	#if defined(IOCB_SYNC)
	if (kiocb->ki_flags & IOCB_SYNC)
	flags \|= O_SYNC;
	#endif
	#if defined(IOCB_APPEND)
	if (kiocb->ki_flags & IOCB_APPEND)
	flags \|= O_APPEND;
	#endif
	#if defined(IOCB_DIRECT)
	if (kiocb->ki_flags & IOCB_DIRECT)
	flags \|= O_DIRECT;
	#endif
	return (flags);
	}

	/*
	* If relatime is enabled, call file_accessed() if zfs_relatime_need_update()
	* is true. This is needed since datasets with inherited "relatime" property
	* aren't necessarily mounted with the MNT_RELATIME flag (e.g. after
	* `zfs set relatime=...`), which is what relatime test in VFS by
	* relatime_need_update() is based on.
	*/
	static inline void
	zpl_file_accessed(struct file *filp)
	{
	struct inode *ip = filp->f_mapping->host;

	if (!IS_NOATIME(ip) && ITOZSB(ip)->z_relatime) {
	if (zfs_relatime_need_update(ip))
	file_accessed(filp);
	} else {
	file_accessed(filp);
	}
	}

	#if defined(HAVE_VFS_RW_ITERATE)

	/*
	* When HAVE_VFS_IOV_ITER is defined the iov_iter structure supports
	* iovecs, kvevs, bvecs and pipes, plus all the required interfaces to
	* manipulate the iov_iter are available. In which case the full iov_iter
	* can be attached to the uio and correctly handled in the lower layers.
	* Otherwise, for older kernels extract the iovec and pass it instead.
	*/
	static void
	zpl_uio_init(zfs_uio_t uio, struct kiocb kiocb, struct iov_iter *to,
	loff_t pos, ssize_t count, size_t skip)
	{
	#if defined(HAVE_VFS_IOV_ITER)
	zfs_uio_iov_iter_init(uio, to, pos, count, skip);
	#else
	zfs_uio_iovec_init(uio, zfs_uio_iter_iov(to), to->nr_segs, pos,
	zfs_uio_iov_iter_type(to) & ITER_KVEC ?
	UIO_SYSSPACE : UIO_USERSPACE,
	count, skip);
	#endif
	}

	static ssize_t
	zpl_iter_read(struct kiocb kiocb, struct iov_iter to)
	{
	cred_t *cr = CRED();
	fstrans_cookie_t cookie;
	struct file *filp = kiocb->ki_filp;
	ssize_t count = iov_iter_count(to);
	zfs_uio_t uio;

	zpl_uio_init(&uio, kiocb, to, kiocb->ki_pos, count, 0);

	crhold(cr);
	cookie = spl_fstrans_mark();

	int error = -zfs_read(ITOZ(filp->f_mapping->host), &uio,
	filp->f_flags \| zfs_io_flags(kiocb), cr);

	spl_fstrans_unmark(cookie);
	crfree(cr);

	if (error < 0)
	return (error);

	ssize_t read = count - uio.uio_resid;
	kiocb->ki_pos += read;

	zpl_file_accessed(filp);

	return (read);
	}

	static inline ssize_t
	zpl_generic_write_checks(struct kiocb kiocb, struct iov_iter from,
	size_t *countp)
	{
	#ifdef HAVE_GENERIC_WRITE_CHECKS_KIOCB
	ssize_t ret = generic_write_checks(kiocb, from);
	if (ret <= 0)
	return (ret);

	*countp = ret;
	#else
	struct file *file = kiocb->ki_filp;
	struct address_space *mapping = file->f_mapping;
	struct inode *ip = mapping->host;
	int isblk = S_ISBLK(ip->i_mode);

	*countp = iov_iter_count(from);
	ssize_t ret = generic_write_checks(file, &kiocb->ki_pos, countp, isblk);
	if (ret)
	return (ret);
	#endif

	return (0);
	}

	static ssize_t
	zpl_iter_write(struct kiocb kiocb, struct iov_iter from)
	{
	cred_t *cr = CRED();
	fstrans_cookie_t cookie;
	struct file *filp = kiocb->ki_filp;
	struct inode *ip = filp->f_mapping->host;
	zfs_uio_t uio;
	size_t count = 0;
	ssize_t ret;

	ret = zpl_generic_write_checks(kiocb, from, &count);
	if (ret)
	return (ret);

	zpl_uio_init(&uio, kiocb, from, kiocb->ki_pos, count, from->iov_offset);

	crhold(cr);
	cookie = spl_fstrans_mark();

	int error = -zfs_write(ITOZ(ip), &uio,
	filp->f_flags \| zfs_io_flags(kiocb), cr);

	spl_fstrans_unmark(cookie);
	crfree(cr);

	if (error < 0)
	return (error);

	ssize_t wrote = count - uio.uio_resid;
	kiocb->ki_pos += wrote;

	return (wrote);
	}

	#else /* !HAVE_VFS_RW_ITERATE */

	static ssize_t
	zpl_aio_read(struct kiocb kiocb, const struct iovec iov,
	unsigned long nr_segs, loff_t pos)
	{
	cred_t *cr = CRED();
	fstrans_cookie_t cookie;
	struct file *filp = kiocb->ki_filp;
	size_t count;
	ssize_t ret;

	ret = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
	if (ret)
	return (ret);

	zfs_uio_t uio;
	zfs_uio_iovec_init(&uio, iov, nr_segs, kiocb->ki_pos, UIO_USERSPACE,
	count, 0);

	crhold(cr);
	cookie = spl_fstrans_mark();

	int error = -zfs_read(ITOZ(filp->f_mapping->host), &uio,
	filp->f_flags \| zfs_io_flags(kiocb), cr);

	spl_fstrans_unmark(cookie);
	crfree(cr);

	if (error < 0)
	return (error);

	ssize_t read = count - uio.uio_resid;
	kiocb->ki_pos += read;

	zpl_file_accessed(filp);

	return (read);
	}

	static ssize_t
	zpl_aio_write(struct kiocb kiocb, const struct iovec iov,
	unsigned long nr_segs, loff_t pos)
	{
	cred_t *cr = CRED();
	fstrans_cookie_t cookie;
	struct file *filp = kiocb->ki_filp;
	struct inode *ip = filp->f_mapping->host;
	size_t count;
	ssize_t ret;

	ret = generic_segment_checks(iov, &nr_segs, &count, VERIFY_READ);
	if (ret)
	return (ret);

	ret = generic_write_checks(filp, &pos, &count, S_ISBLK(ip->i_mode));
	if (ret)
	return (ret);

	kiocb->ki_pos = pos;

	zfs_uio_t uio;
	zfs_uio_iovec_init(&uio, iov, nr_segs, kiocb->ki_pos, UIO_USERSPACE,
	count, 0);

	crhold(cr);
	cookie = spl_fstrans_mark();

	int error = -zfs_write(ITOZ(ip), &uio,
	filp->f_flags \| zfs_io_flags(kiocb), cr);

	spl_fstrans_unmark(cookie);
	crfree(cr);

	if (error < 0)
	return (error);

	ssize_t wrote = count - uio.uio_resid;
	kiocb->ki_pos += wrote;

	return (wrote);
	}
	#endif /* HAVE_VFS_RW_ITERATE */

	#if defined(HAVE_VFS_RW_ITERATE)
	static ssize_t
	zpl_direct_IO_impl(int rw, struct kiocb kiocb, struct iov_iter iter)
	{
	if (rw == WRITE)
	return (zpl_iter_write(kiocb, iter));
	else
	return (zpl_iter_read(kiocb, iter));
	}
	#if defined(HAVE_VFS_DIRECT_IO_ITER)
	static ssize_t
	zpl_direct_IO(struct kiocb kiocb, struct iov_iter iter)
	{
	return (zpl_direct_IO_impl(iov_iter_rw(iter), kiocb, iter));
	}
	#elif defined(HAVE_VFS_DIRECT_IO_ITER_OFFSET)
	static ssize_t
	zpl_direct_IO(struct kiocb kiocb, struct iov_iter iter, loff_t pos)
	{
	ASSERT3S(pos, ==, kiocb->ki_pos);
	return (zpl_direct_IO_impl(iov_iter_rw(iter), kiocb, iter));
	}
	#elif defined(HAVE_VFS_DIRECT_IO_ITER_RW_OFFSET)
	static ssize_t
	zpl_direct_IO(int rw, struct kiocb kiocb, struct iov_iter iter, loff_t pos)
	{
	ASSERT3S(pos, ==, kiocb->ki_pos);
	return (zpl_direct_IO_impl(rw, kiocb, iter));
	}
	#else
	#error "Unknown direct IO interface"
	#endif

	#else /* HAVE_VFS_RW_ITERATE */

	#if defined(HAVE_VFS_DIRECT_IO_IOVEC)
	static ssize_t
	zpl_direct_IO(int rw, struct kiocb kiocb, const struct iovec iov,
	loff_t pos, unsigned long nr_segs)
	{
	if (rw == WRITE)
	return (zpl_aio_write(kiocb, iov, nr_segs, pos));
	else
	return (zpl_aio_read(kiocb, iov, nr_segs, pos));
	}
	#elif defined(HAVE_VFS_DIRECT_IO_ITER_RW_OFFSET)
	static ssize_t
	zpl_direct_IO(int rw, struct kiocb kiocb, struct iov_iter iter, loff_t pos)
	{
	const struct iovec *iovp = iov_iter_iovec(iter);
	unsigned long nr_segs = iter->nr_segs;

	ASSERT3S(pos, ==, kiocb->ki_pos);
	if (rw == WRITE)
	return (zpl_aio_write(kiocb, iovp, nr_segs, pos));
	else
	return (zpl_aio_read(kiocb, iovp, nr_segs, pos));
	}
	#else
	#error "Unknown direct IO interface"
	#endif

	#endif /* HAVE_VFS_RW_ITERATE */

	static loff_t
	zpl_llseek(struct file *filp, loff_t offset, int whence)
	{
	#if defined(SEEK_HOLE) && defined(SEEK_DATA)
	fstrans_cookie_t cookie;

	if (whence == SEEK_DATA \|\| whence == SEEK_HOLE) {
	struct inode *ip = filp->f_mapping->host;
	loff_t maxbytes = ip->i_sb->s_maxbytes;
	loff_t error;

	spl_inode_lock_shared(ip);
	cookie = spl_fstrans_mark();
	error = -zfs_holey(ITOZ(ip), whence, &offset);
	spl_fstrans_unmark(cookie);
	if (error == 0)
	error = lseek_execute(filp, ip, offset, maxbytes);
	spl_inode_unlock_shared(ip);

	return (error);
	}
	#endif /* SEEK_HOLE && SEEK_DATA */

	return (generic_file_llseek(filp, offset, whence));
	}

	/*
	* It's worth taking a moment to describe how mmap is implemented
	* for zfs because it differs considerably from other Linux filesystems.
	* However, this issue is handled the same way under OpenSolaris.
	*
	* The issue is that by design zfs bypasses the Linux page cache and
	* leaves all caching up to the ARC. This has been shown to work
	* well for the common read(2)/write(2) case. However, mmap(2)
	* is problem because it relies on being tightly integrated with the
	* page cache. To handle this we cache mmap'ed files twice, once in
	* the ARC and a second time in the page cache. The code is careful
	* to keep both copies synchronized.
	*
	* When a file with an mmap'ed region is written to using write(2)
	* both the data in the ARC and existing pages in the page cache
	* are updated. For a read(2) data will be read first from the page
	* cache then the ARC if needed. Neither a write(2) or read(2) will
	* will ever result in new pages being added to the page cache.
	*
	* New pages are added to the page cache only via .readpage() which
	* is called when the vfs needs to read a page off disk to back the
	* virtual memory region. These pages may be modified without
	* notifying the ARC and will be written out periodically via
	* .writepage(). This will occur due to either a sync or the usual
	* page aging behavior. Note because a read(2) of a mmap'ed file
	* will always check the page cache first even when the ARC is out
	* of date correct data will still be returned.
	*
	* While this implementation ensures correct behavior it does have
	* have some drawbacks. The most obvious of which is that it
	* increases the required memory footprint when access mmap'ed
	* files. It also adds additional complexity to the code keeping
	* both caches synchronized.
	*
	* Longer term it may be possible to cleanly resolve this wart by
	* mapping page cache pages directly on to the ARC buffers. The
	* Linux address space operations are flexible enough to allow
	* selection of which pages back a particular index. The trick
	* would be working out the details of which subsystem is in
	* charge, the ARC, the page cache, or both. It may also prove
	* helpful to move the ARC buffers to a scatter-gather lists
	* rather than a vmalloc'ed region.
	*/
	static int
	zpl_mmap(struct file filp, struct vm_area_struct vma)
	{
	struct inode *ip = filp->f_mapping->host;
	int error;
	fstrans_cookie_t cookie;

	cookie = spl_fstrans_mark();
	error = -zfs_map(ip, vma->vm_pgoff, (caddr_t *)vma->vm_start,
	(size_t)(vma->vm_end - vma->vm_start), vma->vm_flags);
	spl_fstrans_unmark(cookie);
	if (error)
	return (error);

	error = generic_file_mmap(filp, vma);
	if (error)
	return (error);

	#if !defined(HAVE_FILEMAP_RANGE_HAS_PAGE)
	znode_t *zp = ITOZ(ip);
	mutex_enter(&zp->z_lock);
	zp->z_is_mapped = B_TRUE;
	mutex_exit(&zp->z_lock);
	#endif

	return (error);
	}

	/*
	* Populate a page with data for the Linux page cache. This function is
	* only used to support mmap(2). There will be an identical copy of the
	* data in the ARC which is kept up to date via .write() and .writepage().
	*/
	static inline int
	zpl_readpage_common(struct page *pp)
	{
	fstrans_cookie_t cookie;

	ASSERT(PageLocked(pp));

	cookie = spl_fstrans_mark();
	int error = -zfs_getpage(pp->mapping->host, pp);
	spl_fstrans_unmark(cookie);

	unlock_page(pp);

	return (error);
	}

	#ifdef HAVE_VFS_READ_FOLIO
	static int
	zpl_read_folio(struct file filp, struct folio folio)
	{
	return (zpl_readpage_common(&folio->page));
	}
	#else
	static int
	zpl_readpage(struct file filp, struct page pp)
	{
	return (zpl_readpage_common(pp));
	}
	#endif

	static int
	zpl_readpage_filler(void data, struct page pp)
	{
	return (zpl_readpage_common(pp));
	}

	/*
	* Populate a set of pages with data for the Linux page cache. This
	* function will only be called for read ahead and never for demand
	* paging. For simplicity, the code relies on read_cache_pages() to
	* correctly lock each page for IO and call zpl_readpage().
	*/
	#ifdef HAVE_VFS_READPAGES
	static int
	zpl_readpages(struct file filp, struct address_space mapping,
	struct list_head *pages, unsigned nr_pages)
	{
	return (read_cache_pages(mapping, pages, zpl_readpage_filler, NULL));
	}
	#else
	static void
	zpl_readahead(struct readahead_control *ractl)
	{
	struct page *page;

	while ((page = readahead_page(ractl)) != NULL) {
	int ret;

	ret = zpl_readpage_filler(NULL, page);
	put_page(page);
	if (ret)
	break;
	}
	}
	#endif

	static int
	zpl_putpage(struct page pp, struct writeback_control wbc, void *data)
	{
	boolean_t *for_sync = data;
	fstrans_cookie_t cookie;
	+ int ret;

	ASSERT(PageLocked(pp));
	ASSERT(!PageWriteback(pp));

	cookie = spl_fstrans_mark();
	- (void) zfs_putpage(pp->mapping->host, pp, wbc, *for_sync);
	+ ret = zfs_putpage(pp->mapping->host, pp, wbc, *for_sync);
	spl_fstrans_unmark(cookie);

	- return (0);
	+ return (ret);
	}

	#ifdef HAVE_WRITEPAGE_T_FOLIO
	static int
	zpl_putfolio(struct folio pp, struct writeback_control wbc, void *data)
	{
	- (void) zpl_putpage(&pp->page, wbc, data);
	- return (0);
	+ return (zpl_putpage(&pp->page, wbc, data));
	}
	#endif

	static inline int
	zpl_write_cache_pages(struct address_space *mapping,
	struct writeback_control wbc, void data)
	{
	int result;

	#ifdef HAVE_WRITEPAGE_T_FOLIO
	result = write_cache_pages(mapping, wbc, zpl_putfolio, data);
	#else
	result = write_cache_pages(mapping, wbc, zpl_putpage, data);
	#endif
	return (result);
	}

	static int
	zpl_writepages(struct address_space mapping, struct writeback_control wbc)
	{
	znode_t *zp = ITOZ(mapping->host);
	zfsvfs_t *zfsvfs = ITOZSB(mapping->host);
	enum writeback_sync_modes sync_mode;
	int result;

	if ((result = zpl_enter(zfsvfs, FTAG)) != 0)
	return (result);
	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	wbc->sync_mode = WB_SYNC_ALL;
	zpl_exit(zfsvfs, FTAG);
	sync_mode = wbc->sync_mode;

	/*
	* We don't want to run write_cache_pages() in SYNC mode here, because
	* that would make putpage() wait for a single page to be committed to
	* disk every single time, resulting in atrocious performance. Instead
	* we run it once in non-SYNC mode so that the ZIL gets all the data,
	* and then we commit it all in one go.
	*/
	boolean_t for_sync = (sync_mode == WB_SYNC_ALL);
	wbc->sync_mode = WB_SYNC_NONE;
	result = zpl_write_cache_pages(mapping, wbc, &for_sync);
	if (sync_mode != wbc->sync_mode) {
	if ((result = zpl_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (result);
	if (zfsvfs->z_log != NULL)
	zil_commit(zfsvfs->z_log, zp->z_id);
	zpl_exit(zfsvfs, FTAG);

	/*
	* We need to call write_cache_pages() again (we can't just
	* return after the commit) because the previous call in
	* non-SYNC mode does not guarantee that we got all the dirty
	* pages (see the implementation of write_cache_pages() for
	* details). That being said, this is a no-op in most cases.
	*/
	wbc->sync_mode = sync_mode;
	result = zpl_write_cache_pages(mapping, wbc, &for_sync);
	}
	return (result);
	}

	/*
	* Write out dirty pages to the ARC, this function is only required to
	* support mmap(2). Mapped pages may be dirtied by memory operations
	* which never call .write(). These dirty pages are kept in sync with
	* the ARC buffers via this hook.
	*/
	static int
	zpl_writepage(struct page pp, struct writeback_control wbc)
	{
	if (ITOZSB(pp->mapping->host)->z_os->os_sync == ZFS_SYNC_ALWAYS)
	wbc->sync_mode = WB_SYNC_ALL;

	boolean_t for_sync = (wbc->sync_mode == WB_SYNC_ALL);

	return (zpl_putpage(pp, wbc, &for_sync));
	}

	/*
	* The flag combination which matches the behavior of zfs_space() is
	* FALLOC_FL_KEEP_SIZE \| FALLOC_FL_PUNCH_HOLE. The FALLOC_FL_PUNCH_HOLE
	* flag was introduced in the 2.6.38 kernel.
	*
	* The original mode=0 (allocate space) behavior can be reasonably emulated
	* by checking if enough space exists and creating a sparse file, as real
	* persistent space reservation is not possible due to COW, snapshots, etc.
	*/
	static long
	zpl_fallocate_common(struct inode *ip, int mode, loff_t offset, loff_t len)
	{
	cred_t *cr = CRED();
	loff_t olen;
	fstrans_cookie_t cookie;
	int error = 0;

	int test_mode = FALLOC_FL_PUNCH_HOLE;
	#ifdef HAVE_FALLOC_FL_ZERO_RANGE
	test_mode \|= FALLOC_FL_ZERO_RANGE;
	#endif

	if ((mode & ~(FALLOC_FL_KEEP_SIZE \| test_mode)) != 0)
	return (-EOPNOTSUPP);

	if (offset < 0 \|\| len <= 0)
	return (-EINVAL);

	spl_inode_lock(ip);
	olen = i_size_read(ip);

	crhold(cr);
	cookie = spl_fstrans_mark();
	if (mode & (test_mode)) {
	flock64_t bf;

	if (mode & FALLOC_FL_KEEP_SIZE) {
	if (offset > olen)
	goto out_unmark;

	if (offset + len > olen)
	len = olen - offset;
	}
	bf.l_type = F_WRLCK;
	bf.l_whence = SEEK_SET;
	bf.l_start = offset;
	bf.l_len = len;
	bf.l_pid = 0;

	error = -zfs_space(ITOZ(ip), F_FREESP, &bf, O_RDWR, offset, cr);
	} else if ((mode & ~FALLOC_FL_KEEP_SIZE) == 0) {
	unsigned int percent = zfs_fallocate_reserve_percent;
	struct kstatfs statfs;

	/* Legacy mode, disable fallocate compatibility. */
	if (percent == 0) {
	error = -EOPNOTSUPP;
	goto out_unmark;
	}

	/*
	* Use zfs_statvfs() instead of dmu_objset_space() since it
	* also checks project quota limits, which are relevant here.
	*/
	error = zfs_statvfs(ip, &statfs);
	if (error)
	goto out_unmark;

	/*
	* Shrink available space a bit to account for overhead/races.
	* We know the product previously fit into availbytes from
	* dmu_objset_space(), so the smaller product will also fit.
	*/
	if (len > statfs.f_bavail * (statfs.f_bsize * 100 / percent)) {
	error = -ENOSPC;
	goto out_unmark;
	}
	if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > olen)
	error = zfs_freesp(ITOZ(ip), offset + len, 0, 0, FALSE);
	}
	out_unmark:
	spl_fstrans_unmark(cookie);
	spl_inode_unlock(ip);

	crfree(cr);

	return (error);
	}

	static long
	zpl_fallocate(struct file *filp, int mode, loff_t offset, loff_t len)
	{
	return zpl_fallocate_common(file_inode(filp),
	mode, offset, len);
	}

	static int
	zpl_ioctl_getversion(struct file filp, void __user arg)
	{
	uint32_t generation = file_inode(filp)->i_generation;

	return (copy_to_user(arg, &generation, sizeof (generation)));
	}

	#ifdef HAVE_FILE_FADVISE
	static int
	zpl_fadvise(struct file *filp, loff_t offset, loff_t len, int advice)
	{
	struct inode *ip = file_inode(filp);
	znode_t *zp = ITOZ(ip);
	zfsvfs_t *zfsvfs = ITOZSB(ip);
	objset_t *os = zfsvfs->z_os;
	int error = 0;

	if (S_ISFIFO(ip->i_mode))
	return (-ESPIPE);

	if (offset < 0 \|\| len < 0)
	return (-EINVAL);

	if ((error = zpl_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	switch (advice) {
	case POSIX_FADV_SEQUENTIAL:
	case POSIX_FADV_WILLNEED:
	#ifdef HAVE_GENERIC_FADVISE
	if (zn_has_cached_data(zp, offset, offset + len - 1))
	error = generic_fadvise(filp, offset, len, advice);
	#endif
	/*
	* Pass on the caller's size directly, but note that
	* dmu_prefetch_max will effectively cap it. If there
	* really is a larger sequential access pattern, perhaps
	* dmu_zfetch will detect it.
	*/
	if (len == 0)
	len = i_size_read(ip) - offset;

	dmu_prefetch(os, zp->z_id, 0, offset, len,
	ZIO_PRIORITY_ASYNC_READ);
	break;
	case POSIX_FADV_NORMAL:
	case POSIX_FADV_RANDOM:
	case POSIX_FADV_DONTNEED:
	case POSIX_FADV_NOREUSE:
	/* ignored for now */
	break;
	default:
	error = -EINVAL;
	break;
	}

	zfs_exit(zfsvfs, FTAG);

	return (error);
	}
	#endif /* HAVE_FILE_FADVISE */

	#define ZFS_FL_USER_VISIBLE (FS_FL_USER_VISIBLE \| ZFS_PROJINHERIT_FL)
	#define ZFS_FL_USER_MODIFIABLE (FS_FL_USER_MODIFIABLE \| ZFS_PROJINHERIT_FL)

	static uint32_t
	__zpl_ioctl_getflags(struct inode *ip)
	{
	uint64_t zfs_flags = ITOZ(ip)->z_pflags;
	uint32_t ioctl_flags = 0;

	if (zfs_flags & ZFS_IMMUTABLE)
	ioctl_flags \|= FS_IMMUTABLE_FL;

	if (zfs_flags & ZFS_APPENDONLY)
	ioctl_flags \|= FS_APPEND_FL;

	if (zfs_flags & ZFS_NODUMP)
	ioctl_flags \|= FS_NODUMP_FL;

	if (zfs_flags & ZFS_PROJINHERIT)
	ioctl_flags \|= ZFS_PROJINHERIT_FL;

	return (ioctl_flags & ZFS_FL_USER_VISIBLE);
	}

	/*
	* Map zfs file z_pflags (xvattr_t) to linux file attributes. Only file
	* attributes common to both Linux and Solaris are mapped.
	*/
	static int
	zpl_ioctl_getflags(struct file filp, void __user arg)
	{
	uint32_t flags;
	int err;

	flags = __zpl_ioctl_getflags(file_inode(filp));
	err = copy_to_user(arg, &flags, sizeof (flags));

	return (err);
	}

	/*
	* fchange() is a helper macro to detect if we have been asked to change a
	* flag. This is ugly, but the requirement that we do this is a consequence of
	* how the Linux file attribute interface was designed. Another consequence is
	* that concurrent modification of files suffers from a TOCTOU race. Neither
	* are things we can fix without modifying the kernel-userland interface, which
	* is outside of our jurisdiction.
	*/

	#define fchange(f0, f1, b0, b1) (!((f0) & (b0)) != !((f1) & (b1)))

	static int
	__zpl_ioctl_setflags(struct inode ip, uint32_t ioctl_flags, xvattr_t xva)
	{
	uint64_t zfs_flags = ITOZ(ip)->z_pflags;
	xoptattr_t *xoap;

	if (ioctl_flags & ~(FS_IMMUTABLE_FL \| FS_APPEND_FL \| FS_NODUMP_FL \|
	ZFS_PROJINHERIT_FL))
	return (-EOPNOTSUPP);

	if (ioctl_flags & ~ZFS_FL_USER_MODIFIABLE)
	return (-EACCES);

	if ((fchange(ioctl_flags, zfs_flags, FS_IMMUTABLE_FL, ZFS_IMMUTABLE) \|\|
	fchange(ioctl_flags, zfs_flags, FS_APPEND_FL, ZFS_APPENDONLY)) &&
	!capable(CAP_LINUX_IMMUTABLE))
	return (-EPERM);

	if (!zpl_inode_owner_or_capable(zfs_init_idmap, ip))
	return (-EACCES);

	xva_init(xva);
	xoap = xva_getxoptattr(xva);

	#define FLAG_CHANGE(iflag, zflag, xflag, xfield) do { \
	if (((ioctl_flags & (iflag)) && !(zfs_flags & (zflag))) \|\| \
	((zfs_flags & (zflag)) && !(ioctl_flags & (iflag)))) { \
	XVA_SET_REQ(xva, (xflag)); \
	(xfield) = ((ioctl_flags & (iflag)) != 0); \
	} \
	} while (0)

	FLAG_CHANGE(FS_IMMUTABLE_FL, ZFS_IMMUTABLE, XAT_IMMUTABLE,
	xoap->xoa_immutable);
	FLAG_CHANGE(FS_APPEND_FL, ZFS_APPENDONLY, XAT_APPENDONLY,
	xoap->xoa_appendonly);
	FLAG_CHANGE(FS_NODUMP_FL, ZFS_NODUMP, XAT_NODUMP,
	xoap->xoa_nodump);
	FLAG_CHANGE(ZFS_PROJINHERIT_FL, ZFS_PROJINHERIT, XAT_PROJINHERIT,
	xoap->xoa_projinherit);

	#undef FLAG_CHANGE

	return (0);
	}

	static int
	zpl_ioctl_setflags(struct file filp, void __user arg)
	{
	struct inode *ip = file_inode(filp);
	uint32_t flags;
	cred_t *cr = CRED();
	xvattr_t xva;
	int err;
	fstrans_cookie_t cookie;

	if (copy_from_user(&flags, arg, sizeof (flags)))
	return (-EFAULT);

	err = __zpl_ioctl_setflags(ip, flags, &xva);
	if (err)
	return (err);

	crhold(cr);
	cookie = spl_fstrans_mark();
	err = -zfs_setattr(ITOZ(ip), (vattr_t *)&xva, 0, cr, zfs_init_idmap);
	spl_fstrans_unmark(cookie);
	crfree(cr);

	return (err);
	}

	static int
	zpl_ioctl_getxattr(struct file filp, void __user arg)
	{
	zfsxattr_t fsx = { 0 };
	struct inode *ip = file_inode(filp);
	int err;

	fsx.fsx_xflags = __zpl_ioctl_getflags(ip);
	fsx.fsx_projid = ITOZ(ip)->z_projid;
	err = copy_to_user(arg, &fsx, sizeof (fsx));

	return (err);
	}

	static int
	zpl_ioctl_setxattr(struct file filp, void __user arg)
	{
	struct inode *ip = file_inode(filp);
	zfsxattr_t fsx;
	cred_t *cr = CRED();
	xvattr_t xva;
	xoptattr_t *xoap;
	int err;
	fstrans_cookie_t cookie;

	if (copy_from_user(&fsx, arg, sizeof (fsx)))
	return (-EFAULT);

	if (!zpl_is_valid_projid(fsx.fsx_projid))
	return (-EINVAL);

	err = __zpl_ioctl_setflags(ip, fsx.fsx_xflags, &xva);
	if (err)
	return (err);

	xoap = xva_getxoptattr(&xva);
	XVA_SET_REQ(&xva, XAT_PROJID);
	xoap->xoa_projid = fsx.fsx_projid;

	crhold(cr);
	cookie = spl_fstrans_mark();
	err = -zfs_setattr(ITOZ(ip), (vattr_t *)&xva, 0, cr, zfs_init_idmap);
	spl_fstrans_unmark(cookie);
	crfree(cr);

	return (err);
	}

	/*
	* Expose Additional File Level Attributes of ZFS.
	*/
	static int
	zpl_ioctl_getdosflags(struct file filp, void __user arg)
	{
	struct inode *ip = file_inode(filp);
	uint64_t dosflags = ITOZ(ip)->z_pflags;
	dosflags &= ZFS_DOS_FL_USER_VISIBLE;
	int err = copy_to_user(arg, &dosflags, sizeof (dosflags));

	return (err);
	}

	static int
	__zpl_ioctl_setdosflags(struct inode ip, uint64_t ioctl_flags, xvattr_t xva)
	{
	uint64_t zfs_flags = ITOZ(ip)->z_pflags;
	xoptattr_t *xoap;

	if (ioctl_flags & (~ZFS_DOS_FL_USER_VISIBLE))
	return (-EOPNOTSUPP);

	if ((fchange(ioctl_flags, zfs_flags, ZFS_IMMUTABLE, ZFS_IMMUTABLE) \|\|
	fchange(ioctl_flags, zfs_flags, ZFS_APPENDONLY, ZFS_APPENDONLY)) &&
	!capable(CAP_LINUX_IMMUTABLE))
	return (-EPERM);

	if (!zpl_inode_owner_or_capable(zfs_init_idmap, ip))
	return (-EACCES);

	xva_init(xva);
	xoap = xva_getxoptattr(xva);

	#define FLAG_CHANGE(iflag, xflag, xfield) do { \
	if (((ioctl_flags & (iflag)) && !(zfs_flags & (iflag))) \|\| \
	((zfs_flags & (iflag)) && !(ioctl_flags & (iflag)))) { \
	XVA_SET_REQ(xva, (xflag)); \
	(xfield) = ((ioctl_flags & (iflag)) != 0); \
	} \
	} while (0)

	FLAG_CHANGE(ZFS_IMMUTABLE, XAT_IMMUTABLE, xoap->xoa_immutable);
	FLAG_CHANGE(ZFS_APPENDONLY, XAT_APPENDONLY, xoap->xoa_appendonly);
	FLAG_CHANGE(ZFS_NODUMP, XAT_NODUMP, xoap->xoa_nodump);
	FLAG_CHANGE(ZFS_READONLY, XAT_READONLY, xoap->xoa_readonly);
	FLAG_CHANGE(ZFS_HIDDEN, XAT_HIDDEN, xoap->xoa_hidden);
	FLAG_CHANGE(ZFS_SYSTEM, XAT_SYSTEM, xoap->xoa_system);
	FLAG_CHANGE(ZFS_ARCHIVE, XAT_ARCHIVE, xoap->xoa_archive);
	FLAG_CHANGE(ZFS_NOUNLINK, XAT_NOUNLINK, xoap->xoa_nounlink);
	FLAG_CHANGE(ZFS_REPARSE, XAT_REPARSE, xoap->xoa_reparse);
	FLAG_CHANGE(ZFS_OFFLINE, XAT_OFFLINE, xoap->xoa_offline);
	FLAG_CHANGE(ZFS_SPARSE, XAT_SPARSE, xoap->xoa_sparse);

	#undef FLAG_CHANGE

	return (0);
	}

	/*
	* Set Additional File Level Attributes of ZFS.
	*/
	static int
	zpl_ioctl_setdosflags(struct file filp, void __user arg)
	{
	struct inode *ip = file_inode(filp);
	uint64_t dosflags;
	cred_t *cr = CRED();
	xvattr_t xva;
	int err;
	fstrans_cookie_t cookie;

	if (copy_from_user(&dosflags, arg, sizeof (dosflags)))
	return (-EFAULT);

	err = __zpl_ioctl_setdosflags(ip, dosflags, &xva);
	if (err)
	return (err);

	crhold(cr);
	cookie = spl_fstrans_mark();
	err = -zfs_setattr(ITOZ(ip), (vattr_t *)&xva, 0, cr, zfs_init_idmap);
	spl_fstrans_unmark(cookie);
	crfree(cr);

	return (err);
	}

	static long
	zpl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
	{
	switch (cmd) {
	case FS_IOC_GETVERSION:
	return (zpl_ioctl_getversion(filp, (void *)arg));
	case FS_IOC_GETFLAGS:
	return (zpl_ioctl_getflags(filp, (void *)arg));
	case FS_IOC_SETFLAGS:
	return (zpl_ioctl_setflags(filp, (void *)arg));
	case ZFS_IOC_FSGETXATTR:
	return (zpl_ioctl_getxattr(filp, (void *)arg));
	case ZFS_IOC_FSSETXATTR:
	return (zpl_ioctl_setxattr(filp, (void *)arg));
	case ZFS_IOC_GETDOSFLAGS:
	return (zpl_ioctl_getdosflags(filp, (void *)arg));
	case ZFS_IOC_SETDOSFLAGS:
	return (zpl_ioctl_setdosflags(filp, (void *)arg));
	case ZFS_IOC_COMPAT_FICLONE:
	return (zpl_ioctl_ficlone(filp, (void *)arg));
	case ZFS_IOC_COMPAT_FICLONERANGE:
	return (zpl_ioctl_ficlonerange(filp, (void *)arg));
	case ZFS_IOC_COMPAT_FIDEDUPERANGE:
	return (zpl_ioctl_fideduperange(filp, (void *)arg));
	default:
	return (-ENOTTY);
	}
	}

	#ifdef CONFIG_COMPAT
	static long
	zpl_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
	{
	switch (cmd) {
	case FS_IOC32_GETVERSION:
	cmd = FS_IOC_GETVERSION;
	break;
	case FS_IOC32_GETFLAGS:
	cmd = FS_IOC_GETFLAGS;
	break;
	case FS_IOC32_SETFLAGS:
	cmd = FS_IOC_SETFLAGS;
	break;
	default:
	return (-ENOTTY);
	}
	return (zpl_ioctl(filp, cmd, (unsigned long)compat_ptr(arg)));
	}
	#endif /* CONFIG_COMPAT */

	const struct address_space_operations zpl_address_space_operations = {
	#ifdef HAVE_VFS_READPAGES
	.readpages = zpl_readpages,
	#else
	.readahead = zpl_readahead,
	#endif
	#ifdef HAVE_VFS_READ_FOLIO
	.read_folio = zpl_read_folio,
	#else
	.readpage = zpl_readpage,
	#endif
	.writepage = zpl_writepage,
	.writepages = zpl_writepages,
	.direct_IO = zpl_direct_IO,
	#ifdef HAVE_VFS_SET_PAGE_DIRTY_NOBUFFERS
	.set_page_dirty = __set_page_dirty_nobuffers,
	#endif
	#ifdef HAVE_VFS_FILEMAP_DIRTY_FOLIO
	.dirty_folio = filemap_dirty_folio,
	#endif
	};

	#ifdef HAVE_VFS_FILE_OPERATIONS_EXTEND
	const struct file_operations_extend zpl_file_operations = {
	.kabi_fops = {
	#else
	const struct file_operations zpl_file_operations = {
	#endif
	.open = zpl_open,
	.release = zpl_release,
	.llseek = zpl_llseek,
	#ifdef HAVE_VFS_RW_ITERATE
	#ifdef HAVE_NEW_SYNC_READ
	.read = new_sync_read,
	.write = new_sync_write,
	#endif
	.read_iter = zpl_iter_read,
	.write_iter = zpl_iter_write,
	#ifdef HAVE_VFS_IOV_ITER
	#ifdef HAVE_COPY_SPLICE_READ
	.splice_read = copy_splice_read,
	#else
	.splice_read = generic_file_splice_read,
	#endif
	.splice_write = iter_file_splice_write,
	#endif
	#else
	.read = do_sync_read,
	.write = do_sync_write,
	.aio_read = zpl_aio_read,
	.aio_write = zpl_aio_write,
	#endif
	.mmap = zpl_mmap,
	.fsync = zpl_fsync,
	#ifdef HAVE_FILE_AIO_FSYNC
	.aio_fsync = zpl_aio_fsync,
	#endif
	.fallocate = zpl_fallocate,
	#ifdef HAVE_VFS_COPY_FILE_RANGE
	.copy_file_range = zpl_copy_file_range,
	#endif
	#ifdef HAVE_VFS_CLONE_FILE_RANGE
	.clone_file_range = zpl_clone_file_range,
	#endif
	#ifdef HAVE_VFS_REMAP_FILE_RANGE
	.remap_file_range = zpl_remap_file_range,
	#endif
	#ifdef HAVE_VFS_DEDUPE_FILE_RANGE
	.dedupe_file_range = zpl_dedupe_file_range,
	#endif
	#ifdef HAVE_FILE_FADVISE
	.fadvise = zpl_fadvise,
	#endif
	.unlocked_ioctl = zpl_ioctl,
	#ifdef CONFIG_COMPAT
	.compat_ioctl = zpl_compat_ioctl,
	#endif
	#ifdef HAVE_VFS_FILE_OPERATIONS_EXTEND
	}, /* kabi_fops */
	.copy_file_range = zpl_copy_file_range,
	.clone_file_range = zpl_clone_file_range,
	#endif
	};

	const struct file_operations zpl_dir_file_operations = {
	.llseek = generic_file_llseek,
	.read = generic_read_dir,
	#if defined(HAVE_VFS_ITERATE_SHARED)
	.iterate_shared = zpl_iterate,
	#elif defined(HAVE_VFS_ITERATE)
	.iterate = zpl_iterate,
	#else
	.readdir = zpl_readdir,
	#endif
	.fsync = zpl_fsync,
	.unlocked_ioctl = zpl_ioctl,
	#ifdef CONFIG_COMPAT
	.compat_ioctl = zpl_compat_ioctl,
	#endif
	};

	/* CSTYLED */
	module_param(zfs_fallocate_reserve_percent, uint, 0644);
	MODULE_PARM_DESC(zfs_fallocate_reserve_percent,
	"Percentage of length to use for the available capacity check");
	diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zpl_file_range.c b/sys/contrib/openzfs/module/os/linux/zfs/zpl_file_range.c
	index 3065d54fa9da..64728fdb1187 100644
	--- a/sys/contrib/openzfs/module/os/linux/zfs/zpl_file_range.c
	+++ b/sys/contrib/openzfs/module/os/linux/zfs/zpl_file_range.c
	@@ -1,287 +1,299 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2023, Klara Inc.
	*/

	#ifdef CONFIG_COMPAT
	#include <linux/compat.h>
	#endif
	#include <linux/fs.h>
	+#ifdef HAVE_VFS_SPLICE_COPY_FILE_RANGE
	+#include <linux/splice.h>
	+#endif
	#include <sys/file.h>
	#include <sys/zfs_znode.h>
	#include <sys/zfs_vnops.h>
	#include <sys/zfeature.h>

	/*
	* Clone part of a file via block cloning.
	*
	* Note that we are not required to update file offsets; the kernel will take
	* care of that depending on how it was called.
	*/
	static ssize_t
	zpl_clone_file_range_impl(struct file *src_file, loff_t src_off,
	struct file *dst_file, loff_t dst_off, size_t len)
	{
	struct inode *src_i = file_inode(src_file);
	struct inode *dst_i = file_inode(dst_file);
	uint64_t src_off_o = (uint64_t)src_off;
	uint64_t dst_off_o = (uint64_t)dst_off;
	uint64_t len_o = (uint64_t)len;
	cred_t *cr = CRED();
	fstrans_cookie_t cookie;
	int err;

	if (!zfs_bclone_enabled)
	return (-EOPNOTSUPP);

	if (!spa_feature_is_enabled(
	dmu_objset_spa(ITOZSB(dst_i)->z_os), SPA_FEATURE_BLOCK_CLONING))
	return (-EOPNOTSUPP);

	if (src_i != dst_i)
	spl_inode_lock_shared(src_i);
	spl_inode_lock(dst_i);

	crhold(cr);
	cookie = spl_fstrans_mark();

	err = -zfs_clone_range(ITOZ(src_i), &src_off_o, ITOZ(dst_i),
	&dst_off_o, &len_o, cr);

	spl_fstrans_unmark(cookie);
	crfree(cr);

	spl_inode_unlock(dst_i);
	if (src_i != dst_i)
	spl_inode_unlock_shared(src_i);

	if (err < 0)
	return (err);

	return ((ssize_t)len_o);
	}

	#if defined(HAVE_VFS_COPY_FILE_RANGE) \|\| \
	defined(HAVE_VFS_FILE_OPERATIONS_EXTEND)
	/*
	* Entry point for copy_file_range(). Copy len bytes from src_off in src_file
	* to dst_off in dst_file. We are permitted to do this however we like, so we
	* try to just clone the blocks, and if we can't support it, fall back to the
	* kernel's generic byte copy function.
	*/
	ssize_t
	zpl_copy_file_range(struct file *src_file, loff_t src_off,
	struct file *dst_file, loff_t dst_off, size_t len, unsigned int flags)
	{
	ssize_t ret;

	/* Flags is reserved for future extensions and must be zero. */
	if (flags != 0)
	return (-EINVAL);

	/* Try to do it via zfs_clone_range() and allow shortening. */
	ret = zpl_clone_file_range_impl(src_file, src_off,
	dst_file, dst_off, len);

	-#ifdef HAVE_VFS_GENERIC_COPY_FILE_RANGE
	+#if defined(HAVE_VFS_GENERIC_COPY_FILE_RANGE)
	/*
	* Since Linux 5.3 the filesystem driver is responsible for executing
	* an appropriate fallback, and a generic fallback function is provided.
	*/
	if (ret == -EOPNOTSUPP \|\| ret == -EINVAL \|\| ret == -EXDEV \|\|
	ret == -EAGAIN)
	ret = generic_copy_file_range(src_file, src_off, dst_file,
	dst_off, len, flags);
	+#elif defined(HAVE_VFS_SPLICE_COPY_FILE_RANGE)
	+ /*
	+ * Since 6.8 the fallback function is called splice_copy_file_range
	+ * and has a slightly different signature.
	+ */
	+ if (ret == -EOPNOTSUPP \|\| ret == -EINVAL \|\| ret == -EXDEV \|\|
	+ ret == -EAGAIN)
	+ ret = splice_copy_file_range(src_file, src_off, dst_file,
	+ dst_off, len);
	#else
	/*
	* Before Linux 5.3 the filesystem has to return -EOPNOTSUPP to signal
	* to the kernel that it should fallback to a content copy.
	*/
	if (ret == -EINVAL \|\| ret == -EXDEV \|\| ret == -EAGAIN)
	ret = -EOPNOTSUPP;
	-#endif /* HAVE_VFS_GENERIC_COPY_FILE_RANGE */
	+#endif /* HAVE_VFS_GENERIC_COPY_FILE_RANGE \|\| HAVE_VFS_SPLICE_COPY_FILE_RANGE */

	return (ret);
	}
	#endif /* HAVE_VFS_COPY_FILE_RANGE \|\| HAVE_VFS_FILE_OPERATIONS_EXTEND */

	#ifdef HAVE_VFS_REMAP_FILE_RANGE
	/*
	* Entry point for FICLONE/FICLONERANGE/FIDEDUPERANGE.
	*
	* FICLONE and FICLONERANGE are basically the same as copy_file_range(), except
	* that they must clone - they cannot fall back to copying. FICLONE is exactly
	* FICLONERANGE, for the entire file. We don't need to try to tell them apart;
	* the kernel will sort that out for us.
	*
	* FIDEDUPERANGE is for turning a non-clone into a clone, that is, compare the
	* range in both files and if they're the same, arrange for them to be backed
	* by the same storage.
	*
	* REMAP_FILE_CAN_SHORTEN lets us know we can clone less than the given range
	* if we want. It's designed for filesystems that may need to shorten the
	* length for alignment, EOF, or any other requirement. ZFS may shorten the
	* request when there is outstanding dirty data which hasn't been written.
	*/
	loff_t
	zpl_remap_file_range(struct file *src_file, loff_t src_off,
	struct file *dst_file, loff_t dst_off, loff_t len, unsigned int flags)
	{
	if (flags & ~(REMAP_FILE_DEDUP \| REMAP_FILE_CAN_SHORTEN))
	return (-EINVAL);

	/* No support for dedup yet */
	if (flags & REMAP_FILE_DEDUP)
	return (-EOPNOTSUPP);

	/* Zero length means to clone everything to the end of the file */
	if (len == 0)
	len = i_size_read(file_inode(src_file)) - src_off;

	ssize_t ret = zpl_clone_file_range_impl(src_file, src_off,
	dst_file, dst_off, len);

	if (!(flags & REMAP_FILE_CAN_SHORTEN) && ret >= 0 && ret != len)
	ret = -EINVAL;

	return (ret);
	}
	#endif /* HAVE_VFS_REMAP_FILE_RANGE */

	#if defined(HAVE_VFS_CLONE_FILE_RANGE) \|\| \
	defined(HAVE_VFS_FILE_OPERATIONS_EXTEND)
	/*
	* Entry point for FICLONE and FICLONERANGE, before Linux 4.20.
	*/
	int
	zpl_clone_file_range(struct file *src_file, loff_t src_off,
	struct file *dst_file, loff_t dst_off, uint64_t len)
	{
	/* Zero length means to clone everything to the end of the file */
	if (len == 0)
	len = i_size_read(file_inode(src_file)) - src_off;

	/* The entire length must be cloned or this is an error. */
	ssize_t ret = zpl_clone_file_range_impl(src_file, src_off,
	dst_file, dst_off, len);

	if (ret >= 0 && ret != len)
	ret = -EINVAL;

	return (ret);
	}
	#endif /* HAVE_VFS_CLONE_FILE_RANGE \|\| HAVE_VFS_FILE_OPERATIONS_EXTEND */

	#ifdef HAVE_VFS_DEDUPE_FILE_RANGE
	/*
	* Entry point for FIDEDUPERANGE, before Linux 4.20.
	*/
	int
	zpl_dedupe_file_range(struct file *src_file, loff_t src_off,
	struct file *dst_file, loff_t dst_off, uint64_t len)
	{
	/* No support for dedup yet */
	return (-EOPNOTSUPP);
	}
	#endif /* HAVE_VFS_DEDUPE_FILE_RANGE */

	/* Entry point for FICLONE, before Linux 4.5. */
	long
	zpl_ioctl_ficlone(struct file dst_file, void arg)
	{
	unsigned long sfd = (unsigned long)arg;

	struct file *src_file = fget(sfd);
	if (src_file == NULL)
	return (-EBADF);

	if (dst_file->f_op != src_file->f_op) {
	fput(src_file);
	return (-EXDEV);
	}

	size_t len = i_size_read(file_inode(src_file));

	ssize_t ret = zpl_clone_file_range_impl(src_file, 0, dst_file, 0, len);

	fput(src_file);

	if (ret < 0) {
	if (ret == -EOPNOTSUPP)
	return (-ENOTTY);
	return (ret);
	}

	if (ret != len)
	return (-EINVAL);

	return (0);
	}

	/* Entry point for FICLONERANGE, before Linux 4.5. */
	long
	zpl_ioctl_ficlonerange(struct file dst_file, void __user arg)
	{
	zfs_ioc_compat_file_clone_range_t fcr;

	if (copy_from_user(&fcr, arg, sizeof (fcr)))
	return (-EFAULT);

	struct file *src_file = fget(fcr.fcr_src_fd);
	if (src_file == NULL)
	return (-EBADF);

	if (dst_file->f_op != src_file->f_op) {
	fput(src_file);
	return (-EXDEV);
	}

	size_t len = fcr.fcr_src_length;
	if (len == 0)
	len = i_size_read(file_inode(src_file)) - fcr.fcr_src_offset;

	ssize_t ret = zpl_clone_file_range_impl(src_file, fcr.fcr_src_offset,
	dst_file, fcr.fcr_dest_offset, len);

	fput(src_file);

	if (ret < 0) {
	if (ret == -EOPNOTSUPP)
	return (-ENOTTY);
	return (ret);
	}

	if (ret != len)
	return (-EINVAL);

	return (0);
	}

	/* Entry point for FIDEDUPERANGE, before Linux 4.5. */
	long
	zpl_ioctl_fideduperange(struct file filp, void arg)
	{
	(void) arg;

	/* No support for dedup yet */
	return (-ENOTTY);
	}
	diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c b/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c
	index 8562e989738d..61c8db5d8178 100644
	--- a/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c
	+++ b/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c
	@@ -1,1648 +1,1764 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2012, 2020 by Delphix. All rights reserved.
	*/

	#include <sys/dataset_kstats.h>
	#include <sys/dbuf.h>
	#include <sys/dmu_traverse.h>
	#include <sys/dsl_dataset.h>
	#include <sys/dsl_prop.h>
	#include <sys/dsl_dir.h>
	#include <sys/zap.h>
	#include <sys/zfeature.h>
	#include <sys/zil_impl.h>
	#include <sys/dmu_tx.h>
	#include <sys/zio.h>
	#include <sys/zfs_rlock.h>
	#include <sys/spa_impl.h>
	#include <sys/zvol.h>
	#include <sys/zvol_impl.h>
	+#include <cityhash.h>

	#include <linux/blkdev_compat.h>
	#include <linux/task_io_accounting_ops.h>

	#ifdef HAVE_BLK_MQ
	#include <linux/blk-mq.h>
	#endif

	static void zvol_request_impl(zvol_state_t zv, struct bio bio,
	struct request *rq, boolean_t force_sync);

	static unsigned int zvol_major = ZVOL_MAJOR;
	static unsigned int zvol_request_sync = 0;
	static unsigned int zvol_prefetch_bytes = (128 * 1024);
	static unsigned long zvol_max_discard_blocks = 16384;

	+/*
	+ * Switch taskq at multiple of 512 MB offset. This can be set to a lower value
	+ * to utilize more threads for small files but may affect prefetch hits.
	+ */
	+#define ZVOL_TASKQ_OFFSET_SHIFT 29
	+
	#ifndef HAVE_BLKDEV_GET_ERESTARTSYS
	static unsigned int zvol_open_timeout_ms = 1000;
	#endif

	static unsigned int zvol_threads = 0;
	#ifdef HAVE_BLK_MQ
	static unsigned int zvol_blk_mq_threads = 0;
	static unsigned int zvol_blk_mq_actual_threads;
	static boolean_t zvol_use_blk_mq = B_FALSE;

	/*
	* The maximum number of volblocksize blocks to process per thread. Typically,
	* write heavy workloads preform better with higher values here, and read
	* heavy workloads preform better with lower values, but that's not a hard
	* and fast rule. It's basically a knob to tune between "less overhead with
	* less parallelism" and "more overhead, but more parallelism".
	*
	* '8' was chosen as a reasonable, balanced, default based off of sequential
	* read and write tests to a zvol in an NVMe pool (with 16 CPUs).
	*/
	static unsigned int zvol_blk_mq_blocks_per_thread = 8;
	#endif

	+static unsigned int zvol_num_taskqs = 0;
	+
	#ifndef BLKDEV_DEFAULT_RQ
	/* BLKDEV_MAX_RQ was renamed to BLKDEV_DEFAULT_RQ in the 5.16 kernel */
	#define BLKDEV_DEFAULT_RQ BLKDEV_MAX_RQ
	#endif

	/*
	* Finalize our BIO or request.
	*/
	#ifdef HAVE_BLK_MQ
	#define END_IO(zv, bio, rq, error) do { \
	if (bio) { \
	BIO_END_IO(bio, error); \
	} else { \
	blk_mq_end_request(rq, errno_to_bi_status(error)); \
	} \
	} while (0)
	#else
	#define END_IO(zv, bio, rq, error) BIO_END_IO(bio, error)
	#endif

	#ifdef HAVE_BLK_MQ
	static unsigned int zvol_blk_mq_queue_depth = BLKDEV_DEFAULT_RQ;
	static unsigned int zvol_actual_blk_mq_queue_depth;
	#endif

	struct zvol_state_os {
	struct gendisk zvo_disk; / generic disk */
	struct request_queue zvo_queue; / request queue */
	dev_t zvo_dev; /* device id */

	#ifdef HAVE_BLK_MQ
	struct blk_mq_tag_set tag_set;
	#endif

	/* Set from the global 'zvol_use_blk_mq' at zvol load */
	boolean_t use_blk_mq;
	};

	-static taskq_t *zvol_taskq;
	+typedef struct zv_taskq {
	+ uint_t tqs_cnt;
	+ taskq_t **tqs_taskq;
	+} zv_taskq_t;
	+static zv_taskq_t zvol_taskqs;
	static struct ida zvol_ida;

	typedef struct zv_request_stack {
	zvol_state_t *zv;
	struct bio *bio;
	struct request *rq;
	} zv_request_t;

	typedef struct zv_work {
	struct request *rq;
	struct work_struct work;
	} zv_work_t;

	typedef struct zv_request_task {
	zv_request_t zvr;
	taskq_ent_t ent;
	} zv_request_task_t;

	static zv_request_task_t *
	zv_request_task_create(zv_request_t zvr)
	{
	zv_request_task_t *task;
	task = kmem_alloc(sizeof (zv_request_task_t), KM_SLEEP);
	taskq_init_ent(&task->ent);
	task->zvr = zvr;
	return (task);
	}

	static void
	zv_request_task_free(zv_request_task_t *task)
	{
	kmem_free(task, sizeof (*task));
	}

	#ifdef HAVE_BLK_MQ

	/*
	* This is called when a new block multiqueue request comes in. A request
	* contains one or more BIOs.
	*/
	static blk_status_t zvol_mq_queue_rq(struct blk_mq_hw_ctx *hctx,
	const struct blk_mq_queue_data *bd)
	{
	struct request *rq = bd->rq;
	zvol_state_t *zv = rq->q->queuedata;

	/* Tell the kernel that we are starting to process this request */
	blk_mq_start_request(rq);

	if (blk_rq_is_passthrough(rq)) {
	/* Skip non filesystem request */
	blk_mq_end_request(rq, BLK_STS_IOERR);
	return (BLK_STS_IOERR);
	}

	zvol_request_impl(zv, NULL, rq, 0);

	/* Acknowledge to the kernel that we got this request */
	return (BLK_STS_OK);
	}

	static struct blk_mq_ops zvol_blk_mq_queue_ops = {
	.queue_rq = zvol_mq_queue_rq,
	};

	/* Initialize our blk-mq struct */
	static int zvol_blk_mq_alloc_tag_set(zvol_state_t *zv)
	{
	struct zvol_state_os *zso = zv->zv_zso;

	memset(&zso->tag_set, 0, sizeof (zso->tag_set));

	/* Initialize tag set. */
	zso->tag_set.ops = &zvol_blk_mq_queue_ops;
	zso->tag_set.nr_hw_queues = zvol_blk_mq_actual_threads;
	zso->tag_set.queue_depth = zvol_actual_blk_mq_queue_depth;
	zso->tag_set.numa_node = NUMA_NO_NODE;
	zso->tag_set.cmd_size = 0;

	/*
	* We need BLK_MQ_F_BLOCKING here since we do blocking calls in
	* zvol_request_impl()
	*/
	zso->tag_set.flags = BLK_MQ_F_SHOULD_MERGE \| BLK_MQ_F_BLOCKING;
	zso->tag_set.driver_data = zv;

	return (blk_mq_alloc_tag_set(&zso->tag_set));
	}
	#endif /* HAVE_BLK_MQ */

	/*
	* Given a path, return TRUE if path is a ZVOL.
	*/
	boolean_t
	zvol_os_is_zvol(const char *path)
	{
	dev_t dev = 0;

	if (vdev_lookup_bdev(path, &dev) != 0)
	return (B_FALSE);

	if (MAJOR(dev) == zvol_major)
	return (B_TRUE);

	return (B_FALSE);
	}

	static void
	zvol_write(zv_request_t *zvr)
	{
	struct bio *bio = zvr->bio;
	struct request *rq = zvr->rq;
	int error = 0;
	zfs_uio_t uio;
	zvol_state_t *zv = zvr->zv;
	struct request_queue *q;
	struct gendisk *disk;
	unsigned long start_time = 0;
	boolean_t acct = B_FALSE;

	ASSERT3P(zv, !=, NULL);
	ASSERT3U(zv->zv_open_count, >, 0);
	ASSERT3P(zv->zv_zilog, !=, NULL);

	q = zv->zv_zso->zvo_queue;
	disk = zv->zv_zso->zvo_disk;

	/* bio marked as FLUSH need to flush before write */
	if (io_is_flush(bio, rq))
	zil_commit(zv->zv_zilog, ZVOL_OBJ);

	/* Some requests are just for flush and nothing else. */
	if (io_size(bio, rq) == 0) {
	rw_exit(&zv->zv_suspend_lock);
	END_IO(zv, bio, rq, 0);
	return;
	}

	zfs_uio_bvec_init(&uio, bio, rq);

	ssize_t start_resid = uio.uio_resid;

	/*
	* With use_blk_mq, accounting is done by blk_mq_start_request()
	* and blk_mq_end_request(), so we can skip it here.
	*/
	if (bio) {
	acct = blk_queue_io_stat(q);
	if (acct) {
	start_time = blk_generic_start_io_acct(q, disk, WRITE,
	bio);
	}
	}

	boolean_t sync =
	io_is_fua(bio, rq) \|\| zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS;

	zfs_locked_range_t *lr = zfs_rangelock_enter(&zv->zv_rangelock,
	uio.uio_loffset, uio.uio_resid, RL_WRITER);

	uint64_t volsize = zv->zv_volsize;
	while (uio.uio_resid > 0 && uio.uio_loffset < volsize) {
	uint64_t bytes = MIN(uio.uio_resid, DMU_MAX_ACCESS >> 1);
	uint64_t off = uio.uio_loffset;
	dmu_tx_t *tx = dmu_tx_create(zv->zv_objset);

	if (bytes > volsize - off) /* don't write past the end */
	bytes = volsize - off;

	dmu_tx_hold_write_by_dnode(tx, zv->zv_dn, off, bytes);

	/* This will only fail for ENOSPC */
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error) {
	dmu_tx_abort(tx);
	break;
	}
	error = dmu_write_uio_dnode(zv->zv_dn, &uio, bytes, tx);
	if (error == 0) {
	zvol_log_write(zv, tx, off, bytes, sync);
	}
	dmu_tx_commit(tx);

	if (error)
	break;
	}
	zfs_rangelock_exit(lr);

	int64_t nwritten = start_resid - uio.uio_resid;
	dataset_kstats_update_write_kstats(&zv->zv_kstat, nwritten);
	task_io_account_write(nwritten);

	if (sync)
	zil_commit(zv->zv_zilog, ZVOL_OBJ);

	rw_exit(&zv->zv_suspend_lock);

	if (bio && acct) {
	blk_generic_end_io_acct(q, disk, WRITE, bio, start_time);
	}

	END_IO(zv, bio, rq, -error);
	}

	static void
	zvol_write_task(void *arg)
	{
	zv_request_task_t *task = arg;
	zvol_write(&task->zvr);
	zv_request_task_free(task);
	}

	static void
	zvol_discard(zv_request_t *zvr)
	{
	struct bio *bio = zvr->bio;
	struct request *rq = zvr->rq;
	zvol_state_t *zv = zvr->zv;
	uint64_t start = io_offset(bio, rq);
	uint64_t size = io_size(bio, rq);
	uint64_t end = start + size;
	boolean_t sync;
	int error = 0;
	dmu_tx_t *tx;
	struct request_queue *q = zv->zv_zso->zvo_queue;
	struct gendisk *disk = zv->zv_zso->zvo_disk;
	unsigned long start_time = 0;
	boolean_t acct = B_FALSE;

	ASSERT3P(zv, !=, NULL);
	ASSERT3U(zv->zv_open_count, >, 0);
	ASSERT3P(zv->zv_zilog, !=, NULL);

	if (bio) {
	acct = blk_queue_io_stat(q);
	if (acct) {
	start_time = blk_generic_start_io_acct(q, disk, WRITE,
	bio);
	}
	}

	sync = io_is_fua(bio, rq) \|\| zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS;

	if (end > zv->zv_volsize) {
	error = SET_ERROR(EIO);
	goto unlock;
	}

	/*
	* Align the request to volume block boundaries when a secure erase is
	* not required. This will prevent dnode_free_range() from zeroing out
	* the unaligned parts which is slow (read-modify-write) and useless
	* since we are not freeing any space by doing so.
	*/
	if (!io_is_secure_erase(bio, rq)) {
	start = P2ROUNDUP(start, zv->zv_volblocksize);
	end = P2ALIGN(end, zv->zv_volblocksize);
	size = end - start;
	}

	if (start >= end)
	goto unlock;

	zfs_locked_range_t *lr = zfs_rangelock_enter(&zv->zv_rangelock,
	start, size, RL_WRITER);

	tx = dmu_tx_create(zv->zv_objset);
	dmu_tx_mark_netfree(tx);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error != 0) {
	dmu_tx_abort(tx);
	} else {
	zvol_log_truncate(zv, tx, start, size, B_TRUE);
	dmu_tx_commit(tx);
	error = dmu_free_long_range(zv->zv_objset,
	ZVOL_OBJ, start, size);
	}
	zfs_rangelock_exit(lr);

	if (error == 0 && sync)
	zil_commit(zv->zv_zilog, ZVOL_OBJ);

	unlock:
	rw_exit(&zv->zv_suspend_lock);

	if (bio && acct) {
	blk_generic_end_io_acct(q, disk, WRITE, bio,
	start_time);
	}

	END_IO(zv, bio, rq, -error);
	}

	static void
	zvol_discard_task(void *arg)
	{
	zv_request_task_t *task = arg;
	zvol_discard(&task->zvr);
	zv_request_task_free(task);
	}

	static void
	zvol_read(zv_request_t *zvr)
	{
	struct bio *bio = zvr->bio;
	struct request *rq = zvr->rq;
	int error = 0;
	zfs_uio_t uio;
	boolean_t acct = B_FALSE;
	zvol_state_t *zv = zvr->zv;
	struct request_queue *q;
	struct gendisk *disk;
	unsigned long start_time = 0;

	ASSERT3P(zv, !=, NULL);
	ASSERT3U(zv->zv_open_count, >, 0);

	zfs_uio_bvec_init(&uio, bio, rq);

	q = zv->zv_zso->zvo_queue;
	disk = zv->zv_zso->zvo_disk;

	ssize_t start_resid = uio.uio_resid;

	/*
	* When blk-mq is being used, accounting is done by
	* blk_mq_start_request() and blk_mq_end_request().
	*/
	if (bio) {
	acct = blk_queue_io_stat(q);
	if (acct)
	start_time = blk_generic_start_io_acct(q, disk, READ,
	bio);
	}

	zfs_locked_range_t *lr = zfs_rangelock_enter(&zv->zv_rangelock,
	uio.uio_loffset, uio.uio_resid, RL_READER);

	uint64_t volsize = zv->zv_volsize;

	while (uio.uio_resid > 0 && uio.uio_loffset < volsize) {
	uint64_t bytes = MIN(uio.uio_resid, DMU_MAX_ACCESS >> 1);

	/* don't read past the end */
	if (bytes > volsize - uio.uio_loffset)
	bytes = volsize - uio.uio_loffset;

	error = dmu_read_uio_dnode(zv->zv_dn, &uio, bytes);
	if (error) {
	/* convert checksum errors into IO errors */
	if (error == ECKSUM)
	error = SET_ERROR(EIO);
	break;
	}
	}
	zfs_rangelock_exit(lr);

	int64_t nread = start_resid - uio.uio_resid;
	dataset_kstats_update_read_kstats(&zv->zv_kstat, nread);
	task_io_account_read(nread);

	rw_exit(&zv->zv_suspend_lock);

	if (bio && acct) {
	blk_generic_end_io_acct(q, disk, READ, bio, start_time);
	}

	END_IO(zv, bio, rq, -error);
	}

	static void
	zvol_read_task(void *arg)
	{
	zv_request_task_t *task = arg;
	zvol_read(&task->zvr);
	zv_request_task_free(task);
	}


	/*
	* Process a BIO or request
	*
	* Either 'bio' or 'rq' should be set depending on if we are processing a
	* bio or a request (both should not be set).
	*
	* force_sync: Set to 0 to defer processing to a background taskq
	* Set to 1 to process data synchronously
	*/
	static void
	zvol_request_impl(zvol_state_t zv, struct bio bio, struct request *rq,
	boolean_t force_sync)
	{
	fstrans_cookie_t cookie = spl_fstrans_mark();
	uint64_t offset = io_offset(bio, rq);
	uint64_t size = io_size(bio, rq);
	int rw = io_data_dir(bio, rq);

	if (zvol_request_sync)
	force_sync = 1;

	zv_request_t zvr = {
	.zv = zv,
	.bio = bio,
	.rq = rq,
	};

	if (io_has_data(bio, rq) && offset + size > zv->zv_volsize) {
	printk(KERN_INFO "%s: bad access: offset=%llu, size=%lu\n",
	zv->zv_zso->zvo_disk->disk_name,
	(long long unsigned)offset,
	(long unsigned)size);

	END_IO(zv, bio, rq, -SET_ERROR(EIO));
	goto out;
	}

	zv_request_task_t *task;
	+ zv_taskq_t *ztqs = &zvol_taskqs;
	+ uint_t blk_mq_hw_queue = 0;
	+ uint_t tq_idx;
	+ uint_t taskq_hash;
	+#ifdef HAVE_BLK_MQ
	+ if (rq)
	+#ifdef HAVE_BLK_MQ_RQ_HCTX
	+ blk_mq_hw_queue = rq->mq_hctx->queue_num;
	+#else
	+ blk_mq_hw_queue =
	+ rq->q->queue_hw_ctx[rq->q->mq_map[rq->cpu]]->queue_num;
	+#endif
	+#endif
	+ taskq_hash = cityhash4((uintptr_t)zv, offset >> ZVOL_TASKQ_OFFSET_SHIFT,
	+ blk_mq_hw_queue, 0);
	+ tq_idx = taskq_hash % ztqs->tqs_cnt;

	if (rw == WRITE) {
	if (unlikely(zv->zv_flags & ZVOL_RDONLY)) {
	END_IO(zv, bio, rq, -SET_ERROR(EROFS));
	goto out;
	}

	/*
	* Prevents the zvol from being suspended, or the ZIL being
	* concurrently opened. Will be released after the i/o
	* completes.
	*/
	rw_enter(&zv->zv_suspend_lock, RW_READER);

	/*
	* Open a ZIL if this is the first time we have written to this
	* zvol. We protect zv->zv_zilog with zv_suspend_lock rather
	* than zv_state_lock so that we don't need to acquire an
	* additional lock in this path.
	*/
	if (zv->zv_zilog == NULL) {
	rw_exit(&zv->zv_suspend_lock);
	rw_enter(&zv->zv_suspend_lock, RW_WRITER);
	if (zv->zv_zilog == NULL) {
	zv->zv_zilog = zil_open(zv->zv_objset,
	zvol_get_data, &zv->zv_kstat.dk_zil_sums);
	zv->zv_flags \|= ZVOL_WRITTEN_TO;
	/* replay / destroy done in zvol_create_minor */
	VERIFY0((zv->zv_zilog->zl_header->zh_flags &
	ZIL_REPLAY_NEEDED));
	}
	rw_downgrade(&zv->zv_suspend_lock);
	}

	/*
	* We don't want this thread to be blocked waiting for i/o to
	* complete, so we instead wait from a taskq callback. The
	* i/o may be a ZIL write (via zil_commit()), or a read of an
	* indirect block, or a read of a data block (if this is a
	* partial-block write). We will indicate that the i/o is
	* complete by calling END_IO() from the taskq callback.
	*
	* This design allows the calling thread to continue and
	* initiate more concurrent operations by calling
	* zvol_request() again. There are typically only a small
	* number of threads available to call zvol_request() (e.g.
	* one per iSCSI target), so keeping the latency of
	* zvol_request() low is important for performance.
	*
	* The zvol_request_sync module parameter allows this
	* behavior to be altered, for performance evaluation
	* purposes. If the callback blocks, setting
	* zvol_request_sync=1 will result in much worse performance.
	*
	* We can have up to zvol_threads concurrent i/o's being
	* processed for all zvols on the system. This is typically
	* a vast improvement over the zvol_request_sync=1 behavior
	* of one i/o at a time per zvol. However, an even better
	* design would be for zvol_request() to initiate the zio
	* directly, and then be notified by the zio_done callback,
	* which would call END_IO(). Unfortunately, the DMU/ZIL
	* interfaces lack this functionality (they block waiting for
	* the i/o to complete).
	*/
	if (io_is_discard(bio, rq) \|\| io_is_secure_erase(bio, rq)) {
	if (force_sync) {
	zvol_discard(&zvr);
	} else {
	task = zv_request_task_create(zvr);
	- taskq_dispatch_ent(zvol_taskq,
	+ taskq_dispatch_ent(ztqs->tqs_taskq[tq_idx],
	zvol_discard_task, task, 0, &task->ent);
	}
	} else {
	if (force_sync) {
	zvol_write(&zvr);
	} else {
	task = zv_request_task_create(zvr);
	- taskq_dispatch_ent(zvol_taskq,
	+ taskq_dispatch_ent(ztqs->tqs_taskq[tq_idx],
	zvol_write_task, task, 0, &task->ent);
	}
	}
	} else {
	/*
	* The SCST driver, and possibly others, may issue READ I/Os
	* with a length of zero bytes. These empty I/Os contain no
	* data and require no additional handling.
	*/
	if (size == 0) {
	END_IO(zv, bio, rq, 0);
	goto out;
	}

	rw_enter(&zv->zv_suspend_lock, RW_READER);

	/* See comment in WRITE case above. */
	if (force_sync) {
	zvol_read(&zvr);
	} else {
	task = zv_request_task_create(zvr);
	- taskq_dispatch_ent(zvol_taskq,
	+ taskq_dispatch_ent(ztqs->tqs_taskq[tq_idx],
	zvol_read_task, task, 0, &task->ent);
	}
	}

	out:
	spl_fstrans_unmark(cookie);
	}

	#ifdef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
	#ifdef HAVE_BDEV_SUBMIT_BIO_RETURNS_VOID
	static void
	zvol_submit_bio(struct bio *bio)
	#else
	static blk_qc_t
	zvol_submit_bio(struct bio *bio)
	#endif
	#else
	static MAKE_REQUEST_FN_RET
	zvol_request(struct request_queue q, struct bio bio)
	#endif
	{
	#ifdef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
	#if defined(HAVE_BIO_BDEV_DISK)
	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
	#else
	struct request_queue *q = bio->bi_disk->queue;
	#endif
	#endif
	zvol_state_t *zv = q->queuedata;

	zvol_request_impl(zv, bio, NULL, 0);
	#if defined(HAVE_MAKE_REQUEST_FN_RET_QC) \|\| \
	defined(HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS) && \
	!defined(HAVE_BDEV_SUBMIT_BIO_RETURNS_VOID)
	return (BLK_QC_T_NONE);
	#endif
	}

	static int
	#ifdef HAVE_BLK_MODE_T
	zvol_open(struct gendisk *disk, blk_mode_t flag)
	#else
	zvol_open(struct block_device *bdev, fmode_t flag)
	#endif
	{
	zvol_state_t *zv;
	int error = 0;
	boolean_t drop_suspend = B_FALSE;
	#ifndef HAVE_BLKDEV_GET_ERESTARTSYS
	hrtime_t timeout = MSEC2NSEC(zvol_open_timeout_ms);
	hrtime_t start = gethrtime();

	retry:
	#endif
	rw_enter(&zvol_state_lock, RW_READER);
	/*
	* Obtain a copy of private_data under the zvol_state_lock to make
	* sure that either the result of zvol free code path setting
	* disk->private_data to NULL is observed, or zvol_os_free()
	* is not called on this zv because of the positive zv_open_count.
	*/
	#ifdef HAVE_BLK_MODE_T
	zv = disk->private_data;
	#else
	zv = bdev->bd_disk->private_data;
	#endif
	if (zv == NULL) {
	rw_exit(&zvol_state_lock);
	return (SET_ERROR(-ENXIO));
	}

	mutex_enter(&zv->zv_state_lock);
	/*
	* Make sure zvol is not suspended during first open
	* (hold zv_suspend_lock) and respect proper lock acquisition
	* ordering - zv_suspend_lock before zv_state_lock
	*/
	if (zv->zv_open_count == 0) {
	if (!rw_tryenter(&zv->zv_suspend_lock, RW_READER)) {
	mutex_exit(&zv->zv_state_lock);
	rw_enter(&zv->zv_suspend_lock, RW_READER);
	mutex_enter(&zv->zv_state_lock);
	/* check to see if zv_suspend_lock is needed */
	if (zv->zv_open_count != 0) {
	rw_exit(&zv->zv_suspend_lock);
	} else {
	drop_suspend = B_TRUE;
	}
	} else {
	drop_suspend = B_TRUE;
	}
	}
	rw_exit(&zvol_state_lock);

	ASSERT(MUTEX_HELD(&zv->zv_state_lock));

	if (zv->zv_open_count == 0) {
	boolean_t drop_namespace = B_FALSE;

	ASSERT(RW_READ_HELD(&zv->zv_suspend_lock));

	/*
	* In all other call paths the spa_namespace_lock is taken
	* before the bdev->bd_mutex lock. However, on open(2)
	* the __blkdev_get() function calls fops->open() with the
	* bdev->bd_mutex lock held. This can result in a deadlock
	* when zvols from one pool are used as vdevs in another.
	*
	* To prevent a lock inversion deadlock we preemptively
	* take the spa_namespace_lock. Normally the lock will not
	* be contended and this is safe because spa_open_common()
	* handles the case where the caller already holds the
	* spa_namespace_lock.
	*
	* When the lock cannot be aquired after multiple retries
	* this must be the vdev on zvol deadlock case and we have
	* no choice but to return an error. For 5.12 and older
	* kernels returning -ERESTARTSYS will result in the
	* bdev->bd_mutex being dropped, then reacquired, and
	* fops->open() being called again. This process can be
	* repeated safely until both locks are acquired. For 5.13
	* and newer the -ERESTARTSYS retry logic was removed from
	* the kernel so the only option is to return the error for
	* the caller to handle it.
	*/
	if (!mutex_owned(&spa_namespace_lock)) {
	if (!mutex_tryenter(&spa_namespace_lock)) {
	mutex_exit(&zv->zv_state_lock);
	rw_exit(&zv->zv_suspend_lock);

	#ifdef HAVE_BLKDEV_GET_ERESTARTSYS
	schedule();
	return (SET_ERROR(-ERESTARTSYS));
	#else
	if ((gethrtime() - start) > timeout)
	return (SET_ERROR(-ERESTARTSYS));

	schedule_timeout(MSEC_TO_TICK(10));
	goto retry;
	#endif
	} else {
	drop_namespace = B_TRUE;
	}
	}

	error = -zvol_first_open(zv, !(blk_mode_is_open_write(flag)));

	if (drop_namespace)
	mutex_exit(&spa_namespace_lock);
	}

	if (error == 0) {
	if ((blk_mode_is_open_write(flag)) &&
	(zv->zv_flags & ZVOL_RDONLY)) {
	if (zv->zv_open_count == 0)
	zvol_last_close(zv);

	error = SET_ERROR(-EROFS);
	} else {
	zv->zv_open_count++;
	}
	}

	mutex_exit(&zv->zv_state_lock);
	if (drop_suspend)
	rw_exit(&zv->zv_suspend_lock);

	if (error == 0)
	#ifdef HAVE_BLK_MODE_T
	disk_check_media_change(disk);
	#else
	zfs_check_media_change(bdev);
	#endif

	return (error);
	}

	static void
	#ifdef HAVE_BLOCK_DEVICE_OPERATIONS_RELEASE_1ARG
	zvol_release(struct gendisk *disk)
	#else
	zvol_release(struct gendisk *disk, fmode_t unused)
	#endif
	{
	#if !defined(HAVE_BLOCK_DEVICE_OPERATIONS_RELEASE_1ARG)
	(void) unused;
	#endif
	zvol_state_t *zv;
	boolean_t drop_suspend = B_TRUE;

	rw_enter(&zvol_state_lock, RW_READER);
	zv = disk->private_data;

	mutex_enter(&zv->zv_state_lock);
	ASSERT3U(zv->zv_open_count, >, 0);
	/*
	* make sure zvol is not suspended during last close
	* (hold zv_suspend_lock) and respect proper lock acquisition
	* ordering - zv_suspend_lock before zv_state_lock
	*/
	if (zv->zv_open_count == 1) {
	if (!rw_tryenter(&zv->zv_suspend_lock, RW_READER)) {
	mutex_exit(&zv->zv_state_lock);
	rw_enter(&zv->zv_suspend_lock, RW_READER);
	mutex_enter(&zv->zv_state_lock);
	/* check to see if zv_suspend_lock is needed */
	if (zv->zv_open_count != 1) {
	rw_exit(&zv->zv_suspend_lock);
	drop_suspend = B_FALSE;
	}
	}
	} else {
	drop_suspend = B_FALSE;
	}
	rw_exit(&zvol_state_lock);

	ASSERT(MUTEX_HELD(&zv->zv_state_lock));

	zv->zv_open_count--;
	if (zv->zv_open_count == 0) {
	ASSERT(RW_READ_HELD(&zv->zv_suspend_lock));
	zvol_last_close(zv);
	}

	mutex_exit(&zv->zv_state_lock);

	if (drop_suspend)
	rw_exit(&zv->zv_suspend_lock);
	}

	static int
	zvol_ioctl(struct block_device *bdev, fmode_t mode,
	unsigned int cmd, unsigned long arg)
	{
	zvol_state_t *zv = bdev->bd_disk->private_data;
	int error = 0;

	ASSERT3U(zv->zv_open_count, >, 0);

	switch (cmd) {
	case BLKFLSBUF:
	#ifdef HAVE_FSYNC_BDEV
	fsync_bdev(bdev);
	#elif defined(HAVE_SYNC_BLOCKDEV)
	sync_blockdev(bdev);
	#else
	#error "Neither fsync_bdev() nor sync_blockdev() found"
	#endif
	invalidate_bdev(bdev);
	rw_enter(&zv->zv_suspend_lock, RW_READER);

	if (!(zv->zv_flags & ZVOL_RDONLY))
	txg_wait_synced(dmu_objset_pool(zv->zv_objset), 0);

	rw_exit(&zv->zv_suspend_lock);
	break;

	case BLKZNAME:
	mutex_enter(&zv->zv_state_lock);
	error = copy_to_user((void *)arg, zv->zv_name, MAXNAMELEN);
	mutex_exit(&zv->zv_state_lock);
	break;

	default:
	error = -ENOTTY;
	break;
	}

	return (SET_ERROR(error));
	}

	#ifdef CONFIG_COMPAT
	static int
	zvol_compat_ioctl(struct block_device *bdev, fmode_t mode,
	unsigned cmd, unsigned long arg)
	{
	return (zvol_ioctl(bdev, mode, cmd, arg));
	}
	#else
	#define zvol_compat_ioctl NULL
	#endif

	static unsigned int
	zvol_check_events(struct gendisk *disk, unsigned int clearing)
	{
	unsigned int mask = 0;

	rw_enter(&zvol_state_lock, RW_READER);

	zvol_state_t *zv = disk->private_data;
	if (zv != NULL) {
	mutex_enter(&zv->zv_state_lock);
	mask = zv->zv_changed ? DISK_EVENT_MEDIA_CHANGE : 0;
	zv->zv_changed = 0;
	mutex_exit(&zv->zv_state_lock);
	}

	rw_exit(&zvol_state_lock);

	return (mask);
	}

	static int
	zvol_revalidate_disk(struct gendisk *disk)
	{
	rw_enter(&zvol_state_lock, RW_READER);

	zvol_state_t *zv = disk->private_data;
	if (zv != NULL) {
	mutex_enter(&zv->zv_state_lock);
	set_capacity(zv->zv_zso->zvo_disk,
	zv->zv_volsize >> SECTOR_BITS);
	mutex_exit(&zv->zv_state_lock);
	}

	rw_exit(&zvol_state_lock);

	return (0);
	}

	int
	zvol_os_update_volsize(zvol_state_t *zv, uint64_t volsize)
	{
	struct gendisk *disk = zv->zv_zso->zvo_disk;

	#if defined(HAVE_REVALIDATE_DISK_SIZE)
	revalidate_disk_size(disk, zvol_revalidate_disk(disk) == 0);
	#elif defined(HAVE_REVALIDATE_DISK)
	revalidate_disk(disk);
	#else
	zvol_revalidate_disk(disk);
	#endif
	return (0);
	}

	void
	zvol_os_clear_private(zvol_state_t *zv)
	{
	/*
	* Cleared while holding zvol_state_lock as a writer
	* which will prevent zvol_open() from opening it.
	*/
	zv->zv_zso->zvo_disk->private_data = NULL;
	}

	/*
	* Provide a simple virtual geometry for legacy compatibility. For devices
	* smaller than 1 MiB a small head and sector count is used to allow very
	* tiny devices. For devices over 1 Mib a standard head and sector count
	* is used to keep the cylinders count reasonable.
	*/
	static int
	zvol_getgeo(struct block_device bdev, struct hd_geometry geo)
	{
	zvol_state_t *zv = bdev->bd_disk->private_data;
	sector_t sectors;

	ASSERT3U(zv->zv_open_count, >, 0);

	sectors = get_capacity(zv->zv_zso->zvo_disk);

	if (sectors > 2048) {
	geo->heads = 16;
	geo->sectors = 63;
	} else {
	geo->heads = 2;
	geo->sectors = 4;
	}

	geo->start = 0;
	geo->cylinders = sectors / (geo->heads * geo->sectors);

	return (0);
	}

	/*
	* Why have two separate block_device_operations structs?
	*
	* Normally we'd just have one, and assign 'submit_bio' as needed. However,
	* it's possible the user's kernel is built with CONSTIFY_PLUGIN, meaning we
	* can't just change submit_bio dynamically at runtime. So just create two
	* separate structs to get around this.
	*/
	static const struct block_device_operations zvol_ops_blk_mq = {
	.open = zvol_open,
	.release = zvol_release,
	.ioctl = zvol_ioctl,
	.compat_ioctl = zvol_compat_ioctl,
	.check_events = zvol_check_events,
	#ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
	.revalidate_disk = zvol_revalidate_disk,
	#endif
	.getgeo = zvol_getgeo,
	.owner = THIS_MODULE,
	};

	static const struct block_device_operations zvol_ops = {
	.open = zvol_open,
	.release = zvol_release,
	.ioctl = zvol_ioctl,
	.compat_ioctl = zvol_compat_ioctl,
	.check_events = zvol_check_events,
	#ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
	.revalidate_disk = zvol_revalidate_disk,
	#endif
	.getgeo = zvol_getgeo,
	.owner = THIS_MODULE,
	#ifdef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
	.submit_bio = zvol_submit_bio,
	#endif
	};

	static int
	zvol_alloc_non_blk_mq(struct zvol_state_os *zso)
	{
	#if defined(HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS)
	#if defined(HAVE_BLK_ALLOC_DISK)
	zso->zvo_disk = blk_alloc_disk(NUMA_NO_NODE);
	if (zso->zvo_disk == NULL)
	return (1);

	+ zso->zvo_disk->minors = ZVOL_MINORS;
	+ zso->zvo_queue = zso->zvo_disk->queue;
	+#elif defined(HAVE_BLK_ALLOC_DISK_2ARG)
	+ struct gendisk *disk = blk_alloc_disk(NULL, NUMA_NO_NODE);
	+ if (IS_ERR(disk)) {
	+ zso->zvo_disk = NULL;
	+ return (1);
	+ }
	+
	+ zso->zvo_disk = disk;
	zso->zvo_disk->minors = ZVOL_MINORS;
	zso->zvo_queue = zso->zvo_disk->queue;
	#else
	zso->zvo_queue = blk_alloc_queue(NUMA_NO_NODE);
	if (zso->zvo_queue == NULL)
	return (1);

	zso->zvo_disk = alloc_disk(ZVOL_MINORS);
	if (zso->zvo_disk == NULL) {
	blk_cleanup_queue(zso->zvo_queue);
	return (1);
	}

	zso->zvo_disk->queue = zso->zvo_queue;
	#endif /* HAVE_BLK_ALLOC_DISK */
	#else
	zso->zvo_queue = blk_generic_alloc_queue(zvol_request, NUMA_NO_NODE);
	if (zso->zvo_queue == NULL)
	return (1);

	zso->zvo_disk = alloc_disk(ZVOL_MINORS);
	if (zso->zvo_disk == NULL) {
	blk_cleanup_queue(zso->zvo_queue);
	return (1);
	}

	zso->zvo_disk->queue = zso->zvo_queue;
	#endif /* HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS */
	return (0);

	}

	static int
	zvol_alloc_blk_mq(zvol_state_t *zv)
	{
	#ifdef HAVE_BLK_MQ
	struct zvol_state_os *zso = zv->zv_zso;

	/* Allocate our blk-mq tag_set */
	if (zvol_blk_mq_alloc_tag_set(zv) != 0)
	return (1);

	#if defined(HAVE_BLK_ALLOC_DISK)
	zso->zvo_disk = blk_mq_alloc_disk(&zso->tag_set, zv);
	if (zso->zvo_disk == NULL) {
	blk_mq_free_tag_set(&zso->tag_set);
	return (1);
	}
	zso->zvo_queue = zso->zvo_disk->queue;
	zso->zvo_disk->minors = ZVOL_MINORS;
	+#elif defined(HAVE_BLK_ALLOC_DISK_2ARG)
	+ struct gendisk *disk = blk_mq_alloc_disk(&zso->tag_set, NULL, zv);
	+ if (IS_ERR(disk)) {
	+ zso->zvo_disk = NULL;
	+ blk_mq_free_tag_set(&zso->tag_set);
	+ return (1);
	+ }
	+
	+ zso->zvo_disk = disk;
	+ zso->zvo_queue = zso->zvo_disk->queue;
	+ zso->zvo_disk->minors = ZVOL_MINORS;
	#else
	zso->zvo_disk = alloc_disk(ZVOL_MINORS);
	if (zso->zvo_disk == NULL) {
	blk_cleanup_queue(zso->zvo_queue);
	blk_mq_free_tag_set(&zso->tag_set);
	return (1);
	}
	/* Allocate queue */
	zso->zvo_queue = blk_mq_init_queue(&zso->tag_set);
	if (IS_ERR(zso->zvo_queue)) {
	blk_mq_free_tag_set(&zso->tag_set);
	return (1);
	}

	/* Our queue is now created, assign it to our disk */
	zso->zvo_disk->queue = zso->zvo_queue;

	#endif
	#endif
	return (0);
	}

	/*
	* Allocate memory for a new zvol_state_t and setup the required
	* request queue and generic disk structures for the block device.
	*/
	static zvol_state_t *
	zvol_alloc(dev_t dev, const char *name)
	{
	zvol_state_t *zv;
	struct zvol_state_os *zso;
	uint64_t volmode;
	int ret;

	if (dsl_prop_get_integer(name, "volmode", &volmode, NULL) != 0)
	return (NULL);

	if (volmode == ZFS_VOLMODE_DEFAULT)
	volmode = zvol_volmode;

	if (volmode == ZFS_VOLMODE_NONE)
	return (NULL);

	zv = kmem_zalloc(sizeof (zvol_state_t), KM_SLEEP);
	zso = kmem_zalloc(sizeof (struct zvol_state_os), KM_SLEEP);
	zv->zv_zso = zso;
	zv->zv_volmode = volmode;

	list_link_init(&zv->zv_next);
	mutex_init(&zv->zv_state_lock, NULL, MUTEX_DEFAULT, NULL);

	#ifdef HAVE_BLK_MQ
	zv->zv_zso->use_blk_mq = zvol_use_blk_mq;
	#endif

	/*
	* The block layer has 3 interfaces for getting BIOs:
	*
	* 1. blk-mq request queues (new)
	* 2. submit_bio() (oldest)
	* 3. regular request queues (old).
	*
	* Each of those interfaces has two permutations:
	*
	* a) We have blk_alloc_disk()/blk_mq_alloc_disk(), which allocates
	* both the disk and its queue (5.14 kernel or newer)
	*
	* b) We don't have blk_*alloc_disk(), and have to allocate the
	* disk and the queue separately. (5.13 kernel or older)
	*/
	if (zv->zv_zso->use_blk_mq) {
	ret = zvol_alloc_blk_mq(zv);
	zso->zvo_disk->fops = &zvol_ops_blk_mq;
	} else {
	ret = zvol_alloc_non_blk_mq(zso);
	zso->zvo_disk->fops = &zvol_ops;
	}
	if (ret != 0)
	goto out_kmem;

	blk_queue_set_write_cache(zso->zvo_queue, B_TRUE, B_TRUE);

	/* Limit read-ahead to a single page to prevent over-prefetching. */
	blk_queue_set_read_ahead(zso->zvo_queue, 1);

	if (!zv->zv_zso->use_blk_mq) {
	/* Disable write merging in favor of the ZIO pipeline. */
	blk_queue_flag_set(QUEUE_FLAG_NOMERGES, zso->zvo_queue);
	}

	/* Enable /proc/diskstats */
	blk_queue_flag_set(QUEUE_FLAG_IO_STAT, zso->zvo_queue);

	zso->zvo_queue->queuedata = zv;
	zso->zvo_dev = dev;
	zv->zv_open_count = 0;
	strlcpy(zv->zv_name, name, MAXNAMELEN);

	zfs_rangelock_init(&zv->zv_rangelock, NULL, NULL);
	rw_init(&zv->zv_suspend_lock, NULL, RW_DEFAULT, NULL);

	zso->zvo_disk->major = zvol_major;
	zso->zvo_disk->events = DISK_EVENT_MEDIA_CHANGE;

	/*
	* Setting ZFS_VOLMODE_DEV disables partitioning on ZVOL devices.
	* This is accomplished by limiting the number of minors for the
	* device to one and explicitly disabling partition scanning.
	*/
	if (volmode == ZFS_VOLMODE_DEV) {
	zso->zvo_disk->minors = 1;
	zso->zvo_disk->flags &= ~ZFS_GENHD_FL_EXT_DEVT;
	zso->zvo_disk->flags \|= ZFS_GENHD_FL_NO_PART;
	}

	zso->zvo_disk->first_minor = (dev & MINORMASK);
	zso->zvo_disk->private_data = zv;
	snprintf(zso->zvo_disk->disk_name, DISK_NAME_LEN, "%s%d",
	ZVOL_DEV_NAME, (dev & MINORMASK));

	return (zv);

	out_kmem:
	kmem_free(zso, sizeof (struct zvol_state_os));
	kmem_free(zv, sizeof (zvol_state_t));
	return (NULL);
	}

	/*
	* Cleanup then free a zvol_state_t which was created by zvol_alloc().
	* At this time, the structure is not opened by anyone, is taken off
	* the zvol_state_list, and has its private data set to NULL.
	* The zvol_state_lock is dropped.
	*
	* This function may take many milliseconds to complete (e.g. we've seen
	* it take over 256ms), due to the calls to "blk_cleanup_queue" and
	* "del_gendisk". Thus, consumers need to be careful to account for this
	* latency when calling this function.
	*/
	void
	zvol_os_free(zvol_state_t *zv)
	{

	ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock));
	ASSERT(!MUTEX_HELD(&zv->zv_state_lock));
	ASSERT0(zv->zv_open_count);
	ASSERT3P(zv->zv_zso->zvo_disk->private_data, ==, NULL);

	rw_destroy(&zv->zv_suspend_lock);
	zfs_rangelock_fini(&zv->zv_rangelock);

	del_gendisk(zv->zv_zso->zvo_disk);
	#if defined(HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS) && \
	- defined(HAVE_BLK_ALLOC_DISK)
	+ (defined(HAVE_BLK_ALLOC_DISK) \|\| defined(HAVE_BLK_ALLOC_DISK_2ARG))
	#if defined(HAVE_BLK_CLEANUP_DISK)
	blk_cleanup_disk(zv->zv_zso->zvo_disk);
	#else
	put_disk(zv->zv_zso->zvo_disk);
	#endif
	#else
	blk_cleanup_queue(zv->zv_zso->zvo_queue);
	put_disk(zv->zv_zso->zvo_disk);
	#endif

	#ifdef HAVE_BLK_MQ
	if (zv->zv_zso->use_blk_mq)
	blk_mq_free_tag_set(&zv->zv_zso->tag_set);
	#endif

	ida_simple_remove(&zvol_ida,
	MINOR(zv->zv_zso->zvo_dev) >> ZVOL_MINOR_BITS);

	mutex_destroy(&zv->zv_state_lock);
	dataset_kstats_destroy(&zv->zv_kstat);

	kmem_free(zv->zv_zso, sizeof (struct zvol_state_os));
	kmem_free(zv, sizeof (zvol_state_t));
	}

	void
	zvol_wait_close(zvol_state_t *zv)
	{
	}

	/*
	* Create a block device minor node and setup the linkage between it
	* and the specified volume. Once this function returns the block
	* device is live and ready for use.
	*/
	int
	zvol_os_create_minor(const char *name)
	{
	zvol_state_t *zv;
	objset_t *os;
	dmu_object_info_t *doi;
	uint64_t volsize;
	uint64_t len;
	unsigned minor = 0;
	int error = 0;
	int idx;
	uint64_t hash = zvol_name_hash(name);
	bool replayed_zil = B_FALSE;

	if (zvol_inhibit_dev)
	return (0);

	idx = ida_simple_get(&zvol_ida, 0, 0, kmem_flags_convert(KM_SLEEP));
	if (idx < 0)
	return (SET_ERROR(-idx));
	minor = idx << ZVOL_MINOR_BITS;
	+ if (MINOR(minor) != minor) {
	+ /* too many partitions can cause an overflow */
	+ zfs_dbgmsg("zvol: create minor overflow: %s, minor %u/%u",
	+ name, minor, MINOR(minor));
	+ ida_simple_remove(&zvol_ida, idx);
	+ return (SET_ERROR(EINVAL));
	+ }

	zv = zvol_find_by_name_hash(name, hash, RW_NONE);
	if (zv) {
	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
	mutex_exit(&zv->zv_state_lock);
	ida_simple_remove(&zvol_ida, idx);
	return (SET_ERROR(EEXIST));
	}

	doi = kmem_alloc(sizeof (dmu_object_info_t), KM_SLEEP);

	error = dmu_objset_own(name, DMU_OST_ZVOL, B_TRUE, B_TRUE, FTAG, &os);
	if (error)
	goto out_doi;

	error = dmu_object_info(os, ZVOL_OBJ, doi);
	if (error)
	goto out_dmu_objset_disown;

	error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize);
	if (error)
	goto out_dmu_objset_disown;

	zv = zvol_alloc(MKDEV(zvol_major, minor), name);
	if (zv == NULL) {
	error = SET_ERROR(EAGAIN);
	goto out_dmu_objset_disown;
	}
	zv->zv_hash = hash;

	if (dmu_objset_is_snapshot(os))
	zv->zv_flags \|= ZVOL_RDONLY;

	zv->zv_volblocksize = doi->doi_data_block_size;
	zv->zv_volsize = volsize;
	zv->zv_objset = os;

	set_capacity(zv->zv_zso->zvo_disk, zv->zv_volsize >> 9);

	blk_queue_max_hw_sectors(zv->zv_zso->zvo_queue,
	(DMU_MAX_ACCESS / 4) >> 9);

	if (zv->zv_zso->use_blk_mq) {
	/*
	* IO requests can be really big (1MB). When an IO request
	* comes in, it is passed off to zvol_read() or zvol_write()
	* in a new thread, where it is chunked up into 'volblocksize'
	* sized pieces and processed. So for example, if the request
	* is a 1MB write and your volblocksize is 128k, one zvol_write
	* thread will take that request and sequentially do ten 128k
	* IOs. This is due to the fact that the thread needs to lock
	* each volblocksize sized block. So you might be wondering:
	* "instead of passing the whole 1MB request to one thread,
	* why not pass ten individual 128k chunks to ten threads and
	* process the whole write in parallel?" The short answer is
	* that there's a sweet spot number of chunks that balances
	* the greater parallelism with the added overhead of more
	* threads. The sweet spot can be different depending on if you
	* have a read or write heavy workload. Writes typically want
	* high chunk counts while reads typically want lower ones. On
	* a test pool with 6 NVMe drives in a 3x 2-disk mirror
	* configuration, with volblocksize=8k, the sweet spot for good
	* sequential reads and writes was at 8 chunks.
	*/

	/*
	* Below we tell the kernel how big we want our requests
	* to be. You would think that blk_queue_io_opt() would be
	* used to do this since it is used to "set optimal request
	* size for the queue", but that doesn't seem to do
	* anything - the kernel still gives you huge requests
	* with tons of little PAGE_SIZE segments contained within it.
	*
	* Knowing that the kernel will just give you PAGE_SIZE segments
	* no matter what, you can say "ok, I want PAGE_SIZE byte
	* segments, and I want 'N' of them per request", where N is
	* the correct number of segments for the volblocksize and
	* number of chunks you want.
	*/
	#ifdef HAVE_BLK_MQ
	if (zvol_blk_mq_blocks_per_thread != 0) {
	unsigned int chunks;
	chunks = MIN(zvol_blk_mq_blocks_per_thread, UINT16_MAX);

	blk_queue_max_segment_size(zv->zv_zso->zvo_queue,
	PAGE_SIZE);
	blk_queue_max_segments(zv->zv_zso->zvo_queue,
	(zv->zv_volblocksize * chunks) / PAGE_SIZE);
	} else {
	/*
	* Special case: zvol_blk_mq_blocks_per_thread = 0
	* Max everything out.
	*/
	blk_queue_max_segments(zv->zv_zso->zvo_queue,
	UINT16_MAX);
	blk_queue_max_segment_size(zv->zv_zso->zvo_queue,
	UINT_MAX);
	}
	#endif
	} else {
	blk_queue_max_segments(zv->zv_zso->zvo_queue, UINT16_MAX);
	blk_queue_max_segment_size(zv->zv_zso->zvo_queue, UINT_MAX);
	}

	blk_queue_physical_block_size(zv->zv_zso->zvo_queue,
	zv->zv_volblocksize);
	blk_queue_io_opt(zv->zv_zso->zvo_queue, zv->zv_volblocksize);
	blk_queue_max_discard_sectors(zv->zv_zso->zvo_queue,
	(zvol_max_discard_blocks * zv->zv_volblocksize) >> 9);
	blk_queue_discard_granularity(zv->zv_zso->zvo_queue,
	zv->zv_volblocksize);
	#ifdef QUEUE_FLAG_DISCARD
	blk_queue_flag_set(QUEUE_FLAG_DISCARD, zv->zv_zso->zvo_queue);
	#endif
	#ifdef QUEUE_FLAG_NONROT
	blk_queue_flag_set(QUEUE_FLAG_NONROT, zv->zv_zso->zvo_queue);
	#endif
	#ifdef QUEUE_FLAG_ADD_RANDOM
	blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, zv->zv_zso->zvo_queue);
	#endif
	/* This flag was introduced in kernel version 4.12. */
	#ifdef QUEUE_FLAG_SCSI_PASSTHROUGH
	blk_queue_flag_set(QUEUE_FLAG_SCSI_PASSTHROUGH, zv->zv_zso->zvo_queue);
	#endif

	ASSERT3P(zv->zv_kstat.dk_kstats, ==, NULL);
	error = dataset_kstats_create(&zv->zv_kstat, zv->zv_objset);
	if (error)
	goto out_dmu_objset_disown;
	ASSERT3P(zv->zv_zilog, ==, NULL);
	zv->zv_zilog = zil_open(os, zvol_get_data, &zv->zv_kstat.dk_zil_sums);
	if (spa_writeable(dmu_objset_spa(os))) {
	if (zil_replay_disable)
	replayed_zil = zil_destroy(zv->zv_zilog, B_FALSE);
	else
	replayed_zil = zil_replay(os, zv, zvol_replay_vector);
	}
	if (replayed_zil)
	zil_close(zv->zv_zilog);
	zv->zv_zilog = NULL;

	/*
	* When udev detects the addition of the device it will immediately
	* invoke blkid(8) to determine the type of content on the device.
	* Prefetching the blocks commonly scanned by blkid(8) will speed
	* up this process.
	*/
	len = MIN(zvol_prefetch_bytes, SPA_MAXBLOCKSIZE);
	if (len > 0) {
	dmu_prefetch(os, ZVOL_OBJ, 0, 0, len, ZIO_PRIORITY_SYNC_READ);
	dmu_prefetch(os, ZVOL_OBJ, 0, volsize - len, len,
	ZIO_PRIORITY_SYNC_READ);
	}

	zv->zv_objset = NULL;
	out_dmu_objset_disown:
	dmu_objset_disown(os, B_TRUE, FTAG);
	out_doi:
	kmem_free(doi, sizeof (dmu_object_info_t));

	/*
	* Keep in mind that once add_disk() is called, the zvol is
	* announced to the world, and zvol_open()/zvol_release() can
	* be called at any time. Incidentally, add_disk() itself calls
	* zvol_open()->zvol_first_open() and zvol_release()->zvol_last_close()
	* directly as well.
	*/
	if (error == 0) {
	rw_enter(&zvol_state_lock, RW_WRITER);
	zvol_insert(zv);
	rw_exit(&zvol_state_lock);
	#ifdef HAVE_ADD_DISK_RET
	error = add_disk(zv->zv_zso->zvo_disk);
	#else
	add_disk(zv->zv_zso->zvo_disk);
	#endif
	} else {
	ida_simple_remove(&zvol_ida, idx);
	}

	return (error);
	}

	void
	zvol_os_rename_minor(zvol_state_t zv, const char newname)
	{
	int readonly = get_disk_ro(zv->zv_zso->zvo_disk);

	ASSERT(RW_LOCK_HELD(&zvol_state_lock));
	ASSERT(MUTEX_HELD(&zv->zv_state_lock));

	strlcpy(zv->zv_name, newname, sizeof (zv->zv_name));

	/* move to new hashtable entry */
	- zv->zv_hash = zvol_name_hash(zv->zv_name);
	+ zv->zv_hash = zvol_name_hash(newname);
	hlist_del(&zv->zv_hlink);
	hlist_add_head(&zv->zv_hlink, ZVOL_HT_HEAD(zv->zv_hash));

	/*
	* The block device's read-only state is briefly changed causing
	* a KOBJ_CHANGE uevent to be issued. This ensures udev detects
	* the name change and fixes the symlinks. This does not change
	* ZVOL_RDONLY in zv->zv_flags so the actual read-only state never
	* changes. This would normally be done using kobject_uevent() but
	* that is a GPL-only symbol which is why we need this workaround.
	*/
	set_disk_ro(zv->zv_zso->zvo_disk, !readonly);
	set_disk_ro(zv->zv_zso->zvo_disk, readonly);

	dataset_kstats_rename(&zv->zv_kstat, newname);
	}

	void
	zvol_os_set_disk_ro(zvol_state_t *zv, int flags)
	{

	set_disk_ro(zv->zv_zso->zvo_disk, flags);
	}

	void
	zvol_os_set_capacity(zvol_state_t *zv, uint64_t capacity)
	{

	set_capacity(zv->zv_zso->zvo_disk, capacity);
	}

	int
	zvol_init(void)
	{
	int error;

	/*
	* zvol_threads is the module param the user passes in.
	*
	* zvol_actual_threads is what we use internally, since the user can
	* pass zvol_thread = 0 to mean "use all the CPUs" (the default).
	*/
	static unsigned int zvol_actual_threads;

	if (zvol_threads == 0) {
	/*
	* See dde9380a1 for why 32 was chosen here. This should
	* probably be refined to be some multiple of the number
	* of CPUs.
	*/
	zvol_actual_threads = MAX(num_online_cpus(), 32);
	} else {
	zvol_actual_threads = MIN(MAX(zvol_threads, 1), 1024);
	}

	+ /*
	+ * Use atleast 32 zvol_threads but for many core system,
	+ * prefer 6 threads per taskq, but no more taskqs
	+ * than threads in them on large systems.
	+ *
	+ * taskq total
	+ * cpus taskqs threads threads
	+ * ------- ------- ------- -------
	+ * 1 1 32 32
	+ * 2 1 32 32
	+ * 4 1 32 32
	+ * 8 2 16 32
	+ * 16 3 11 33
	+ * 32 5 7 35
	+ * 64 8 8 64
	+ * 128 11 12 132
	+ * 256 16 16 256
	+ */
	+ zv_taskq_t *ztqs = &zvol_taskqs;
	+ uint_t num_tqs = MIN(num_online_cpus(), zvol_num_taskqs);
	+ if (num_tqs == 0) {
	+ num_tqs = 1 + num_online_cpus() / 6;
	+ while (num_tqs * num_tqs > zvol_actual_threads)
	+ num_tqs--;
	+ }
	+ uint_t per_tq_thread = zvol_actual_threads / num_tqs;
	+ if (per_tq_thread * num_tqs < zvol_actual_threads)
	+ per_tq_thread++;
	+ ztqs->tqs_cnt = num_tqs;
	+ ztqs->tqs_taskq = kmem_alloc(num_tqs * sizeof (taskq_t *), KM_SLEEP);
	error = register_blkdev(zvol_major, ZVOL_DRIVER);
	if (error) {
	+ kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt * sizeof (taskq_t *));
	+ ztqs->tqs_taskq = NULL;
	printk(KERN_INFO "ZFS: register_blkdev() failed %d\n", error);
	return (error);
	}

	#ifdef HAVE_BLK_MQ
	if (zvol_blk_mq_queue_depth == 0) {
	zvol_actual_blk_mq_queue_depth = BLKDEV_DEFAULT_RQ;
	} else {
	zvol_actual_blk_mq_queue_depth =
	MAX(zvol_blk_mq_queue_depth, BLKDEV_MIN_RQ);
	}

	if (zvol_blk_mq_threads == 0) {
	zvol_blk_mq_actual_threads = num_online_cpus();
	} else {
	zvol_blk_mq_actual_threads = MIN(MAX(zvol_blk_mq_threads, 1),
	1024);
	}
	#endif
	- zvol_taskq = taskq_create(ZVOL_DRIVER, zvol_actual_threads, maxclsyspri,
	- zvol_actual_threads, INT_MAX, TASKQ_PREPOPULATE \| TASKQ_DYNAMIC);
	- if (zvol_taskq == NULL) {
	- unregister_blkdev(zvol_major, ZVOL_DRIVER);
	- return (-ENOMEM);
	+ for (uint_t i = 0; i < num_tqs; i++) {
	+ char name[32];
	+ (void) snprintf(name, sizeof (name), "%s_tq-%u",
	+ ZVOL_DRIVER, i);
	+ ztqs->tqs_taskq[i] = taskq_create(name, per_tq_thread,
	+ maxclsyspri, per_tq_thread, INT_MAX,
	+ TASKQ_PREPOPULATE \| TASKQ_DYNAMIC);
	+ if (ztqs->tqs_taskq[i] == NULL) {
	+ for (int j = i - 1; j >= 0; j--)
	+ taskq_destroy(ztqs->tqs_taskq[j]);
	+ unregister_blkdev(zvol_major, ZVOL_DRIVER);
	+ kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt *
	+ sizeof (taskq_t *));
	+ ztqs->tqs_taskq = NULL;
	+ return (-ENOMEM);
	+ }
	}

	zvol_init_impl();
	ida_init(&zvol_ida);
	return (0);
	}

	void
	zvol_fini(void)
	{
	+ zv_taskq_t *ztqs = &zvol_taskqs;
	zvol_fini_impl();
	unregister_blkdev(zvol_major, ZVOL_DRIVER);
	- taskq_destroy(zvol_taskq);
	+
	+ if (ztqs->tqs_taskq == NULL) {
	+ ASSERT3U(ztqs->tqs_cnt, ==, 0);
	+ } else {
	+ for (uint_t i = 0; i < ztqs->tqs_cnt; i++) {
	+ ASSERT3P(ztqs->tqs_taskq[i], !=, NULL);
	+ taskq_destroy(ztqs->tqs_taskq[i]);
	+ }
	+ kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt *
	+ sizeof (taskq_t *));
	+ ztqs->tqs_taskq = NULL;
	+ }
	+
	ida_destroy(&zvol_ida);
	}

	/* BEGIN CSTYLED */
	module_param(zvol_inhibit_dev, uint, 0644);
	MODULE_PARM_DESC(zvol_inhibit_dev, "Do not create zvol device nodes");

	module_param(zvol_major, uint, 0444);
	MODULE_PARM_DESC(zvol_major, "Major number for zvol device");

	module_param(zvol_threads, uint, 0444);
	MODULE_PARM_DESC(zvol_threads, "Number of threads to handle I/O requests. Set"
	"to 0 to use all active CPUs");

	module_param(zvol_request_sync, uint, 0644);
	MODULE_PARM_DESC(zvol_request_sync, "Synchronously handle bio requests");

	module_param(zvol_max_discard_blocks, ulong, 0444);
	MODULE_PARM_DESC(zvol_max_discard_blocks, "Max number of blocks to discard");

	+module_param(zvol_num_taskqs, uint, 0444);
	+MODULE_PARM_DESC(zvol_num_taskqs, "Number of zvol taskqs");
	+
	module_param(zvol_prefetch_bytes, uint, 0644);
	MODULE_PARM_DESC(zvol_prefetch_bytes, "Prefetch N bytes at zvol start+end");

	module_param(zvol_volmode, uint, 0644);
	MODULE_PARM_DESC(zvol_volmode, "Default volmode property value");

	#ifdef HAVE_BLK_MQ
	module_param(zvol_blk_mq_queue_depth, uint, 0644);
	MODULE_PARM_DESC(zvol_blk_mq_queue_depth, "Default blk-mq queue depth");

	module_param(zvol_use_blk_mq, uint, 0644);
	MODULE_PARM_DESC(zvol_use_blk_mq, "Use the blk-mq API for zvols");

	module_param(zvol_blk_mq_blocks_per_thread, uint, 0644);
	MODULE_PARM_DESC(zvol_blk_mq_blocks_per_thread,
	"Process volblocksize blocks per thread");
	#endif

	#ifndef HAVE_BLKDEV_GET_ERESTARTSYS
	module_param(zvol_open_timeout_ms, uint, 0644);
	MODULE_PARM_DESC(zvol_open_timeout_ms, "Timeout for ZVOL open retries");
	#endif

	/* END CSTYLED */
	diff --git a/sys/contrib/openzfs/module/zcommon/zfs_prop.c b/sys/contrib/openzfs/module/zcommon/zfs_prop.c
	index 3db6fd13f4ae..29764674a31b 100644
	--- a/sys/contrib/openzfs/module/zcommon/zfs_prop.c
	+++ b/sys/contrib/openzfs/module/zcommon/zfs_prop.c
	@@ -1,1101 +1,1112 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2011, 2018 by Delphix. All rights reserved.
	* Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
	* Copyright 2016, Joyent, Inc.
	* Copyright (c) 2019, Klara Inc.
	* Copyright (c) 2019, Allan Jude
	* Copyright (c) 2022 Hewlett Packard Enterprise Development LP.
	*/

	/* Portions Copyright 2010 Robert Milkowski */

	#if defined(_KERNEL)
	#include <sys/simd.h>
	#endif

	#include <sys/zio.h>
	#include <sys/spa.h>
	#include <sys/u8_textprep.h>
	#include <sys/zfs_acl.h>
	#include <sys/zfs_ioctl.h>
	#include <sys/zfs_znode.h>
	#include <sys/dsl_crypt.h>

	#include "zfs_prop.h"
	#include "zfs_deleg.h"
	#include "zfs_fletcher.h"

	#if !defined(_KERNEL)
	#include <stdlib.h>
	#include <string.h>
	#include <ctype.h>
	#endif

	static zprop_desc_t zfs_prop_table[ZFS_NUM_PROPS];

	/* Note this is indexed by zfs_userquota_prop_t, keep the order the same */
	const char *const zfs_userquota_prop_prefixes[] = {
	"userused@",
	"userquota@",
	"groupused@",
	"groupquota@",
	"userobjused@",
	"userobjquota@",
	"groupobjused@",
	"groupobjquota@",
	"projectused@",
	"projectquota@",
	"projectobjused@",
	"projectobjquota@"
	};

	zprop_desc_t *
	zfs_prop_get_table(void)
	{
	return (zfs_prop_table);
	}

	void
	zfs_prop_init(void)
	{
	static const zprop_index_t checksum_table[] = {
	{ "on", ZIO_CHECKSUM_ON },
	{ "off", ZIO_CHECKSUM_OFF },
	{ "fletcher2", ZIO_CHECKSUM_FLETCHER_2 },
	{ "fletcher4", ZIO_CHECKSUM_FLETCHER_4 },
	{ "sha256", ZIO_CHECKSUM_SHA256 },
	{ "noparity", ZIO_CHECKSUM_NOPARITY },
	{ "sha512", ZIO_CHECKSUM_SHA512 },
	{ "skein", ZIO_CHECKSUM_SKEIN },
	{ "edonr", ZIO_CHECKSUM_EDONR },
	{ "blake3", ZIO_CHECKSUM_BLAKE3 },
	{ NULL }
	};

	static const zprop_index_t dedup_table[] = {
	{ "on", ZIO_CHECKSUM_ON },
	{ "off", ZIO_CHECKSUM_OFF },
	{ "verify", ZIO_CHECKSUM_ON \| ZIO_CHECKSUM_VERIFY },
	{ "sha256", ZIO_CHECKSUM_SHA256 },
	{ "sha256,verify",
	ZIO_CHECKSUM_SHA256 \| ZIO_CHECKSUM_VERIFY },
	{ "sha512", ZIO_CHECKSUM_SHA512 },
	{ "sha512,verify",
	ZIO_CHECKSUM_SHA512 \| ZIO_CHECKSUM_VERIFY },
	{ "skein", ZIO_CHECKSUM_SKEIN },
	{ "skein,verify",
	ZIO_CHECKSUM_SKEIN \| ZIO_CHECKSUM_VERIFY },
	{ "edonr,verify",
	ZIO_CHECKSUM_EDONR \| ZIO_CHECKSUM_VERIFY },
	{ "blake3", ZIO_CHECKSUM_BLAKE3 },
	{ "blake3,verify",
	ZIO_CHECKSUM_BLAKE3 \| ZIO_CHECKSUM_VERIFY },
	{ NULL }
	};

	static const zprop_index_t compress_table[] = {
	{ "on", ZIO_COMPRESS_ON },
	{ "off", ZIO_COMPRESS_OFF },
	{ "lzjb", ZIO_COMPRESS_LZJB },
	{ "gzip", ZIO_COMPRESS_GZIP_6 }, /* gzip default */
	{ "gzip-1", ZIO_COMPRESS_GZIP_1 },
	{ "gzip-2", ZIO_COMPRESS_GZIP_2 },
	{ "gzip-3", ZIO_COMPRESS_GZIP_3 },
	{ "gzip-4", ZIO_COMPRESS_GZIP_4 },
	{ "gzip-5", ZIO_COMPRESS_GZIP_5 },
	{ "gzip-6", ZIO_COMPRESS_GZIP_6 },
	{ "gzip-7", ZIO_COMPRESS_GZIP_7 },
	{ "gzip-8", ZIO_COMPRESS_GZIP_8 },
	{ "gzip-9", ZIO_COMPRESS_GZIP_9 },
	{ "zle", ZIO_COMPRESS_ZLE },
	{ "lz4", ZIO_COMPRESS_LZ4 },
	{ "zstd", ZIO_COMPRESS_ZSTD },
	{ "zstd-fast",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_DEFAULT) },

	/*
	* ZSTD 1-19 are synthetic. We store the compression level in a
	* separate hidden property to avoid wasting a large amount of
	* space in the ZIO_COMPRESS enum.
	*
	* The compression level is also stored within the header of the
	* compressed block since we may need it for later recompression
	* to avoid checksum errors (L2ARC).
	*
	* Note that the level here is defined as bit shifted mask on
	* top of the method.
	*/
	{ "zstd-1", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_1) },
	{ "zstd-2", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_2) },
	{ "zstd-3", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_3) },
	{ "zstd-4", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_4) },
	{ "zstd-5", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_5) },
	{ "zstd-6", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_6) },
	{ "zstd-7", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_7) },
	{ "zstd-8", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_8) },
	{ "zstd-9", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_9) },
	{ "zstd-10", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_10) },
	{ "zstd-11", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_11) },
	{ "zstd-12", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_12) },
	{ "zstd-13", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_13) },
	{ "zstd-14", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_14) },
	{ "zstd-15", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_15) },
	{ "zstd-16", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_16) },
	{ "zstd-17", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_17) },
	{ "zstd-18", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_18) },
	{ "zstd-19", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_19) },

	/*
	* The ZSTD-Fast levels are also synthetic.
	*/
	{ "zstd-fast-1",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_1) },
	{ "zstd-fast-2",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_2) },
	{ "zstd-fast-3",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_3) },
	{ "zstd-fast-4",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_4) },
	{ "zstd-fast-5",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_5) },
	{ "zstd-fast-6",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_6) },
	{ "zstd-fast-7",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_7) },
	{ "zstd-fast-8",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_8) },
	{ "zstd-fast-9",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_9) },
	{ "zstd-fast-10",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_10) },
	{ "zstd-fast-20",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_20) },
	{ "zstd-fast-30",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_30) },
	{ "zstd-fast-40",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_40) },
	{ "zstd-fast-50",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_50) },
	{ "zstd-fast-60",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_60) },
	{ "zstd-fast-70",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_70) },
	{ "zstd-fast-80",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_80) },
	{ "zstd-fast-90",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_90) },
	{ "zstd-fast-100",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_100) },
	{ "zstd-fast-500",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_500) },
	{ "zstd-fast-1000",
	ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_1000) },
	{ NULL }
	};

	static const zprop_index_t crypto_table[] = {
	{ "on", ZIO_CRYPT_ON },
	{ "off", ZIO_CRYPT_OFF },
	{ "aes-128-ccm", ZIO_CRYPT_AES_128_CCM },
	{ "aes-192-ccm", ZIO_CRYPT_AES_192_CCM },
	{ "aes-256-ccm", ZIO_CRYPT_AES_256_CCM },
	{ "aes-128-gcm", ZIO_CRYPT_AES_128_GCM },
	{ "aes-192-gcm", ZIO_CRYPT_AES_192_GCM },
	{ "aes-256-gcm", ZIO_CRYPT_AES_256_GCM },
	{ NULL }
	};

	static const zprop_index_t keyformat_table[] = {
	{ "none", ZFS_KEYFORMAT_NONE },
	{ "raw", ZFS_KEYFORMAT_RAW },
	{ "hex", ZFS_KEYFORMAT_HEX },
	{ "passphrase", ZFS_KEYFORMAT_PASSPHRASE },
	{ NULL }
	};

	static const zprop_index_t snapdir_table[] = {
	{ "hidden", ZFS_SNAPDIR_HIDDEN },
	{ "visible", ZFS_SNAPDIR_VISIBLE },
	{ NULL }
	};

	static const zprop_index_t snapdev_table[] = {
	{ "hidden", ZFS_SNAPDEV_HIDDEN },
	{ "visible", ZFS_SNAPDEV_VISIBLE },
	{ NULL }
	};

	static const zprop_index_t acl_mode_table[] = {
	{ "discard", ZFS_ACL_DISCARD },
	{ "groupmask", ZFS_ACL_GROUPMASK },
	{ "passthrough", ZFS_ACL_PASSTHROUGH },
	{ "restricted", ZFS_ACL_RESTRICTED },
	{ NULL }
	};

	static const zprop_index_t acltype_table[] = {
	{ "off", ZFS_ACLTYPE_OFF },
	{ "posix", ZFS_ACLTYPE_POSIX },
	{ "nfsv4", ZFS_ACLTYPE_NFSV4 },
	{ "disabled", ZFS_ACLTYPE_OFF }, /* bkwrd compatibility */
	{ "noacl", ZFS_ACLTYPE_OFF }, /* bkwrd compatibility */
	{ "posixacl", ZFS_ACLTYPE_POSIX }, /* bkwrd compatibility */
	{ NULL }
	};

	static const zprop_index_t acl_inherit_table[] = {
	{ "discard", ZFS_ACL_DISCARD },
	{ "noallow", ZFS_ACL_NOALLOW },
	{ "restricted", ZFS_ACL_RESTRICTED },
	{ "passthrough", ZFS_ACL_PASSTHROUGH },
	{ "secure", ZFS_ACL_RESTRICTED }, /* bkwrd compatibility */
	{ "passthrough-x", ZFS_ACL_PASSTHROUGH_X },
	{ NULL }
	};

	static const zprop_index_t case_table[] = {
	{ "sensitive", ZFS_CASE_SENSITIVE },
	{ "insensitive", ZFS_CASE_INSENSITIVE },
	{ "mixed", ZFS_CASE_MIXED },
	{ NULL }
	};

	static const zprop_index_t copies_table[] = {
	{ "1", 1 },
	{ "2", 2 },
	{ "3", 3 },
	{ NULL }
	};

	/*
	* Use the unique flags we have to send to u8_strcmp() and/or
	* u8_textprep() to represent the various normalization property
	* values.
	*/
	static const zprop_index_t normalize_table[] = {
	{ "none", 0 },
	{ "formD", U8_TEXTPREP_NFD },
	{ "formKC", U8_TEXTPREP_NFKC },
	{ "formC", U8_TEXTPREP_NFC },
	{ "formKD", U8_TEXTPREP_NFKD },
	{ NULL }
	};

	static const zprop_index_t version_table[] = {
	{ "1", 1 },
	{ "2", 2 },
	{ "3", 3 },
	{ "4", 4 },
	{ "5", 5 },
	{ "current", ZPL_VERSION },
	{ NULL }
	};

	static const zprop_index_t boolean_table[] = {
	{ "off", 0 },
	{ "on", 1 },
	{ NULL }
	};

	static const zprop_index_t keystatus_table[] = {
	{ "none", ZFS_KEYSTATUS_NONE},
	{ "unavailable", ZFS_KEYSTATUS_UNAVAILABLE},
	{ "available", ZFS_KEYSTATUS_AVAILABLE},
	{ NULL }
	};

	static const zprop_index_t logbias_table[] = {
	{ "latency", ZFS_LOGBIAS_LATENCY },
	{ "throughput", ZFS_LOGBIAS_THROUGHPUT },
	{ NULL }
	};

	static const zprop_index_t canmount_table[] = {
	{ "off", ZFS_CANMOUNT_OFF },
	{ "on", ZFS_CANMOUNT_ON },
	{ "noauto", ZFS_CANMOUNT_NOAUTO },
	{ NULL }
	};

	static const zprop_index_t cache_table[] = {
	{ "none", ZFS_CACHE_NONE },
	{ "metadata", ZFS_CACHE_METADATA },
	{ "all", ZFS_CACHE_ALL },
	{ NULL }
	};

	+ static const zprop_index_t prefetch_table[] = {
	+ { "none", ZFS_PREFETCH_NONE },
	+ { "metadata", ZFS_PREFETCH_METADATA },
	+ { "all", ZFS_PREFETCH_ALL },
	+ { NULL }
	+ };
	+
	static const zprop_index_t sync_table[] = {
	{ "standard", ZFS_SYNC_STANDARD },
	{ "always", ZFS_SYNC_ALWAYS },
	{ "disabled", ZFS_SYNC_DISABLED },
	{ NULL }
	};

	static const zprop_index_t xattr_table[] = {
	{ "off", ZFS_XATTR_OFF },
	{ "on", ZFS_XATTR_DIR },
	{ "sa", ZFS_XATTR_SA },
	{ "dir", ZFS_XATTR_DIR },
	{ NULL }
	};

	static const zprop_index_t dnsize_table[] = {
	{ "legacy", ZFS_DNSIZE_LEGACY },
	{ "auto", ZFS_DNSIZE_AUTO },
	{ "1k", ZFS_DNSIZE_1K },
	{ "2k", ZFS_DNSIZE_2K },
	{ "4k", ZFS_DNSIZE_4K },
	{ "8k", ZFS_DNSIZE_8K },
	{ "16k", ZFS_DNSIZE_16K },
	{ NULL }
	};

	static const zprop_index_t redundant_metadata_table[] = {
	{ "all", ZFS_REDUNDANT_METADATA_ALL },
	{ "most", ZFS_REDUNDANT_METADATA_MOST },
	{ "some", ZFS_REDUNDANT_METADATA_SOME },
	{ "none", ZFS_REDUNDANT_METADATA_NONE },
	{ NULL }
	};

	static const zprop_index_t volmode_table[] = {
	{ "default", ZFS_VOLMODE_DEFAULT },
	{ "full", ZFS_VOLMODE_GEOM },
	{ "geom", ZFS_VOLMODE_GEOM },
	{ "dev", ZFS_VOLMODE_DEV },
	{ "none", ZFS_VOLMODE_NONE },
	{ NULL }
	};

	struct zfs_mod_supported_features *sfeatures =
	zfs_mod_list_supported(ZFS_SYSFS_DATASET_PROPERTIES);

	/* inherit index properties */
	zprop_register_index(ZFS_PROP_REDUNDANT_METADATA, "redundant_metadata",
	ZFS_REDUNDANT_METADATA_ALL,
	PROP_INHERIT, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME,
	"all \| most \| some \| none", "REDUND_MD",
	redundant_metadata_table, sfeatures);
	zprop_register_index(ZFS_PROP_SYNC, "sync", ZFS_SYNC_STANDARD,
	PROP_INHERIT, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME,
	"standard \| always \| disabled", "SYNC",
	sync_table, sfeatures);
	zprop_register_index(ZFS_PROP_CHECKSUM, "checksum",
	ZIO_CHECKSUM_DEFAULT, PROP_INHERIT, ZFS_TYPE_FILESYSTEM \|
	ZFS_TYPE_VOLUME,
	"on \| off \| fletcher2 \| fletcher4 \| sha256 \| sha512 \| skein"
	" \| edonr \| blake3",
	"CHECKSUM", checksum_table, sfeatures);
	zprop_register_index(ZFS_PROP_DEDUP, "dedup", ZIO_CHECKSUM_OFF,
	PROP_INHERIT, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME,
	"on \| off \| verify \| sha256[,verify] \| sha512[,verify] \| "
	"skein[,verify] \| edonr,verify \| blake3[,verify]",
	"DEDUP", dedup_table, sfeatures);
	zprop_register_index(ZFS_PROP_COMPRESSION, "compression",
	ZIO_COMPRESS_DEFAULT, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME,
	"on \| off \| lzjb \| gzip \| gzip-[1-9] \| zle \| lz4 \| "
	"zstd \| zstd-[1-19] \| "
	"zstd-fast \| zstd-fast-[1-10,20,30,40,50,60,70,80,90,100,500,1000]",
	"COMPRESS", compress_table, sfeatures);
	zprop_register_index(ZFS_PROP_SNAPDIR, "snapdir", ZFS_SNAPDIR_HIDDEN,
	PROP_INHERIT, ZFS_TYPE_FILESYSTEM,
	"hidden \| visible", "SNAPDIR", snapdir_table, sfeatures);
	zprop_register_index(ZFS_PROP_SNAPDEV, "snapdev", ZFS_SNAPDEV_HIDDEN,
	PROP_INHERIT, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME,
	"hidden \| visible", "SNAPDEV", snapdev_table, sfeatures);
	zprop_register_index(ZFS_PROP_ACLMODE, "aclmode", ZFS_ACL_DISCARD,
	PROP_INHERIT, ZFS_TYPE_FILESYSTEM,
	"discard \| groupmask \| passthrough \| restricted", "ACLMODE",
	acl_mode_table, sfeatures);
	zprop_register_index(ZFS_PROP_ACLTYPE, "acltype",
	#ifdef __linux__
	/* Linux doesn't natively support ZFS's NFSv4-style ACLs. */
	ZFS_ACLTYPE_OFF,
	#else
	ZFS_ACLTYPE_NFSV4,
	#endif
	PROP_INHERIT, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_SNAPSHOT,
	"off \| nfsv4 \| posix", "ACLTYPE", acltype_table, sfeatures);
	zprop_register_index(ZFS_PROP_ACLINHERIT, "aclinherit",
	ZFS_ACL_RESTRICTED, PROP_INHERIT, ZFS_TYPE_FILESYSTEM,
	"discard \| noallow \| restricted \| passthrough \| passthrough-x",
	"ACLINHERIT", acl_inherit_table, sfeatures);
	zprop_register_index(ZFS_PROP_COPIES, "copies", 1, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME,
	"1 \| 2 \| 3", "COPIES", copies_table, sfeatures);
	zprop_register_index(ZFS_PROP_PRIMARYCACHE, "primarycache",
	ZFS_CACHE_ALL, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_SNAPSHOT \| ZFS_TYPE_VOLUME,
	"all \| none \| metadata", "PRIMARYCACHE", cache_table, sfeatures);
	zprop_register_index(ZFS_PROP_SECONDARYCACHE, "secondarycache",
	ZFS_CACHE_ALL, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_SNAPSHOT \| ZFS_TYPE_VOLUME,
	"all \| none \| metadata", "SECONDARYCACHE", cache_table, sfeatures);
	+ zprop_register_index(ZFS_PROP_PREFETCH, "prefetch",
	+ ZFS_PREFETCH_ALL, PROP_INHERIT,
	+ ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_SNAPSHOT \| ZFS_TYPE_VOLUME,
	+ "none \| metadata \| all", "PREFETCH", prefetch_table, sfeatures);
	zprop_register_index(ZFS_PROP_LOGBIAS, "logbias", ZFS_LOGBIAS_LATENCY,
	PROP_INHERIT, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME,
	"latency \| throughput", "LOGBIAS", logbias_table, sfeatures);
	zprop_register_index(ZFS_PROP_XATTR, "xattr", ZFS_XATTR_DIR,
	PROP_INHERIT, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_SNAPSHOT,
	"on \| off \| dir \| sa", "XATTR", xattr_table, sfeatures);
	zprop_register_index(ZFS_PROP_DNODESIZE, "dnodesize",
	ZFS_DNSIZE_LEGACY, PROP_INHERIT, ZFS_TYPE_FILESYSTEM,
	"legacy \| auto \| 1k \| 2k \| 4k \| 8k \| 16k", "DNSIZE", dnsize_table,
	sfeatures);
	zprop_register_index(ZFS_PROP_VOLMODE, "volmode",
	ZFS_VOLMODE_DEFAULT, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME,
	"default \| full \| geom \| dev \| none", "VOLMODE", volmode_table,
	sfeatures);

	/* inherit index (boolean) properties */
	zprop_register_index(ZFS_PROP_ATIME, "atime", 1, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM, "on \| off", "ATIME", boolean_table, sfeatures);
	zprop_register_index(ZFS_PROP_RELATIME, "relatime", 1, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM, "on \| off", "RELATIME", boolean_table,
	sfeatures);
	zprop_register_index(ZFS_PROP_DEVICES, "devices", 1, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_SNAPSHOT, "on \| off", "DEVICES",
	boolean_table, sfeatures);
	zprop_register_index(ZFS_PROP_EXEC, "exec", 1, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_SNAPSHOT, "on \| off", "EXEC",
	boolean_table, sfeatures);
	zprop_register_index(ZFS_PROP_SETUID, "setuid", 1, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_SNAPSHOT, "on \| off", "SETUID",
	boolean_table, sfeatures);
	zprop_register_index(ZFS_PROP_READONLY, "readonly", 0, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME, "on \| off", "RDONLY",
	boolean_table, sfeatures);
	#ifdef __FreeBSD__
	zprop_register_index(ZFS_PROP_ZONED, "jailed", 0, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM, "on \| off", "JAILED", boolean_table,
	sfeatures);
	#else
	zprop_register_index(ZFS_PROP_ZONED, "zoned", 0, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM, "on \| off", "ZONED", boolean_table, sfeatures);
	#endif
	zprop_register_index(ZFS_PROP_VSCAN, "vscan", 0, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM, "on \| off", "VSCAN", boolean_table, sfeatures);
	zprop_register_index(ZFS_PROP_NBMAND, "nbmand", 0, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_SNAPSHOT, "on \| off", "NBMAND",
	boolean_table, sfeatures);
	zprop_register_index(ZFS_PROP_OVERLAY, "overlay", 1, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM, "on \| off", "OVERLAY", boolean_table,
	sfeatures);

	/* default index properties */
	zprop_register_index(ZFS_PROP_VERSION, "version", 0, PROP_DEFAULT,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_SNAPSHOT,
	"1 \| 2 \| 3 \| 4 \| 5 \| current", "VERSION", version_table, sfeatures);
	zprop_register_index(ZFS_PROP_CANMOUNT, "canmount", ZFS_CANMOUNT_ON,
	PROP_DEFAULT, ZFS_TYPE_FILESYSTEM, "on \| off \| noauto",
	"CANMOUNT", canmount_table, sfeatures);

	/* readonly index properties */
	zprop_register_index(ZFS_PROP_MOUNTED, "mounted", 0, PROP_READONLY,
	ZFS_TYPE_FILESYSTEM, "yes \| no", "MOUNTED", boolean_table,
	sfeatures);
	zprop_register_index(ZFS_PROP_DEFER_DESTROY, "defer_destroy", 0,
	PROP_READONLY, ZFS_TYPE_SNAPSHOT, "yes \| no", "DEFER_DESTROY",
	boolean_table, sfeatures);
	zprop_register_index(ZFS_PROP_KEYSTATUS, "keystatus",
	ZFS_KEYSTATUS_NONE, PROP_READONLY, ZFS_TYPE_DATASET,
	"none \| unavailable \| available",
	"KEYSTATUS", keystatus_table, sfeatures);

	/* set once index properties */
	zprop_register_index(ZFS_PROP_NORMALIZE, "normalization", 0,
	PROP_ONETIME, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_SNAPSHOT,
	"none \| formC \| formD \| formKC \| formKD", "NORMALIZATION",
	normalize_table, sfeatures);
	zprop_register_index(ZFS_PROP_CASE, "casesensitivity",
	ZFS_CASE_SENSITIVE, PROP_ONETIME, ZFS_TYPE_FILESYSTEM \|
	ZFS_TYPE_SNAPSHOT,
	"sensitive \| insensitive \| mixed", "CASE", case_table, sfeatures);
	zprop_register_index(ZFS_PROP_KEYFORMAT, "keyformat",
	ZFS_KEYFORMAT_NONE, PROP_ONETIME_DEFAULT,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME,
	"none \| raw \| hex \| passphrase", "KEYFORMAT", keyformat_table,
	sfeatures);
	zprop_register_index(ZFS_PROP_ENCRYPTION, "encryption",
	ZIO_CRYPT_DEFAULT, PROP_ONETIME, ZFS_TYPE_DATASET,
	"on \| off \| aes-128-ccm \| aes-192-ccm \| aes-256-ccm \| "
	"aes-128-gcm \| aes-192-gcm \| aes-256-gcm", "ENCRYPTION",
	crypto_table, sfeatures);

	/* set once index (boolean) properties */
	zprop_register_index(ZFS_PROP_UTF8ONLY, "utf8only", 0, PROP_ONETIME,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_SNAPSHOT,
	"on \| off", "UTF8ONLY", boolean_table, sfeatures);

	/* string properties */
	zprop_register_string(ZFS_PROP_ORIGIN, "origin", NULL, PROP_READONLY,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME, "<snapshot>", "ORIGIN",
	sfeatures);
	zprop_register_string(ZFS_PROP_CLONES, "clones", NULL, PROP_READONLY,
	ZFS_TYPE_SNAPSHOT, "<dataset>[,...]", "CLONES", sfeatures);
	zprop_register_string(ZFS_PROP_MOUNTPOINT, "mountpoint", "/",
	PROP_INHERIT, ZFS_TYPE_FILESYSTEM, "<path> \| legacy \| none",
	"MOUNTPOINT", sfeatures);
	zprop_register_string(ZFS_PROP_SHARENFS, "sharenfs", "off",
	PROP_INHERIT, ZFS_TYPE_FILESYSTEM, "on \| off \| NFS share options",
	"SHARENFS", sfeatures);
	zprop_register_string(ZFS_PROP_TYPE, "type", NULL, PROP_READONLY,
	ZFS_TYPE_DATASET \| ZFS_TYPE_BOOKMARK,
	"filesystem \| volume \| snapshot \| bookmark", "TYPE", sfeatures);
	zprop_register_string(ZFS_PROP_SHARESMB, "sharesmb", "off",
	PROP_INHERIT, ZFS_TYPE_FILESYSTEM,
	"on \| off \| SMB share options", "SHARESMB", sfeatures);
	zprop_register_string(ZFS_PROP_MLSLABEL, "mlslabel",
	ZFS_MLSLABEL_DEFAULT, PROP_INHERIT, ZFS_TYPE_DATASET,
	"<sensitivity label>", "MLSLABEL", sfeatures);
	zprop_register_string(ZFS_PROP_SELINUX_CONTEXT, "context",
	"none", PROP_DEFAULT, ZFS_TYPE_DATASET, "<selinux context>",
	"CONTEXT", sfeatures);
	zprop_register_string(ZFS_PROP_SELINUX_FSCONTEXT, "fscontext",
	"none", PROP_DEFAULT, ZFS_TYPE_DATASET, "<selinux fscontext>",
	"FSCONTEXT", sfeatures);
	zprop_register_string(ZFS_PROP_SELINUX_DEFCONTEXT, "defcontext",
	"none", PROP_DEFAULT, ZFS_TYPE_DATASET, "<selinux defcontext>",
	"DEFCONTEXT", sfeatures);
	zprop_register_string(ZFS_PROP_SELINUX_ROOTCONTEXT, "rootcontext",
	"none", PROP_DEFAULT, ZFS_TYPE_DATASET, "<selinux rootcontext>",
	"ROOTCONTEXT", sfeatures);
	zprop_register_string(ZFS_PROP_RECEIVE_RESUME_TOKEN,
	"receive_resume_token",
	NULL, PROP_READONLY, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME,
	"<string token>", "RESUMETOK", sfeatures);
	zprop_register_string(ZFS_PROP_ENCRYPTION_ROOT, "encryptionroot", NULL,
	PROP_READONLY, ZFS_TYPE_DATASET, "<filesystem \| volume>",
	"ENCROOT", sfeatures);
	zprop_register_string(ZFS_PROP_KEYLOCATION, "keylocation",
	"none", PROP_DEFAULT, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME,
	"prompt \| <file URI> \| <https URL> \| <http URL>", "KEYLOCATION",
	sfeatures);
	zprop_register_string(ZFS_PROP_REDACT_SNAPS,
	"redact_snaps", NULL, PROP_READONLY,
	ZFS_TYPE_DATASET \| ZFS_TYPE_BOOKMARK, "<snapshot>[,...]",
	"RSNAPS", sfeatures);

	/* readonly number properties */
	zprop_register_number(ZFS_PROP_USED, "used", 0, PROP_READONLY,
	ZFS_TYPE_DATASET, "<size>", "USED", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_AVAILABLE, "available", 0, PROP_READONLY,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME, "<size>", "AVAIL",
	B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_REFERENCED, "referenced", 0,
	PROP_READONLY, ZFS_TYPE_DATASET \| ZFS_TYPE_BOOKMARK, "<size>",
	"REFER", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_COMPRESSRATIO, "compressratio", 0,
	PROP_READONLY, ZFS_TYPE_DATASET \| ZFS_TYPE_BOOKMARK,
	"<1.00x or higher if compressed>", "RATIO", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_REFRATIO, "refcompressratio", 0,
	PROP_READONLY, ZFS_TYPE_DATASET,
	"<1.00x or higher if compressed>", "REFRATIO", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_VOLBLOCKSIZE, "volblocksize",
	ZVOL_DEFAULT_BLOCKSIZE, PROP_ONETIME,
	ZFS_TYPE_VOLUME, "512 to 128k, power of 2", "VOLBLOCK", B_FALSE,
	sfeatures);
	zprop_register_number(ZFS_PROP_USEDSNAP, "usedbysnapshots", 0,
	PROP_READONLY, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME, "<size>",
	"USEDSNAP", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_USEDDS, "usedbydataset", 0,
	PROP_READONLY, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME, "<size>",
	"USEDDS", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_USEDCHILD, "usedbychildren", 0,
	PROP_READONLY, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME, "<size>",
	"USEDCHILD", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_USEDREFRESERV, "usedbyrefreservation", 0,
	PROP_READONLY,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME, "<size>", "USEDREFRESERV",
	B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_USERREFS, "userrefs", 0, PROP_READONLY,
	ZFS_TYPE_SNAPSHOT, "<count>", "USERREFS", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_WRITTEN, "written", 0, PROP_READONLY,
	ZFS_TYPE_DATASET, "<size>", "WRITTEN", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_LOGICALUSED, "logicalused", 0,
	PROP_READONLY, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME, "<size>",
	"LUSED", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_LOGICALREFERENCED, "logicalreferenced",
	0, PROP_READONLY, ZFS_TYPE_DATASET \| ZFS_TYPE_BOOKMARK, "<size>",
	"LREFER", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_FILESYSTEM_COUNT, "filesystem_count",
	UINT64_MAX, PROP_READONLY, ZFS_TYPE_FILESYSTEM,
	"<count>", "FSCOUNT", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_SNAPSHOT_COUNT, "snapshot_count",
	UINT64_MAX, PROP_READONLY, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME,
	"<count>", "SSCOUNT", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_GUID, "guid", 0, PROP_READONLY,
	ZFS_TYPE_DATASET \| ZFS_TYPE_BOOKMARK, "<uint64>", "GUID",
	B_TRUE, sfeatures);
	zprop_register_number(ZFS_PROP_CREATETXG, "createtxg", 0, PROP_READONLY,
	ZFS_TYPE_DATASET \| ZFS_TYPE_BOOKMARK, "<uint64>", "CREATETXG",
	B_TRUE, sfeatures);
	zprop_register_number(ZFS_PROP_PBKDF2_ITERS, "pbkdf2iters",
	0, PROP_ONETIME_DEFAULT, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME,
	"<iters>", "PBKDF2ITERS", B_TRUE, sfeatures);
	zprop_register_number(ZFS_PROP_OBJSETID, "objsetid", 0,
	PROP_READONLY, ZFS_TYPE_DATASET, "<uint64>", "OBJSETID", B_TRUE,
	sfeatures);

	/* default number properties */
	zprop_register_number(ZFS_PROP_QUOTA, "quota", 0, PROP_DEFAULT,
	ZFS_TYPE_FILESYSTEM, "<size> \| none", "QUOTA", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_RESERVATION, "reservation", 0,
	PROP_DEFAULT, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME,
	"<size> \| none", "RESERV", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_VOLSIZE, "volsize", 0, PROP_DEFAULT,
	ZFS_TYPE_SNAPSHOT \| ZFS_TYPE_VOLUME, "<size>", "VOLSIZE",
	B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_REFQUOTA, "refquota", 0, PROP_DEFAULT,
	ZFS_TYPE_FILESYSTEM, "<size> \| none", "REFQUOTA", B_FALSE,
	sfeatures);
	zprop_register_number(ZFS_PROP_REFRESERVATION, "refreservation", 0,
	PROP_DEFAULT, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME,
	"<size> \| none", "REFRESERV", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_FILESYSTEM_LIMIT, "filesystem_limit",
	UINT64_MAX, PROP_DEFAULT, ZFS_TYPE_FILESYSTEM,
	"<count> \| none", "FSLIMIT", B_FALSE, sfeatures);
	zprop_register_number(ZFS_PROP_SNAPSHOT_LIMIT, "snapshot_limit",
	UINT64_MAX, PROP_DEFAULT, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME,
	"<count> \| none", "SSLIMIT", B_FALSE, sfeatures);

	/* inherit number properties */
	zprop_register_number(ZFS_PROP_RECORDSIZE, "recordsize",
	SPA_OLD_MAXBLOCKSIZE, PROP_INHERIT,
	ZFS_TYPE_FILESYSTEM, "512 to 1M, power of 2", "RECSIZE", B_FALSE,
	sfeatures);
	zprop_register_number(ZFS_PROP_SPECIAL_SMALL_BLOCKS,
	"special_small_blocks", 0, PROP_INHERIT, ZFS_TYPE_FILESYSTEM,
	"zero or 512 to 1M, power of 2", "SPECIAL_SMALL_BLOCKS", B_FALSE,
	sfeatures);

	/* hidden properties */
	zprop_register_hidden(ZFS_PROP_NUMCLONES, "numclones", PROP_TYPE_NUMBER,
	PROP_READONLY, ZFS_TYPE_SNAPSHOT, "NUMCLONES", B_FALSE, sfeatures);
	zprop_register_hidden(ZFS_PROP_NAME, "name", PROP_TYPE_STRING,
	PROP_READONLY, ZFS_TYPE_DATASET \| ZFS_TYPE_BOOKMARK, "NAME",
	B_TRUE, sfeatures);
	zprop_register_hidden(ZFS_PROP_ISCSIOPTIONS, "iscsioptions",
	PROP_TYPE_STRING, PROP_INHERIT, ZFS_TYPE_VOLUME, "ISCSIOPTIONS",
	B_TRUE, sfeatures);
	zprop_register_hidden(ZFS_PROP_STMF_SHAREINFO, "stmf_sbd_lu",
	PROP_TYPE_STRING, PROP_INHERIT, ZFS_TYPE_VOLUME,
	"STMF_SBD_LU", B_TRUE, sfeatures);
	zprop_register_hidden(ZFS_PROP_USERACCOUNTING, "useraccounting",
	PROP_TYPE_NUMBER, PROP_READONLY, ZFS_TYPE_DATASET,
	"USERACCOUNTING", B_FALSE, sfeatures);
	zprop_register_hidden(ZFS_PROP_UNIQUE, "unique", PROP_TYPE_NUMBER,
	PROP_READONLY, ZFS_TYPE_DATASET, "UNIQUE", B_FALSE, sfeatures);
	zprop_register_hidden(ZFS_PROP_INCONSISTENT, "inconsistent",
	PROP_TYPE_NUMBER, PROP_READONLY, ZFS_TYPE_DATASET, "INCONSISTENT",
	B_FALSE, sfeatures);
	zprop_register_hidden(ZFS_PROP_IVSET_GUID, "ivsetguid",
	PROP_TYPE_NUMBER, PROP_READONLY,
	ZFS_TYPE_DATASET \| ZFS_TYPE_BOOKMARK, "IVSETGUID", B_TRUE,
	sfeatures);
	zprop_register_hidden(ZFS_PROP_PREV_SNAP, "prevsnap", PROP_TYPE_STRING,
	PROP_READONLY, ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME, "PREVSNAP",
	B_TRUE, sfeatures);
	zprop_register_hidden(ZFS_PROP_PBKDF2_SALT, "pbkdf2salt",
	PROP_TYPE_NUMBER, PROP_ONETIME_DEFAULT,
	ZFS_TYPE_FILESYSTEM \| ZFS_TYPE_VOLUME, "PBKDF2SALT", B_FALSE,
	sfeatures);
	zprop_register_hidden(ZFS_PROP_KEY_GUID, "keyguid", PROP_TYPE_NUMBER,
	PROP_READONLY, ZFS_TYPE_DATASET, "KEYGUID", B_TRUE, sfeatures);
	zprop_register_hidden(ZFS_PROP_REDACTED, "redacted", PROP_TYPE_NUMBER,
	PROP_READONLY, ZFS_TYPE_DATASET, "REDACTED", B_FALSE, sfeatures);

	/*
	* Properties that are obsolete and not used. These are retained so
	* that we don't have to change the values of the zfs_prop_t enum, or
	* have NULL pointers in the zfs_prop_table[].
	*/
	zprop_register_hidden(ZFS_PROP_REMAPTXG, "remaptxg", PROP_TYPE_NUMBER,
	PROP_READONLY, ZFS_TYPE_DATASET, "REMAPTXG", B_FALSE, sfeatures);

	/* oddball properties */
	/* 'creation' is a number but displayed as human-readable => flex */
	zprop_register_impl(ZFS_PROP_CREATION, "creation", PROP_TYPE_NUMBER, 0,
	NULL, PROP_READONLY, ZFS_TYPE_DATASET \| ZFS_TYPE_BOOKMARK,
	"<date>", "CREATION", B_FALSE, B_TRUE, B_TRUE, NULL, sfeatures);

	zprop_register_impl(ZFS_PROP_SNAPSHOTS_CHANGED, "snapshots_changed",
	PROP_TYPE_NUMBER, 0, NULL, PROP_READONLY, ZFS_TYPE_FILESYSTEM \|
	ZFS_TYPE_VOLUME, "<date>", "SNAPSHOTS_CHANGED", B_FALSE, B_TRUE,
	B_TRUE, NULL, sfeatures);

	zfs_mod_list_supported_free(sfeatures);
	}

	boolean_t
	zfs_prop_delegatable(zfs_prop_t prop)
	{
	ASSERT3S(prop, >=, 0);
	ASSERT3S(prop, <, ZFS_NUM_PROPS);
	zprop_desc_t *pd = &zfs_prop_table[prop];

	/* The mlslabel property is never delegatable. */
	if (prop == ZFS_PROP_MLSLABEL)
	return (B_FALSE);

	return (pd->pd_attr != PROP_READONLY);
	}

	/*
	* Given a zfs dataset property name, returns the corresponding property ID.
	*/
	zfs_prop_t
	zfs_name_to_prop(const char *propname)
	{
	return (zprop_name_to_prop(propname, ZFS_TYPE_DATASET));
	}

	/*
	* Returns true if this is a valid user-defined property (one with a ':').
	*/
	boolean_t
	zfs_prop_user(const char *name)
	{
	int i;
	char c;
	boolean_t foundsep = B_FALSE;

	for (i = 0; i < strlen(name); i++) {
	c = name[i];
	if (!zprop_valid_char(c))
	return (B_FALSE);
	if (c == ':')
	foundsep = B_TRUE;
	}

	if (!foundsep)
	return (B_FALSE);

	return (B_TRUE);
	}

	/*
	* Returns true if this is a valid userspace-type property (one with a '@').
	* Note that after the @, any character is valid (eg, another @, for SID
	* user@domain).
	*/
	boolean_t
	zfs_prop_userquota(const char *name)
	{
	zfs_userquota_prop_t prop;

	for (prop = 0; prop < ZFS_NUM_USERQUOTA_PROPS; prop++) {
	if (strncmp(name, zfs_userquota_prop_prefixes[prop],
	strlen(zfs_userquota_prop_prefixes[prop])) == 0) {
	return (B_TRUE);
	}
	}

	return (B_FALSE);
	}

	/*
	* Returns true if this is a valid written@ property.
	* Note that after the @, any character is valid (eg, another @, for
	* written@pool/fs@origin).
	*/
	boolean_t
	zfs_prop_written(const char *name)
	{
	static const char *prop_prefix = "written@";
	static const char *book_prefix = "written#";
	return (strncmp(name, prop_prefix, strlen(prop_prefix)) == 0 \|\|
	strncmp(name, book_prefix, strlen(book_prefix)) == 0);
	}

	/*
	* Tables of index types, plus functions to convert between the user view
	* (strings) and internal representation (uint64_t).
	*/
	int
	zfs_prop_string_to_index(zfs_prop_t prop, const char string, uint64_t index)
	{
	return (zprop_string_to_index(prop, string, index, ZFS_TYPE_DATASET));
	}

	int
	zfs_prop_index_to_string(zfs_prop_t prop, uint64_t index, const char **string)
	{
	return (zprop_index_to_string(prop, index, string, ZFS_TYPE_DATASET));
	}

	uint64_t
	zfs_prop_random_value(zfs_prop_t prop, uint64_t seed)
	{
	return (zprop_random_value(prop, seed, ZFS_TYPE_DATASET));
	}

	/*
	* Returns TRUE if the property applies to any of the given dataset types.
	*/
	boolean_t
	zfs_prop_valid_for_type(int prop, zfs_type_t types, boolean_t headcheck)
	{
	return (zprop_valid_for_type(prop, types, headcheck));
	}

	zprop_type_t
	zfs_prop_get_type(zfs_prop_t prop)
	{
	ASSERT3S(prop, >=, 0);
	ASSERT3S(prop, <, ZFS_NUM_PROPS);
	return (zfs_prop_table[prop].pd_proptype);
	}

	/*
	* Returns TRUE if the property is readonly.
	*/
	boolean_t
	zfs_prop_readonly(zfs_prop_t prop)
	{
	ASSERT3S(prop, >=, 0);
	ASSERT3S(prop, <, ZFS_NUM_PROPS);
	return (zfs_prop_table[prop].pd_attr == PROP_READONLY \|\|
	zfs_prop_table[prop].pd_attr == PROP_ONETIME \|\|
	zfs_prop_table[prop].pd_attr == PROP_ONETIME_DEFAULT);
	}

	/*
	* Returns TRUE if the property is visible (not hidden).
	*/
	boolean_t
	zfs_prop_visible(zfs_prop_t prop)
	{
	ASSERT3S(prop, >=, 0);
	ASSERT3S(prop, <, ZFS_NUM_PROPS);
	return (zfs_prop_table[prop].pd_visible &&
	zfs_prop_table[prop].pd_zfs_mod_supported);
	}

	/*
	* Returns TRUE if the property is only allowed to be set once.
	*/
	boolean_t
	zfs_prop_setonce(zfs_prop_t prop)
	{
	ASSERT3S(prop, >=, 0);
	ASSERT3S(prop, <, ZFS_NUM_PROPS);
	return (zfs_prop_table[prop].pd_attr == PROP_ONETIME \|\|
	zfs_prop_table[prop].pd_attr == PROP_ONETIME_DEFAULT);
	}

	const char *
	zfs_prop_default_string(zfs_prop_t prop)
	{
	ASSERT3S(prop, >=, 0);
	ASSERT3S(prop, <, ZFS_NUM_PROPS);
	return (zfs_prop_table[prop].pd_strdefault);
	}

	uint64_t
	zfs_prop_default_numeric(zfs_prop_t prop)
	{
	ASSERT3S(prop, >=, 0);
	ASSERT3S(prop, <, ZFS_NUM_PROPS);
	return (zfs_prop_table[prop].pd_numdefault);
	}

	/*
	* Given a dataset property ID, returns the corresponding name.
	* Assuming the zfs dataset property ID is valid.
	*/
	const char *
	zfs_prop_to_name(zfs_prop_t prop)
	{
	ASSERT3S(prop, >=, 0);
	ASSERT3S(prop, <, ZFS_NUM_PROPS);
	return (zfs_prop_table[prop].pd_name);
	}

	/*
	* Returns TRUE if the property is inheritable.
	*/
	boolean_t
	zfs_prop_inheritable(zfs_prop_t prop)
	{
	ASSERT3S(prop, >=, 0);
	ASSERT3S(prop, <, ZFS_NUM_PROPS);
	return (zfs_prop_table[prop].pd_attr == PROP_INHERIT \|\|
	zfs_prop_table[prop].pd_attr == PROP_ONETIME);
	}

	/*
	* Returns TRUE if property is one of the encryption properties that requires
	* a loaded encryption key to modify.
	*/
	boolean_t
	zfs_prop_encryption_key_param(zfs_prop_t prop)
	{
	/*
	* keylocation does not count as an encryption property. It can be
	* changed at will without needing the master keys.
	*/
	return (prop == ZFS_PROP_PBKDF2_SALT \|\| prop == ZFS_PROP_PBKDF2_ITERS \|\|
	prop == ZFS_PROP_KEYFORMAT);
	}

	/*
	* Helper function used by both kernelspace and userspace to check the
	* keylocation property. If encrypted is set, the keylocation must be valid
	* for an encrypted dataset.
	*/
	boolean_t
	zfs_prop_valid_keylocation(const char *str, boolean_t encrypted)
	{
	if (strcmp("none", str) == 0)
	return (!encrypted);
	else if (strcmp("prompt", str) == 0)
	return (B_TRUE);
	else if (strlen(str) > 8 && strncmp("file:///", str, 8) == 0)
	return (B_TRUE);
	else if (strlen(str) > 8 && strncmp("https://", str, 8) == 0)
	return (B_TRUE);
	else if (strlen(str) > 7 && strncmp("http://", str, 7) == 0)
	return (B_TRUE);

	return (B_FALSE);
	}


	#ifndef _KERNEL
	#include <libzfs.h>

	/*
	* Returns a string describing the set of acceptable values for the given
	* zfs property, or NULL if it cannot be set.
	*/
	const char *
	zfs_prop_values(zfs_prop_t prop)
	{
	ASSERT3S(prop, >=, 0);
	ASSERT3S(prop, <, ZFS_NUM_PROPS);
	return (zfs_prop_table[prop].pd_values);
	}

	/*
	* Returns TRUE if this property is a string type. Note that index types
	* (compression, checksum) are treated as strings in userland, even though they
	* are stored numerically on disk.
	*/
	int
	zfs_prop_is_string(zfs_prop_t prop)
	{
	ASSERT3S(prop, >=, 0);
	ASSERT3S(prop, <, ZFS_NUM_PROPS);
	return (zfs_prop_table[prop].pd_proptype == PROP_TYPE_STRING \|\|
	zfs_prop_table[prop].pd_proptype == PROP_TYPE_INDEX);
	}

	/*
	* Returns the column header for the given property. Used only in
	* 'zfs list -o', but centralized here with the other property information.
	*/
	const char *
	zfs_prop_column_name(zfs_prop_t prop)
	{
	ASSERT3S(prop, >=, 0);
	ASSERT3S(prop, <, ZFS_NUM_PROPS);
	return (zfs_prop_table[prop].pd_colname);
	}

	/*
	* Returns whether the given property should be displayed right-justified for
	* 'zfs list'.
	*/
	boolean_t
	zfs_prop_align_right(zfs_prop_t prop)
	{
	ASSERT3S(prop, >=, 0);
	ASSERT3S(prop, <, ZFS_NUM_PROPS);
	return (zfs_prop_table[prop].pd_rightalign);
	}

	#endif

	#if defined(_KERNEL)

	#if defined(HAVE_KERNEL_FPU_INTERNAL)
	uint8_t **zfs_kfpu_fpregs;
	EXPORT_SYMBOL(zfs_kfpu_fpregs);
	#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */

	extern int __init zcommon_init(void);
	extern void zcommon_fini(void);

	int __init
	zcommon_init(void)
	{
	int error = kfpu_init();
	if (error)
	return (error);

	fletcher_4_init();

	return (0);
	}

	void
	zcommon_fini(void)
	{
	fletcher_4_fini();
	kfpu_fini();
	}

	#ifdef __FreeBSD__
	module_init_early(zcommon_init);
	module_exit(zcommon_fini);
	#endif

	#endif

	/* zfs dataset property functions */
	EXPORT_SYMBOL(zfs_userquota_prop_prefixes);
	EXPORT_SYMBOL(zfs_prop_init);
	EXPORT_SYMBOL(zfs_prop_get_type);
	EXPORT_SYMBOL(zfs_prop_get_table);
	EXPORT_SYMBOL(zfs_prop_delegatable);
	EXPORT_SYMBOL(zfs_prop_visible);

	/* Dataset property functions shared between libzfs and kernel. */
	EXPORT_SYMBOL(zfs_prop_default_string);
	EXPORT_SYMBOL(zfs_prop_default_numeric);
	EXPORT_SYMBOL(zfs_prop_readonly);
	EXPORT_SYMBOL(zfs_prop_inheritable);
	EXPORT_SYMBOL(zfs_prop_encryption_key_param);
	EXPORT_SYMBOL(zfs_prop_valid_keylocation);
	EXPORT_SYMBOL(zfs_prop_setonce);
	EXPORT_SYMBOL(zfs_prop_to_name);
	EXPORT_SYMBOL(zfs_name_to_prop);
	EXPORT_SYMBOL(zfs_prop_user);
	EXPORT_SYMBOL(zfs_prop_userquota);
	EXPORT_SYMBOL(zfs_prop_index_to_string);
	EXPORT_SYMBOL(zfs_prop_string_to_index);
	EXPORT_SYMBOL(zfs_prop_valid_for_type);
	EXPORT_SYMBOL(zfs_prop_written);
	diff --git a/sys/contrib/openzfs/module/zcommon/zpool_prop.c b/sys/contrib/openzfs/module/zcommon/zpool_prop.c
	index c4aca04a96bd..ff70c0e3c35b 100644
	--- a/sys/contrib/openzfs/module/zcommon/zpool_prop.c
	+++ b/sys/contrib/openzfs/module/zcommon/zpool_prop.c
	@@ -1,600 +1,606 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright 2011 Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2012, 2018 by Delphix. All rights reserved.
	* Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
	* Copyright (c) 2021, Klara Inc.
	*/

	#include <sys/zio.h>
	#include <sys/spa.h>
	#include <sys/zfs_acl.h>
	#include <sys/zfs_ioctl.h>
	#include <sys/fs/zfs.h>

	#include "zfs_prop.h"

	#if !defined(_KERNEL)
	#include <stdlib.h>
	#include <string.h>
	#include <ctype.h>
	#endif

	static zprop_desc_t zpool_prop_table[ZPOOL_NUM_PROPS];
	static zprop_desc_t vdev_prop_table[VDEV_NUM_PROPS];

	zprop_desc_t *
	zpool_prop_get_table(void)
	{
	return (zpool_prop_table);
	}

	void
	zpool_prop_init(void)
	{
	static const zprop_index_t boolean_table[] = {
	{ "off", 0},
	{ "on", 1},
	{ NULL }
	};

	static const zprop_index_t failuremode_table[] = {
	{ "wait", ZIO_FAILURE_MODE_WAIT },
	{ "continue", ZIO_FAILURE_MODE_CONTINUE },
	{ "panic", ZIO_FAILURE_MODE_PANIC },
	{ NULL }
	};

	struct zfs_mod_supported_features *sfeatures =
	zfs_mod_list_supported(ZFS_SYSFS_POOL_PROPERTIES);

	/* string properties */
	zprop_register_string(ZPOOL_PROP_ALTROOT, "altroot", NULL, PROP_DEFAULT,
	ZFS_TYPE_POOL, "<path>", "ALTROOT", sfeatures);
	zprop_register_string(ZPOOL_PROP_BOOTFS, "bootfs", NULL, PROP_DEFAULT,
	ZFS_TYPE_POOL, "<filesystem>", "BOOTFS", sfeatures);
	zprop_register_string(ZPOOL_PROP_CACHEFILE, "cachefile", NULL,
	PROP_DEFAULT, ZFS_TYPE_POOL, "<file> \| none", "CACHEFILE",
	sfeatures);
	zprop_register_string(ZPOOL_PROP_COMMENT, "comment", NULL,
	PROP_DEFAULT, ZFS_TYPE_POOL, "<comment-string>", "COMMENT",
	sfeatures);
	zprop_register_string(ZPOOL_PROP_COMPATIBILITY, "compatibility",
	"off", PROP_DEFAULT, ZFS_TYPE_POOL,
	"<file[,file...]> \| off \| legacy", "COMPATIBILITY", sfeatures);

	/* readonly number properties */
	zprop_register_number(ZPOOL_PROP_SIZE, "size", 0, PROP_READONLY,
	ZFS_TYPE_POOL, "<size>", "SIZE", B_FALSE, sfeatures);
	zprop_register_number(ZPOOL_PROP_FREE, "free", 0, PROP_READONLY,
	ZFS_TYPE_POOL, "<size>", "FREE", B_FALSE, sfeatures);
	zprop_register_number(ZPOOL_PROP_FREEING, "freeing", 0, PROP_READONLY,
	ZFS_TYPE_POOL, "<size>", "FREEING", B_FALSE, sfeatures);
	zprop_register_number(ZPOOL_PROP_CHECKPOINT, "checkpoint", 0,
	PROP_READONLY, ZFS_TYPE_POOL, "<size>", "CKPOINT", B_FALSE,
	sfeatures);
	zprop_register_number(ZPOOL_PROP_LEAKED, "leaked", 0, PROP_READONLY,
	ZFS_TYPE_POOL, "<size>", "LEAKED", B_FALSE, sfeatures);
	zprop_register_number(ZPOOL_PROP_ALLOCATED, "allocated", 0,
	PROP_READONLY, ZFS_TYPE_POOL, "<size>", "ALLOC", B_FALSE,
	sfeatures);
	zprop_register_number(ZPOOL_PROP_EXPANDSZ, "expandsize", 0,
	PROP_READONLY, ZFS_TYPE_POOL, "<size>", "EXPANDSZ", B_FALSE,
	sfeatures);
	zprop_register_number(ZPOOL_PROP_FRAGMENTATION, "fragmentation", 0,
	PROP_READONLY, ZFS_TYPE_POOL, "<percent>", "FRAG", B_FALSE,
	sfeatures);
	zprop_register_number(ZPOOL_PROP_CAPACITY, "capacity", 0, PROP_READONLY,
	ZFS_TYPE_POOL, "<size>", "CAP", B_FALSE, sfeatures);
	zprop_register_number(ZPOOL_PROP_GUID, "guid", 0, PROP_READONLY,
	ZFS_TYPE_POOL, "<guid>", "GUID", B_TRUE, sfeatures);
	zprop_register_number(ZPOOL_PROP_LOAD_GUID, "load_guid", 0,
	PROP_READONLY, ZFS_TYPE_POOL, "<load_guid>", "LOAD_GUID",
	B_TRUE, sfeatures);
	zprop_register_number(ZPOOL_PROP_HEALTH, "health", 0, PROP_READONLY,
	ZFS_TYPE_POOL, "<state>", "HEALTH", B_FALSE, sfeatures);
	zprop_register_number(ZPOOL_PROP_DEDUPRATIO, "dedupratio", 0,
	PROP_READONLY, ZFS_TYPE_POOL, "<1.00x or higher if deduped>",
	"DEDUP", B_FALSE, sfeatures);
	zprop_register_number(ZPOOL_PROP_BCLONEUSED, "bcloneused", 0,
	PROP_READONLY, ZFS_TYPE_POOL, "<size>",
	"BCLONE_USED", B_FALSE, sfeatures);
	zprop_register_number(ZPOOL_PROP_BCLONESAVED, "bclonesaved", 0,
	PROP_READONLY, ZFS_TYPE_POOL, "<size>",
	"BCLONE_SAVED", B_FALSE, sfeatures);
	zprop_register_number(ZPOOL_PROP_BCLONERATIO, "bcloneratio", 0,
	PROP_READONLY, ZFS_TYPE_POOL, "<1.00x or higher if cloned>",
	"BCLONE_RATIO", B_FALSE, sfeatures);

	/* default number properties */
	zprop_register_number(ZPOOL_PROP_VERSION, "version", SPA_VERSION,
	PROP_DEFAULT, ZFS_TYPE_POOL, "<version>", "VERSION", B_FALSE,
	sfeatures);
	zprop_register_number(ZPOOL_PROP_ASHIFT, "ashift", 0, PROP_DEFAULT,
	ZFS_TYPE_POOL, "<ashift, 9-16, or 0=default>", "ASHIFT", B_FALSE,
	sfeatures);

	/* default index (boolean) properties */
	zprop_register_index(ZPOOL_PROP_DELEGATION, "delegation", 1,
	PROP_DEFAULT, ZFS_TYPE_POOL, "on \| off", "DELEGATION",
	boolean_table, sfeatures);
	zprop_register_index(ZPOOL_PROP_AUTOREPLACE, "autoreplace", 0,
	PROP_DEFAULT, ZFS_TYPE_POOL, "on \| off", "REPLACE", boolean_table,
	sfeatures);
	zprop_register_index(ZPOOL_PROP_LISTSNAPS, "listsnapshots", 0,
	PROP_DEFAULT, ZFS_TYPE_POOL, "on \| off", "LISTSNAPS",
	boolean_table, sfeatures);
	zprop_register_index(ZPOOL_PROP_AUTOEXPAND, "autoexpand", 0,
	PROP_DEFAULT, ZFS_TYPE_POOL, "on \| off", "EXPAND", boolean_table,
	sfeatures);
	zprop_register_index(ZPOOL_PROP_READONLY, "readonly", 0,
	PROP_DEFAULT, ZFS_TYPE_POOL, "on \| off", "RDONLY", boolean_table,
	sfeatures);
	zprop_register_index(ZPOOL_PROP_MULTIHOST, "multihost", 0,
	PROP_DEFAULT, ZFS_TYPE_POOL, "on \| off", "MULTIHOST",
	boolean_table, sfeatures);

	/* default index properties */
	zprop_register_index(ZPOOL_PROP_FAILUREMODE, "failmode",
	ZIO_FAILURE_MODE_WAIT, PROP_DEFAULT, ZFS_TYPE_POOL,
	"wait \| continue \| panic", "FAILMODE", failuremode_table,
	sfeatures);
	zprop_register_index(ZPOOL_PROP_AUTOTRIM, "autotrim",
	SPA_AUTOTRIM_OFF, PROP_DEFAULT, ZFS_TYPE_POOL,
	"on \| off", "AUTOTRIM", boolean_table, sfeatures);

	/* hidden properties */
	zprop_register_hidden(ZPOOL_PROP_NAME, "name", PROP_TYPE_STRING,
	PROP_READONLY, ZFS_TYPE_POOL, "NAME", B_TRUE, sfeatures);
	zprop_register_hidden(ZPOOL_PROP_MAXBLOCKSIZE, "maxblocksize",
	PROP_TYPE_NUMBER, PROP_READONLY, ZFS_TYPE_POOL, "MAXBLOCKSIZE",
	B_FALSE, sfeatures);
	zprop_register_hidden(ZPOOL_PROP_TNAME, "tname", PROP_TYPE_STRING,
	PROP_ONETIME, ZFS_TYPE_POOL, "TNAME", B_TRUE, sfeatures);
	zprop_register_hidden(ZPOOL_PROP_MAXDNODESIZE, "maxdnodesize",
	PROP_TYPE_NUMBER, PROP_READONLY, ZFS_TYPE_POOL, "MAXDNODESIZE",
	B_FALSE, sfeatures);
	zprop_register_hidden(ZPOOL_PROP_DEDUPDITTO, "dedupditto",
	PROP_TYPE_NUMBER, PROP_DEFAULT, ZFS_TYPE_POOL, "DEDUPDITTO",
	B_FALSE, sfeatures);

	zfs_mod_list_supported_free(sfeatures);
	}

	/*
	* Given a property name and its type, returns the corresponding property ID.
	*/
	zpool_prop_t
	zpool_name_to_prop(const char *propname)
	{
	return (zprop_name_to_prop(propname, ZFS_TYPE_POOL));
	}

	/*
	* Given a pool property ID, returns the corresponding name.
	* Assuming the pool property ID is valid.
	*/
	const char *
	zpool_prop_to_name(zpool_prop_t prop)
	{
	return (zpool_prop_table[prop].pd_name);
	}

	zprop_type_t
	zpool_prop_get_type(zpool_prop_t prop)
	{
	return (zpool_prop_table[prop].pd_proptype);
	}

	boolean_t
	zpool_prop_readonly(zpool_prop_t prop)
	{
	return (zpool_prop_table[prop].pd_attr == PROP_READONLY);
	}

	boolean_t
	zpool_prop_setonce(zpool_prop_t prop)
	{
	return (zpool_prop_table[prop].pd_attr == PROP_ONETIME);
	}

	const char *
	zpool_prop_default_string(zpool_prop_t prop)
	{
	return (zpool_prop_table[prop].pd_strdefault);
	}

	uint64_t
	zpool_prop_default_numeric(zpool_prop_t prop)
	{
	return (zpool_prop_table[prop].pd_numdefault);
	}

	/*
	* Returns true if this is a valid feature@ property.
	*/
	boolean_t
	zpool_prop_feature(const char *name)
	{
	static const char *prefix = "feature@";
	return (strncmp(name, prefix, strlen(prefix)) == 0);
	}

	/*
	* Returns true if this is a valid unsupported@ property.
	*/
	boolean_t
	zpool_prop_unsupported(const char *name)
	{
	static const char *prefix = "unsupported@";
	return (strncmp(name, prefix, strlen(prefix)) == 0);
	}

	int
	zpool_prop_string_to_index(zpool_prop_t prop, const char *string,
	uint64_t *index)
	{
	return (zprop_string_to_index(prop, string, index, ZFS_TYPE_POOL));
	}

	int
	zpool_prop_index_to_string(zpool_prop_t prop, uint64_t index,
	const char **string)
	{
	return (zprop_index_to_string(prop, index, string, ZFS_TYPE_POOL));
	}

	uint64_t
	zpool_prop_random_value(zpool_prop_t prop, uint64_t seed)
	{
	return (zprop_random_value(prop, seed, ZFS_TYPE_POOL));
	}

	#ifndef _KERNEL
	#include <libzfs.h>

	const char *
	zpool_prop_values(zpool_prop_t prop)
	{
	return (zpool_prop_table[prop].pd_values);
	}

	const char *
	zpool_prop_column_name(zpool_prop_t prop)
	{
	return (zpool_prop_table[prop].pd_colname);
	}

	boolean_t
	zpool_prop_align_right(zpool_prop_t prop)
	{
	return (zpool_prop_table[prop].pd_rightalign);
	}
	#endif

	zprop_desc_t *
	vdev_prop_get_table(void)
	{
	return (vdev_prop_table);
	}

	void
	vdev_prop_init(void)
	{
	static const zprop_index_t boolean_table[] = {
	{ "off", 0},
	{ "on", 1},
	{ NULL }
	};
	static const zprop_index_t boolean_na_table[] = {
	{ "off", 0},
	{ "on", 1},
	{ "-", 2}, /* ZPROP_BOOLEAN_NA */
	{ NULL }
	};

	struct zfs_mod_supported_features *sfeatures =
	zfs_mod_list_supported(ZFS_SYSFS_VDEV_PROPERTIES);

	/* string properties */
	zprop_register_string(VDEV_PROP_COMMENT, "comment", NULL,
	PROP_DEFAULT, ZFS_TYPE_VDEV, "<comment-string>", "COMMENT",
	sfeatures);
	zprop_register_string(VDEV_PROP_PATH, "path", NULL,
	PROP_DEFAULT, ZFS_TYPE_VDEV, "<device-path>", "PATH", sfeatures);
	zprop_register_string(VDEV_PROP_DEVID, "devid", NULL,
	PROP_READONLY, ZFS_TYPE_VDEV, "<devid>", "DEVID", sfeatures);
	zprop_register_string(VDEV_PROP_PHYS_PATH, "physpath", NULL,
	PROP_READONLY, ZFS_TYPE_VDEV, "<physpath>", "PHYSPATH", sfeatures);
	zprop_register_string(VDEV_PROP_ENC_PATH, "encpath", NULL,
	PROP_READONLY, ZFS_TYPE_VDEV, "<encpath>", "ENCPATH", sfeatures);
	zprop_register_string(VDEV_PROP_FRU, "fru", NULL,
	PROP_READONLY, ZFS_TYPE_VDEV, "<fru>", "FRU", sfeatures);
	zprop_register_string(VDEV_PROP_PARENT, "parent", NULL,
	PROP_READONLY, ZFS_TYPE_VDEV, "<parent>", "PARENT", sfeatures);
	zprop_register_string(VDEV_PROP_CHILDREN, "children", NULL,
	PROP_READONLY, ZFS_TYPE_VDEV, "<child[,...]>", "CHILDREN",
	sfeatures);

	/* readonly number properties */
	zprop_register_number(VDEV_PROP_SIZE, "size", 0, PROP_READONLY,
	ZFS_TYPE_VDEV, "<size>", "SIZE", B_FALSE, sfeatures);
	zprop_register_number(VDEV_PROP_FREE, "free", 0, PROP_READONLY,
	ZFS_TYPE_VDEV, "<size>", "FREE", B_FALSE, sfeatures);
	zprop_register_number(VDEV_PROP_ALLOCATED, "allocated", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<size>", "ALLOC", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_EXPANDSZ, "expandsize", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<size>", "EXPANDSZ", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_FRAGMENTATION, "fragmentation", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<percent>", "FRAG", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_CAPACITY, "capacity", 0, PROP_READONLY,
	ZFS_TYPE_VDEV, "<size>", "CAP", B_FALSE, sfeatures);
	zprop_register_number(VDEV_PROP_GUID, "guid", 0, PROP_READONLY,
	ZFS_TYPE_VDEV, "<guid>", "GUID", B_TRUE, sfeatures);
	zprop_register_number(VDEV_PROP_STATE, "state", 0, PROP_READONLY,
	ZFS_TYPE_VDEV, "<state>", "STATE", B_FALSE, sfeatures);
	zprop_register_number(VDEV_PROP_BOOTSIZE, "bootsize", 0, PROP_READONLY,
	ZFS_TYPE_VDEV, "<size>", "BOOTSIZE", B_FALSE, sfeatures);
	zprop_register_number(VDEV_PROP_ASIZE, "asize", 0, PROP_READONLY,
	ZFS_TYPE_VDEV, "<asize>", "ASIZE", B_FALSE, sfeatures);
	zprop_register_number(VDEV_PROP_PSIZE, "psize", 0, PROP_READONLY,
	ZFS_TYPE_VDEV, "<psize>", "PSIZE", B_FALSE, sfeatures);
	zprop_register_number(VDEV_PROP_ASHIFT, "ashift", 0, PROP_READONLY,
	ZFS_TYPE_VDEV, "<ashift>", "ASHIFT", B_FALSE, sfeatures);
	zprop_register_number(VDEV_PROP_PARITY, "parity", 0, PROP_READONLY,
	ZFS_TYPE_VDEV, "<parity>", "PARITY", B_FALSE, sfeatures);
	zprop_register_number(VDEV_PROP_NUMCHILDREN, "numchildren", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<number-of-children>", "NUMCHILD",
	B_FALSE, sfeatures);
	zprop_register_number(VDEV_PROP_READ_ERRORS, "read_errors", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<errors>", "RDERR", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_WRITE_ERRORS, "write_errors", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<errors>", "WRERR", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_CHECKSUM_ERRORS, "checksum_errors", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<errors>", "CKERR", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_INITIALIZE_ERRORS,
	"initialize_errors", 0, PROP_READONLY, ZFS_TYPE_VDEV, "<errors>",
	"INITERR", B_FALSE, sfeatures);
	zprop_register_number(VDEV_PROP_OPS_NULL, "null_ops", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<operations>", "NULLOP", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_OPS_READ, "read_ops", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<operations>", "READOP", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_OPS_WRITE, "write_ops", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<operations>", "WRITEOP", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_OPS_FREE, "free_ops", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<operations>", "FREEOP", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_OPS_CLAIM, "claim_ops", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<operations>", "CLAIMOP", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_OPS_TRIM, "trim_ops", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<operations>", "TRIMOP", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_BYTES_NULL, "null_bytes", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<bytes>", "NULLBYTE", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_BYTES_READ, "read_bytes", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<bytes>", "READBYTE", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_BYTES_WRITE, "write_bytes", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<bytes>", "WRITEBYTE", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_BYTES_FREE, "free_bytes", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<bytes>", "FREEBYTE", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_BYTES_CLAIM, "claim_bytes", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<bytes>", "CLAIMBYTE", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_BYTES_TRIM, "trim_bytes", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "<bytes>", "TRIMBYTE", B_FALSE,
	sfeatures);

	/* default numeric properties */
	zprop_register_number(VDEV_PROP_CHECKSUM_N, "checksum_n", UINT64_MAX,
	PROP_DEFAULT, ZFS_TYPE_VDEV, "<events>", "CKSUM_N", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_CHECKSUM_T, "checksum_t", UINT64_MAX,
	PROP_DEFAULT, ZFS_TYPE_VDEV, "<seconds>", "CKSUM_T", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_IO_N, "io_n", UINT64_MAX,
	PROP_DEFAULT, ZFS_TYPE_VDEV, "<events>", "IO_N", B_FALSE,
	sfeatures);
	zprop_register_number(VDEV_PROP_IO_T, "io_t", UINT64_MAX,
	PROP_DEFAULT, ZFS_TYPE_VDEV, "<seconds>", "IO_T", B_FALSE,
	sfeatures);
	+ zprop_register_number(VDEV_PROP_SLOW_IO_N, "slow_io_n", UINT64_MAX,
	+ PROP_DEFAULT, ZFS_TYPE_VDEV, "<events>", "SLOW_IO_N", B_FALSE,
	+ sfeatures);
	+ zprop_register_number(VDEV_PROP_SLOW_IO_T, "slow_io_t", UINT64_MAX,
	+ PROP_DEFAULT, ZFS_TYPE_VDEV, "<seconds>", "SLOW_IO_T", B_FALSE,
	+ sfeatures);

	/* default index (boolean) properties */
	zprop_register_index(VDEV_PROP_REMOVING, "removing", 0,
	PROP_READONLY, ZFS_TYPE_VDEV, "on \| off", "REMOVING",
	boolean_table, sfeatures);
	zprop_register_index(VDEV_PROP_ALLOCATING, "allocating", 1,
	PROP_DEFAULT, ZFS_TYPE_VDEV, "on \| off", "ALLOCATING",
	boolean_na_table, sfeatures);

	/* default index properties */
	zprop_register_index(VDEV_PROP_FAILFAST, "failfast", B_TRUE,
	PROP_DEFAULT, ZFS_TYPE_VDEV, "on \| off", "FAILFAST", boolean_table,
	sfeatures);

	/* hidden properties */
	zprop_register_hidden(VDEV_PROP_NAME, "name", PROP_TYPE_STRING,
	PROP_READONLY, ZFS_TYPE_VDEV, "NAME", B_TRUE, sfeatures);

	zfs_mod_list_supported_free(sfeatures);
	}

	/*
	* Given a property name and its type, returns the corresponding property ID.
	*/
	vdev_prop_t
	vdev_name_to_prop(const char *propname)
	{
	return (zprop_name_to_prop(propname, ZFS_TYPE_VDEV));
	}

	/*
	* Returns true if this is a valid user-defined property (one with a ':').
	*/
	boolean_t
	vdev_prop_user(const char *name)
	{
	int i;
	char c;
	boolean_t foundsep = B_FALSE;

	for (i = 0; i < strlen(name); i++) {
	c = name[i];
	if (!zprop_valid_char(c))
	return (B_FALSE);
	if (c == ':')
	foundsep = B_TRUE;
	}

	return (foundsep);
	}

	/*
	* Given a pool property ID, returns the corresponding name.
	* Assuming the pool property ID is valid.
	*/
	const char *
	vdev_prop_to_name(vdev_prop_t prop)
	{
	return (vdev_prop_table[prop].pd_name);
	}

	zprop_type_t
	vdev_prop_get_type(vdev_prop_t prop)
	{
	return (vdev_prop_table[prop].pd_proptype);
	}

	boolean_t
	vdev_prop_readonly(vdev_prop_t prop)
	{
	return (vdev_prop_table[prop].pd_attr == PROP_READONLY);
	}

	const char *
	vdev_prop_default_string(vdev_prop_t prop)
	{
	return (vdev_prop_table[prop].pd_strdefault);
	}

	uint64_t
	vdev_prop_default_numeric(vdev_prop_t prop)
	{
	return (vdev_prop_table[prop].pd_numdefault);
	}

	int
	vdev_prop_string_to_index(vdev_prop_t prop, const char *string,
	uint64_t *index)
	{
	return (zprop_string_to_index(prop, string, index, ZFS_TYPE_VDEV));
	}

	int
	vdev_prop_index_to_string(vdev_prop_t prop, uint64_t index,
	const char **string)
	{
	return (zprop_index_to_string(prop, index, string, ZFS_TYPE_VDEV));
	}

	/*
	* Returns true if this is a valid vdev property.
	*/
	boolean_t
	zpool_prop_vdev(const char *name)
	{
	return (vdev_name_to_prop(name) != VDEV_PROP_INVAL);
	}

	uint64_t
	vdev_prop_random_value(vdev_prop_t prop, uint64_t seed)
	{
	return (zprop_random_value(prop, seed, ZFS_TYPE_VDEV));
	}

	#ifndef _KERNEL
	const char *
	vdev_prop_values(vdev_prop_t prop)
	{
	return (vdev_prop_table[prop].pd_values);
	}

	const char *
	vdev_prop_column_name(vdev_prop_t prop)
	{
	return (vdev_prop_table[prop].pd_colname);
	}

	boolean_t
	vdev_prop_align_right(vdev_prop_t prop)
	{
	return (vdev_prop_table[prop].pd_rightalign);
	}
	#endif

	#if defined(_KERNEL)
	/* zpool property functions */
	EXPORT_SYMBOL(zpool_prop_init);
	EXPORT_SYMBOL(zpool_prop_get_type);
	EXPORT_SYMBOL(zpool_prop_get_table);

	/* vdev property functions */
	EXPORT_SYMBOL(vdev_prop_init);
	EXPORT_SYMBOL(vdev_prop_get_type);
	EXPORT_SYMBOL(vdev_prop_get_table);

	/* Pool property functions shared between libzfs and kernel. */
	EXPORT_SYMBOL(zpool_name_to_prop);
	EXPORT_SYMBOL(zpool_prop_to_name);
	EXPORT_SYMBOL(zpool_prop_default_string);
	EXPORT_SYMBOL(zpool_prop_default_numeric);
	EXPORT_SYMBOL(zpool_prop_readonly);
	EXPORT_SYMBOL(zpool_prop_feature);
	EXPORT_SYMBOL(zpool_prop_unsupported);
	EXPORT_SYMBOL(zpool_prop_index_to_string);
	EXPORT_SYMBOL(zpool_prop_string_to_index);
	EXPORT_SYMBOL(zpool_prop_vdev);

	/* vdev property functions shared between libzfs and kernel. */
	EXPORT_SYMBOL(vdev_name_to_prop);
	EXPORT_SYMBOL(vdev_prop_user);
	EXPORT_SYMBOL(vdev_prop_to_name);
	EXPORT_SYMBOL(vdev_prop_default_string);
	EXPORT_SYMBOL(vdev_prop_default_numeric);
	EXPORT_SYMBOL(vdev_prop_readonly);
	EXPORT_SYMBOL(vdev_prop_index_to_string);
	EXPORT_SYMBOL(vdev_prop_string_to_index);
	#endif
	diff --git a/sys/contrib/openzfs/module/zfs/abd.c b/sys/contrib/openzfs/module/zfs/abd.c
	index d982f201c930..3388e2357305 100644
	--- a/sys/contrib/openzfs/module/zfs/abd.c
	+++ b/sys/contrib/openzfs/module/zfs/abd.c
	@@ -1,1177 +1,1219 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2014 by Chunwei Chen. All rights reserved.
	* Copyright (c) 2019 by Delphix. All rights reserved.
	*/

	/*
	* ARC buffer data (ABD).
	*
	* ABDs are an abstract data structure for the ARC which can use two
	* different ways of storing the underlying data:
	*
	* (a) Linear buffer. In this case, all the data in the ABD is stored in one
	* contiguous buffer in memory (from a zio_[data_]buf_* kmem cache).
	*
	* +-------------------+
	* \| ABD (linear) \|
	* \| abd_flags = ... \|
	* \| abd_size = ... \| +--------------------------------+
	* \| abd_buf ------------->\| raw buffer of size abd_size \|
	* +-------------------+ +--------------------------------+
	* no abd_chunks
	*
	* (b) Scattered buffer. In this case, the data in the ABD is split into
	* equal-sized chunks (from the abd_chunk_cache kmem_cache), with pointers
	* to the chunks recorded in an array at the end of the ABD structure.
	*
	* +-------------------+
	* \| ABD (scattered) \|
	* \| abd_flags = ... \|
	* \| abd_size = ... \|
	* \| abd_offset = 0 \| +-----------+
	* \| abd_chunks[0] ----------------------------->\| chunk 0 \|
	* \| abd_chunks[1] ---------------------+ +-----------+
	* \| ... \| \| +-----------+
	* \| abd_chunks[N-1] ---------+ +------->\| chunk 1 \|
	* +-------------------+ \| +-----------+
	* \| ...
	* \| +-----------+
	* +----------------->\| chunk N-1 \|
	* +-----------+
	*
	* In addition to directly allocating a linear or scattered ABD, it is also
	* possible to create an ABD by requesting the "sub-ABD" starting at an offset
	* within an existing ABD. In linear buffers this is simple (set abd_buf of
	* the new ABD to the starting point within the original raw buffer), but
	* scattered ABDs are a little more complex. The new ABD makes a copy of the
	* relevant abd_chunks pointers (but not the underlying data). However, to
	* provide arbitrary rather than only chunk-aligned starting offsets, it also
	* tracks an abd_offset field which represents the starting point of the data
	* within the first chunk in abd_chunks. For both linear and scattered ABDs,
	* creating an offset ABD marks the original ABD as the offset's parent, and the
	* original ABD's abd_children refcount is incremented. This data allows us to
	* ensure the root ABD isn't deleted before its children.
	*
	* Most consumers should never need to know what type of ABD they're using --
	* the ABD public API ensures that it's possible to transparently switch from
	* using a linear ABD to a scattered one when doing so would be beneficial.
	*
	* If you need to use the data within an ABD directly, if you know it's linear
	* (because you allocated it) you can use abd_to_buf() to access the underlying
	* raw buffer. Otherwise, you should use one of the abd_borrow_buf* functions
	* which will allocate a raw buffer if necessary. Use the abd_return_buf*
	* functions to return any raw buffers that are no longer necessary when you're
	* done using them.
	*
	* There are a variety of ABD APIs that implement basic buffer operations:
	* compare, copy, read, write, and fill with zeroes. If you need a custom
	* function which progressively accesses the whole ABD, use the abd_iterate_*
	* functions.
	*
	* As an additional feature, linear and scatter ABD's can be stitched together
	* by using the gang ABD type (abd_alloc_gang_abd()). This allows for
	* multiple ABDs to be viewed as a singular ABD.
	*
	* It is possible to make all ABDs linear by setting zfs_abd_scatter_enabled to
	* B_FALSE.
	*/

	#include <sys/abd_impl.h>
	#include <sys/param.h>
	#include <sys/zio.h>
	#include <sys/zfs_context.h>
	#include <sys/zfs_znode.h>

	/* see block comment above for description */
	int zfs_abd_scatter_enabled = B_TRUE;

	void
	abd_verify(abd_t *abd)
	{
	#ifdef ZFS_DEBUG
	ASSERT3U(abd->abd_size, <=, SPA_MAXBLOCKSIZE);
	ASSERT3U(abd->abd_flags, ==, abd->abd_flags & (ABD_FLAG_LINEAR \|
	ABD_FLAG_OWNER \| ABD_FLAG_META \| ABD_FLAG_MULTI_ZONE \|
	ABD_FLAG_MULTI_CHUNK \| ABD_FLAG_LINEAR_PAGE \| ABD_FLAG_GANG \|
	ABD_FLAG_GANG_FREE \| ABD_FLAG_ZEROS \| ABD_FLAG_ALLOCD));
	IMPLY(abd->abd_parent != NULL, !(abd->abd_flags & ABD_FLAG_OWNER));
	IMPLY(abd->abd_flags & ABD_FLAG_META, abd->abd_flags & ABD_FLAG_OWNER);
	if (abd_is_linear(abd)) {
	ASSERT3U(abd->abd_size, >, 0);
	ASSERT3P(ABD_LINEAR_BUF(abd), !=, NULL);
	} else if (abd_is_gang(abd)) {
	uint_t child_sizes = 0;
	for (abd_t *cabd = list_head(&ABD_GANG(abd).abd_gang_chain);
	cabd != NULL;
	cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
	ASSERT(list_link_active(&cabd->abd_gang_link));
	child_sizes += cabd->abd_size;
	abd_verify(cabd);
	}
	ASSERT3U(abd->abd_size, ==, child_sizes);
	} else {
	ASSERT3U(abd->abd_size, >, 0);
	abd_verify_scatter(abd);
	}
	#endif
	}

	static void
	abd_init_struct(abd_t *abd)
	{
	list_link_init(&abd->abd_gang_link);
	mutex_init(&abd->abd_mtx, NULL, MUTEX_DEFAULT, NULL);
	abd->abd_flags = 0;
	#ifdef ZFS_DEBUG
	zfs_refcount_create(&abd->abd_children);
	abd->abd_parent = NULL;
	#endif
	abd->abd_size = 0;
	}

	static void
	abd_fini_struct(abd_t *abd)
	{
	mutex_destroy(&abd->abd_mtx);
	ASSERT(!list_link_active(&abd->abd_gang_link));
	#ifdef ZFS_DEBUG
	zfs_refcount_destroy(&abd->abd_children);
	#endif
	}

	abd_t *
	abd_alloc_struct(size_t size)
	{
	abd_t *abd = abd_alloc_struct_impl(size);
	abd_init_struct(abd);
	abd->abd_flags \|= ABD_FLAG_ALLOCD;
	return (abd);
	}

	void
	abd_free_struct(abd_t *abd)
	{
	abd_fini_struct(abd);
	abd_free_struct_impl(abd);
	}

	/*
	* Allocate an ABD, along with its own underlying data buffers. Use this if you
	* don't care whether the ABD is linear or not.
	*/
	abd_t *
	abd_alloc(size_t size, boolean_t is_metadata)
	{
	if (abd_size_alloc_linear(size))
	return (abd_alloc_linear(size, is_metadata));

	VERIFY3U(size, <=, SPA_MAXBLOCKSIZE);

	abd_t *abd = abd_alloc_struct(size);
	abd->abd_flags \|= ABD_FLAG_OWNER;
	abd->abd_u.abd_scatter.abd_offset = 0;
	abd_alloc_chunks(abd, size);

	if (is_metadata) {
	abd->abd_flags \|= ABD_FLAG_META;
	}
	abd->abd_size = size;

	abd_update_scatter_stats(abd, ABDSTAT_INCR);

	return (abd);
	}

	/*
	* Allocate an ABD that must be linear, along with its own underlying data
	* buffer. Only use this when it would be very annoying to write your ABD
	* consumer with a scattered ABD.
	*/
	abd_t *
	abd_alloc_linear(size_t size, boolean_t is_metadata)
	{
	abd_t *abd = abd_alloc_struct(0);

	VERIFY3U(size, <=, SPA_MAXBLOCKSIZE);

	abd->abd_flags \|= ABD_FLAG_LINEAR \| ABD_FLAG_OWNER;
	if (is_metadata) {
	abd->abd_flags \|= ABD_FLAG_META;
	}
	abd->abd_size = size;

	if (is_metadata) {
	ABD_LINEAR_BUF(abd) = zio_buf_alloc(size);
	} else {
	ABD_LINEAR_BUF(abd) = zio_data_buf_alloc(size);
	}

	abd_update_linear_stats(abd, ABDSTAT_INCR);

	return (abd);
	}

	static void
	abd_free_linear(abd_t *abd)
	{
	if (abd_is_linear_page(abd)) {
	abd_free_linear_page(abd);
	return;
	}
	if (abd->abd_flags & ABD_FLAG_META) {
	zio_buf_free(ABD_LINEAR_BUF(abd), abd->abd_size);
	} else {
	zio_data_buf_free(ABD_LINEAR_BUF(abd), abd->abd_size);
	}

	abd_update_linear_stats(abd, ABDSTAT_DECR);
	}

	static void
	abd_free_gang(abd_t *abd)
	{
	ASSERT(abd_is_gang(abd));
	abd_t *cabd;

	while ((cabd = list_head(&ABD_GANG(abd).abd_gang_chain)) != NULL) {
	/*
	* We must acquire the child ABDs mutex to ensure that if it
	* is being added to another gang ABD we will set the link
	* as inactive when removing it from this gang ABD and before
	* adding it to the other gang ABD.
	*/
	mutex_enter(&cabd->abd_mtx);
	ASSERT(list_link_active(&cabd->abd_gang_link));
	list_remove(&ABD_GANG(abd).abd_gang_chain, cabd);
	mutex_exit(&cabd->abd_mtx);
	if (cabd->abd_flags & ABD_FLAG_GANG_FREE)
	abd_free(cabd);
	}
	list_destroy(&ABD_GANG(abd).abd_gang_chain);
	}

	static void
	abd_free_scatter(abd_t *abd)
	{
	abd_free_chunks(abd);
	abd_update_scatter_stats(abd, ABDSTAT_DECR);
	}

	/*
	* Free an ABD. Use with any kind of abd: those created with abd_alloc_*()
	* and abd_get_*(), including abd_get_offset_struct().
	*
	* If the ABD was created with abd_alloc_*(), the underlying data
	* (scatterlist or linear buffer) will also be freed. (Subject to ownership
	* changes via abd_*_ownership_of_buf().)
	*
	* Unless the ABD was created with abd_get_offset_struct(), the abd_t will
	* also be freed.
	*/
	void
	abd_free(abd_t *abd)
	{
	if (abd == NULL)
	return;

	abd_verify(abd);
	#ifdef ZFS_DEBUG
	IMPLY(abd->abd_flags & ABD_FLAG_OWNER, abd->abd_parent == NULL);
	#endif

	if (abd_is_gang(abd)) {
	abd_free_gang(abd);
	} else if (abd_is_linear(abd)) {
	if (abd->abd_flags & ABD_FLAG_OWNER)
	abd_free_linear(abd);
	} else {
	if (abd->abd_flags & ABD_FLAG_OWNER)
	abd_free_scatter(abd);
	}

	#ifdef ZFS_DEBUG
	if (abd->abd_parent != NULL) {
	(void) zfs_refcount_remove_many(&abd->abd_parent->abd_children,
	abd->abd_size, abd);
	}
	#endif

	abd_fini_struct(abd);
	if (abd->abd_flags & ABD_FLAG_ALLOCD)
	abd_free_struct_impl(abd);
	}

	/*
	* Allocate an ABD of the same format (same metadata flag, same scatterize
	* setting) as another ABD.
	*/
	abd_t *
	abd_alloc_sametype(abd_t *sabd, size_t size)
	{
	boolean_t is_metadata = (sabd->abd_flags & ABD_FLAG_META) != 0;
	if (abd_is_linear(sabd) &&
	!abd_is_linear_page(sabd)) {
	return (abd_alloc_linear(size, is_metadata));
	} else {
	return (abd_alloc(size, is_metadata));
	}
	}

	/*
	* Create gang ABD that will be the head of a list of ABD's. This is used
	* to "chain" scatter/gather lists together when constructing aggregated
	* IO's. To free this abd, abd_free() must be called.
	*/
	abd_t *
	abd_alloc_gang(void)
	{
	abd_t *abd = abd_alloc_struct(0);
	abd->abd_flags \|= ABD_FLAG_GANG \| ABD_FLAG_OWNER;
	list_create(&ABD_GANG(abd).abd_gang_chain,
	sizeof (abd_t), offsetof(abd_t, abd_gang_link));
	return (abd);
	}

	/*
	* Add a child gang ABD to a parent gang ABDs chained list.
	*/
	static void
	abd_gang_add_gang(abd_t pabd, abd_t cabd, boolean_t free_on_free)
	{
	ASSERT(abd_is_gang(pabd));
	ASSERT(abd_is_gang(cabd));

	if (free_on_free) {
	/*
	* If the parent is responsible for freeing the child gang
	* ABD we will just splice the child's children ABD list to
	* the parent's list and immediately free the child gang ABD
	* struct. The parent gang ABDs children from the child gang
	* will retain all the free_on_free settings after being
	* added to the parents list.
	*/
	#ifdef ZFS_DEBUG
	/*
	* If cabd had abd_parent, we have to drop it here. We can't
	* transfer it to pabd, nor we can clear abd_size leaving it.
	*/
	if (cabd->abd_parent != NULL) {
	(void) zfs_refcount_remove_many(
	&cabd->abd_parent->abd_children,
	cabd->abd_size, cabd);
	cabd->abd_parent = NULL;
	}
	#endif
	pabd->abd_size += cabd->abd_size;
	cabd->abd_size = 0;
	list_move_tail(&ABD_GANG(pabd).abd_gang_chain,
	&ABD_GANG(cabd).abd_gang_chain);
	ASSERT(list_is_empty(&ABD_GANG(cabd).abd_gang_chain));
	abd_verify(pabd);
	abd_free(cabd);
	} else {
	for (abd_t *child = list_head(&ABD_GANG(cabd).abd_gang_chain);
	child != NULL;
	child = list_next(&ABD_GANG(cabd).abd_gang_chain, child)) {
	/*
	* We always pass B_FALSE for free_on_free as it is the
	* original child gang ABDs responsibility to determine
	* if any of its child ABDs should be free'd on the call
	* to abd_free().
	*/
	abd_gang_add(pabd, child, B_FALSE);
	}
	abd_verify(pabd);
	}
	}

	/*
	* Add a child ABD to a gang ABD's chained list.
	*/
	void
	abd_gang_add(abd_t pabd, abd_t cabd, boolean_t free_on_free)
	{
	ASSERT(abd_is_gang(pabd));
	abd_t *child_abd = NULL;

	/*
	* If the child being added is a gang ABD, we will add the
	* child's ABDs to the parent gang ABD. This allows us to account
	* for the offset correctly in the parent gang ABD.
	*/
	if (abd_is_gang(cabd)) {
	ASSERT(!list_link_active(&cabd->abd_gang_link));
	return (abd_gang_add_gang(pabd, cabd, free_on_free));
	}
	ASSERT(!abd_is_gang(cabd));

	/*
	* In order to verify that an ABD is not already part of
	* another gang ABD, we must lock the child ABD's abd_mtx
	* to check its abd_gang_link status. We unlock the abd_mtx
	* only after it is has been added to a gang ABD, which
	* will update the abd_gang_link's status. See comment below
	* for how an ABD can be in multiple gang ABD's simultaneously.
	*/
	mutex_enter(&cabd->abd_mtx);
	if (list_link_active(&cabd->abd_gang_link)) {
	/*
	* If the child ABD is already part of another
	* gang ABD then we must allocate a new
	* ABD to use a separate link. We mark the newly
	* allocated ABD with ABD_FLAG_GANG_FREE, before
	* adding it to the gang ABD's list, to make the
	* gang ABD aware that it is responsible to call
	* abd_free(). We use abd_get_offset() in order
	* to just allocate a new ABD but avoid copying the
	* data over into the newly allocated ABD.
	*
	* An ABD may become part of multiple gang ABD's. For
	* example, when writing ditto bocks, the same ABD
	* is used to write 2 or 3 locations with 2 or 3
	* zio_t's. Each of the zio's may be aggregated with
	* different adjacent zio's. zio aggregation uses gang
	* zio's, so the single ABD can become part of multiple
	* gang zio's.
	*
	* The ASSERT below is to make sure that if
	* free_on_free is passed as B_TRUE, the ABD can
	* not be in multiple gang ABD's. The gang ABD
	* can not be responsible for cleaning up the child
	* ABD memory allocation if the ABD can be in
	* multiple gang ABD's at one time.
	*/
	ASSERT3B(free_on_free, ==, B_FALSE);
	child_abd = abd_get_offset(cabd, 0);
	child_abd->abd_flags \|= ABD_FLAG_GANG_FREE;
	} else {
	child_abd = cabd;
	if (free_on_free)
	child_abd->abd_flags \|= ABD_FLAG_GANG_FREE;
	}
	ASSERT3P(child_abd, !=, NULL);

	list_insert_tail(&ABD_GANG(pabd).abd_gang_chain, child_abd);
	mutex_exit(&cabd->abd_mtx);
	pabd->abd_size += child_abd->abd_size;
	}

	/*
	* Locate the ABD for the supplied offset in the gang ABD.
	* Return a new offset relative to the returned ABD.
	*/
	abd_t *
	abd_gang_get_offset(abd_t abd, size_t off)
	{
	abd_t *cabd;

	ASSERT(abd_is_gang(abd));
	ASSERT3U(*off, <, abd->abd_size);
	for (cabd = list_head(&ABD_GANG(abd).abd_gang_chain); cabd != NULL;
	cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
	if (*off >= cabd->abd_size)
	*off -= cabd->abd_size;
	else
	return (cabd);
	}
	VERIFY3P(cabd, !=, NULL);
	return (cabd);
	}

	/*
	* Allocate a new ABD, using the provided struct (if non-NULL, and if
	* circumstances allow - otherwise allocate the struct). The returned ABD will
	* point to offset off of sabd. It shares the underlying buffer data with sabd.
	* Use abd_free() to free. sabd must not be freed while any derived ABDs exist.
	*/
	static abd_t *
	abd_get_offset_impl(abd_t abd, abd_t sabd, size_t off, size_t size)
	{
	abd_verify(sabd);
	ASSERT3U(off + size, <=, sabd->abd_size);

	if (abd_is_linear(sabd)) {
	if (abd == NULL)
	abd = abd_alloc_struct(0);
	/*
	* Even if this buf is filesystem metadata, we only track that
	* if we own the underlying data buffer, which is not true in
	* this case. Therefore, we don't ever use ABD_FLAG_META here.
	*/
	abd->abd_flags \|= ABD_FLAG_LINEAR;

	ABD_LINEAR_BUF(abd) = (char *)ABD_LINEAR_BUF(sabd) + off;
	} else if (abd_is_gang(sabd)) {
	size_t left = size;
	if (abd == NULL) {
	abd = abd_alloc_gang();
	} else {
	abd->abd_flags \|= ABD_FLAG_GANG;
	list_create(&ABD_GANG(abd).abd_gang_chain,
	sizeof (abd_t), offsetof(abd_t, abd_gang_link));
	}

	abd->abd_flags &= ~ABD_FLAG_OWNER;
	for (abd_t *cabd = abd_gang_get_offset(sabd, &off);
	cabd != NULL && left > 0;
	cabd = list_next(&ABD_GANG(sabd).abd_gang_chain, cabd)) {
	int csize = MIN(left, cabd->abd_size - off);

	abd_t *nabd = abd_get_offset_size(cabd, off, csize);
	abd_gang_add(abd, nabd, B_TRUE);
	left -= csize;
	off = 0;
	}
	ASSERT3U(left, ==, 0);
	} else {
	abd = abd_get_offset_scatter(abd, sabd, off, size);
	}

	ASSERT3P(abd, !=, NULL);
	abd->abd_size = size;
	#ifdef ZFS_DEBUG
	abd->abd_parent = sabd;
	(void) zfs_refcount_add_many(&sabd->abd_children, abd->abd_size, abd);
	#endif
	return (abd);
	}

	/*
	* Like abd_get_offset_size(), but memory for the abd_t is provided by the
	* caller. Using this routine can improve performance by avoiding the cost
	* of allocating memory for the abd_t struct, and updating the abd stats.
	* Usually, the provided abd is returned, but in some circumstances (FreeBSD,
	* if sabd is scatter and size is more than 2 pages) a new abd_t may need to
	* be allocated. Therefore callers should be careful to use the returned
	* abd_t*.
	*/
	abd_t *
	abd_get_offset_struct(abd_t abd, abd_t sabd, size_t off, size_t size)
	{
	abd_t *result;
	abd_init_struct(abd);
	result = abd_get_offset_impl(abd, sabd, off, size);
	if (result != abd)
	abd_fini_struct(abd);
	return (result);
	}

	abd_t *
	abd_get_offset(abd_t *sabd, size_t off)
	{
	size_t size = sabd->abd_size > off ? sabd->abd_size - off : 0;
	VERIFY3U(size, >, 0);
	return (abd_get_offset_impl(NULL, sabd, off, size));
	}

	abd_t *
	abd_get_offset_size(abd_t *sabd, size_t off, size_t size)
	{
	ASSERT3U(off + size, <=, sabd->abd_size);
	return (abd_get_offset_impl(NULL, sabd, off, size));
	}

	/*
	* Return a size scatter ABD containing only zeros.
	*/
	abd_t *
	abd_get_zeros(size_t size)
	{
	ASSERT3P(abd_zero_scatter, !=, NULL);
	ASSERT3U(size, <=, SPA_MAXBLOCKSIZE);
	return (abd_get_offset_size(abd_zero_scatter, 0, size));
	}

	/*
	* Allocate a linear ABD structure for buf.
	*/
	abd_t *
	abd_get_from_buf(void *buf, size_t size)
	{
	abd_t *abd = abd_alloc_struct(0);

	VERIFY3U(size, <=, SPA_MAXBLOCKSIZE);

	/*
	* Even if this buf is filesystem metadata, we only track that if we
	* own the underlying data buffer, which is not true in this case.
	* Therefore, we don't ever use ABD_FLAG_META here.
	*/
	abd->abd_flags \|= ABD_FLAG_LINEAR;
	abd->abd_size = size;

	ABD_LINEAR_BUF(abd) = buf;

	return (abd);
	}

	/*
	* Get the raw buffer associated with a linear ABD.
	*/
	void *
	abd_to_buf(abd_t *abd)
	{
	ASSERT(abd_is_linear(abd));
	abd_verify(abd);
	return (ABD_LINEAR_BUF(abd));
	}

	/*
	* Borrow a raw buffer from an ABD without copying the contents of the ABD
	* into the buffer. If the ABD is scattered, this will allocate a raw buffer
	* whose contents are undefined. To copy over the existing data in the ABD, use
	* abd_borrow_buf_copy() instead.
	*/
	void *
	abd_borrow_buf(abd_t *abd, size_t n)
	{
	void *buf;
	abd_verify(abd);
	ASSERT3U(abd->abd_size, >=, n);
	if (abd_is_linear(abd)) {
	buf = abd_to_buf(abd);
	} else {
	buf = zio_buf_alloc(n);
	}
	#ifdef ZFS_DEBUG
	(void) zfs_refcount_add_many(&abd->abd_children, n, buf);
	#endif
	return (buf);
	}

	void *
	abd_borrow_buf_copy(abd_t *abd, size_t n)
	{
	void *buf = abd_borrow_buf(abd, n);
	if (!abd_is_linear(abd)) {
	abd_copy_to_buf(buf, abd, n);
	}
	return (buf);
	}

	/*
	* Return a borrowed raw buffer to an ABD. If the ABD is scattered, this will
	* not change the contents of the ABD and will ASSERT that you didn't modify
	* the buffer since it was borrowed. If you want any changes you made to buf to
	* be copied back to abd, use abd_return_buf_copy() instead.
	*/
	void
	abd_return_buf(abd_t abd, void buf, size_t n)
	{
	abd_verify(abd);
	ASSERT3U(abd->abd_size, >=, n);
	#ifdef ZFS_DEBUG
	(void) zfs_refcount_remove_many(&abd->abd_children, n, buf);
	#endif
	if (abd_is_linear(abd)) {
	ASSERT3P(buf, ==, abd_to_buf(abd));
	} else {
	ASSERT0(abd_cmp_buf(abd, buf, n));
	zio_buf_free(buf, n);
	}
	}

	void
	abd_return_buf_copy(abd_t abd, void buf, size_t n)
	{
	if (!abd_is_linear(abd)) {
	abd_copy_from_buf(abd, buf, n);
	}
	abd_return_buf(abd, buf, n);
	}

	void
	abd_release_ownership_of_buf(abd_t *abd)
	{
	ASSERT(abd_is_linear(abd));
	ASSERT(abd->abd_flags & ABD_FLAG_OWNER);

	/*
	* abd_free() needs to handle LINEAR_PAGE ABD's specially.
	* Since that flag does not survive the
	* abd_release_ownership_of_buf() -> abd_get_from_buf() ->
	* abd_take_ownership_of_buf() sequence, we don't allow releasing
	* these "linear but not zio_[data_]buf_alloc()'ed" ABD's.
	*/
	ASSERT(!abd_is_linear_page(abd));

	abd_verify(abd);

	abd->abd_flags &= ~ABD_FLAG_OWNER;
	/* Disable this flag since we no longer own the data buffer */
	abd->abd_flags &= ~ABD_FLAG_META;

	abd_update_linear_stats(abd, ABDSTAT_DECR);
	}


	/*
	* Give this ABD ownership of the buffer that it's storing. Can only be used on
	* linear ABDs which were allocated via abd_get_from_buf(), or ones allocated
	* with abd_alloc_linear() which subsequently released ownership of their buf
	* with abd_release_ownership_of_buf().
	*/
	void
	abd_take_ownership_of_buf(abd_t *abd, boolean_t is_metadata)
	{
	ASSERT(abd_is_linear(abd));
	ASSERT(!(abd->abd_flags & ABD_FLAG_OWNER));
	abd_verify(abd);

	abd->abd_flags \|= ABD_FLAG_OWNER;
	if (is_metadata) {
	abd->abd_flags \|= ABD_FLAG_META;
	}

	abd_update_linear_stats(abd, ABDSTAT_INCR);
	}

	/*
	* Initializes an abd_iter based on whether the abd is a gang ABD
	* or just a single ABD.
	*/
	static inline abd_t *
	abd_init_abd_iter(abd_t abd, struct abd_iter aiter, size_t off)
	{
	abd_t *cabd = NULL;

	if (abd_is_gang(abd)) {
	cabd = abd_gang_get_offset(abd, &off);
	if (cabd) {
	abd_iter_init(aiter, cabd);
	abd_iter_advance(aiter, off);
	}
	} else {
	abd_iter_init(aiter, abd);
	abd_iter_advance(aiter, off);
	}
	return (cabd);
	}

	/*
	* Advances an abd_iter. We have to be careful with gang ABD as
	* advancing could mean that we are at the end of a particular ABD and
	* must grab the ABD in the gang ABD's list.
	*/
	static inline abd_t *
	abd_advance_abd_iter(abd_t abd, abd_t cabd, struct abd_iter *aiter,
	size_t len)
	{
	abd_iter_advance(aiter, len);
	if (abd_is_gang(abd) && abd_iter_at_end(aiter)) {
	ASSERT3P(cabd, !=, NULL);
	cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd);
	if (cabd) {
	abd_iter_init(aiter, cabd);
	abd_iter_advance(aiter, 0);
	}
	}
	return (cabd);
	}

	int
	abd_iterate_func(abd_t *abd, size_t off, size_t size,
	abd_iter_func_t func, void private)
	{
	struct abd_iter aiter;
	int ret = 0;

	if (size == 0)
	return (0);

	abd_verify(abd);
	ASSERT3U(off + size, <=, abd->abd_size);

	abd_t *c_abd = abd_init_abd_iter(abd, &aiter, off);

	while (size > 0) {
	IMPLY(abd_is_gang(abd), c_abd != NULL);

	abd_iter_map(&aiter);

	size_t len = MIN(aiter.iter_mapsize, size);
	ASSERT3U(len, >, 0);

	ret = func(aiter.iter_mapaddr, len, private);

	abd_iter_unmap(&aiter);

	if (ret != 0)
	break;

	size -= len;
	c_abd = abd_advance_abd_iter(abd, c_abd, &aiter, len);
	}

	return (ret);
	}

	+#if defined(__linux__) && defined(_KERNEL)
	+int
	+abd_iterate_page_func(abd_t *abd, size_t off, size_t size,
	+ abd_iter_page_func_t func, void private)
	+{
	+ struct abd_iter aiter;
	+ int ret = 0;
	+
	+ if (size == 0)
	+ return (0);
	+
	+ abd_verify(abd);
	+ ASSERT3U(off + size, <=, abd->abd_size);
	+
	+ abd_t *c_abd = abd_init_abd_iter(abd, &aiter, off);
	+
	+ while (size > 0) {
	+ IMPLY(abd_is_gang(abd), c_abd != NULL);
	+
	+ abd_iter_page(&aiter);
	+
	+ size_t len = MIN(aiter.iter_page_dsize, size);
	+ ASSERT3U(len, >, 0);
	+
	+ ret = func(aiter.iter_page, aiter.iter_page_doff,
	+ len, private);
	+
	+ aiter.iter_page = NULL;
	+ aiter.iter_page_doff = 0;
	+ aiter.iter_page_dsize = 0;
	+
	+ if (ret != 0)
	+ break;
	+
	+ size -= len;
	+ c_abd = abd_advance_abd_iter(abd, c_abd, &aiter, len);
	+ }
	+
	+ return (ret);
	+}
	+#endif
	+
	struct buf_arg {
	void *arg_buf;
	};

	static int
	abd_copy_to_buf_off_cb(void buf, size_t size, void private)
	{
	struct buf_arg *ba_ptr = private;

	(void) memcpy(ba_ptr->arg_buf, buf, size);
	ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size;

	return (0);
	}

	/*
	* Copy abd to buf. (off is the offset in abd.)
	*/
	void
	abd_copy_to_buf_off(void buf, abd_t abd, size_t off, size_t size)
	{
	struct buf_arg ba_ptr = { buf };

	(void) abd_iterate_func(abd, off, size, abd_copy_to_buf_off_cb,
	&ba_ptr);
	}

	static int
	abd_cmp_buf_off_cb(void buf, size_t size, void private)
	{
	int ret;
	struct buf_arg *ba_ptr = private;

	ret = memcmp(buf, ba_ptr->arg_buf, size);
	ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size;

	return (ret);
	}

	/*
	* Compare the contents of abd to buf. (off is the offset in abd.)
	*/
	int
	abd_cmp_buf_off(abd_t abd, const void buf, size_t off, size_t size)
	{
	struct buf_arg ba_ptr = { (void *) buf };

	return (abd_iterate_func(abd, off, size, abd_cmp_buf_off_cb, &ba_ptr));
	}

	static int
	abd_copy_from_buf_off_cb(void buf, size_t size, void private)
	{
	struct buf_arg *ba_ptr = private;

	(void) memcpy(buf, ba_ptr->arg_buf, size);
	ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size;

	return (0);
	}

	/*
	* Copy from buf to abd. (off is the offset in abd.)
	*/
	void
	abd_copy_from_buf_off(abd_t abd, const void buf, size_t off, size_t size)
	{
	struct buf_arg ba_ptr = { (void *) buf };

	(void) abd_iterate_func(abd, off, size, abd_copy_from_buf_off_cb,
	&ba_ptr);
	}

	static int
	abd_zero_off_cb(void buf, size_t size, void private)
	{
	(void) private;
	(void) memset(buf, 0, size);
	return (0);
	}

	/*
	* Zero out the abd from a particular offset to the end.
	*/
	void
	abd_zero_off(abd_t *abd, size_t off, size_t size)
	{
	(void) abd_iterate_func(abd, off, size, abd_zero_off_cb, NULL);
	}

	/*
	* Iterate over two ABDs and call func incrementally on the two ABDs' data in
	* equal-sized chunks (passed to func as raw buffers). func could be called many
	* times during this iteration.
	*/
	int
	abd_iterate_func2(abd_t dabd, abd_t sabd, size_t doff, size_t soff,
	size_t size, abd_iter_func2_t func, void private)
	{
	int ret = 0;
	struct abd_iter daiter, saiter;
	abd_t c_dabd, c_sabd;

	if (size == 0)
	return (0);

	abd_verify(dabd);
	abd_verify(sabd);

	ASSERT3U(doff + size, <=, dabd->abd_size);
	ASSERT3U(soff + size, <=, sabd->abd_size);

	c_dabd = abd_init_abd_iter(dabd, &daiter, doff);
	c_sabd = abd_init_abd_iter(sabd, &saiter, soff);

	while (size > 0) {
	IMPLY(abd_is_gang(dabd), c_dabd != NULL);
	IMPLY(abd_is_gang(sabd), c_sabd != NULL);

	abd_iter_map(&daiter);
	abd_iter_map(&saiter);

	size_t dlen = MIN(daiter.iter_mapsize, size);
	size_t slen = MIN(saiter.iter_mapsize, size);
	size_t len = MIN(dlen, slen);
	ASSERT(dlen > 0 \|\| slen > 0);

	ret = func(daiter.iter_mapaddr, saiter.iter_mapaddr, len,
	private);

	abd_iter_unmap(&saiter);
	abd_iter_unmap(&daiter);

	if (ret != 0)
	break;

	size -= len;
	c_dabd =
	abd_advance_abd_iter(dabd, c_dabd, &daiter, len);
	c_sabd =
	abd_advance_abd_iter(sabd, c_sabd, &saiter, len);
	}

	return (ret);
	}

	static int
	abd_copy_off_cb(void dbuf, void sbuf, size_t size, void *private)
	{
	(void) private;
	(void) memcpy(dbuf, sbuf, size);
	return (0);
	}

	/*
	* Copy from sabd to dabd starting from soff and doff.
	*/
	void
	abd_copy_off(abd_t dabd, abd_t sabd, size_t doff, size_t soff, size_t size)
	{
	(void) abd_iterate_func2(dabd, sabd, doff, soff, size,
	abd_copy_off_cb, NULL);
	}

	static int
	abd_cmp_cb(void bufa, void bufb, size_t size, void *private)
	{
	(void) private;
	return (memcmp(bufa, bufb, size));
	}

	/*
	* Compares the contents of two ABDs.
	*/
	int
	abd_cmp(abd_t dabd, abd_t sabd)
	{
	ASSERT3U(dabd->abd_size, ==, sabd->abd_size);
	return (abd_iterate_func2(dabd, sabd, 0, 0, dabd->abd_size,
	abd_cmp_cb, NULL));
	}

	/*
	* Iterate over code ABDs and a data ABD and call @func_raidz_gen.
	*
	* @cabds parity ABDs, must have equal size
	* @dabd data ABD. Can be NULL (in this case @dsize = 0)
	* @func_raidz_gen should be implemented so that its behaviour
	* is the same when taking linear and when taking scatter
	*/
	void
	abd_raidz_gen_iterate(abd_t *cabds, abd_t dabd,
	ssize_t csize, ssize_t dsize, const unsigned parity,
	void (func_raidz_gen)(void , const void , size_t, size_t))
	{
	int i;
	ssize_t len, dlen;
	struct abd_iter caiters[3];
	struct abd_iter daiter;
	void *caddrs[3];
	unsigned long flags __maybe_unused = 0;
	abd_t *c_cabds[3];
	abd_t *c_dabd = NULL;

	ASSERT3U(parity, <=, 3);
	for (i = 0; i < parity; i++) {
	abd_verify(cabds[i]);
	ASSERT3U(csize, <=, cabds[i]->abd_size);
	c_cabds[i] = abd_init_abd_iter(cabds[i], &caiters[i], 0);
	}

	ASSERT3S(dsize, >=, 0);
	if (dsize > 0) {
	ASSERT(dabd);
	abd_verify(dabd);
	ASSERT3U(dsize, <=, dabd->abd_size);
	c_dabd = abd_init_abd_iter(dabd, &daiter, 0);
	}

	abd_enter_critical(flags);
	while (csize > 0) {
	len = csize;
	for (i = 0; i < parity; i++) {
	IMPLY(abd_is_gang(cabds[i]), c_cabds[i] != NULL);
	abd_iter_map(&caiters[i]);
	caddrs[i] = caiters[i].iter_mapaddr;
	len = MIN(caiters[i].iter_mapsize, len);
	}

	if (dsize > 0) {
	IMPLY(abd_is_gang(dabd), c_dabd != NULL);
	abd_iter_map(&daiter);
	len = MIN(daiter.iter_mapsize, len);
	dlen = len;
	} else
	dlen = 0;

	/* must be progressive */
	ASSERT3S(len, >, 0);
	/*
	* The iterated function likely will not do well if each
	* segment except the last one is not multiple of 512 (raidz).
	*/
	ASSERT3U(((uint64_t)len & 511ULL), ==, 0);

	func_raidz_gen(caddrs, daiter.iter_mapaddr, len, dlen);

	for (i = parity-1; i >= 0; i--) {
	abd_iter_unmap(&caiters[i]);
	c_cabds[i] =
	abd_advance_abd_iter(cabds[i], c_cabds[i],
	&caiters[i], len);
	}

	if (dsize > 0) {
	abd_iter_unmap(&daiter);
	c_dabd =
	abd_advance_abd_iter(dabd, c_dabd, &daiter,
	dlen);
	dsize -= dlen;
	}

	csize -= len;

	ASSERT3S(dsize, >=, 0);
	ASSERT3S(csize, >=, 0);
	}
	abd_exit_critical(flags);
	}

	/*
	* Iterate over code ABDs and data reconstruction target ABDs and call
	* @func_raidz_rec. Function maps at most 6 pages atomically.
	*
	* @cabds parity ABDs, must have equal size
	* @tabds rec target ABDs, at most 3
	* @tsize size of data target columns
	* @func_raidz_rec expects syndrome data in target columns. Function
	* reconstructs data and overwrites target columns.
	*/
	void
	abd_raidz_rec_iterate(abd_t cabds, abd_t tabds,
	ssize_t tsize, const unsigned parity,
	void (func_raidz_rec)(void t, const size_t tsize, void *c,
	const unsigned *mul),
	const unsigned *mul)
	{
	int i;
	ssize_t len;
	struct abd_iter citers[3];
	struct abd_iter xiters[3];
	void caddrs[3], xaddrs[3];
	unsigned long flags __maybe_unused = 0;
	abd_t *c_cabds[3];
	abd_t *c_tabds[3];

	ASSERT3U(parity, <=, 3);

	for (i = 0; i < parity; i++) {
	abd_verify(cabds[i]);
	abd_verify(tabds[i]);
	ASSERT3U(tsize, <=, cabds[i]->abd_size);
	ASSERT3U(tsize, <=, tabds[i]->abd_size);
	c_cabds[i] =
	abd_init_abd_iter(cabds[i], &citers[i], 0);
	c_tabds[i] =
	abd_init_abd_iter(tabds[i], &xiters[i], 0);
	}

	abd_enter_critical(flags);
	while (tsize > 0) {
	len = tsize;
	for (i = 0; i < parity; i++) {
	IMPLY(abd_is_gang(cabds[i]), c_cabds[i] != NULL);
	IMPLY(abd_is_gang(tabds[i]), c_tabds[i] != NULL);
	abd_iter_map(&citers[i]);
	abd_iter_map(&xiters[i]);
	caddrs[i] = citers[i].iter_mapaddr;
	xaddrs[i] = xiters[i].iter_mapaddr;
	len = MIN(citers[i].iter_mapsize, len);
	len = MIN(xiters[i].iter_mapsize, len);
	}

	/* must be progressive */
	ASSERT3S(len, >, 0);
	/*
	* The iterated function likely will not do well if each
	* segment except the last one is not multiple of 512 (raidz).
	*/
	ASSERT3U(((uint64_t)len & 511ULL), ==, 0);

	func_raidz_rec(xaddrs, len, caddrs, mul);

	for (i = parity-1; i >= 0; i--) {
	abd_iter_unmap(&xiters[i]);
	abd_iter_unmap(&citers[i]);
	c_tabds[i] =
	abd_advance_abd_iter(tabds[i], c_tabds[i],
	&xiters[i], len);
	c_cabds[i] =
	abd_advance_abd_iter(cabds[i], c_cabds[i],
	&citers[i], len);
	}

	tsize -= len;
	ASSERT3S(tsize, >=, 0);
	}
	abd_exit_critical(flags);
	}
	diff --git a/sys/contrib/openzfs/module/zfs/arc.c b/sys/contrib/openzfs/module/zfs/arc.c
	index 4db6c06148b1..1953640139b3 100644
	--- a/sys/contrib/openzfs/module/zfs/arc.c
	+++ b/sys/contrib/openzfs/module/zfs/arc.c
	@@ -1,10756 +1,10763 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2018, Joyent, Inc.
	* Copyright (c) 2011, 2020, Delphix. All rights reserved.
	* Copyright (c) 2014, Saso Kiselkov. All rights reserved.
	* Copyright (c) 2017, Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
	* Copyright (c) 2020, George Amanakis. All rights reserved.
	* Copyright (c) 2019, Klara Inc.
	* Copyright (c) 2019, Allan Jude
	* Copyright (c) 2020, The FreeBSD Foundation [1]
	*
	* [1] Portions of this software were developed by Allan Jude
	* under sponsorship from the FreeBSD Foundation.
	*/

	/*
	* DVA-based Adjustable Replacement Cache
	*
	* While much of the theory of operation used here is
	* based on the self-tuning, low overhead replacement cache
	* presented by Megiddo and Modha at FAST 2003, there are some
	* significant differences:
	*
	* 1. The Megiddo and Modha model assumes any page is evictable.
	* Pages in its cache cannot be "locked" into memory. This makes
	* the eviction algorithm simple: evict the last page in the list.
	* This also make the performance characteristics easy to reason
	* about. Our cache is not so simple. At any given moment, some
	* subset of the blocks in the cache are un-evictable because we
	* have handed out a reference to them. Blocks are only evictable
	* when there are no external references active. This makes
	* eviction far more problematic: we choose to evict the evictable
	* blocks that are the "lowest" in the list.
	*
	* There are times when it is not possible to evict the requested
	* space. In these circumstances we are unable to adjust the cache
	* size. To prevent the cache growing unbounded at these times we
	* implement a "cache throttle" that slows the flow of new data
	* into the cache until we can make space available.
	*
	* 2. The Megiddo and Modha model assumes a fixed cache size.
	* Pages are evicted when the cache is full and there is a cache
	* miss. Our model has a variable sized cache. It grows with
	* high use, but also tries to react to memory pressure from the
	* operating system: decreasing its size when system memory is
	* tight.
	*
	* 3. The Megiddo and Modha model assumes a fixed page size. All
	* elements of the cache are therefore exactly the same size. So
	* when adjusting the cache size following a cache miss, its simply
	* a matter of choosing a single page to evict. In our model, we
	* have variable sized cache blocks (ranging from 512 bytes to
	* 128K bytes). We therefore choose a set of blocks to evict to make
	* space for a cache miss that approximates as closely as possible
	* the space used by the new block.
	*
	* See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache"
	* by N. Megiddo & D. Modha, FAST 2003
	*/

	/*
	* The locking model:
	*
	* A new reference to a cache buffer can be obtained in two
	* ways: 1) via a hash table lookup using the DVA as a key,
	* or 2) via one of the ARC lists. The arc_read() interface
	* uses method 1, while the internal ARC algorithms for
	* adjusting the cache use method 2. We therefore provide two
	* types of locks: 1) the hash table lock array, and 2) the
	* ARC list locks.
	*
	* Buffers do not have their own mutexes, rather they rely on the
	* hash table mutexes for the bulk of their protection (i.e. most
	* fields in the arc_buf_hdr_t are protected by these mutexes).
	*
	* buf_hash_find() returns the appropriate mutex (held) when it
	* locates the requested buffer in the hash table. It returns
	* NULL for the mutex if the buffer was not in the table.
	*
	* buf_hash_remove() expects the appropriate hash mutex to be
	* already held before it is invoked.
	*
	* Each ARC state also has a mutex which is used to protect the
	* buffer list associated with the state. When attempting to
	* obtain a hash table lock while holding an ARC list lock you
	* must use: mutex_tryenter() to avoid deadlock. Also note that
	* the active state mutex must be held before the ghost state mutex.
	*
	* It as also possible to register a callback which is run when the
	* metadata limit is reached and no buffers can be safely evicted. In
	* this case the arc user should drop a reference on some arc buffers so
	* they can be reclaimed. For example, when using the ZPL each dentry
	* holds a references on a znode. These dentries must be pruned before
	* the arc buffer holding the znode can be safely evicted.
	*
	* Note that the majority of the performance stats are manipulated
	* with atomic operations.
	*
	* The L2ARC uses the l2ad_mtx on each vdev for the following:
	*
	* - L2ARC buflist creation
	* - L2ARC buflist eviction
	* - L2ARC write completion, which walks L2ARC buflists
	* - ARC header destruction, as it removes from L2ARC buflists
	* - ARC header release, as it removes from L2ARC buflists
	*/

	/*
	* ARC operation:
	*
	* Every block that is in the ARC is tracked by an arc_buf_hdr_t structure.
	* This structure can point either to a block that is still in the cache or to
	* one that is only accessible in an L2 ARC device, or it can provide
	* information about a block that was recently evicted. If a block is
	* only accessible in the L2ARC, then the arc_buf_hdr_t only has enough
	* information to retrieve it from the L2ARC device. This information is
	* stored in the l2arc_buf_hdr_t sub-structure of the arc_buf_hdr_t. A block
	* that is in this state cannot access the data directly.
	*
	* Blocks that are actively being referenced or have not been evicted
	* are cached in the L1ARC. The L1ARC (l1arc_buf_hdr_t) is a structure within
	* the arc_buf_hdr_t that will point to the data block in memory. A block can
	* only be read by a consumer if it has an l1arc_buf_hdr_t. The L1ARC
	* caches data in two ways -- in a list of ARC buffers (arc_buf_t) and
	* also in the arc_buf_hdr_t's private physical data block pointer (b_pabd).
	*
	* The L1ARC's data pointer may or may not be uncompressed. The ARC has the
	* ability to store the physical data (b_pabd) associated with the DVA of the
	* arc_buf_hdr_t. Since the b_pabd is a copy of the on-disk physical block,
	* it will match its on-disk compression characteristics. This behavior can be
	* disabled by setting 'zfs_compressed_arc_enabled' to B_FALSE. When the
	* compressed ARC functionality is disabled, the b_pabd will point to an
	* uncompressed version of the on-disk data.
	*
	* Data in the L1ARC is not accessed by consumers of the ARC directly. Each
	* arc_buf_hdr_t can have multiple ARC buffers (arc_buf_t) which reference it.
	* Each ARC buffer (arc_buf_t) is being actively accessed by a specific ARC
	* consumer. The ARC will provide references to this data and will keep it
	* cached until it is no longer in use. The ARC caches only the L1ARC's physical
	* data block and will evict any arc_buf_t that is no longer referenced. The
	* amount of memory consumed by the arc_buf_ts' data buffers can be seen via the
	* "overhead_size" kstat.
	*
	* Depending on the consumer, an arc_buf_t can be requested in uncompressed or
	* compressed form. The typical case is that consumers will want uncompressed
	* data, and when that happens a new data buffer is allocated where the data is
	* decompressed for them to use. Currently the only consumer who wants
	* compressed arc_buf_t's is "zfs send", when it streams data exactly as it
	* exists on disk. When this happens, the arc_buf_t's data buffer is shared
	* with the arc_buf_hdr_t.
	*
	* Here is a diagram showing an arc_buf_hdr_t referenced by two arc_buf_t's. The
	* first one is owned by a compressed send consumer (and therefore references
	* the same compressed data buffer as the arc_buf_hdr_t) and the second could be
	* used by any other consumer (and has its own uncompressed copy of the data
	* buffer).
	*
	* arc_buf_hdr_t
	* +-----------+
	* \| fields \|
	* \| common to \|
	* \| L1- and \|
	* \| L2ARC \|
	* +-----------+
	* \| l2arc_buf_hdr_t
	* \| \|
	* +-----------+
	* \| l1arc_buf_hdr_t
	* \| \| arc_buf_t
	* \| b_buf +------------>+-----------+ arc_buf_t
	* \| b_pabd +-+ \|b_next +---->+-----------+
	* +-----------+ \| \|-----------\| \|b_next +-->NULL
	* \| \|b_comp = T \| +-----------+
	* \| \|b_data +-+ \|b_comp = F \|
	* \| +-----------+ \| \|b_data +-+
	* +->+------+ \| +-----------+ \|
	* compressed \| \| \| \|
	* data \| \|<--------------+ \| uncompressed
	* +------+ compressed, \| data
	* shared +-->+------+
	* data \| \|
	* \| \|
	* +------+
	*
	* When a consumer reads a block, the ARC must first look to see if the
	* arc_buf_hdr_t is cached. If the hdr is cached then the ARC allocates a new
	* arc_buf_t and either copies uncompressed data into a new data buffer from an
	* existing uncompressed arc_buf_t, decompresses the hdr's b_pabd buffer into a
	* new data buffer, or shares the hdr's b_pabd buffer, depending on whether the
	* hdr is compressed and the desired compression characteristics of the
	* arc_buf_t consumer. If the arc_buf_t ends up sharing data with the
	* arc_buf_hdr_t and both of them are uncompressed then the arc_buf_t must be
	* the last buffer in the hdr's b_buf list, however a shared compressed buf can
	* be anywhere in the hdr's list.
	*
	* The diagram below shows an example of an uncompressed ARC hdr that is
	* sharing its data with an arc_buf_t (note that the shared uncompressed buf is
	* the last element in the buf list):
	*
	* arc_buf_hdr_t
	* +-----------+
	* \| \|
	* \| \|
	* \| \|
	* +-----------+
	* l2arc_buf_hdr_t\| \|
	* \| \|
	* +-----------+
	* l1arc_buf_hdr_t\| \|
	* \| \| arc_buf_t (shared)
	* \| b_buf +------------>+---------+ arc_buf_t
	* \| \| \|b_next +---->+---------+
	* \| b_pabd +-+ \|---------\| \|b_next +-->NULL
	* +-----------+ \| \| \| +---------+
	* \| \|b_data +-+ \| \|
	* \| +---------+ \| \|b_data +-+
	* +->+------+ \| +---------+ \|
	* \| \| \| \|
	* uncompressed \| \| \| \|
	* data +------+ \| \|
	* ^ +->+------+ \|
	* \| uncompressed \| \| \|
	* \| data \| \| \|
	* \| +------+ \|
	* +---------------------------------+
	*
	* Writing to the ARC requires that the ARC first discard the hdr's b_pabd
	* since the physical block is about to be rewritten. The new data contents
	* will be contained in the arc_buf_t. As the I/O pipeline performs the write,
	* it may compress the data before writing it to disk. The ARC will be called
	* with the transformed data and will memcpy the transformed on-disk block into
	* a newly allocated b_pabd. Writes are always done into buffers which have
	* either been loaned (and hence are new and don't have other readers) or
	* buffers which have been released (and hence have their own hdr, if there
	* were originally other readers of the buf's original hdr). This ensures that
	* the ARC only needs to update a single buf and its hdr after a write occurs.
	*
	* When the L2ARC is in use, it will also take advantage of the b_pabd. The
	* L2ARC will always write the contents of b_pabd to the L2ARC. This means
	* that when compressed ARC is enabled that the L2ARC blocks are identical
	* to the on-disk block in the main data pool. This provides a significant
	* advantage since the ARC can leverage the bp's checksum when reading from the
	* L2ARC to determine if the contents are valid. However, if the compressed
	* ARC is disabled, then the L2ARC's block must be transformed to look
	* like the physical block in the main data pool before comparing the
	* checksum and determining its validity.
	*
	* The L1ARC has a slightly different system for storing encrypted data.
	* Raw (encrypted + possibly compressed) data has a few subtle differences from
	* data that is just compressed. The biggest difference is that it is not
	* possible to decrypt encrypted data (or vice-versa) if the keys aren't loaded.
	* The other difference is that encryption cannot be treated as a suggestion.
	* If a caller would prefer compressed data, but they actually wind up with
	* uncompressed data the worst thing that could happen is there might be a
	* performance hit. If the caller requests encrypted data, however, we must be
	* sure they actually get it or else secret information could be leaked. Raw
	* data is stored in hdr->b_crypt_hdr.b_rabd. An encrypted header, therefore,
	* may have both an encrypted version and a decrypted version of its data at
	* once. When a caller needs a raw arc_buf_t, it is allocated and the data is
	* copied out of this header. To avoid complications with b_pabd, raw buffers
	* cannot be shared.
	*/

	#include <sys/spa.h>
	#include <sys/zio.h>
	#include <sys/spa_impl.h>
	#include <sys/zio_compress.h>
	#include <sys/zio_checksum.h>
	#include <sys/zfs_context.h>
	#include <sys/arc.h>
	#include <sys/zfs_refcount.h>
	#include <sys/vdev.h>
	#include <sys/vdev_impl.h>
	#include <sys/dsl_pool.h>
	#include <sys/multilist.h>
	#include <sys/abd.h>
	#include <sys/zil.h>
	#include <sys/fm/fs/zfs.h>
	#include <sys/callb.h>
	#include <sys/kstat.h>
	#include <sys/zthr.h>
	#include <zfs_fletcher.h>
	#include <sys/arc_impl.h>
	#include <sys/trace_zfs.h>
	#include <sys/aggsum.h>
	#include <sys/wmsum.h>
	#include <cityhash.h>
	#include <sys/vdev_trim.h>
	#include <sys/zfs_racct.h>
	#include <sys/zstd/zstd.h>

	#ifndef _KERNEL
	/* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
	boolean_t arc_watch = B_FALSE;
	#endif

	/*
	* This thread's job is to keep enough free memory in the system, by
	* calling arc_kmem_reap_soon() plus arc_reduce_target_size(), which improves
	* arc_available_memory().
	*/
	static zthr_t *arc_reap_zthr;

	/*
	* This thread's job is to keep arc_size under arc_c, by calling
	* arc_evict(), which improves arc_is_overflowing().
	*/
	static zthr_t *arc_evict_zthr;
	static arc_buf_hdr_t **arc_state_evict_markers;
	static int arc_state_evict_marker_count;

	static kmutex_t arc_evict_lock;
	static boolean_t arc_evict_needed = B_FALSE;
	static clock_t arc_last_uncached_flush;

	/*
	* Count of bytes evicted since boot.
	*/
	static uint64_t arc_evict_count;

	/*
	* List of arc_evict_waiter_t's, representing threads waiting for the
	* arc_evict_count to reach specific values.
	*/
	static list_t arc_evict_waiters;

	/*
	* When arc_is_overflowing(), arc_get_data_impl() waits for this percent of
	* the requested amount of data to be evicted. For example, by default for
	* every 2KB that's evicted, 1KB of it may be "reused" by a new allocation.
	* Since this is above 100%, it ensures that progress is made towards getting
	* arc_size under arc_c. Since this is finite, it ensures that allocations
	* can still happen, even during the potentially long time that arc_size is
	* more than arc_c.
	*/
	static uint_t zfs_arc_eviction_pct = 200;

	/*
	* The number of headers to evict in arc_evict_state_impl() before
	* dropping the sublist lock and evicting from another sublist. A lower
	* value means we're more likely to evict the "correct" header (i.e. the
	* oldest header in the arc state), but comes with higher overhead
	* (i.e. more invocations of arc_evict_state_impl()).
	*/
	static uint_t zfs_arc_evict_batch_limit = 10;

	/* number of seconds before growing cache again */
	uint_t arc_grow_retry = 5;

	/*
	* Minimum time between calls to arc_kmem_reap_soon().
	*/
	static const int arc_kmem_cache_reap_retry_ms = 1000;

	/* shift of arc_c for calculating overflow limit in arc_get_data_impl */
	static int zfs_arc_overflow_shift = 8;

	/* log2(fraction of arc to reclaim) */
	uint_t arc_shrink_shift = 7;

	/* percent of pagecache to reclaim arc to */
	#ifdef _KERNEL
	uint_t zfs_arc_pc_percent = 0;
	#endif

	/*
	* log2(fraction of ARC which must be free to allow growing).
	* I.e. If there is less than arc_c >> arc_no_grow_shift free memory,
	* when reading a new block into the ARC, we will evict an equal-sized block
	* from the ARC.
	*
	* This must be less than arc_shrink_shift, so that when we shrink the ARC,
	* we will still not allow it to grow.
	*/
	uint_t arc_no_grow_shift = 5;


	/*
	* minimum lifespan of a prefetch block in clock ticks
	* (initialized in arc_init())
	*/
	static uint_t arc_min_prefetch_ms;
	static uint_t arc_min_prescient_prefetch_ms;

	/*
	* If this percent of memory is free, don't throttle.
	*/
	uint_t arc_lotsfree_percent = 10;

	/*
	* The arc has filled available memory and has now warmed up.
	*/
	boolean_t arc_warm;

	/*
	* These tunables are for performance analysis.
	*/
	uint64_t zfs_arc_max = 0;
	uint64_t zfs_arc_min = 0;
	static uint64_t zfs_arc_dnode_limit = 0;
	static uint_t zfs_arc_dnode_reduce_percent = 10;
	static uint_t zfs_arc_grow_retry = 0;
	static uint_t zfs_arc_shrink_shift = 0;
	uint_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */

	/*
	* ARC dirty data constraints for arc_tempreserve_space() throttle:
	* * total dirty data limit
	* * anon block dirty limit
	* * each pool's anon allowance
	*/
	static const unsigned long zfs_arc_dirty_limit_percent = 50;
	static const unsigned long zfs_arc_anon_limit_percent = 25;
	static const unsigned long zfs_arc_pool_dirty_percent = 20;

	/*
	* Enable or disable compressed arc buffers.
	*/
	int zfs_compressed_arc_enabled = B_TRUE;

	/*
	* Balance between metadata and data on ghost hits. Values above 100
	* increase metadata caching by proportionally reducing effect of ghost
	* data hits on target data/metadata rate.
	*/
	static uint_t zfs_arc_meta_balance = 500;

	/*
	* Percentage that can be consumed by dnodes of ARC meta buffers.
	*/
	static uint_t zfs_arc_dnode_limit_percent = 10;

	/*
	* These tunables are Linux-specific
	*/
	static uint64_t zfs_arc_sys_free = 0;
	static uint_t zfs_arc_min_prefetch_ms = 0;
	static uint_t zfs_arc_min_prescient_prefetch_ms = 0;
	static uint_t zfs_arc_lotsfree_percent = 10;

	/*
	* Number of arc_prune threads
	*/
	static int zfs_arc_prune_task_threads = 1;

	/* The 7 states: */
	arc_state_t ARC_anon;
	arc_state_t ARC_mru;
	arc_state_t ARC_mru_ghost;
	arc_state_t ARC_mfu;
	arc_state_t ARC_mfu_ghost;
	arc_state_t ARC_l2c_only;
	arc_state_t ARC_uncached;

	arc_stats_t arc_stats = {
	{ "hits", KSTAT_DATA_UINT64 },
	{ "iohits", KSTAT_DATA_UINT64 },
	{ "misses", KSTAT_DATA_UINT64 },
	{ "demand_data_hits", KSTAT_DATA_UINT64 },
	{ "demand_data_iohits", KSTAT_DATA_UINT64 },
	{ "demand_data_misses", KSTAT_DATA_UINT64 },
	{ "demand_metadata_hits", KSTAT_DATA_UINT64 },
	{ "demand_metadata_iohits", KSTAT_DATA_UINT64 },
	{ "demand_metadata_misses", KSTAT_DATA_UINT64 },
	{ "prefetch_data_hits", KSTAT_DATA_UINT64 },
	{ "prefetch_data_iohits", KSTAT_DATA_UINT64 },
	{ "prefetch_data_misses", KSTAT_DATA_UINT64 },
	{ "prefetch_metadata_hits", KSTAT_DATA_UINT64 },
	{ "prefetch_metadata_iohits", KSTAT_DATA_UINT64 },
	{ "prefetch_metadata_misses", KSTAT_DATA_UINT64 },
	{ "mru_hits", KSTAT_DATA_UINT64 },
	{ "mru_ghost_hits", KSTAT_DATA_UINT64 },
	{ "mfu_hits", KSTAT_DATA_UINT64 },
	{ "mfu_ghost_hits", KSTAT_DATA_UINT64 },
	{ "uncached_hits", KSTAT_DATA_UINT64 },
	{ "deleted", KSTAT_DATA_UINT64 },
	{ "mutex_miss", KSTAT_DATA_UINT64 },
	{ "access_skip", KSTAT_DATA_UINT64 },
	{ "evict_skip", KSTAT_DATA_UINT64 },
	{ "evict_not_enough", KSTAT_DATA_UINT64 },
	{ "evict_l2_cached", KSTAT_DATA_UINT64 },
	{ "evict_l2_eligible", KSTAT_DATA_UINT64 },
	{ "evict_l2_eligible_mfu", KSTAT_DATA_UINT64 },
	{ "evict_l2_eligible_mru", KSTAT_DATA_UINT64 },
	{ "evict_l2_ineligible", KSTAT_DATA_UINT64 },
	{ "evict_l2_skip", KSTAT_DATA_UINT64 },
	{ "hash_elements", KSTAT_DATA_UINT64 },
	{ "hash_elements_max", KSTAT_DATA_UINT64 },
	{ "hash_collisions", KSTAT_DATA_UINT64 },
	{ "hash_chains", KSTAT_DATA_UINT64 },
	{ "hash_chain_max", KSTAT_DATA_UINT64 },
	{ "meta", KSTAT_DATA_UINT64 },
	{ "pd", KSTAT_DATA_UINT64 },
	{ "pm", KSTAT_DATA_UINT64 },
	{ "c", KSTAT_DATA_UINT64 },
	{ "c_min", KSTAT_DATA_UINT64 },
	{ "c_max", KSTAT_DATA_UINT64 },
	{ "size", KSTAT_DATA_UINT64 },
	{ "compressed_size", KSTAT_DATA_UINT64 },
	{ "uncompressed_size", KSTAT_DATA_UINT64 },
	{ "overhead_size", KSTAT_DATA_UINT64 },
	{ "hdr_size", KSTAT_DATA_UINT64 },
	{ "data_size", KSTAT_DATA_UINT64 },
	{ "metadata_size", KSTAT_DATA_UINT64 },
	{ "dbuf_size", KSTAT_DATA_UINT64 },
	{ "dnode_size", KSTAT_DATA_UINT64 },
	{ "bonus_size", KSTAT_DATA_UINT64 },
	#if defined(COMPAT_FREEBSD11)
	{ "other_size", KSTAT_DATA_UINT64 },
	#endif
	{ "anon_size", KSTAT_DATA_UINT64 },
	{ "anon_data", KSTAT_DATA_UINT64 },
	{ "anon_metadata", KSTAT_DATA_UINT64 },
	{ "anon_evictable_data", KSTAT_DATA_UINT64 },
	{ "anon_evictable_metadata", KSTAT_DATA_UINT64 },
	{ "mru_size", KSTAT_DATA_UINT64 },
	{ "mru_data", KSTAT_DATA_UINT64 },
	{ "mru_metadata", KSTAT_DATA_UINT64 },
	{ "mru_evictable_data", KSTAT_DATA_UINT64 },
	{ "mru_evictable_metadata", KSTAT_DATA_UINT64 },
	{ "mru_ghost_size", KSTAT_DATA_UINT64 },
	{ "mru_ghost_data", KSTAT_DATA_UINT64 },
	{ "mru_ghost_metadata", KSTAT_DATA_UINT64 },
	{ "mru_ghost_evictable_data", KSTAT_DATA_UINT64 },
	{ "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
	{ "mfu_size", KSTAT_DATA_UINT64 },
	{ "mfu_data", KSTAT_DATA_UINT64 },
	{ "mfu_metadata", KSTAT_DATA_UINT64 },
	{ "mfu_evictable_data", KSTAT_DATA_UINT64 },
	{ "mfu_evictable_metadata", KSTAT_DATA_UINT64 },
	{ "mfu_ghost_size", KSTAT_DATA_UINT64 },
	{ "mfu_ghost_data", KSTAT_DATA_UINT64 },
	{ "mfu_ghost_metadata", KSTAT_DATA_UINT64 },
	{ "mfu_ghost_evictable_data", KSTAT_DATA_UINT64 },
	{ "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
	{ "uncached_size", KSTAT_DATA_UINT64 },
	{ "uncached_data", KSTAT_DATA_UINT64 },
	{ "uncached_metadata", KSTAT_DATA_UINT64 },
	{ "uncached_evictable_data", KSTAT_DATA_UINT64 },
	{ "uncached_evictable_metadata", KSTAT_DATA_UINT64 },
	{ "l2_hits", KSTAT_DATA_UINT64 },
	{ "l2_misses", KSTAT_DATA_UINT64 },
	{ "l2_prefetch_asize", KSTAT_DATA_UINT64 },
	{ "l2_mru_asize", KSTAT_DATA_UINT64 },
	{ "l2_mfu_asize", KSTAT_DATA_UINT64 },
	{ "l2_bufc_data_asize", KSTAT_DATA_UINT64 },
	{ "l2_bufc_metadata_asize", KSTAT_DATA_UINT64 },
	{ "l2_feeds", KSTAT_DATA_UINT64 },
	{ "l2_rw_clash", KSTAT_DATA_UINT64 },
	{ "l2_read_bytes", KSTAT_DATA_UINT64 },
	{ "l2_write_bytes", KSTAT_DATA_UINT64 },
	{ "l2_writes_sent", KSTAT_DATA_UINT64 },
	{ "l2_writes_done", KSTAT_DATA_UINT64 },
	{ "l2_writes_error", KSTAT_DATA_UINT64 },
	{ "l2_writes_lock_retry", KSTAT_DATA_UINT64 },
	{ "l2_evict_lock_retry", KSTAT_DATA_UINT64 },
	{ "l2_evict_reading", KSTAT_DATA_UINT64 },
	{ "l2_evict_l1cached", KSTAT_DATA_UINT64 },
	{ "l2_free_on_write", KSTAT_DATA_UINT64 },
	{ "l2_abort_lowmem", KSTAT_DATA_UINT64 },
	{ "l2_cksum_bad", KSTAT_DATA_UINT64 },
	{ "l2_io_error", KSTAT_DATA_UINT64 },
	{ "l2_size", KSTAT_DATA_UINT64 },
	{ "l2_asize", KSTAT_DATA_UINT64 },
	{ "l2_hdr_size", KSTAT_DATA_UINT64 },
	{ "l2_log_blk_writes", KSTAT_DATA_UINT64 },
	{ "l2_log_blk_avg_asize", KSTAT_DATA_UINT64 },
	{ "l2_log_blk_asize", KSTAT_DATA_UINT64 },
	{ "l2_log_blk_count", KSTAT_DATA_UINT64 },
	{ "l2_data_to_meta_ratio", KSTAT_DATA_UINT64 },
	{ "l2_rebuild_success", KSTAT_DATA_UINT64 },
	{ "l2_rebuild_unsupported", KSTAT_DATA_UINT64 },
	{ "l2_rebuild_io_errors", KSTAT_DATA_UINT64 },
	{ "l2_rebuild_dh_errors", KSTAT_DATA_UINT64 },
	{ "l2_rebuild_cksum_lb_errors", KSTAT_DATA_UINT64 },
	{ "l2_rebuild_lowmem", KSTAT_DATA_UINT64 },
	{ "l2_rebuild_size", KSTAT_DATA_UINT64 },
	{ "l2_rebuild_asize", KSTAT_DATA_UINT64 },
	{ "l2_rebuild_bufs", KSTAT_DATA_UINT64 },
	{ "l2_rebuild_bufs_precached", KSTAT_DATA_UINT64 },
	{ "l2_rebuild_log_blks", KSTAT_DATA_UINT64 },
	{ "memory_throttle_count", KSTAT_DATA_UINT64 },
	{ "memory_direct_count", KSTAT_DATA_UINT64 },
	{ "memory_indirect_count", KSTAT_DATA_UINT64 },
	{ "memory_all_bytes", KSTAT_DATA_UINT64 },
	{ "memory_free_bytes", KSTAT_DATA_UINT64 },
	{ "memory_available_bytes", KSTAT_DATA_INT64 },
	{ "arc_no_grow", KSTAT_DATA_UINT64 },
	{ "arc_tempreserve", KSTAT_DATA_UINT64 },
	{ "arc_loaned_bytes", KSTAT_DATA_UINT64 },
	{ "arc_prune", KSTAT_DATA_UINT64 },
	{ "arc_meta_used", KSTAT_DATA_UINT64 },
	{ "arc_dnode_limit", KSTAT_DATA_UINT64 },
	{ "async_upgrade_sync", KSTAT_DATA_UINT64 },
	{ "predictive_prefetch", KSTAT_DATA_UINT64 },
	{ "demand_hit_predictive_prefetch", KSTAT_DATA_UINT64 },
	{ "demand_iohit_predictive_prefetch", KSTAT_DATA_UINT64 },
	{ "prescient_prefetch", KSTAT_DATA_UINT64 },
	{ "demand_hit_prescient_prefetch", KSTAT_DATA_UINT64 },
	{ "demand_iohit_prescient_prefetch", KSTAT_DATA_UINT64 },
	{ "arc_need_free", KSTAT_DATA_UINT64 },
	{ "arc_sys_free", KSTAT_DATA_UINT64 },
	{ "arc_raw_size", KSTAT_DATA_UINT64 },
	{ "cached_only_in_progress", KSTAT_DATA_UINT64 },
	{ "abd_chunk_waste_size", KSTAT_DATA_UINT64 },
	};

	arc_sums_t arc_sums;

	#define ARCSTAT_MAX(stat, val) { \
	uint64_t m; \
	while ((val) > (m = arc_stats.stat.value.ui64) && \
	(m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val)))) \
	continue; \
	}

	/*
	* We define a macro to allow ARC hits/misses to be easily broken down by
	* two separate conditions, giving a total of four different subtypes for
	* each of hits and misses (so eight statistics total).
	*/
	#define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
	if (cond1) { \
	if (cond2) { \
	ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
	} else { \
	ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
	} \
	} else { \
	if (cond2) { \
	ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
	} else { \
	ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
	} \
	}

	/*
	* This macro allows us to use kstats as floating averages. Each time we
	* update this kstat, we first factor it and the update value by
	* ARCSTAT_AVG_FACTOR to shrink the new value's contribution to the overall
	* average. This macro assumes that integer loads and stores are atomic, but
	* is not safe for multiple writers updating the kstat in parallel (only the
	* last writer's update will remain).
	*/
	#define ARCSTAT_F_AVG_FACTOR 3
	#define ARCSTAT_F_AVG(stat, value) \
	do { \
	uint64_t x = ARCSTAT(stat); \
	x = x - x / ARCSTAT_F_AVG_FACTOR + \
	(value) / ARCSTAT_F_AVG_FACTOR; \
	ARCSTAT(stat) = x; \
	} while (0)

	static kstat_t *arc_ksp;

	/*
	* There are several ARC variables that are critical to export as kstats --
	* but we don't want to have to grovel around in the kstat whenever we wish to
	* manipulate them. For these variables, we therefore define them to be in
	* terms of the statistic variable. This assures that we are not introducing
	* the possibility of inconsistency by having shadow copies of the variables,
	* while still allowing the code to be readable.
	*/
	#define arc_tempreserve ARCSTAT(arcstat_tempreserve)
	#define arc_loaned_bytes ARCSTAT(arcstat_loaned_bytes)
	#define arc_dnode_limit ARCSTAT(arcstat_dnode_limit) /* max size for dnodes */
	#define arc_need_free ARCSTAT(arcstat_need_free) /* waiting to be evicted */

	hrtime_t arc_growtime;
	list_t arc_prune_list;
	kmutex_t arc_prune_mtx;
	taskq_t *arc_prune_taskq;

	#define GHOST_STATE(state) \
	((state) == arc_mru_ghost \|\| (state) == arc_mfu_ghost \|\| \
	(state) == arc_l2c_only)

	#define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
	#define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
	#define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR)
	#define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH)
	#define HDR_PRESCIENT_PREFETCH(hdr) \
	((hdr)->b_flags & ARC_FLAG_PRESCIENT_PREFETCH)
	#define HDR_COMPRESSION_ENABLED(hdr) \
	((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)

	#define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE)
	#define HDR_UNCACHED(hdr) ((hdr)->b_flags & ARC_FLAG_UNCACHED)
	#define HDR_L2_READING(hdr) \
	(((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \
	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
	#define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING)
	#define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
	#define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
	#define HDR_PROTECTED(hdr) ((hdr)->b_flags & ARC_FLAG_PROTECTED)
	#define HDR_NOAUTH(hdr) ((hdr)->b_flags & ARC_FLAG_NOAUTH)
	#define HDR_SHARED_DATA(hdr) ((hdr)->b_flags & ARC_FLAG_SHARED_DATA)

	#define HDR_ISTYPE_METADATA(hdr) \
	((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
	#define HDR_ISTYPE_DATA(hdr) (!HDR_ISTYPE_METADATA(hdr))

	#define HDR_HAS_L1HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
	#define HDR_HAS_L2HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
	#define HDR_HAS_RABD(hdr) \
	(HDR_HAS_L1HDR(hdr) && HDR_PROTECTED(hdr) && \
	(hdr)->b_crypt_hdr.b_rabd != NULL)
	#define HDR_ENCRYPTED(hdr) \
	(HDR_PROTECTED(hdr) && DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
	#define HDR_AUTHENTICATED(hdr) \
	(HDR_PROTECTED(hdr) && !DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))

	/* For storing compression mode in b_flags */
	#define HDR_COMPRESS_OFFSET (highbit64(ARC_FLAG_COMPRESS_0) - 1)

	#define HDR_GET_COMPRESS(hdr) ((enum zio_compress)BF32_GET((hdr)->b_flags, \
	HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS))
	#define HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \
	HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS, (cmp));

	#define ARC_BUF_LAST(buf) ((buf)->b_next == NULL)
	#define ARC_BUF_SHARED(buf) ((buf)->b_flags & ARC_BUF_FLAG_SHARED)
	#define ARC_BUF_COMPRESSED(buf) ((buf)->b_flags & ARC_BUF_FLAG_COMPRESSED)
	#define ARC_BUF_ENCRYPTED(buf) ((buf)->b_flags & ARC_BUF_FLAG_ENCRYPTED)

	/*
	* Other sizes
	*/

	#define HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
	#define HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr))

	/*
	* Hash table routines
	*/

	#define BUF_LOCKS 2048
	typedef struct buf_hash_table {
	uint64_t ht_mask;
	arc_buf_hdr_t **ht_table;
	kmutex_t ht_locks[BUF_LOCKS] ____cacheline_aligned;
	} buf_hash_table_t;

	static buf_hash_table_t buf_hash_table;

	#define BUF_HASH_INDEX(spa, dva, birth) \
	(buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
	#define BUF_HASH_LOCK(idx) (&buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
	#define HDR_LOCK(hdr) \
	(BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))

	uint64_t zfs_crc64_table[256];

	/*
	* Level 2 ARC
	*/

	#define L2ARC_WRITE_SIZE (8 * 1024 * 1024) /* initial write max */
	#define L2ARC_HEADROOM 2 /* num of writes */

	/*
	* If we discover during ARC scan any buffers to be compressed, we boost
	* our headroom for the next scanning cycle by this percentage multiple.
	*/
	#define L2ARC_HEADROOM_BOOST 200
	#define L2ARC_FEED_SECS 1 /* caching interval secs */
	#define L2ARC_FEED_MIN_MS 200 /* min caching interval ms */

	/*
	* We can feed L2ARC from two states of ARC buffers, mru and mfu,
	* and each of the state has two types: data and metadata.
	*/
	#define L2ARC_FEED_TYPES 4

	/* L2ARC Performance Tunables */
	uint64_t l2arc_write_max = L2ARC_WRITE_SIZE; /* def max write size */
	uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE; /* extra warmup write */
	uint64_t l2arc_headroom = L2ARC_HEADROOM; /* # of dev writes */
	uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
	uint64_t l2arc_feed_secs = L2ARC_FEED_SECS; /* interval seconds */
	uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS; /* min interval msecs */
	int l2arc_noprefetch = B_TRUE; /* don't cache prefetch bufs */
	int l2arc_feed_again = B_TRUE; /* turbo warmup */
	int l2arc_norw = B_FALSE; /* no reads during writes */
	static uint_t l2arc_meta_percent = 33; /* limit on headers size */

	/*
	* L2ARC Internals
	*/
	static list_t L2ARC_dev_list; /* device list */
	static list_t l2arc_dev_list; / device list pointer */
	static kmutex_t l2arc_dev_mtx; /* device list mutex */
	static l2arc_dev_t l2arc_dev_last; / last device used */
	static list_t L2ARC_free_on_write; /* free after write buf list */
	static list_t l2arc_free_on_write; / free after write list ptr */
	static kmutex_t l2arc_free_on_write_mtx; /* mutex for list */
	static uint64_t l2arc_ndev; /* number of devices */

	typedef struct l2arc_read_callback {
	arc_buf_hdr_t l2rcb_hdr; / read header */
	blkptr_t l2rcb_bp; /* original blkptr */
	zbookmark_phys_t l2rcb_zb; /* original bookmark */
	int l2rcb_flags; /* original flags */
	abd_t l2rcb_abd; / temporary buffer */
	} l2arc_read_callback_t;

	typedef struct l2arc_data_free {
	/* protected by l2arc_free_on_write_mtx */
	abd_t *l2df_abd;
	size_t l2df_size;
	arc_buf_contents_t l2df_type;
	list_node_t l2df_list_node;
	} l2arc_data_free_t;

	typedef enum arc_fill_flags {
	ARC_FILL_LOCKED = 1 << 0, /* hdr lock is held */
	ARC_FILL_COMPRESSED = 1 << 1, /* fill with compressed data */
	ARC_FILL_ENCRYPTED = 1 << 2, /* fill with encrypted data */
	ARC_FILL_NOAUTH = 1 << 3, /* don't attempt to authenticate */
	ARC_FILL_IN_PLACE = 1 << 4 /* fill in place (special case) */
	} arc_fill_flags_t;

	typedef enum arc_ovf_level {
	ARC_OVF_NONE, /* ARC within target size. */
	ARC_OVF_SOME, /* ARC is slightly overflowed. */
	ARC_OVF_SEVERE /* ARC is severely overflowed. */
	} arc_ovf_level_t;

	static kmutex_t l2arc_feed_thr_lock;
	static kcondvar_t l2arc_feed_thr_cv;
	static uint8_t l2arc_thread_exit;

	static kmutex_t l2arc_rebuild_thr_lock;
	static kcondvar_t l2arc_rebuild_thr_cv;

	enum arc_hdr_alloc_flags {
	ARC_HDR_ALLOC_RDATA = 0x1,
	ARC_HDR_USE_RESERVE = 0x4,
	ARC_HDR_ALLOC_LINEAR = 0x8,
	};


	static abd_t arc_get_data_abd(arc_buf_hdr_t , uint64_t, const void *, int);
	static void arc_get_data_buf(arc_buf_hdr_t , uint64_t, const void *);
	static void arc_get_data_impl(arc_buf_hdr_t , uint64_t, const void , int);
	static void arc_free_data_abd(arc_buf_hdr_t , abd_t , uint64_t, const void *);
	static void arc_free_data_buf(arc_buf_hdr_t , void , uint64_t, const void *);
	static void arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size,
	const void *tag);
	static void arc_hdr_free_abd(arc_buf_hdr_t *, boolean_t);
	static void arc_hdr_alloc_abd(arc_buf_hdr_t *, int);
	static void arc_hdr_destroy(arc_buf_hdr_t *);
	static void arc_access(arc_buf_hdr_t *, arc_flags_t, boolean_t);
	static void arc_buf_watch(arc_buf_t *);
	static void arc_change_state(arc_state_t , arc_buf_hdr_t );

	static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
	static uint32_t arc_bufc_to_flags(arc_buf_contents_t);
	static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
	static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);

	static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
	static void l2arc_read_done(zio_t *);
	static void l2arc_do_free_on_write(void);
	static void l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr,
	boolean_t state_only);

	static void arc_prune_async(uint64_t adjust);

	#define l2arc_hdr_arcstats_increment(hdr) \
	l2arc_hdr_arcstats_update((hdr), B_TRUE, B_FALSE)
	#define l2arc_hdr_arcstats_decrement(hdr) \
	l2arc_hdr_arcstats_update((hdr), B_FALSE, B_FALSE)
	#define l2arc_hdr_arcstats_increment_state(hdr) \
	l2arc_hdr_arcstats_update((hdr), B_TRUE, B_TRUE)
	#define l2arc_hdr_arcstats_decrement_state(hdr) \
	l2arc_hdr_arcstats_update((hdr), B_FALSE, B_TRUE)

	/*
	* l2arc_exclude_special : A zfs module parameter that controls whether buffers
	* present on special vdevs are eligibile for caching in L2ARC. If
	* set to 1, exclude dbufs on special vdevs from being cached to
	* L2ARC.
	*/
	int l2arc_exclude_special = 0;

	/*
	* l2arc_mfuonly : A ZFS module parameter that controls whether only MFU
	* metadata and data are cached from ARC into L2ARC.
	*/
	static int l2arc_mfuonly = 0;

	/*
	* L2ARC TRIM
	* l2arc_trim_ahead : A ZFS module parameter that controls how much ahead of
	* the current write size (l2arc_write_max) we should TRIM if we
	* have filled the device. It is defined as a percentage of the
	* write size. If set to 100 we trim twice the space required to
	* accommodate upcoming writes. A minimum of 64MB will be trimmed.
	* It also enables TRIM of the whole L2ARC device upon creation or
	* addition to an existing pool or if the header of the device is
	* invalid upon importing a pool or onlining a cache device. The
	* default is 0, which disables TRIM on L2ARC altogether as it can
	* put significant stress on the underlying storage devices. This
	* will vary depending of how well the specific device handles
	* these commands.
	*/
	static uint64_t l2arc_trim_ahead = 0;

	/*
	* Performance tuning of L2ARC persistence:
	*
	* l2arc_rebuild_enabled : A ZFS module parameter that controls whether adding
	* an L2ARC device (either at pool import or later) will attempt
	* to rebuild L2ARC buffer contents.
	* l2arc_rebuild_blocks_min_l2size : A ZFS module parameter that controls
	* whether log blocks are written to the L2ARC device. If the L2ARC
	* device is less than 1GB, the amount of data l2arc_evict()
	* evicts is significant compared to the amount of restored L2ARC
	* data. In this case do not write log blocks in L2ARC in order
	* not to waste space.
	*/
	static int l2arc_rebuild_enabled = B_TRUE;
	static uint64_t l2arc_rebuild_blocks_min_l2size = 1024 * 1024 * 1024;

	/* L2ARC persistence rebuild control routines. */
	void l2arc_rebuild_vdev(vdev_t *vd, boolean_t reopen);
	static __attribute__((noreturn)) void l2arc_dev_rebuild_thread(void *arg);
	static int l2arc_rebuild(l2arc_dev_t *dev);

	/* L2ARC persistence read I/O routines. */
	static int l2arc_dev_hdr_read(l2arc_dev_t *dev);
	static int l2arc_log_blk_read(l2arc_dev_t *dev,
	const l2arc_log_blkptr_t this_lp, const l2arc_log_blkptr_t next_lp,
	l2arc_log_blk_phys_t this_lb, l2arc_log_blk_phys_t next_lb,
	zio_t this_io, zio_t *next_io);
	static zio_t l2arc_log_blk_fetch(vdev_t vd,
	const l2arc_log_blkptr_t lp, l2arc_log_blk_phys_t lb);
	static void l2arc_log_blk_fetch_abort(zio_t *zio);

	/* L2ARC persistence block restoration routines. */
	static void l2arc_log_blk_restore(l2arc_dev_t *dev,
	const l2arc_log_blk_phys_t *lb, uint64_t lb_asize);
	static void l2arc_hdr_restore(const l2arc_log_ent_phys_t *le,
	l2arc_dev_t *dev);

	/* L2ARC persistence write I/O routines. */
	static uint64_t l2arc_log_blk_commit(l2arc_dev_t dev, zio_t pio,
	l2arc_write_callback_t *cb);

	/* L2ARC persistence auxiliary routines. */
	boolean_t l2arc_log_blkptr_valid(l2arc_dev_t *dev,
	const l2arc_log_blkptr_t *lbp);
	static boolean_t l2arc_log_blk_insert(l2arc_dev_t *dev,
	const arc_buf_hdr_t *ab);
	boolean_t l2arc_range_check_overlap(uint64_t bottom,
	uint64_t top, uint64_t check);
	static void l2arc_blk_fetch_done(zio_t *zio);
	static inline uint64_t
	l2arc_log_blk_overhead(uint64_t write_sz, l2arc_dev_t *dev);

	/*
	* We use Cityhash for this. It's fast, and has good hash properties without
	* requiring any large static buffers.
	*/
	static uint64_t
	buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
	{
	return (cityhash4(spa, dva->dva_word[0], dva->dva_word[1], birth));
	}

	#define HDR_EMPTY(hdr) \
	((hdr)->b_dva.dva_word[0] == 0 && \
	(hdr)->b_dva.dva_word[1] == 0)

	#define HDR_EMPTY_OR_LOCKED(hdr) \
	(HDR_EMPTY(hdr) \|\| MUTEX_HELD(HDR_LOCK(hdr)))

	#define HDR_EQUAL(spa, dva, birth, hdr) \
	((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \
	((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \
	((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)

	static void
	buf_discard_identity(arc_buf_hdr_t *hdr)
	{
	hdr->b_dva.dva_word[0] = 0;
	hdr->b_dva.dva_word[1] = 0;
	hdr->b_birth = 0;
	}

	static arc_buf_hdr_t *
	buf_hash_find(uint64_t spa, const blkptr_t bp, kmutex_t *lockp)
	{
	const dva_t *dva = BP_IDENTITY(bp);
	uint64_t birth = BP_PHYSICAL_BIRTH(bp);
	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
	arc_buf_hdr_t *hdr;

	mutex_enter(hash_lock);
	for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
	hdr = hdr->b_hash_next) {
	if (HDR_EQUAL(spa, dva, birth, hdr)) {
	*lockp = hash_lock;
	return (hdr);
	}
	}
	mutex_exit(hash_lock);
	*lockp = NULL;
	return (NULL);
	}

	/*
	* Insert an entry into the hash table. If there is already an element
	* equal to elem in the hash table, then the already existing element
	* will be returned and the new element will not be inserted.
	* Otherwise returns NULL.
	* If lockp == NULL, the caller is assumed to already hold the hash lock.
	*/
	static arc_buf_hdr_t *
	buf_hash_insert(arc_buf_hdr_t hdr, kmutex_t *lockp)
	{
	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
	arc_buf_hdr_t *fhdr;
	uint32_t i;

	ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
	ASSERT(hdr->b_birth != 0);
	ASSERT(!HDR_IN_HASH_TABLE(hdr));

	if (lockp != NULL) {
	*lockp = hash_lock;
	mutex_enter(hash_lock);
	} else {
	ASSERT(MUTEX_HELD(hash_lock));
	}

	for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL;
	fhdr = fhdr->b_hash_next, i++) {
	if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
	return (fhdr);
	}

	hdr->b_hash_next = buf_hash_table.ht_table[idx];
	buf_hash_table.ht_table[idx] = hdr;
	arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);

	/* collect some hash table performance data */
	if (i > 0) {
	ARCSTAT_BUMP(arcstat_hash_collisions);
	if (i == 1)
	ARCSTAT_BUMP(arcstat_hash_chains);

	ARCSTAT_MAX(arcstat_hash_chain_max, i);
	}
	uint64_t he = atomic_inc_64_nv(
	&arc_stats.arcstat_hash_elements.value.ui64);
	ARCSTAT_MAX(arcstat_hash_elements_max, he);

	return (NULL);
	}

	static void
	buf_hash_remove(arc_buf_hdr_t *hdr)
	{
	arc_buf_hdr_t fhdr, *hdrp;
	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);

	ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
	ASSERT(HDR_IN_HASH_TABLE(hdr));

	hdrp = &buf_hash_table.ht_table[idx];
	while ((fhdr = *hdrp) != hdr) {
	ASSERT3P(fhdr, !=, NULL);
	hdrp = &fhdr->b_hash_next;
	}
	*hdrp = hdr->b_hash_next;
	hdr->b_hash_next = NULL;
	arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE);

	/* collect some hash table performance data */
	atomic_dec_64(&arc_stats.arcstat_hash_elements.value.ui64);

	if (buf_hash_table.ht_table[idx] &&
	buf_hash_table.ht_table[idx]->b_hash_next == NULL)
	ARCSTAT_BUMPDOWN(arcstat_hash_chains);
	}

	/*
	* Global data structures and functions for the buf kmem cache.
	*/

	static kmem_cache_t *hdr_full_cache;
	static kmem_cache_t *hdr_l2only_cache;
	static kmem_cache_t *buf_cache;

	static void
	buf_fini(void)
	{
	#if defined(_KERNEL)
	/*
	* Large allocations which do not require contiguous pages
	* should be using vmem_free() in the linux kernel\
	*/
	vmem_free(buf_hash_table.ht_table,
	(buf_hash_table.ht_mask + 1) * sizeof (void *));
	#else
	kmem_free(buf_hash_table.ht_table,
	(buf_hash_table.ht_mask + 1) * sizeof (void *));
	#endif
	for (int i = 0; i < BUF_LOCKS; i++)
	mutex_destroy(BUF_HASH_LOCK(i));
	kmem_cache_destroy(hdr_full_cache);
	kmem_cache_destroy(hdr_l2only_cache);
	kmem_cache_destroy(buf_cache);
	}

	/*
	* Constructor callback - called when the cache is empty
	* and a new buf is requested.
	*/
	static int
	hdr_full_cons(void vbuf, void unused, int kmflag)
	{
	(void) unused, (void) kmflag;
	arc_buf_hdr_t *hdr = vbuf;

	memset(hdr, 0, HDR_FULL_SIZE);
	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
	zfs_refcount_create(&hdr->b_l1hdr.b_refcnt);
	#ifdef ZFS_DEBUG
	mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
	#endif
	multilist_link_init(&hdr->b_l1hdr.b_arc_node);
	list_link_init(&hdr->b_l2hdr.b_l2node);
	arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);

	return (0);
	}

	static int
	hdr_l2only_cons(void vbuf, void unused, int kmflag)
	{
	(void) unused, (void) kmflag;
	arc_buf_hdr_t *hdr = vbuf;

	memset(hdr, 0, HDR_L2ONLY_SIZE);
	arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);

	return (0);
	}

	static int
	buf_cons(void vbuf, void unused, int kmflag)
	{
	(void) unused, (void) kmflag;
	arc_buf_t *buf = vbuf;

	memset(buf, 0, sizeof (arc_buf_t));
	arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);

	return (0);
	}

	/*
	* Destructor callback - called when a cached buf is
	* no longer required.
	*/
	static void
	hdr_full_dest(void vbuf, void unused)
	{
	(void) unused;
	arc_buf_hdr_t *hdr = vbuf;

	ASSERT(HDR_EMPTY(hdr));
	zfs_refcount_destroy(&hdr->b_l1hdr.b_refcnt);
	#ifdef ZFS_DEBUG
	mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
	#endif
	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
	arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
	}

	static void
	hdr_l2only_dest(void vbuf, void unused)
	{
	(void) unused;
	arc_buf_hdr_t *hdr = vbuf;

	ASSERT(HDR_EMPTY(hdr));
	arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
	}

	static void
	buf_dest(void vbuf, void unused)
	{
	(void) unused;
	(void) vbuf;

	arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
	}

	static void
	buf_init(void)
	{
	uint64_t *ct = NULL;
	uint64_t hsize = 1ULL << 12;
	int i, j;

	/*
	* The hash table is big enough to fill all of physical memory
	* with an average block size of zfs_arc_average_blocksize (default 8K).
	* By default, the table will take up
	* totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
	*/
	while (hsize * zfs_arc_average_blocksize < arc_all_memory())
	hsize <<= 1;
	retry:
	buf_hash_table.ht_mask = hsize - 1;
	#if defined(_KERNEL)
	/*
	* Large allocations which do not require contiguous pages
	* should be using vmem_alloc() in the linux kernel
	*/
	buf_hash_table.ht_table =
	vmem_zalloc(hsize * sizeof (void*), KM_SLEEP);
	#else
	buf_hash_table.ht_table =
	kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
	#endif
	if (buf_hash_table.ht_table == NULL) {
	ASSERT(hsize > (1ULL << 8));
	hsize >>= 1;
	goto retry;
	}

	hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE,
	0, hdr_full_cons, hdr_full_dest, NULL, NULL, NULL, 0);
	hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only",
	HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, NULL,
	NULL, NULL, 0);
	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
	0, buf_cons, buf_dest, NULL, NULL, NULL, 0);

	for (i = 0; i < 256; i++)
	for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
	ct = (ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);

	for (i = 0; i < BUF_LOCKS; i++)
	mutex_init(BUF_HASH_LOCK(i), NULL, MUTEX_DEFAULT, NULL);
	}

	#define ARC_MINTIME (hz>>4) /* 62 ms */

	/*
	* This is the size that the buf occupies in memory. If the buf is compressed,
	* it will correspond to the compressed size. You should use this method of
	* getting the buf size unless you explicitly need the logical size.
	*/
	uint64_t
	arc_buf_size(arc_buf_t *buf)
	{
	return (ARC_BUF_COMPRESSED(buf) ?
	HDR_GET_PSIZE(buf->b_hdr) : HDR_GET_LSIZE(buf->b_hdr));
	}

	uint64_t
	arc_buf_lsize(arc_buf_t *buf)
	{
	return (HDR_GET_LSIZE(buf->b_hdr));
	}

	/*
	* This function will return B_TRUE if the buffer is encrypted in memory.
	* This buffer can be decrypted by calling arc_untransform().
	*/
	boolean_t
	arc_is_encrypted(arc_buf_t *buf)
	{
	return (ARC_BUF_ENCRYPTED(buf) != 0);
	}

	/*
	* Returns B_TRUE if the buffer represents data that has not had its MAC
	* verified yet.
	*/
	boolean_t
	arc_is_unauthenticated(arc_buf_t *buf)
	{
	return (HDR_NOAUTH(buf->b_hdr) != 0);
	}

	void
	arc_get_raw_params(arc_buf_t buf, boolean_t byteorder, uint8_t *salt,
	uint8_t iv, uint8_t mac)
	{
	arc_buf_hdr_t *hdr = buf->b_hdr;

	ASSERT(HDR_PROTECTED(hdr));

	memcpy(salt, hdr->b_crypt_hdr.b_salt, ZIO_DATA_SALT_LEN);
	memcpy(iv, hdr->b_crypt_hdr.b_iv, ZIO_DATA_IV_LEN);
	memcpy(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN);
	*byteorder = (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?
	ZFS_HOST_BYTEORDER : !ZFS_HOST_BYTEORDER;
	}

	/*
	* Indicates how this buffer is compressed in memory. If it is not compressed
	* the value will be ZIO_COMPRESS_OFF. It can be made normally readable with
	* arc_untransform() as long as it is also unencrypted.
	*/
	enum zio_compress
	arc_get_compression(arc_buf_t *buf)
	{
	return (ARC_BUF_COMPRESSED(buf) ?
	HDR_GET_COMPRESS(buf->b_hdr) : ZIO_COMPRESS_OFF);
	}

	/*
	* Return the compression algorithm used to store this data in the ARC. If ARC
	* compression is enabled or this is an encrypted block, this will be the same
	* as what's used to store it on-disk. Otherwise, this will be ZIO_COMPRESS_OFF.
	*/
	static inline enum zio_compress
	arc_hdr_get_compress(arc_buf_hdr_t *hdr)
	{
	return (HDR_COMPRESSION_ENABLED(hdr) ?
	HDR_GET_COMPRESS(hdr) : ZIO_COMPRESS_OFF);
	}

	uint8_t
	arc_get_complevel(arc_buf_t *buf)
	{
	return (buf->b_hdr->b_complevel);
	}

	static inline boolean_t
	arc_buf_is_shared(arc_buf_t *buf)
	{
	boolean_t shared = (buf->b_data != NULL &&
	buf->b_hdr->b_l1hdr.b_pabd != NULL &&
	abd_is_linear(buf->b_hdr->b_l1hdr.b_pabd) &&
	buf->b_data == abd_to_buf(buf->b_hdr->b_l1hdr.b_pabd));
	IMPLY(shared, HDR_SHARED_DATA(buf->b_hdr));
	EQUIV(shared, ARC_BUF_SHARED(buf));
	IMPLY(shared, ARC_BUF_COMPRESSED(buf) \|\| ARC_BUF_LAST(buf));

	/*
	* It would be nice to assert arc_can_share() too, but the "hdr isn't
	* already being shared" requirement prevents us from doing that.
	*/

	return (shared);
	}

	/*
	* Free the checksum associated with this header. If there is no checksum, this
	* is a no-op.
	*/
	static inline void
	arc_cksum_free(arc_buf_hdr_t *hdr)
	{
	#ifdef ZFS_DEBUG
	ASSERT(HDR_HAS_L1HDR(hdr));

	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
	if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
	kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t));
	hdr->b_l1hdr.b_freeze_cksum = NULL;
	}
	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
	#endif
	}

	/*
	* Return true iff at least one of the bufs on hdr is not compressed.
	* Encrypted buffers count as compressed.
	*/
	static boolean_t
	arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr)
	{
	ASSERT(hdr->b_l1hdr.b_state == arc_anon \|\| HDR_EMPTY_OR_LOCKED(hdr));

	for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) {
	if (!ARC_BUF_COMPRESSED(b)) {
	return (B_TRUE);
	}
	}
	return (B_FALSE);
	}


	/*
	* If we've turned on the ZFS_DEBUG_MODIFY flag, verify that the buf's data
	* matches the checksum that is stored in the hdr. If there is no checksum,
	* or if the buf is compressed, this is a no-op.
	*/
	static void
	arc_cksum_verify(arc_buf_t *buf)
	{
	#ifdef ZFS_DEBUG
	arc_buf_hdr_t *hdr = buf->b_hdr;
	zio_cksum_t zc;

	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
	return;

	if (ARC_BUF_COMPRESSED(buf))
	return;

	ASSERT(HDR_HAS_L1HDR(hdr));

	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);

	if (hdr->b_l1hdr.b_freeze_cksum == NULL \|\| HDR_IO_ERROR(hdr)) {
	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
	return;
	}

	fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, &zc);
	if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc))
	panic("buffer modified while frozen!");
	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
	#endif
	}

	/*
	* This function makes the assumption that data stored in the L2ARC
	* will be transformed exactly as it is in the main pool. Because of
	* this we can verify the checksum against the reading process's bp.
	*/
	static boolean_t
	arc_cksum_is_equal(arc_buf_hdr_t hdr, zio_t zio)
	{
	ASSERT(!BP_IS_EMBEDDED(zio->io_bp));
	VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr));

	/*
	* Block pointers always store the checksum for the logical data.
	* If the block pointer has the gang bit set, then the checksum
	* it represents is for the reconstituted data and not for an
	* individual gang member. The zio pipeline, however, must be able to
	* determine the checksum of each of the gang constituents so it
	* treats the checksum comparison differently than what we need
	* for l2arc blocks. This prevents us from using the
	* zio_checksum_error() interface directly. Instead we must call the
	* zio_checksum_error_impl() so that we can ensure the checksum is
	* generated using the correct checksum algorithm and accounts for the
	* logical I/O size and not just a gang fragment.
	*/
	return (zio_checksum_error_impl(zio->io_spa, zio->io_bp,
	BP_GET_CHECKSUM(zio->io_bp), zio->io_abd, zio->io_size,
	zio->io_offset, NULL) == 0);
	}

	/*
	* Given a buf full of data, if ZFS_DEBUG_MODIFY is enabled this computes a
	* checksum and attaches it to the buf's hdr so that we can ensure that the buf
	* isn't modified later on. If buf is compressed or there is already a checksum
	* on the hdr, this is a no-op (we only checksum uncompressed bufs).
	*/
	static void
	arc_cksum_compute(arc_buf_t *buf)
	{
	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
	return;

	#ifdef ZFS_DEBUG
	arc_buf_hdr_t *hdr = buf->b_hdr;
	ASSERT(HDR_HAS_L1HDR(hdr));
	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
	if (hdr->b_l1hdr.b_freeze_cksum != NULL \|\| ARC_BUF_COMPRESSED(buf)) {
	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
	return;
	}

	ASSERT(!ARC_BUF_ENCRYPTED(buf));
	ASSERT(!ARC_BUF_COMPRESSED(buf));
	hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t),
	KM_SLEEP);
	fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL,
	hdr->b_l1hdr.b_freeze_cksum);
	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
	#endif
	arc_buf_watch(buf);
	}

	#ifndef _KERNEL
	void
	arc_buf_sigsegv(int sig, siginfo_t si, void unused)
	{
	(void) sig, (void) unused;
	panic("Got SIGSEGV at address: 0x%lx\n", (long)si->si_addr);
	}
	#endif

	static void
	arc_buf_unwatch(arc_buf_t *buf)
	{
	#ifndef _KERNEL
	if (arc_watch) {
	ASSERT0(mprotect(buf->b_data, arc_buf_size(buf),
	PROT_READ \| PROT_WRITE));
	}
	#else
	(void) buf;
	#endif
	}

	static void
	arc_buf_watch(arc_buf_t *buf)
	{
	#ifndef _KERNEL
	if (arc_watch)
	ASSERT0(mprotect(buf->b_data, arc_buf_size(buf),
	PROT_READ));
	#else
	(void) buf;
	#endif
	}

	static arc_buf_contents_t
	arc_buf_type(arc_buf_hdr_t *hdr)
	{
	arc_buf_contents_t type;
	if (HDR_ISTYPE_METADATA(hdr)) {
	type = ARC_BUFC_METADATA;
	} else {
	type = ARC_BUFC_DATA;
	}
	VERIFY3U(hdr->b_type, ==, type);
	return (type);
	}

	boolean_t
	arc_is_metadata(arc_buf_t *buf)
	{
	return (HDR_ISTYPE_METADATA(buf->b_hdr) != 0);
	}

	static uint32_t
	arc_bufc_to_flags(arc_buf_contents_t type)
	{
	switch (type) {
	case ARC_BUFC_DATA:
	/* metadata field is 0 if buffer contains normal data */
	return (0);
	case ARC_BUFC_METADATA:
	return (ARC_FLAG_BUFC_METADATA);
	default:
	break;
	}
	panic("undefined ARC buffer type!");
	return ((uint32_t)-1);
	}

	void
	arc_buf_thaw(arc_buf_t *buf)
	{
	arc_buf_hdr_t *hdr = buf->b_hdr;

	ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
	ASSERT(!HDR_IO_IN_PROGRESS(hdr));

	arc_cksum_verify(buf);

	/*
	* Compressed buffers do not manipulate the b_freeze_cksum.
	*/
	if (ARC_BUF_COMPRESSED(buf))
	return;

	ASSERT(HDR_HAS_L1HDR(hdr));
	arc_cksum_free(hdr);
	arc_buf_unwatch(buf);
	}

	void
	arc_buf_freeze(arc_buf_t *buf)
	{
	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
	return;

	if (ARC_BUF_COMPRESSED(buf))
	return;

	ASSERT(HDR_HAS_L1HDR(buf->b_hdr));
	arc_cksum_compute(buf);
	}

	/*
	* The arc_buf_hdr_t's b_flags should never be modified directly. Instead,
	* the following functions should be used to ensure that the flags are
	* updated in a thread-safe way. When manipulating the flags either
	* the hash_lock must be held or the hdr must be undiscoverable. This
	* ensures that we're not racing with any other threads when updating
	* the flags.
	*/
	static inline void
	arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
	{
	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
	hdr->b_flags \|= flags;
	}

	static inline void
	arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
	{
	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
	hdr->b_flags &= ~flags;
	}

	/*
	* Setting the compression bits in the arc_buf_hdr_t's b_flags is
	* done in a special way since we have to clear and set bits
	* at the same time. Consumers that wish to set the compression bits
	* must use this function to ensure that the flags are updated in
	* thread-safe manner.
	*/
	static void
	arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp)
	{
	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));

	/*
	* Holes and embedded blocks will always have a psize = 0 so
	* we ignore the compression of the blkptr and set the
	* want to uncompress them. Mark them as uncompressed.
	*/
	if (!zfs_compressed_arc_enabled \|\| HDR_GET_PSIZE(hdr) == 0) {
	arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
	ASSERT(!HDR_COMPRESSION_ENABLED(hdr));
	} else {
	arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
	ASSERT(HDR_COMPRESSION_ENABLED(hdr));
	}

	HDR_SET_COMPRESS(hdr, cmp);
	ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp);
	}

	/*
	* Looks for another buf on the same hdr which has the data decompressed, copies
	* from it, and returns true. If no such buf exists, returns false.
	*/
	static boolean_t
	arc_buf_try_copy_decompressed_data(arc_buf_t *buf)
	{
	arc_buf_hdr_t *hdr = buf->b_hdr;
	boolean_t copied = B_FALSE;

	ASSERT(HDR_HAS_L1HDR(hdr));
	ASSERT3P(buf->b_data, !=, NULL);
	ASSERT(!ARC_BUF_COMPRESSED(buf));

	for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL;
	from = from->b_next) {
	/* can't use our own data buffer */
	if (from == buf) {
	continue;
	}

	if (!ARC_BUF_COMPRESSED(from)) {
	memcpy(buf->b_data, from->b_data, arc_buf_size(buf));
	copied = B_TRUE;
	break;
	}
	}

	#ifdef ZFS_DEBUG
	/*
	* There were no decompressed bufs, so there should not be a
	* checksum on the hdr either.
	*/
	if (zfs_flags & ZFS_DEBUG_MODIFY)
	EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL);
	#endif

	return (copied);
	}

	/*
	* Allocates an ARC buf header that's in an evicted & L2-cached state.
	* This is used during l2arc reconstruction to make empty ARC buffers
	* which circumvent the regular disk->arc->l2arc path and instead come
	* into being in the reverse order, i.e. l2arc->arc.
	*/
	static arc_buf_hdr_t *
	arc_buf_alloc_l2only(size_t size, arc_buf_contents_t type, l2arc_dev_t *dev,
	dva_t dva, uint64_t daddr, int32_t psize, uint64_t birth,
	enum zio_compress compress, uint8_t complevel, boolean_t protected,
	boolean_t prefetch, arc_state_type_t arcs_state)
	{
	arc_buf_hdr_t *hdr;

	ASSERT(size != 0);
	hdr = kmem_cache_alloc(hdr_l2only_cache, KM_SLEEP);
	hdr->b_birth = birth;
	hdr->b_type = type;
	hdr->b_flags = 0;
	arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) \| ARC_FLAG_HAS_L2HDR);
	HDR_SET_LSIZE(hdr, size);
	HDR_SET_PSIZE(hdr, psize);
	arc_hdr_set_compress(hdr, compress);
	hdr->b_complevel = complevel;
	if (protected)
	arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
	if (prefetch)
	arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
	hdr->b_spa = spa_load_guid(dev->l2ad_vdev->vdev_spa);

	hdr->b_dva = dva;

	hdr->b_l2hdr.b_dev = dev;
	hdr->b_l2hdr.b_daddr = daddr;
	hdr->b_l2hdr.b_arcs_state = arcs_state;

	return (hdr);
	}

	/*
	* Return the size of the block, b_pabd, that is stored in the arc_buf_hdr_t.
	*/
	static uint64_t
	arc_hdr_size(arc_buf_hdr_t *hdr)
	{
	uint64_t size;

	if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF &&
	HDR_GET_PSIZE(hdr) > 0) {
	size = HDR_GET_PSIZE(hdr);
	} else {
	ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0);
	size = HDR_GET_LSIZE(hdr);
	}
	return (size);
	}

	static int
	arc_hdr_authenticate(arc_buf_hdr_t hdr, spa_t spa, uint64_t dsobj)
	{
	int ret;
	uint64_t csize;
	uint64_t lsize = HDR_GET_LSIZE(hdr);
	uint64_t psize = HDR_GET_PSIZE(hdr);
	void *tmpbuf = NULL;
	abd_t *abd = hdr->b_l1hdr.b_pabd;

	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
	ASSERT(HDR_AUTHENTICATED(hdr));
	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);

	/*
	* The MAC is calculated on the compressed data that is stored on disk.
	* However, if compressed arc is disabled we will only have the
	* decompressed data available to us now. Compress it into a temporary
	* abd so we can verify the MAC. The performance overhead of this will
	* be relatively low, since most objects in an encrypted objset will
	* be encrypted (instead of authenticated) anyway.
	*/
	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
	!HDR_COMPRESSION_ENABLED(hdr)) {

	csize = zio_compress_data(HDR_GET_COMPRESS(hdr),
	hdr->b_l1hdr.b_pabd, &tmpbuf, lsize, hdr->b_complevel);
	ASSERT3P(tmpbuf, !=, NULL);
	ASSERT3U(csize, <=, psize);
	abd = abd_get_from_buf(tmpbuf, lsize);
	abd_take_ownership_of_buf(abd, B_TRUE);
	abd_zero_off(abd, csize, psize - csize);
	}

	/*
	* Authentication is best effort. We authenticate whenever the key is
	* available. If we succeed we clear ARC_FLAG_NOAUTH.
	*/
	if (hdr->b_crypt_hdr.b_ot == DMU_OT_OBJSET) {
	ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
	ASSERT3U(lsize, ==, psize);
	ret = spa_do_crypt_objset_mac_abd(B_FALSE, spa, dsobj, abd,
	psize, hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
	} else {
	ret = spa_do_crypt_mac_abd(B_FALSE, spa, dsobj, abd, psize,
	hdr->b_crypt_hdr.b_mac);
	}

	if (ret == 0)
	arc_hdr_clear_flags(hdr, ARC_FLAG_NOAUTH);
	else if (ret != ENOENT)
	goto error;

	if (tmpbuf != NULL)
	abd_free(abd);

	return (0);

	error:
	if (tmpbuf != NULL)
	abd_free(abd);

	return (ret);
	}

	/*
	* This function will take a header that only has raw encrypted data in
	* b_crypt_hdr.b_rabd and decrypt it into a new buffer which is stored in
	* b_l1hdr.b_pabd. If designated in the header flags, this function will
	* also decompress the data.
	*/
	static int
	arc_hdr_decrypt(arc_buf_hdr_t hdr, spa_t spa, const zbookmark_phys_t *zb)
	{
	int ret;
	abd_t *cabd = NULL;
	void *tmp = NULL;
	boolean_t no_crypt = B_FALSE;
	boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);

	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
	ASSERT(HDR_ENCRYPTED(hdr));

	arc_hdr_alloc_abd(hdr, 0);

	ret = spa_do_crypt_abd(B_FALSE, spa, zb, hdr->b_crypt_hdr.b_ot,
	B_FALSE, bswap, hdr->b_crypt_hdr.b_salt, hdr->b_crypt_hdr.b_iv,
	hdr->b_crypt_hdr.b_mac, HDR_GET_PSIZE(hdr), hdr->b_l1hdr.b_pabd,
	hdr->b_crypt_hdr.b_rabd, &no_crypt);
	if (ret != 0)
	goto error;

	if (no_crypt) {
	abd_copy(hdr->b_l1hdr.b_pabd, hdr->b_crypt_hdr.b_rabd,
	HDR_GET_PSIZE(hdr));
	}

	/*
	* If this header has disabled arc compression but the b_pabd is
	* compressed after decrypting it, we need to decompress the newly
	* decrypted data.
	*/
	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
	!HDR_COMPRESSION_ENABLED(hdr)) {
	/*
	* We want to make sure that we are correctly honoring the
	* zfs_abd_scatter_enabled setting, so we allocate an abd here
	* and then loan a buffer from it, rather than allocating a
	* linear buffer and wrapping it in an abd later.
	*/
	cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, 0);
	tmp = abd_borrow_buf(cabd, arc_hdr_size(hdr));

	ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
	hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr),
	HDR_GET_LSIZE(hdr), &hdr->b_complevel);
	if (ret != 0) {
	abd_return_buf(cabd, tmp, arc_hdr_size(hdr));
	goto error;
	}

	abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr));
	arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
	arc_hdr_size(hdr), hdr);
	hdr->b_l1hdr.b_pabd = cabd;
	}

	return (0);

	error:
	arc_hdr_free_abd(hdr, B_FALSE);
	if (cabd != NULL)
	arc_free_data_buf(hdr, cabd, arc_hdr_size(hdr), hdr);

	return (ret);
	}

	/*
	* This function is called during arc_buf_fill() to prepare the header's
	* abd plaintext pointer for use. This involves authenticated protected
	* data and decrypting encrypted data into the plaintext abd.
	*/
	static int
	arc_fill_hdr_crypt(arc_buf_hdr_t hdr, kmutex_t hash_lock, spa_t *spa,
	const zbookmark_phys_t *zb, boolean_t noauth)
	{
	int ret;

	ASSERT(HDR_PROTECTED(hdr));

	if (hash_lock != NULL)
	mutex_enter(hash_lock);

	if (HDR_NOAUTH(hdr) && !noauth) {
	/*
	* The caller requested authenticated data but our data has
	* not been authenticated yet. Verify the MAC now if we can.
	*/
	ret = arc_hdr_authenticate(hdr, spa, zb->zb_objset);
	if (ret != 0)
	goto error;
	} else if (HDR_HAS_RABD(hdr) && hdr->b_l1hdr.b_pabd == NULL) {
	/*
	* If we only have the encrypted version of the data, but the
	* unencrypted version was requested we take this opportunity
	* to store the decrypted version in the header for future use.
	*/
	ret = arc_hdr_decrypt(hdr, spa, zb);
	if (ret != 0)
	goto error;
	}

	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);

	if (hash_lock != NULL)
	mutex_exit(hash_lock);

	return (0);

	error:
	if (hash_lock != NULL)
	mutex_exit(hash_lock);

	return (ret);
	}

	/*
	* This function is used by the dbuf code to decrypt bonus buffers in place.
	* The dbuf code itself doesn't have any locking for decrypting a shared dnode
	* block, so we use the hash lock here to protect against concurrent calls to
	* arc_buf_fill().
	*/
	static void
	arc_buf_untransform_in_place(arc_buf_t *buf)
	{
	arc_buf_hdr_t *hdr = buf->b_hdr;

	ASSERT(HDR_ENCRYPTED(hdr));
	ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE);
	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);

	zio_crypt_copy_dnode_bonus(hdr->b_l1hdr.b_pabd, buf->b_data,
	arc_buf_size(buf));
	buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
	buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
	}

	/*
	* Given a buf that has a data buffer attached to it, this function will
	* efficiently fill the buf with data of the specified compression setting from
	* the hdr and update the hdr's b_freeze_cksum if necessary. If the buf and hdr
	* are already sharing a data buf, no copy is performed.
	*
	* If the buf is marked as compressed but uncompressed data was requested, this
	* will allocate a new data buffer for the buf, remove that flag, and fill the
	* buf with uncompressed data. You can't request a compressed buf on a hdr with
	* uncompressed data, and (since we haven't added support for it yet) if you
	* want compressed data your buf must already be marked as compressed and have
	* the correct-sized data buffer.
	*/
	static int
	arc_buf_fill(arc_buf_t buf, spa_t spa, const zbookmark_phys_t *zb,
	arc_fill_flags_t flags)
	{
	int error = 0;
	arc_buf_hdr_t *hdr = buf->b_hdr;
	boolean_t hdr_compressed =
	(arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
	boolean_t compressed = (flags & ARC_FILL_COMPRESSED) != 0;
	boolean_t encrypted = (flags & ARC_FILL_ENCRYPTED) != 0;
	dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap;
	kmutex_t *hash_lock = (flags & ARC_FILL_LOCKED) ? NULL : HDR_LOCK(hdr);

	ASSERT3P(buf->b_data, !=, NULL);
	IMPLY(compressed, hdr_compressed \|\| ARC_BUF_ENCRYPTED(buf));
	IMPLY(compressed, ARC_BUF_COMPRESSED(buf));
	IMPLY(encrypted, HDR_ENCRYPTED(hdr));
	IMPLY(encrypted, ARC_BUF_ENCRYPTED(buf));
	IMPLY(encrypted, ARC_BUF_COMPRESSED(buf));
	IMPLY(encrypted, !arc_buf_is_shared(buf));

	/*
	* If the caller wanted encrypted data we just need to copy it from
	* b_rabd and potentially byteswap it. We won't be able to do any
	* further transforms on it.
	*/
	if (encrypted) {
	ASSERT(HDR_HAS_RABD(hdr));
	abd_copy_to_buf(buf->b_data, hdr->b_crypt_hdr.b_rabd,
	HDR_GET_PSIZE(hdr));
	goto byteswap;
	}

	/*
	* Adjust encrypted and authenticated headers to accommodate
	* the request if needed. Dnode blocks (ARC_FILL_IN_PLACE) are
	* allowed to fail decryption due to keys not being loaded
	* without being marked as an IO error.
	*/
	if (HDR_PROTECTED(hdr)) {
	error = arc_fill_hdr_crypt(hdr, hash_lock, spa,
	zb, !!(flags & ARC_FILL_NOAUTH));
	if (error == EACCES && (flags & ARC_FILL_IN_PLACE) != 0) {
	return (error);
	} else if (error != 0) {
	if (hash_lock != NULL)
	mutex_enter(hash_lock);
	arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
	if (hash_lock != NULL)
	mutex_exit(hash_lock);
	return (error);
	}
	}

	/*
	* There is a special case here for dnode blocks which are
	* decrypting their bonus buffers. These blocks may request to
	* be decrypted in-place. This is necessary because there may
	* be many dnodes pointing into this buffer and there is
	* currently no method to synchronize replacing the backing
	* b_data buffer and updating all of the pointers. Here we use
	* the hash lock to ensure there are no races. If the need
	* arises for other types to be decrypted in-place, they must
	* add handling here as well.
	*/
	if ((flags & ARC_FILL_IN_PLACE) != 0) {
	ASSERT(!hdr_compressed);
	ASSERT(!compressed);
	ASSERT(!encrypted);

	if (HDR_ENCRYPTED(hdr) && ARC_BUF_ENCRYPTED(buf)) {
	ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE);

	if (hash_lock != NULL)
	mutex_enter(hash_lock);
	arc_buf_untransform_in_place(buf);
	if (hash_lock != NULL)
	mutex_exit(hash_lock);

	/* Compute the hdr's checksum if necessary */
	arc_cksum_compute(buf);
	}

	return (0);
	}

	if (hdr_compressed == compressed) {
	if (ARC_BUF_SHARED(buf)) {
	ASSERT(arc_buf_is_shared(buf));
	} else {
	abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd,
	arc_buf_size(buf));
	}
	} else {
	ASSERT(hdr_compressed);
	ASSERT(!compressed);

	/*
	* If the buf is sharing its data with the hdr, unlink it and
	* allocate a new data buffer for the buf.
	*/
	if (ARC_BUF_SHARED(buf)) {
	ASSERT(ARC_BUF_COMPRESSED(buf));

	/* We need to give the buf its own b_data */
	buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
	buf->b_data =
	arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
	arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);

	/* Previously overhead was 0; just add new overhead */
	ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr));
	} else if (ARC_BUF_COMPRESSED(buf)) {
	ASSERT(!arc_buf_is_shared(buf));

	/* We need to reallocate the buf's b_data */
	arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr),
	buf);
	buf->b_data =
	arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);

	/* We increased the size of b_data; update overhead */
	ARCSTAT_INCR(arcstat_overhead_size,
	HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr));
	}

	/*
	* Regardless of the buf's previous compression settings, it
	* should not be compressed at the end of this function.
	*/
	buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;

	/*
	* Try copying the data from another buf which already has a
	* decompressed version. If that's not possible, it's time to
	* bite the bullet and decompress the data from the hdr.
	*/
	if (arc_buf_try_copy_decompressed_data(buf)) {
	/* Skip byteswapping and checksumming (already done) */
	return (0);
	} else {
	error = zio_decompress_data(HDR_GET_COMPRESS(hdr),
	hdr->b_l1hdr.b_pabd, buf->b_data,
	HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr),
	&hdr->b_complevel);

	/*
	* Absent hardware errors or software bugs, this should
	* be impossible, but log it anyway so we can debug it.
	*/
	if (error != 0) {
	zfs_dbgmsg(
	"hdr %px, compress %d, psize %d, lsize %d",
	hdr, arc_hdr_get_compress(hdr),
	HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
	if (hash_lock != NULL)
	mutex_enter(hash_lock);
	arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
	if (hash_lock != NULL)
	mutex_exit(hash_lock);
	return (SET_ERROR(EIO));
	}
	}
	}

	byteswap:
	/* Byteswap the buf's data if necessary */
	if (bswap != DMU_BSWAP_NUMFUNCS) {
	ASSERT(!HDR_SHARED_DATA(hdr));
	ASSERT3U(bswap, <, DMU_BSWAP_NUMFUNCS);
	dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr));
	}

	/* Compute the hdr's checksum if necessary */
	arc_cksum_compute(buf);

	return (0);
	}

	/*
	* If this function is being called to decrypt an encrypted buffer or verify an
	* authenticated one, the key must be loaded and a mapping must be made
	* available in the keystore via spa_keystore_create_mapping() or one of its
	* callers.
	*/
	int
	arc_untransform(arc_buf_t buf, spa_t spa, const zbookmark_phys_t *zb,
	boolean_t in_place)
	{
	int ret;
	arc_fill_flags_t flags = 0;

	if (in_place)
	flags \|= ARC_FILL_IN_PLACE;

	ret = arc_buf_fill(buf, spa, zb, flags);
	if (ret == ECKSUM) {
	/*
	* Convert authentication and decryption errors to EIO
	* (and generate an ereport) before leaving the ARC.
	*/
	ret = SET_ERROR(EIO);
	spa_log_error(spa, zb, &buf->b_hdr->b_birth);
	(void) zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION,
	spa, NULL, zb, NULL, 0);
	}

	return (ret);
	}

	/*
	* Increment the amount of evictable space in the arc_state_t's refcount.
	* We account for the space used by the hdr and the arc buf individually
	* so that we can add and remove them from the refcount individually.
	*/
	static void
	arc_evictable_space_increment(arc_buf_hdr_t hdr, arc_state_t state)
	{
	arc_buf_contents_t type = arc_buf_type(hdr);

	ASSERT(HDR_HAS_L1HDR(hdr));

	if (GHOST_STATE(state)) {
	ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
	ASSERT(!HDR_HAS_RABD(hdr));
	(void) zfs_refcount_add_many(&state->arcs_esize[type],
	HDR_GET_LSIZE(hdr), hdr);
	return;
	}

	if (hdr->b_l1hdr.b_pabd != NULL) {
	(void) zfs_refcount_add_many(&state->arcs_esize[type],
	arc_hdr_size(hdr), hdr);
	}
	if (HDR_HAS_RABD(hdr)) {
	(void) zfs_refcount_add_many(&state->arcs_esize[type],
	HDR_GET_PSIZE(hdr), hdr);
	}

	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
	buf = buf->b_next) {
	if (ARC_BUF_SHARED(buf))
	continue;
	(void) zfs_refcount_add_many(&state->arcs_esize[type],
	arc_buf_size(buf), buf);
	}
	}

	/*
	* Decrement the amount of evictable space in the arc_state_t's refcount.
	* We account for the space used by the hdr and the arc buf individually
	* so that we can add and remove them from the refcount individually.
	*/
	static void
	arc_evictable_space_decrement(arc_buf_hdr_t hdr, arc_state_t state)
	{
	arc_buf_contents_t type = arc_buf_type(hdr);

	ASSERT(HDR_HAS_L1HDR(hdr));

	if (GHOST_STATE(state)) {
	ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
	ASSERT(!HDR_HAS_RABD(hdr));
	(void) zfs_refcount_remove_many(&state->arcs_esize[type],
	HDR_GET_LSIZE(hdr), hdr);
	return;
	}

	if (hdr->b_l1hdr.b_pabd != NULL) {
	(void) zfs_refcount_remove_many(&state->arcs_esize[type],
	arc_hdr_size(hdr), hdr);
	}
	if (HDR_HAS_RABD(hdr)) {
	(void) zfs_refcount_remove_many(&state->arcs_esize[type],
	HDR_GET_PSIZE(hdr), hdr);
	}

	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
	buf = buf->b_next) {
	if (ARC_BUF_SHARED(buf))
	continue;
	(void) zfs_refcount_remove_many(&state->arcs_esize[type],
	arc_buf_size(buf), buf);
	}
	}

	/*
	* Add a reference to this hdr indicating that someone is actively
	* referencing that memory. When the refcount transitions from 0 to 1,
	* we remove it from the respective arc_state_t list to indicate that
	* it is not evictable.
	*/
	static void
	add_reference(arc_buf_hdr_t hdr, const void tag)
	{
	arc_state_t *state = hdr->b_l1hdr.b_state;

	ASSERT(HDR_HAS_L1HDR(hdr));
	if (!HDR_EMPTY(hdr) && !MUTEX_HELD(HDR_LOCK(hdr))) {
	ASSERT(state == arc_anon);
	ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
	ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
	}

	if ((zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
	state != arc_anon && state != arc_l2c_only) {
	/* We don't use the L2-only state list. */
	multilist_remove(&state->arcs_list[arc_buf_type(hdr)], hdr);
	arc_evictable_space_decrement(hdr, state);
	}
	}

	/*
	* Remove a reference from this hdr. When the reference transitions from
	* 1 to 0 and we're not anonymous, then we add this hdr to the arc_state_t's
	* list making it eligible for eviction.
	*/
	static int
	remove_reference(arc_buf_hdr_t hdr, const void tag)
	{
	int cnt;
	arc_state_t *state = hdr->b_l1hdr.b_state;

	ASSERT(HDR_HAS_L1HDR(hdr));
	ASSERT(state == arc_anon \|\| MUTEX_HELD(HDR_LOCK(hdr)));
	ASSERT(!GHOST_STATE(state)); /* arc_l2c_only counts as a ghost. */

	if ((cnt = zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) != 0)
	return (cnt);

	if (state == arc_anon) {
	arc_hdr_destroy(hdr);
	return (0);
	}
	if (state == arc_uncached && !HDR_PREFETCH(hdr)) {
	arc_change_state(arc_anon, hdr);
	arc_hdr_destroy(hdr);
	return (0);
	}
	multilist_insert(&state->arcs_list[arc_buf_type(hdr)], hdr);
	arc_evictable_space_increment(hdr, state);
	return (0);
	}

	/*
	* Returns detailed information about a specific arc buffer. When the
	* state_index argument is set the function will calculate the arc header
	* list position for its arc state. Since this requires a linear traversal
	* callers are strongly encourage not to do this. However, it can be helpful
	* for targeted analysis so the functionality is provided.
	*/
	void
	arc_buf_info(arc_buf_t ab, arc_buf_info_t abi, int state_index)
	{
	(void) state_index;
	arc_buf_hdr_t *hdr = ab->b_hdr;
	l1arc_buf_hdr_t *l1hdr = NULL;
	l2arc_buf_hdr_t *l2hdr = NULL;
	arc_state_t *state = NULL;

	memset(abi, 0, sizeof (arc_buf_info_t));

	if (hdr == NULL)
	return;

	abi->abi_flags = hdr->b_flags;

	if (HDR_HAS_L1HDR(hdr)) {
	l1hdr = &hdr->b_l1hdr;
	state = l1hdr->b_state;
	}
	if (HDR_HAS_L2HDR(hdr))
	l2hdr = &hdr->b_l2hdr;

	if (l1hdr) {
	abi->abi_bufcnt = 0;
	for (arc_buf_t *buf = l1hdr->b_buf; buf; buf = buf->b_next)
	abi->abi_bufcnt++;
	abi->abi_access = l1hdr->b_arc_access;
	abi->abi_mru_hits = l1hdr->b_mru_hits;
	abi->abi_mru_ghost_hits = l1hdr->b_mru_ghost_hits;
	abi->abi_mfu_hits = l1hdr->b_mfu_hits;
	abi->abi_mfu_ghost_hits = l1hdr->b_mfu_ghost_hits;
	abi->abi_holds = zfs_refcount_count(&l1hdr->b_refcnt);
	}

	if (l2hdr) {
	abi->abi_l2arc_dattr = l2hdr->b_daddr;
	abi->abi_l2arc_hits = l2hdr->b_hits;
	}

	abi->abi_state_type = state ? state->arcs_state : ARC_STATE_ANON;
	abi->abi_state_contents = arc_buf_type(hdr);
	abi->abi_size = arc_hdr_size(hdr);
	}

	/*
	* Move the supplied buffer to the indicated state. The hash lock
	* for the buffer must be held by the caller.
	*/
	static void
	arc_change_state(arc_state_t new_state, arc_buf_hdr_t hdr)
	{
	arc_state_t *old_state;
	int64_t refcnt;
	boolean_t update_old, update_new;
	arc_buf_contents_t type = arc_buf_type(hdr);

	/*
	* We almost always have an L1 hdr here, since we call arc_hdr_realloc()
	* in arc_read() when bringing a buffer out of the L2ARC. However, the
	* L1 hdr doesn't always exist when we change state to arc_anon before
	* destroying a header, in which case reallocating to add the L1 hdr is
	* pointless.
	*/
	if (HDR_HAS_L1HDR(hdr)) {
	old_state = hdr->b_l1hdr.b_state;
	refcnt = zfs_refcount_count(&hdr->b_l1hdr.b_refcnt);
	update_old = (hdr->b_l1hdr.b_buf != NULL \|\|
	hdr->b_l1hdr.b_pabd != NULL \|\| HDR_HAS_RABD(hdr));

	IMPLY(GHOST_STATE(old_state), hdr->b_l1hdr.b_buf == NULL);
	IMPLY(GHOST_STATE(new_state), hdr->b_l1hdr.b_buf == NULL);
	IMPLY(old_state == arc_anon, hdr->b_l1hdr.b_buf == NULL \|\|
	ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
	} else {
	old_state = arc_l2c_only;
	refcnt = 0;
	update_old = B_FALSE;
	}
	update_new = update_old;
	if (GHOST_STATE(old_state))
	update_old = B_TRUE;
	if (GHOST_STATE(new_state))
	update_new = B_TRUE;

	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
	ASSERT3P(new_state, !=, old_state);

	/*
	* If this buffer is evictable, transfer it from the
	* old state list to the new state list.
	*/
	if (refcnt == 0) {
	if (old_state != arc_anon && old_state != arc_l2c_only) {
	ASSERT(HDR_HAS_L1HDR(hdr));
	/* remove_reference() saves on insert. */
	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
	multilist_remove(&old_state->arcs_list[type],
	hdr);
	arc_evictable_space_decrement(hdr, old_state);
	}
	}
	if (new_state != arc_anon && new_state != arc_l2c_only) {
	/*
	* An L1 header always exists here, since if we're
	* moving to some L1-cached state (i.e. not l2c_only or
	* anonymous), we realloc the header to add an L1hdr
	* beforehand.
	*/
	ASSERT(HDR_HAS_L1HDR(hdr));
	multilist_insert(&new_state->arcs_list[type], hdr);
	arc_evictable_space_increment(hdr, new_state);
	}
	}

	ASSERT(!HDR_EMPTY(hdr));
	if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
	buf_hash_remove(hdr);

	/* adjust state sizes (ignore arc_l2c_only) */

	if (update_new && new_state != arc_l2c_only) {
	ASSERT(HDR_HAS_L1HDR(hdr));
	if (GHOST_STATE(new_state)) {

	/*
	* When moving a header to a ghost state, we first
	* remove all arc buffers. Thus, we'll have no arc
	* buffer to use for the reference. As a result, we
	* use the arc header pointer for the reference.
	*/
	(void) zfs_refcount_add_many(
	&new_state->arcs_size[type],
	HDR_GET_LSIZE(hdr), hdr);
	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
	ASSERT(!HDR_HAS_RABD(hdr));
	} else {

	/*
	* Each individual buffer holds a unique reference,
	* thus we must remove each of these references one
	* at a time.
	*/
	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
	buf = buf->b_next) {

	/*
	* When the arc_buf_t is sharing the data
	* block with the hdr, the owner of the
	* reference belongs to the hdr. Only
	* add to the refcount if the arc_buf_t is
	* not shared.
	*/
	if (ARC_BUF_SHARED(buf))
	continue;

	(void) zfs_refcount_add_many(
	&new_state->arcs_size[type],
	arc_buf_size(buf), buf);
	}

	if (hdr->b_l1hdr.b_pabd != NULL) {
	(void) zfs_refcount_add_many(
	&new_state->arcs_size[type],
	arc_hdr_size(hdr), hdr);
	}

	if (HDR_HAS_RABD(hdr)) {
	(void) zfs_refcount_add_many(
	&new_state->arcs_size[type],
	HDR_GET_PSIZE(hdr), hdr);
	}
	}
	}

	if (update_old && old_state != arc_l2c_only) {
	ASSERT(HDR_HAS_L1HDR(hdr));
	if (GHOST_STATE(old_state)) {
	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
	ASSERT(!HDR_HAS_RABD(hdr));

	/*
	* When moving a header off of a ghost state,
	* the header will not contain any arc buffers.
	* We use the arc header pointer for the reference
	* which is exactly what we did when we put the
	* header on the ghost state.
	*/

	(void) zfs_refcount_remove_many(
	&old_state->arcs_size[type],
	HDR_GET_LSIZE(hdr), hdr);
	} else {

	/*
	* Each individual buffer holds a unique reference,
	* thus we must remove each of these references one
	* at a time.
	*/
	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
	buf = buf->b_next) {

	/*
	* When the arc_buf_t is sharing the data
	* block with the hdr, the owner of the
	* reference belongs to the hdr. Only
	* add to the refcount if the arc_buf_t is
	* not shared.
	*/
	if (ARC_BUF_SHARED(buf))
	continue;

	(void) zfs_refcount_remove_many(
	&old_state->arcs_size[type],
	arc_buf_size(buf), buf);
	}
	ASSERT(hdr->b_l1hdr.b_pabd != NULL \|\|
	HDR_HAS_RABD(hdr));

	if (hdr->b_l1hdr.b_pabd != NULL) {
	(void) zfs_refcount_remove_many(
	&old_state->arcs_size[type],
	arc_hdr_size(hdr), hdr);
	}

	if (HDR_HAS_RABD(hdr)) {
	(void) zfs_refcount_remove_many(
	&old_state->arcs_size[type],
	HDR_GET_PSIZE(hdr), hdr);
	}
	}
	}

	if (HDR_HAS_L1HDR(hdr)) {
	hdr->b_l1hdr.b_state = new_state;

	if (HDR_HAS_L2HDR(hdr) && new_state != arc_l2c_only) {
	l2arc_hdr_arcstats_decrement_state(hdr);
	hdr->b_l2hdr.b_arcs_state = new_state->arcs_state;
	l2arc_hdr_arcstats_increment_state(hdr);
	}
	}
	}

	void
	arc_space_consume(uint64_t space, arc_space_type_t type)
	{
	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);

	switch (type) {
	default:
	break;
	case ARC_SPACE_DATA:
	ARCSTAT_INCR(arcstat_data_size, space);
	break;
	case ARC_SPACE_META:
	ARCSTAT_INCR(arcstat_metadata_size, space);
	break;
	case ARC_SPACE_BONUS:
	ARCSTAT_INCR(arcstat_bonus_size, space);
	break;
	case ARC_SPACE_DNODE:
	ARCSTAT_INCR(arcstat_dnode_size, space);
	break;
	case ARC_SPACE_DBUF:
	ARCSTAT_INCR(arcstat_dbuf_size, space);
	break;
	case ARC_SPACE_HDRS:
	ARCSTAT_INCR(arcstat_hdr_size, space);
	break;
	case ARC_SPACE_L2HDRS:
	aggsum_add(&arc_sums.arcstat_l2_hdr_size, space);
	break;
	case ARC_SPACE_ABD_CHUNK_WASTE:
	/*
	* Note: this includes space wasted by all scatter ABD's, not
	* just those allocated by the ARC. But the vast majority of
	* scatter ABD's come from the ARC, because other users are
	* very short-lived.
	*/
	ARCSTAT_INCR(arcstat_abd_chunk_waste_size, space);
	break;
	}

	if (type != ARC_SPACE_DATA && type != ARC_SPACE_ABD_CHUNK_WASTE)
	ARCSTAT_INCR(arcstat_meta_used, space);

	aggsum_add(&arc_sums.arcstat_size, space);
	}

	void
	arc_space_return(uint64_t space, arc_space_type_t type)
	{
	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);

	switch (type) {
	default:
	break;
	case ARC_SPACE_DATA:
	ARCSTAT_INCR(arcstat_data_size, -space);
	break;
	case ARC_SPACE_META:
	ARCSTAT_INCR(arcstat_metadata_size, -space);
	break;
	case ARC_SPACE_BONUS:
	ARCSTAT_INCR(arcstat_bonus_size, -space);
	break;
	case ARC_SPACE_DNODE:
	ARCSTAT_INCR(arcstat_dnode_size, -space);
	break;
	case ARC_SPACE_DBUF:
	ARCSTAT_INCR(arcstat_dbuf_size, -space);
	break;
	case ARC_SPACE_HDRS:
	ARCSTAT_INCR(arcstat_hdr_size, -space);
	break;
	case ARC_SPACE_L2HDRS:
	aggsum_add(&arc_sums.arcstat_l2_hdr_size, -space);
	break;
	case ARC_SPACE_ABD_CHUNK_WASTE:
	ARCSTAT_INCR(arcstat_abd_chunk_waste_size, -space);
	break;
	}

	if (type != ARC_SPACE_DATA && type != ARC_SPACE_ABD_CHUNK_WASTE)
	ARCSTAT_INCR(arcstat_meta_used, -space);

	ASSERT(aggsum_compare(&arc_sums.arcstat_size, space) >= 0);
	aggsum_add(&arc_sums.arcstat_size, -space);
	}

	/*
	* Given a hdr and a buf, returns whether that buf can share its b_data buffer
	* with the hdr's b_pabd.
	*/
	static boolean_t
	arc_can_share(arc_buf_hdr_t hdr, arc_buf_t buf)
	{
	/*
	* The criteria for sharing a hdr's data are:
	* 1. the buffer is not encrypted
	* 2. the hdr's compression matches the buf's compression
	* 3. the hdr doesn't need to be byteswapped
	* 4. the hdr isn't already being shared
	* 5. the buf is either compressed or it is the last buf in the hdr list
	*
	* Criterion #5 maintains the invariant that shared uncompressed
	* bufs must be the final buf in the hdr's b_buf list. Reading this, you
	* might ask, "if a compressed buf is allocated first, won't that be the
	* last thing in the list?", but in that case it's impossible to create
	* a shared uncompressed buf anyway (because the hdr must be compressed
	* to have the compressed buf). You might also think that #3 is
	* sufficient to make this guarantee, however it's possible
	* (specifically in the rare L2ARC write race mentioned in
	* arc_buf_alloc_impl()) there will be an existing uncompressed buf that
	* is shareable, but wasn't at the time of its allocation. Rather than
	* allow a new shared uncompressed buf to be created and then shuffle
	* the list around to make it the last element, this simply disallows
	* sharing if the new buf isn't the first to be added.
	*/
	ASSERT3P(buf->b_hdr, ==, hdr);
	boolean_t hdr_compressed =
	arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF;
	boolean_t buf_compressed = ARC_BUF_COMPRESSED(buf) != 0;
	return (!ARC_BUF_ENCRYPTED(buf) &&
	buf_compressed == hdr_compressed &&
	hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS &&
	!HDR_SHARED_DATA(hdr) &&
	(ARC_BUF_LAST(buf) \|\| ARC_BUF_COMPRESSED(buf)));
	}

	/*
	* Allocate a buf for this hdr. If you care about the data that's in the hdr,
	* or if you want a compressed buffer, pass those flags in. Returns 0 if the
	* copy was made successfully, or an error code otherwise.
	*/
	static int
	arc_buf_alloc_impl(arc_buf_hdr_t hdr, spa_t spa, const zbookmark_phys_t *zb,
	const void *tag, boolean_t encrypted, boolean_t compressed,
	boolean_t noauth, boolean_t fill, arc_buf_t **ret)
	{
	arc_buf_t *buf;
	arc_fill_flags_t flags = ARC_FILL_LOCKED;

	ASSERT(HDR_HAS_L1HDR(hdr));
	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
	VERIFY(hdr->b_type == ARC_BUFC_DATA \|\|
	hdr->b_type == ARC_BUFC_METADATA);
	ASSERT3P(ret, !=, NULL);
	ASSERT3P(*ret, ==, NULL);
	IMPLY(encrypted, compressed);

	buf = *ret = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
	buf->b_hdr = hdr;
	buf->b_data = NULL;
	buf->b_next = hdr->b_l1hdr.b_buf;
	buf->b_flags = 0;

	add_reference(hdr, tag);

	/*
	* We're about to change the hdr's b_flags. We must either
	* hold the hash_lock or be undiscoverable.
	*/
	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));

	/*
	* Only honor requests for compressed bufs if the hdr is actually
	* compressed. This must be overridden if the buffer is encrypted since
	* encrypted buffers cannot be decompressed.
	*/
	if (encrypted) {
	buf->b_flags \|= ARC_BUF_FLAG_COMPRESSED;
	buf->b_flags \|= ARC_BUF_FLAG_ENCRYPTED;
	flags \|= ARC_FILL_COMPRESSED \| ARC_FILL_ENCRYPTED;
	} else if (compressed &&
	arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) {
	buf->b_flags \|= ARC_BUF_FLAG_COMPRESSED;
	flags \|= ARC_FILL_COMPRESSED;
	}

	if (noauth) {
	ASSERT0(encrypted);
	flags \|= ARC_FILL_NOAUTH;
	}

	/*
	* If the hdr's data can be shared then we share the data buffer and
	* set the appropriate bit in the hdr's b_flags to indicate the hdr is
	* sharing it's b_pabd with the arc_buf_t. Otherwise, we allocate a new
	* buffer to store the buf's data.
	*
	* There are two additional restrictions here because we're sharing
	* hdr -> buf instead of the usual buf -> hdr. First, the hdr can't be
	* actively involved in an L2ARC write, because if this buf is used by
	* an arc_write() then the hdr's data buffer will be released when the
	* write completes, even though the L2ARC write might still be using it.
	* Second, the hdr's ABD must be linear so that the buf's user doesn't
	* need to be ABD-aware. It must be allocated via
	* zio_[data_]buf_alloc(), not as a page, because we need to be able
	* to abd_release_ownership_of_buf(), which isn't allowed on "linear
	* page" buffers because the ABD code needs to handle freeing them
	* specially.
	*/
	boolean_t can_share = arc_can_share(hdr, buf) &&
	!HDR_L2_WRITING(hdr) &&
	hdr->b_l1hdr.b_pabd != NULL &&
	abd_is_linear(hdr->b_l1hdr.b_pabd) &&
	!abd_is_linear_page(hdr->b_l1hdr.b_pabd);

	/* Set up b_data and sharing */
	if (can_share) {
	buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd);
	buf->b_flags \|= ARC_BUF_FLAG_SHARED;
	arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
	} else {
	buf->b_data =
	arc_get_data_buf(hdr, arc_buf_size(buf), buf);
	ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
	}
	VERIFY3P(buf->b_data, !=, NULL);

	hdr->b_l1hdr.b_buf = buf;

	/*
	* If the user wants the data from the hdr, we need to either copy or
	* decompress the data.
	*/
	if (fill) {
	ASSERT3P(zb, !=, NULL);
	return (arc_buf_fill(buf, spa, zb, flags));
	}

	return (0);
	}

	static const char *arc_onloan_tag = "onloan";

	static inline void
	arc_loaned_bytes_update(int64_t delta)
	{
	atomic_add_64(&arc_loaned_bytes, delta);

	/* assert that it did not wrap around */
	ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0);
	}

	/*
	* Loan out an anonymous arc buffer. Loaned buffers are not counted as in
	* flight data by arc_tempreserve_space() until they are "returned". Loaned
	* buffers must be returned to the arc before they can be used by the DMU or
	* freed.
	*/
	arc_buf_t *
	arc_loan_buf(spa_t *spa, boolean_t is_metadata, int size)
	{
	arc_buf_t *buf = arc_alloc_buf(spa, arc_onloan_tag,
	is_metadata ? ARC_BUFC_METADATA : ARC_BUFC_DATA, size);

	arc_loaned_bytes_update(arc_buf_size(buf));

	return (buf);
	}

	arc_buf_t *
	arc_loan_compressed_buf(spa_t *spa, uint64_t psize, uint64_t lsize,
	enum zio_compress compression_type, uint8_t complevel)
	{
	arc_buf_t *buf = arc_alloc_compressed_buf(spa, arc_onloan_tag,
	psize, lsize, compression_type, complevel);

	arc_loaned_bytes_update(arc_buf_size(buf));

	return (buf);
	}

	arc_buf_t *
	arc_loan_raw_buf(spa_t *spa, uint64_t dsobj, boolean_t byteorder,
	const uint8_t salt, const uint8_t iv, const uint8_t *mac,
	dmu_object_type_t ot, uint64_t psize, uint64_t lsize,
	enum zio_compress compression_type, uint8_t complevel)
	{
	arc_buf_t *buf = arc_alloc_raw_buf(spa, arc_onloan_tag, dsobj,
	byteorder, salt, iv, mac, ot, psize, lsize, compression_type,
	complevel);

	atomic_add_64(&arc_loaned_bytes, psize);
	return (buf);
	}


	/*
	* Return a loaned arc buffer to the arc.
	*/
	void
	arc_return_buf(arc_buf_t buf, const void tag)
	{
	arc_buf_hdr_t *hdr = buf->b_hdr;

	ASSERT3P(buf->b_data, !=, NULL);
	ASSERT(HDR_HAS_L1HDR(hdr));
	(void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
	(void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);

	arc_loaned_bytes_update(-arc_buf_size(buf));
	}

	/* Detach an arc_buf from a dbuf (tag) */
	void
	arc_loan_inuse_buf(arc_buf_t buf, const void tag)
	{
	arc_buf_hdr_t *hdr = buf->b_hdr;

	ASSERT3P(buf->b_data, !=, NULL);
	ASSERT(HDR_HAS_L1HDR(hdr));
	(void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
	(void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);

	arc_loaned_bytes_update(arc_buf_size(buf));
	}

	static void
	l2arc_free_abd_on_write(abd_t *abd, size_t size, arc_buf_contents_t type)
	{
	l2arc_data_free_t df = kmem_alloc(sizeof (df), KM_SLEEP);

	df->l2df_abd = abd;
	df->l2df_size = size;
	df->l2df_type = type;
	mutex_enter(&l2arc_free_on_write_mtx);
	list_insert_head(l2arc_free_on_write, df);
	mutex_exit(&l2arc_free_on_write_mtx);
	}

	static void
	arc_hdr_free_on_write(arc_buf_hdr_t *hdr, boolean_t free_rdata)
	{
	arc_state_t *state = hdr->b_l1hdr.b_state;
	arc_buf_contents_t type = arc_buf_type(hdr);
	uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr);

	/* protected by hash lock, if in the hash table */
	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
	ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
	ASSERT(state != arc_anon && state != arc_l2c_only);

	(void) zfs_refcount_remove_many(&state->arcs_esize[type],
	size, hdr);
	}
	(void) zfs_refcount_remove_many(&state->arcs_size[type], size, hdr);
	if (type == ARC_BUFC_METADATA) {
	arc_space_return(size, ARC_SPACE_META);
	} else {
	ASSERT(type == ARC_BUFC_DATA);
	arc_space_return(size, ARC_SPACE_DATA);
	}

	if (free_rdata) {
	l2arc_free_abd_on_write(hdr->b_crypt_hdr.b_rabd, size, type);
	} else {
	l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type);
	}
	}

	/*
	* Share the arc_buf_t's data with the hdr. Whenever we are sharing the
	* data buffer, we transfer the refcount ownership to the hdr and update
	* the appropriate kstats.
	*/
	static void
	arc_share_buf(arc_buf_hdr_t hdr, arc_buf_t buf)
	{
	ASSERT(arc_can_share(hdr, buf));
	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
	ASSERT(!ARC_BUF_ENCRYPTED(buf));
	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));

	/*
	* Start sharing the data buffer. We transfer the
	* refcount ownership to the hdr since it always owns
	* the refcount whenever an arc_buf_t is shared.
	*/
	zfs_refcount_transfer_ownership_many(
	&hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)],
	arc_hdr_size(hdr), buf, hdr);
	hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf));
	abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd,
	HDR_ISTYPE_METADATA(hdr));
	arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
	buf->b_flags \|= ARC_BUF_FLAG_SHARED;

	/*
	* Since we've transferred ownership to the hdr we need
	* to increment its compressed and uncompressed kstats and
	* decrement the overhead size.
	*/
	ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
	ARCSTAT_INCR(arcstat_overhead_size, -arc_buf_size(buf));
	}

	static void
	arc_unshare_buf(arc_buf_hdr_t hdr, arc_buf_t buf)
	{
	ASSERT(arc_buf_is_shared(buf));
	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));

	/*
	* We are no longer sharing this buffer so we need
	* to transfer its ownership to the rightful owner.
	*/
	zfs_refcount_transfer_ownership_many(
	&hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)],
	arc_hdr_size(hdr), hdr, buf);
	arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
	abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd);
	abd_free(hdr->b_l1hdr.b_pabd);
	hdr->b_l1hdr.b_pabd = NULL;
	buf->b_flags &= ~ARC_BUF_FLAG_SHARED;

	/*
	* Since the buffer is no longer shared between
	* the arc buf and the hdr, count it as overhead.
	*/
	ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
	ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
	}

	/*
	* Remove an arc_buf_t from the hdr's buf list and return the last
	* arc_buf_t on the list. If no buffers remain on the list then return
	* NULL.
	*/
	static arc_buf_t *
	arc_buf_remove(arc_buf_hdr_t hdr, arc_buf_t buf)
	{
	ASSERT(HDR_HAS_L1HDR(hdr));
	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));

	arc_buf_t **bufp = &hdr->b_l1hdr.b_buf;
	arc_buf_t *lastbuf = NULL;

	/*
	* Remove the buf from the hdr list and locate the last
	* remaining buffer on the list.
	*/
	while (*bufp != NULL) {
	if (*bufp == buf)
	*bufp = buf->b_next;

	/*
	* If we've removed a buffer in the middle of
	* the list then update the lastbuf and update
	* bufp.
	*/
	if (*bufp != NULL) {
	lastbuf = *bufp;
	bufp = &(*bufp)->b_next;
	}
	}
	buf->b_next = NULL;
	ASSERT3P(lastbuf, !=, buf);
	IMPLY(lastbuf != NULL, ARC_BUF_LAST(lastbuf));

	return (lastbuf);
	}

	/*
	* Free up buf->b_data and pull the arc_buf_t off of the arc_buf_hdr_t's
	* list and free it.
	*/
	static void
	arc_buf_destroy_impl(arc_buf_t *buf)
	{
	arc_buf_hdr_t *hdr = buf->b_hdr;

	/*
	* Free up the data associated with the buf but only if we're not
	* sharing this with the hdr. If we are sharing it with the hdr, the
	* hdr is responsible for doing the free.
	*/
	if (buf->b_data != NULL) {
	/*
	* We're about to change the hdr's b_flags. We must either
	* hold the hash_lock or be undiscoverable.
	*/
	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));

	arc_cksum_verify(buf);
	arc_buf_unwatch(buf);

	if (ARC_BUF_SHARED(buf)) {
	arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
	} else {
	ASSERT(!arc_buf_is_shared(buf));
	uint64_t size = arc_buf_size(buf);
	arc_free_data_buf(hdr, buf->b_data, size, buf);
	ARCSTAT_INCR(arcstat_overhead_size, -size);
	}
	buf->b_data = NULL;

	/*
	* If we have no more encrypted buffers and we've already
	* gotten a copy of the decrypted data we can free b_rabd
	* to save some space.
	*/
	if (ARC_BUF_ENCRYPTED(buf) && HDR_HAS_RABD(hdr) &&
	hdr->b_l1hdr.b_pabd != NULL && !HDR_IO_IN_PROGRESS(hdr)) {
	arc_buf_t *b;
	for (b = hdr->b_l1hdr.b_buf; b; b = b->b_next) {
	if (b != buf && ARC_BUF_ENCRYPTED(b))
	break;
	}
	if (b == NULL)
	arc_hdr_free_abd(hdr, B_TRUE);
	}
	}

	arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);

	if (ARC_BUF_SHARED(buf) && !ARC_BUF_COMPRESSED(buf)) {
	/*
	* If the current arc_buf_t is sharing its data buffer with the
	* hdr, then reassign the hdr's b_pabd to share it with the new
	* buffer at the end of the list. The shared buffer is always
	* the last one on the hdr's buffer list.
	*
	* There is an equivalent case for compressed bufs, but since
	* they aren't guaranteed to be the last buf in the list and
	* that is an exceedingly rare case, we just allow that space be
	* wasted temporarily. We must also be careful not to share
	* encrypted buffers, since they cannot be shared.
	*/
	if (lastbuf != NULL && !ARC_BUF_ENCRYPTED(lastbuf)) {
	/* Only one buf can be shared at once */
	ASSERT(!arc_buf_is_shared(lastbuf));
	/* hdr is uncompressed so can't have compressed buf */
	ASSERT(!ARC_BUF_COMPRESSED(lastbuf));

	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
	arc_hdr_free_abd(hdr, B_FALSE);

	/*
	* We must setup a new shared block between the
	* last buffer and the hdr. The data would have
	* been allocated by the arc buf so we need to transfer
	* ownership to the hdr since it's now being shared.
	*/
	arc_share_buf(hdr, lastbuf);
	}
	} else if (HDR_SHARED_DATA(hdr)) {
	/*
	* Uncompressed shared buffers are always at the end
	* of the list. Compressed buffers don't have the
	* same requirements. This makes it hard to
	* simply assert that the lastbuf is shared so
	* we rely on the hdr's compression flags to determine
	* if we have a compressed, shared buffer.
	*/
	ASSERT3P(lastbuf, !=, NULL);
	ASSERT(arc_buf_is_shared(lastbuf) \|\|
	arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
	}

	/*
	* Free the checksum if we're removing the last uncompressed buf from
	* this hdr.
	*/
	if (!arc_hdr_has_uncompressed_buf(hdr)) {
	arc_cksum_free(hdr);
	}

	/* clean up the buf */
	buf->b_hdr = NULL;
	kmem_cache_free(buf_cache, buf);
	}

	static void
	arc_hdr_alloc_abd(arc_buf_hdr_t *hdr, int alloc_flags)
	{
	uint64_t size;
	boolean_t alloc_rdata = ((alloc_flags & ARC_HDR_ALLOC_RDATA) != 0);

	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
	ASSERT(HDR_HAS_L1HDR(hdr));
	ASSERT(!HDR_SHARED_DATA(hdr) \|\| alloc_rdata);
	IMPLY(alloc_rdata, HDR_PROTECTED(hdr));

	if (alloc_rdata) {
	size = HDR_GET_PSIZE(hdr);
	ASSERT3P(hdr->b_crypt_hdr.b_rabd, ==, NULL);
	hdr->b_crypt_hdr.b_rabd = arc_get_data_abd(hdr, size, hdr,
	alloc_flags);
	ASSERT3P(hdr->b_crypt_hdr.b_rabd, !=, NULL);
	ARCSTAT_INCR(arcstat_raw_size, size);
	} else {
	size = arc_hdr_size(hdr);
	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
	hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, size, hdr,
	alloc_flags);
	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
	}

	ARCSTAT_INCR(arcstat_compressed_size, size);
	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
	}

	static void
	arc_hdr_free_abd(arc_buf_hdr_t *hdr, boolean_t free_rdata)
	{
	uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr);

	ASSERT(HDR_HAS_L1HDR(hdr));
	ASSERT(hdr->b_l1hdr.b_pabd != NULL \|\| HDR_HAS_RABD(hdr));
	IMPLY(free_rdata, HDR_HAS_RABD(hdr));

	/*
	* If the hdr is currently being written to the l2arc then
	* we defer freeing the data by adding it to the l2arc_free_on_write
	* list. The l2arc will free the data once it's finished
	* writing it to the l2arc device.
	*/
	if (HDR_L2_WRITING(hdr)) {
	arc_hdr_free_on_write(hdr, free_rdata);
	ARCSTAT_BUMP(arcstat_l2_free_on_write);
	} else if (free_rdata) {
	arc_free_data_abd(hdr, hdr->b_crypt_hdr.b_rabd, size, hdr);
	} else {
	arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, size, hdr);
	}

	if (free_rdata) {
	hdr->b_crypt_hdr.b_rabd = NULL;
	ARCSTAT_INCR(arcstat_raw_size, -size);
	} else {
	hdr->b_l1hdr.b_pabd = NULL;
	}

	if (hdr->b_l1hdr.b_pabd == NULL && !HDR_HAS_RABD(hdr))
	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;

	ARCSTAT_INCR(arcstat_compressed_size, -size);
	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
	}

	/*
	* Allocate empty anonymous ARC header. The header will get its identity
	* assigned and buffers attached later as part of read or write operations.
	*
	* In case of read arc_read() assigns header its identify (b_dva + b_birth),
	* inserts it into ARC hash to become globally visible and allocates physical
	* (b_pabd) or raw (b_rabd) ABD buffer to read into from disk. On disk read
	* completion arc_read_done() allocates ARC buffer(s) as needed, potentially
	* sharing one of them with the physical ABD buffer.
	*
	* In case of write arc_alloc_buf() allocates ARC buffer to be filled with
	* data. Then after compression and/or encryption arc_write_ready() allocates
	* and fills (or potentially shares) physical (b_pabd) or raw (b_rabd) ABD
	* buffer. On disk write completion arc_write_done() assigns the header its
	* new identity (b_dva + b_birth) and inserts into ARC hash.
	*
	* In case of partial overwrite the old data is read first as described. Then
	* arc_release() either allocates new anonymous ARC header and moves the ARC
	* buffer to it, or reuses the old ARC header by discarding its identity and
	* removing it from ARC hash. After buffer modification normal write process
	* follows as described.
	*/
	static arc_buf_hdr_t *
	arc_hdr_alloc(uint64_t spa, int32_t psize, int32_t lsize,
	boolean_t protected, enum zio_compress compression_type, uint8_t complevel,
	arc_buf_contents_t type)
	{
	arc_buf_hdr_t *hdr;

	VERIFY(type == ARC_BUFC_DATA \|\| type == ARC_BUFC_METADATA);
	hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);

	ASSERT(HDR_EMPTY(hdr));
	#ifdef ZFS_DEBUG
	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
	#endif
	HDR_SET_PSIZE(hdr, psize);
	HDR_SET_LSIZE(hdr, lsize);
	hdr->b_spa = spa;
	hdr->b_type = type;
	hdr->b_flags = 0;
	arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) \| ARC_FLAG_HAS_L1HDR);
	arc_hdr_set_compress(hdr, compression_type);
	hdr->b_complevel = complevel;
	if (protected)
	arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);

	hdr->b_l1hdr.b_state = arc_anon;
	hdr->b_l1hdr.b_arc_access = 0;
	hdr->b_l1hdr.b_mru_hits = 0;
	hdr->b_l1hdr.b_mru_ghost_hits = 0;
	hdr->b_l1hdr.b_mfu_hits = 0;
	hdr->b_l1hdr.b_mfu_ghost_hits = 0;
	hdr->b_l1hdr.b_buf = NULL;

	ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));

	return (hdr);
	}

	/*
	* Transition between the two allocation states for the arc_buf_hdr struct.
	* The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without
	* (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller
	* version is used when a cache buffer is only in the L2ARC in order to reduce
	* memory usage.
	*/
	static arc_buf_hdr_t *
	arc_hdr_realloc(arc_buf_hdr_t hdr, kmem_cache_t old, kmem_cache_t *new)
	{
	ASSERT(HDR_HAS_L2HDR(hdr));

	arc_buf_hdr_t *nhdr;
	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;

	ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) \|\|
	(old == hdr_l2only_cache && new == hdr_full_cache));

	nhdr = kmem_cache_alloc(new, KM_PUSHPAGE);

	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
	buf_hash_remove(hdr);

	memcpy(nhdr, hdr, HDR_L2ONLY_SIZE);

	if (new == hdr_full_cache) {
	arc_hdr_set_flags(nhdr, ARC_FLAG_HAS_L1HDR);
	/*
	* arc_access and arc_change_state need to be aware that a
	* header has just come out of L2ARC, so we set its state to
	* l2c_only even though it's about to change.
	*/
	nhdr->b_l1hdr.b_state = arc_l2c_only;

	/* Verify previous threads set to NULL before freeing */
	ASSERT3P(nhdr->b_l1hdr.b_pabd, ==, NULL);
	ASSERT(!HDR_HAS_RABD(hdr));
	} else {
	ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
	#ifdef ZFS_DEBUG
	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
	#endif

	/*
	* If we've reached here, We must have been called from
	* arc_evict_hdr(), as such we should have already been
	* removed from any ghost list we were previously on
	* (which protects us from racing with arc_evict_state),
	* thus no locking is needed during this check.
	*/
	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));

	/*
	* A buffer must not be moved into the arc_l2c_only
	* state if it's not finished being written out to the
	* l2arc device. Otherwise, the b_l1hdr.b_pabd field
	* might try to be accessed, even though it was removed.
	*/
	VERIFY(!HDR_L2_WRITING(hdr));
	VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL);
	ASSERT(!HDR_HAS_RABD(hdr));

	arc_hdr_clear_flags(nhdr, ARC_FLAG_HAS_L1HDR);
	}
	/*
	* The header has been reallocated so we need to re-insert it into any
	* lists it was on.
	*/
	(void) buf_hash_insert(nhdr, NULL);

	ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));

	mutex_enter(&dev->l2ad_mtx);

	/*
	* We must place the realloc'ed header back into the list at
	* the same spot. Otherwise, if it's placed earlier in the list,
	* l2arc_write_buffers() could find it during the function's
	* write phase, and try to write it out to the l2arc.
	*/
	list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
	list_remove(&dev->l2ad_buflist, hdr);

	mutex_exit(&dev->l2ad_mtx);

	/*
	* Since we're using the pointer address as the tag when
	* incrementing and decrementing the l2ad_alloc refcount, we
	* must remove the old pointer (that we're about to destroy) and
	* add the new pointer to the refcount. Otherwise we'd remove
	* the wrong pointer address when calling arc_hdr_destroy() later.
	*/

	(void) zfs_refcount_remove_many(&dev->l2ad_alloc,
	arc_hdr_size(hdr), hdr);
	(void) zfs_refcount_add_many(&dev->l2ad_alloc,
	arc_hdr_size(nhdr), nhdr);

	buf_discard_identity(hdr);
	kmem_cache_free(old, hdr);

	return (nhdr);
	}

	/*
	* This function is used by the send / receive code to convert a newly
	* allocated arc_buf_t to one that is suitable for a raw encrypted write. It
	* is also used to allow the root objset block to be updated without altering
	* its embedded MACs. Both block types will always be uncompressed so we do not
	* have to worry about compression type or psize.
	*/
	void
	arc_convert_to_raw(arc_buf_t *buf, uint64_t dsobj, boolean_t byteorder,
	dmu_object_type_t ot, const uint8_t salt, const uint8_t iv,
	const uint8_t *mac)
	{
	arc_buf_hdr_t *hdr = buf->b_hdr;

	ASSERT(ot == DMU_OT_DNODE \|\| ot == DMU_OT_OBJSET);
	ASSERT(HDR_HAS_L1HDR(hdr));
	ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);

	buf->b_flags \|= (ARC_BUF_FLAG_COMPRESSED \| ARC_BUF_FLAG_ENCRYPTED);
	arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
	hdr->b_crypt_hdr.b_dsobj = dsobj;
	hdr->b_crypt_hdr.b_ot = ot;
	hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ?
	DMU_BSWAP_NUMFUNCS : DMU_OT_BYTESWAP(ot);
	if (!arc_hdr_has_uncompressed_buf(hdr))
	arc_cksum_free(hdr);

	if (salt != NULL)
	memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN);
	if (iv != NULL)
	memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN);
	if (mac != NULL)
	memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN);
	}

	/*
	* Allocate a new arc_buf_hdr_t and arc_buf_t and return the buf to the caller.
	* The buf is returned thawed since we expect the consumer to modify it.
	*/
	arc_buf_t *
	arc_alloc_buf(spa_t spa, const void tag, arc_buf_contents_t type,
	int32_t size)
	{
	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size,
	B_FALSE, ZIO_COMPRESS_OFF, 0, type);

	arc_buf_t *buf = NULL;
	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE, B_FALSE,
	B_FALSE, B_FALSE, &buf));
	arc_buf_thaw(buf);

	return (buf);
	}

	/*
	* Allocate a compressed buf in the same manner as arc_alloc_buf. Don't use this
	* for bufs containing metadata.
	*/
	arc_buf_t *
	arc_alloc_compressed_buf(spa_t spa, const void tag, uint64_t psize,
	uint64_t lsize, enum zio_compress compression_type, uint8_t complevel)
	{
	ASSERT3U(lsize, >, 0);
	ASSERT3U(lsize, >=, psize);
	ASSERT3U(compression_type, >, ZIO_COMPRESS_OFF);
	ASSERT3U(compression_type, <, ZIO_COMPRESS_FUNCTIONS);

	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
	B_FALSE, compression_type, complevel, ARC_BUFC_DATA);

	arc_buf_t *buf = NULL;
	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE,
	B_TRUE, B_FALSE, B_FALSE, &buf));
	arc_buf_thaw(buf);

	/*
	* To ensure that the hdr has the correct data in it if we call
	* arc_untransform() on this buf before it's been written to disk,
	* it's easiest if we just set up sharing between the buf and the hdr.
	*/
	arc_share_buf(hdr, buf);

	return (buf);
	}

	arc_buf_t *
	arc_alloc_raw_buf(spa_t spa, const void tag, uint64_t dsobj,
	boolean_t byteorder, const uint8_t salt, const uint8_t iv,
	const uint8_t *mac, dmu_object_type_t ot, uint64_t psize, uint64_t lsize,
	enum zio_compress compression_type, uint8_t complevel)
	{
	arc_buf_hdr_t *hdr;
	arc_buf_t *buf;
	arc_buf_contents_t type = DMU_OT_IS_METADATA(ot) ?
	ARC_BUFC_METADATA : ARC_BUFC_DATA;

	ASSERT3U(lsize, >, 0);
	ASSERT3U(lsize, >=, psize);
	ASSERT3U(compression_type, >=, ZIO_COMPRESS_OFF);
	ASSERT3U(compression_type, <, ZIO_COMPRESS_FUNCTIONS);

	hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, B_TRUE,
	compression_type, complevel, type);

	hdr->b_crypt_hdr.b_dsobj = dsobj;
	hdr->b_crypt_hdr.b_ot = ot;
	hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ?
	DMU_BSWAP_NUMFUNCS : DMU_OT_BYTESWAP(ot);
	memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN);
	memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN);
	memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN);

	/*
	* This buffer will be considered encrypted even if the ot is not an
	* encrypted type. It will become authenticated instead in
	* arc_write_ready().
	*/
	buf = NULL;
	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_TRUE, B_TRUE,
	B_FALSE, B_FALSE, &buf));
	arc_buf_thaw(buf);

	return (buf);
	}

	static void
	l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr,
	boolean_t state_only)
	{
	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
	l2arc_dev_t *dev = l2hdr->b_dev;
	uint64_t lsize = HDR_GET_LSIZE(hdr);
	uint64_t psize = HDR_GET_PSIZE(hdr);
	uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev, psize);
	arc_buf_contents_t type = hdr->b_type;
	int64_t lsize_s;
	int64_t psize_s;
	int64_t asize_s;

	if (incr) {
	lsize_s = lsize;
	psize_s = psize;
	asize_s = asize;
	} else {
	lsize_s = -lsize;
	psize_s = -psize;
	asize_s = -asize;
	}

	/* If the buffer is a prefetch, count it as such. */
	if (HDR_PREFETCH(hdr)) {
	ARCSTAT_INCR(arcstat_l2_prefetch_asize, asize_s);
	} else {
	/*
	* We use the value stored in the L2 header upon initial
	* caching in L2ARC. This value will be updated in case
	* an MRU/MRU_ghost buffer transitions to MFU but the L2ARC
	* metadata (log entry) cannot currently be updated. Having
	* the ARC state in the L2 header solves the problem of a
	* possibly absent L1 header (apparent in buffers restored
	* from persistent L2ARC).
	*/
	switch (hdr->b_l2hdr.b_arcs_state) {
	case ARC_STATE_MRU_GHOST:
	case ARC_STATE_MRU:
	ARCSTAT_INCR(arcstat_l2_mru_asize, asize_s);
	break;
	case ARC_STATE_MFU_GHOST:
	case ARC_STATE_MFU:
	ARCSTAT_INCR(arcstat_l2_mfu_asize, asize_s);
	break;
	default:
	break;
	}
	}

	if (state_only)
	return;

	ARCSTAT_INCR(arcstat_l2_psize, psize_s);
	ARCSTAT_INCR(arcstat_l2_lsize, lsize_s);

	switch (type) {
	case ARC_BUFC_DATA:
	ARCSTAT_INCR(arcstat_l2_bufc_data_asize, asize_s);
	break;
	case ARC_BUFC_METADATA:
	ARCSTAT_INCR(arcstat_l2_bufc_metadata_asize, asize_s);
	break;
	default:
	break;
	}
	}


	static void
	arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
	{
	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
	l2arc_dev_t *dev = l2hdr->b_dev;
	uint64_t psize = HDR_GET_PSIZE(hdr);
	uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev, psize);

	ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
	ASSERT(HDR_HAS_L2HDR(hdr));

	list_remove(&dev->l2ad_buflist, hdr);

	l2arc_hdr_arcstats_decrement(hdr);
	vdev_space_update(dev->l2ad_vdev, -asize, 0, 0);

	(void) zfs_refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr),
	hdr);
	arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
	}

	static void
	arc_hdr_destroy(arc_buf_hdr_t *hdr)
	{
	if (HDR_HAS_L1HDR(hdr)) {
	ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
	ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
	}
	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
	ASSERT(!HDR_IN_HASH_TABLE(hdr));

	if (HDR_HAS_L2HDR(hdr)) {
	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
	boolean_t buflist_held = MUTEX_HELD(&dev->l2ad_mtx);

	if (!buflist_held)
	mutex_enter(&dev->l2ad_mtx);

	/*
	* Even though we checked this conditional above, we
	* need to check this again now that we have the
	* l2ad_mtx. This is because we could be racing with
	* another thread calling l2arc_evict() which might have
	* destroyed this header's L2 portion as we were waiting
	* to acquire the l2ad_mtx. If that happens, we don't
	* want to re-destroy the header's L2 portion.
	*/
	if (HDR_HAS_L2HDR(hdr)) {

	if (!HDR_EMPTY(hdr))
	buf_discard_identity(hdr);

	arc_hdr_l2hdr_destroy(hdr);
	}

	if (!buflist_held)
	mutex_exit(&dev->l2ad_mtx);
	}

	/*
	* The header's identify can only be safely discarded once it is no
	* longer discoverable. This requires removing it from the hash table
	* and the l2arc header list. After this point the hash lock can not
	* be used to protect the header.
	*/
	if (!HDR_EMPTY(hdr))
	buf_discard_identity(hdr);

	if (HDR_HAS_L1HDR(hdr)) {
	arc_cksum_free(hdr);

	while (hdr->b_l1hdr.b_buf != NULL)
	arc_buf_destroy_impl(hdr->b_l1hdr.b_buf);

	if (hdr->b_l1hdr.b_pabd != NULL)
	arc_hdr_free_abd(hdr, B_FALSE);

	if (HDR_HAS_RABD(hdr))
	arc_hdr_free_abd(hdr, B_TRUE);
	}

	ASSERT3P(hdr->b_hash_next, ==, NULL);
	if (HDR_HAS_L1HDR(hdr)) {
	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
	#ifdef ZFS_DEBUG
	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
	#endif
	kmem_cache_free(hdr_full_cache, hdr);
	} else {
	kmem_cache_free(hdr_l2only_cache, hdr);
	}
	}

	void
	arc_buf_destroy(arc_buf_t buf, const void tag)
	{
	arc_buf_hdr_t *hdr = buf->b_hdr;

	if (hdr->b_l1hdr.b_state == arc_anon) {
	ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
	ASSERT(ARC_BUF_LAST(buf));
	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
	VERIFY0(remove_reference(hdr, tag));
	return;
	}

	kmutex_t *hash_lock = HDR_LOCK(hdr);
	mutex_enter(hash_lock);

	ASSERT3P(hdr, ==, buf->b_hdr);
	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
	ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon);
	ASSERT3P(buf->b_data, !=, NULL);

	arc_buf_destroy_impl(buf);
	(void) remove_reference(hdr, tag);
	mutex_exit(hash_lock);
	}

	/*
	* Evict the arc_buf_hdr that is provided as a parameter. The resultant
	* state of the header is dependent on its state prior to entering this
	* function. The following transitions are possible:
	*
	* - arc_mru -> arc_mru_ghost
	* - arc_mfu -> arc_mfu_ghost
	* - arc_mru_ghost -> arc_l2c_only
	* - arc_mru_ghost -> deleted
	* - arc_mfu_ghost -> arc_l2c_only
	* - arc_mfu_ghost -> deleted
	* - arc_uncached -> deleted
	*
	* Return total size of evicted data buffers for eviction progress tracking.
	* When evicting from ghost states return logical buffer size to make eviction
	* progress at the same (or at least comparable) rate as from non-ghost states.
	*
	* Return *real_evicted for actual ARC size reduction to wake up threads
	* waiting for it. For non-ghost states it includes size of evicted data
	* buffers (the headers are not freed there). For ghost states it includes
	* only the evicted headers size.
	*/
	static int64_t
	arc_evict_hdr(arc_buf_hdr_t hdr, uint64_t real_evicted)
	{
	arc_state_t evicted_state, state;
	int64_t bytes_evicted = 0;
	uint_t min_lifetime = HDR_PRESCIENT_PREFETCH(hdr) ?
	arc_min_prescient_prefetch_ms : arc_min_prefetch_ms;

	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
	ASSERT(HDR_HAS_L1HDR(hdr));
	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
	ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
	ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt));

	*real_evicted = 0;
	state = hdr->b_l1hdr.b_state;
	if (GHOST_STATE(state)) {

	/*
	* l2arc_write_buffers() relies on a header's L1 portion
	* (i.e. its b_pabd field) during it's write phase.
	* Thus, we cannot push a header onto the arc_l2c_only
	* state (removing its L1 piece) until the header is
	* done being written to the l2arc.
	*/
	if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
	ARCSTAT_BUMP(arcstat_evict_l2_skip);
	return (bytes_evicted);
	}

	ARCSTAT_BUMP(arcstat_deleted);
	bytes_evicted += HDR_GET_LSIZE(hdr);

	DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);

	if (HDR_HAS_L2HDR(hdr)) {
	ASSERT(hdr->b_l1hdr.b_pabd == NULL);
	ASSERT(!HDR_HAS_RABD(hdr));
	/*
	* This buffer is cached on the 2nd Level ARC;
	* don't destroy the header.
	*/
	arc_change_state(arc_l2c_only, hdr);
	/*
	* dropping from L1+L2 cached to L2-only,
	* realloc to remove the L1 header.
	*/
	(void) arc_hdr_realloc(hdr, hdr_full_cache,
	hdr_l2only_cache);
	*real_evicted += HDR_FULL_SIZE - HDR_L2ONLY_SIZE;
	} else {
	arc_change_state(arc_anon, hdr);
	arc_hdr_destroy(hdr);
	*real_evicted += HDR_FULL_SIZE;
	}
	return (bytes_evicted);
	}

	ASSERT(state == arc_mru \|\| state == arc_mfu \|\| state == arc_uncached);
	evicted_state = (state == arc_uncached) ? arc_anon :
	((state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost);

	/* prefetch buffers have a minimum lifespan */
	if ((hdr->b_flags & (ARC_FLAG_PREFETCH \| ARC_FLAG_INDIRECT)) &&
	ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
	MSEC_TO_TICK(min_lifetime)) {
	ARCSTAT_BUMP(arcstat_evict_skip);
	return (bytes_evicted);
	}

	if (HDR_HAS_L2HDR(hdr)) {
	ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr));
	} else {
	if (l2arc_write_eligible(hdr->b_spa, hdr)) {
	ARCSTAT_INCR(arcstat_evict_l2_eligible,
	HDR_GET_LSIZE(hdr));

	switch (state->arcs_state) {
	case ARC_STATE_MRU:
	ARCSTAT_INCR(
	arcstat_evict_l2_eligible_mru,
	HDR_GET_LSIZE(hdr));
	break;
	case ARC_STATE_MFU:
	ARCSTAT_INCR(
	arcstat_evict_l2_eligible_mfu,
	HDR_GET_LSIZE(hdr));
	break;
	default:
	break;
	}
	} else {
	ARCSTAT_INCR(arcstat_evict_l2_ineligible,
	HDR_GET_LSIZE(hdr));
	}
	}

	bytes_evicted += arc_hdr_size(hdr);
	*real_evicted += arc_hdr_size(hdr);

	/*
	* If this hdr is being evicted and has a compressed buffer then we
	* discard it here before we change states. This ensures that the
	* accounting is updated correctly in arc_free_data_impl().
	*/
	if (hdr->b_l1hdr.b_pabd != NULL)
	arc_hdr_free_abd(hdr, B_FALSE);

	if (HDR_HAS_RABD(hdr))
	arc_hdr_free_abd(hdr, B_TRUE);

	arc_change_state(evicted_state, hdr);
	DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
	if (evicted_state == arc_anon) {
	arc_hdr_destroy(hdr);
	*real_evicted += HDR_FULL_SIZE;
	} else {
	ASSERT(HDR_IN_HASH_TABLE(hdr));
	}

	return (bytes_evicted);
	}

	static void
	arc_set_need_free(void)
	{
	ASSERT(MUTEX_HELD(&arc_evict_lock));
	int64_t remaining = arc_free_memory() - arc_sys_free / 2;
	arc_evict_waiter_t *aw = list_tail(&arc_evict_waiters);
	if (aw == NULL) {
	arc_need_free = MAX(-remaining, 0);
	} else {
	arc_need_free =
	MAX(-remaining, (int64_t)(aw->aew_count - arc_evict_count));
	}
	}

	static uint64_t
	arc_evict_state_impl(multilist_t ml, int idx, arc_buf_hdr_t marker,
	uint64_t spa, uint64_t bytes)
	{
	multilist_sublist_t *mls;
	uint64_t bytes_evicted = 0, real_evicted = 0;
	arc_buf_hdr_t *hdr;
	kmutex_t *hash_lock;
	uint_t evict_count = zfs_arc_evict_batch_limit;

	ASSERT3P(marker, !=, NULL);

	- mls = multilist_sublist_lock(ml, idx);
	+ mls = multilist_sublist_lock_idx(ml, idx);

	for (hdr = multilist_sublist_prev(mls, marker); likely(hdr != NULL);
	hdr = multilist_sublist_prev(mls, marker)) {
	if ((evict_count == 0) \|\| (bytes_evicted >= bytes))
	break;

	/*
	* To keep our iteration location, move the marker
	* forward. Since we're not holding hdr's hash lock, we
	* must be very careful and not remove 'hdr' from the
	* sublist. Otherwise, other consumers might mistake the
	* 'hdr' as not being on a sublist when they call the
	* multilist_link_active() function (they all rely on
	* the hash lock protecting concurrent insertions and
	* removals). multilist_sublist_move_forward() was
	* specifically implemented to ensure this is the case
	* (only 'marker' will be removed and re-inserted).
	*/
	multilist_sublist_move_forward(mls, marker);

	/*
	* The only case where the b_spa field should ever be
	* zero, is the marker headers inserted by
	* arc_evict_state(). It's possible for multiple threads
	* to be calling arc_evict_state() concurrently (e.g.
	* dsl_pool_close() and zio_inject_fault()), so we must
	* skip any markers we see from these other threads.
	*/
	if (hdr->b_spa == 0)
	continue;

	/* we're only interested in evicting buffers of a certain spa */
	if (spa != 0 && hdr->b_spa != spa) {
	ARCSTAT_BUMP(arcstat_evict_skip);
	continue;
	}

	hash_lock = HDR_LOCK(hdr);

	/*
	* We aren't calling this function from any code path
	* that would already be holding a hash lock, so we're
	* asserting on this assumption to be defensive in case
	* this ever changes. Without this check, it would be
	* possible to incorrectly increment arcstat_mutex_miss
	* below (e.g. if the code changed such that we called
	* this function with a hash lock held).
	*/
	ASSERT(!MUTEX_HELD(hash_lock));

	if (mutex_tryenter(hash_lock)) {
	uint64_t revicted;
	uint64_t evicted = arc_evict_hdr(hdr, &revicted);
	mutex_exit(hash_lock);

	bytes_evicted += evicted;
	real_evicted += revicted;

	/*
	* If evicted is zero, arc_evict_hdr() must have
	* decided to skip this header, don't increment
	* evict_count in this case.
	*/
	if (evicted != 0)
	evict_count--;

	} else {
	ARCSTAT_BUMP(arcstat_mutex_miss);
	}
	}

	multilist_sublist_unlock(mls);

	/*
	* Increment the count of evicted bytes, and wake up any threads that
	* are waiting for the count to reach this value. Since the list is
	* ordered by ascending aew_count, we pop off the beginning of the
	* list until we reach the end, or a waiter that's past the current
	* "count". Doing this outside the loop reduces the number of times
	* we need to acquire the global arc_evict_lock.
	*
	* Only wake when there's sufficient free memory in the system
	* (specifically, arc_sys_free/2, which by default is a bit more than
	* 1/64th of RAM). See the comments in arc_wait_for_eviction().
	*/
	mutex_enter(&arc_evict_lock);
	arc_evict_count += real_evicted;

	if (arc_free_memory() > arc_sys_free / 2) {
	arc_evict_waiter_t *aw;
	while ((aw = list_head(&arc_evict_waiters)) != NULL &&
	aw->aew_count <= arc_evict_count) {
	list_remove(&arc_evict_waiters, aw);
	cv_broadcast(&aw->aew_cv);
	}
	}
	arc_set_need_free();
	mutex_exit(&arc_evict_lock);

	/*
	* If the ARC size is reduced from arc_c_max to arc_c_min (especially
	* if the average cached block is small), eviction can be on-CPU for
	* many seconds. To ensure that other threads that may be bound to
	* this CPU are able to make progress, make a voluntary preemption
	* call here.
	*/
	kpreempt(KPREEMPT_SYNC);

	return (bytes_evicted);
	}

	+static arc_buf_hdr_t *
	+arc_state_alloc_marker(void)
	+{
	+ arc_buf_hdr_t *marker = kmem_cache_alloc(hdr_full_cache, KM_SLEEP);
	+
	+ /*
	+ * A b_spa of 0 is used to indicate that this header is
	+ * a marker. This fact is used in arc_evict_state_impl().
	+ */
	+ marker->b_spa = 0;
	+
	+ return (marker);
	+}
	+
	+static void
	+arc_state_free_marker(arc_buf_hdr_t *marker)
	+{
	+ kmem_cache_free(hdr_full_cache, marker);
	+}
	+
	/*
	* Allocate an array of buffer headers used as placeholders during arc state
	* eviction.
	*/
	static arc_buf_hdr_t **
	arc_state_alloc_markers(int count)
	{
	arc_buf_hdr_t **markers;

	markers = kmem_zalloc(sizeof (markers) count, KM_SLEEP);
	- for (int i = 0; i < count; i++) {
	- markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP);
	-
	- /*
	- * A b_spa of 0 is used to indicate that this header is
	- * a marker. This fact is used in arc_evict_state_impl().
	- */
	- markers[i]->b_spa = 0;
	-
	- }
	+ for (int i = 0; i < count; i++)
	+ markers[i] = arc_state_alloc_marker();
	return (markers);
	}

	static void
	arc_state_free_markers(arc_buf_hdr_t **markers, int count)
	{
	for (int i = 0; i < count; i++)
	- kmem_cache_free(hdr_full_cache, markers[i]);
	+ arc_state_free_marker(markers[i]);
	kmem_free(markers, sizeof (markers) count);
	}

	/*
	* Evict buffers from the given arc state, until we've removed the
	* specified number of bytes. Move the removed buffers to the
	* appropriate evict state.
	*
	* This function makes a "best effort". It skips over any buffers
	* it can't get a hash_lock on, and so, may not catch all candidates.
	* It may also return without evicting as much space as requested.
	*
	* If bytes is specified using the special value ARC_EVICT_ALL, this
	* will evict all available (i.e. unlocked and evictable) buffers from
	* the given arc state; which is used by arc_flush().
	*/
	static uint64_t
	arc_evict_state(arc_state_t *state, arc_buf_contents_t type, uint64_t spa,
	uint64_t bytes)
	{
	uint64_t total_evicted = 0;
	multilist_t *ml = &state->arcs_list[type];
	int num_sublists;
	arc_buf_hdr_t **markers;

	num_sublists = multilist_get_num_sublists(ml);

	/*
	* If we've tried to evict from each sublist, made some
	* progress, but still have not hit the target number of bytes
	* to evict, we want to keep trying. The markers allow us to
	* pick up where we left off for each individual sublist, rather
	* than starting from the tail each time.
	*/
	if (zthr_iscurthread(arc_evict_zthr)) {
	markers = arc_state_evict_markers;
	ASSERT3S(num_sublists, <=, arc_state_evict_marker_count);
	} else {
	markers = arc_state_alloc_markers(num_sublists);
	}
	for (int i = 0; i < num_sublists; i++) {
	multilist_sublist_t *mls;

	- mls = multilist_sublist_lock(ml, i);
	+ mls = multilist_sublist_lock_idx(ml, i);
	multilist_sublist_insert_tail(mls, markers[i]);
	multilist_sublist_unlock(mls);
	}

	/*
	* While we haven't hit our target number of bytes to evict, or
	* we're evicting all available buffers.
	*/
	while (total_evicted < bytes) {
	int sublist_idx = multilist_get_random_index(ml);
	uint64_t scan_evicted = 0;

	/*
	* Start eviction using a randomly selected sublist,
	* this is to try and evenly balance eviction across all
	* sublists. Always starting at the same sublist
	* (e.g. index 0) would cause evictions to favor certain
	* sublists over others.
	*/
	for (int i = 0; i < num_sublists; i++) {
	uint64_t bytes_remaining;
	uint64_t bytes_evicted;

	if (total_evicted < bytes)
	bytes_remaining = bytes - total_evicted;
	else
	break;

	bytes_evicted = arc_evict_state_impl(ml, sublist_idx,
	markers[sublist_idx], spa, bytes_remaining);

	scan_evicted += bytes_evicted;
	total_evicted += bytes_evicted;

	/* we've reached the end, wrap to the beginning */
	if (++sublist_idx >= num_sublists)
	sublist_idx = 0;
	}

	/*
	* If we didn't evict anything during this scan, we have
	* no reason to believe we'll evict more during another
	* scan, so break the loop.
	*/
	if (scan_evicted == 0) {
	/* This isn't possible, let's make that obvious */
	ASSERT3S(bytes, !=, 0);

	/*
	* When bytes is ARC_EVICT_ALL, the only way to
	* break the loop is when scan_evicted is zero.
	* In that case, we actually have evicted enough,
	* so we don't want to increment the kstat.
	*/
	if (bytes != ARC_EVICT_ALL) {
	ASSERT3S(total_evicted, <, bytes);
	ARCSTAT_BUMP(arcstat_evict_not_enough);
	}

	break;
	}
	}

	for (int i = 0; i < num_sublists; i++) {
	- multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
	+ multilist_sublist_t *mls = multilist_sublist_lock_idx(ml, i);
	multilist_sublist_remove(mls, markers[i]);
	multilist_sublist_unlock(mls);
	}
	if (markers != arc_state_evict_markers)
	arc_state_free_markers(markers, num_sublists);

	return (total_evicted);
	}

	/*
	* Flush all "evictable" data of the given type from the arc state
	* specified. This will not evict any "active" buffers (i.e. referenced).
	*
	* When 'retry' is set to B_FALSE, the function will make a single pass
	* over the state and evict any buffers that it can. Since it doesn't
	* continually retry the eviction, it might end up leaving some buffers
	* in the ARC due to lock misses.
	*
	* When 'retry' is set to B_TRUE, the function will continually retry the
	* eviction until all evictable buffers have been removed from the
	* state. As a result, if concurrent insertions into the state are
	* allowed (e.g. if the ARC isn't shutting down), this function might
	* wind up in an infinite loop, continually trying to evict buffers.
	*/
	static uint64_t
	arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type,
	boolean_t retry)
	{
	uint64_t evicted = 0;

	while (zfs_refcount_count(&state->arcs_esize[type]) != 0) {
	evicted += arc_evict_state(state, type, spa, ARC_EVICT_ALL);

	if (!retry)
	break;
	}

	return (evicted);
	}

	/*
	* Evict the specified number of bytes from the state specified. This
	* function prevents us from trying to evict more from a state's list
	* than is "evictable", and to skip evicting altogether when passed a
	* negative value for "bytes". In contrast, arc_evict_state() will
	* evict everything it can, when passed a negative value for "bytes".
	*/
	static uint64_t
	arc_evict_impl(arc_state_t *state, arc_buf_contents_t type, int64_t bytes)
	{
	uint64_t delta;

	if (bytes > 0 && zfs_refcount_count(&state->arcs_esize[type]) > 0) {
	delta = MIN(zfs_refcount_count(&state->arcs_esize[type]),
	bytes);
	return (arc_evict_state(state, type, 0, delta));
	}

	return (0);
	}

	/*
	* Adjust specified fraction, taking into account initial ghost state(s) size,
	* ghost hit bytes towards increasing the fraction, ghost hit bytes towards
	* decreasing it, plus a balance factor, controlling the decrease rate, used
	* to balance metadata vs data.
	*/
	static uint64_t
	arc_evict_adj(uint64_t frac, uint64_t total, uint64_t up, uint64_t down,
	uint_t balance)
	{
	if (total < 8 \|\| up + down == 0)
	return (frac);

	/*
	* We should not have more ghost hits than ghost size, but they
	* may get close. Restrict maximum adjustment in that case.
	*/
	if (up + down >= total / 4) {
	uint64_t scale = (up + down) / (total / 8);
	up /= scale;
	down /= scale;
	}

	/* Get maximal dynamic range by choosing optimal shifts. */
	int s = highbit64(total);
	s = MIN(64 - s, 32);

	uint64_t ofrac = (1ULL << 32) - frac;

	if (frac >= 4 * ofrac)
	up /= frac / (2 * ofrac + 1);
	up = (up << s) / (total >> (32 - s));
	if (ofrac >= 4 * frac)
	down /= ofrac / (2 * frac + 1);
	down = (down << s) / (total >> (32 - s));
	down = down * 100 / balance;

	return (frac + up - down);
	}

	/*
	* Evict buffers from the cache, such that arcstat_size is capped by arc_c.
	*/
	static uint64_t
	arc_evict(void)
	{
	uint64_t asize, bytes, total_evicted = 0;
	int64_t e, mrud, mrum, mfud, mfum, w;
	static uint64_t ogrd, ogrm, ogfd, ogfm;
	static uint64_t gsrd, gsrm, gsfd, gsfm;
	uint64_t ngrd, ngrm, ngfd, ngfm;

	/* Get current size of ARC states we can evict from. */
	mrud = zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_DATA]) +
	zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_DATA]);
	mrum = zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_METADATA]) +
	zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
	mfud = zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
	mfum = zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
	uint64_t d = mrud + mfud;
	uint64_t m = mrum + mfum;
	uint64_t t = d + m;

	/* Get ARC ghost hits since last eviction. */
	ngrd = wmsum_value(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA]);
	uint64_t grd = ngrd - ogrd;
	ogrd = ngrd;
	ngrm = wmsum_value(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA]);
	uint64_t grm = ngrm - ogrm;
	ogrm = ngrm;
	ngfd = wmsum_value(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA]);
	uint64_t gfd = ngfd - ogfd;
	ogfd = ngfd;
	ngfm = wmsum_value(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA]);
	uint64_t gfm = ngfm - ogfm;
	ogfm = ngfm;

	/* Adjust ARC states balance based on ghost hits. */
	arc_meta = arc_evict_adj(arc_meta, gsrd + gsrm + gsfd + gsfm,
	grm + gfm, grd + gfd, zfs_arc_meta_balance);
	arc_pd = arc_evict_adj(arc_pd, gsrd + gsfd, grd, gfd, 100);
	arc_pm = arc_evict_adj(arc_pm, gsrm + gsfm, grm, gfm, 100);

	asize = aggsum_value(&arc_sums.arcstat_size);
	int64_t wt = t - (asize - arc_c);

	/*
	* Try to reduce pinned dnodes if more than 3/4 of wanted metadata
	* target is not evictable or if they go over arc_dnode_limit.
	*/
	int64_t prune = 0;
	int64_t dn = wmsum_value(&arc_sums.arcstat_dnode_size);
	w = wt * (int64_t)(arc_meta >> 16) >> 16;
	if (zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_METADATA]) +
	zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_METADATA]) -
	zfs_refcount_count(&arc_mru->arcs_esize[ARC_BUFC_METADATA]) -
	zfs_refcount_count(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]) >
	w * 3 / 4) {
	prune = dn / sizeof (dnode_t) *
	zfs_arc_dnode_reduce_percent / 100;
	} else if (dn > arc_dnode_limit) {
	prune = (dn - arc_dnode_limit) / sizeof (dnode_t) *
	zfs_arc_dnode_reduce_percent / 100;
	}
	if (prune > 0)
	arc_prune_async(prune);

	/* Evict MRU metadata. */
	w = wt * (int64_t)(arc_meta * arc_pm >> 48) >> 16;
	e = MIN((int64_t)(asize - arc_c), (int64_t)(mrum - w));
	bytes = arc_evict_impl(arc_mru, ARC_BUFC_METADATA, e);
	total_evicted += bytes;
	mrum -= bytes;
	asize -= bytes;

	/* Evict MFU metadata. */
	w = wt * (int64_t)(arc_meta >> 16) >> 16;
	e = MIN((int64_t)(asize - arc_c), (int64_t)(m - w));
	bytes = arc_evict_impl(arc_mfu, ARC_BUFC_METADATA, e);
	total_evicted += bytes;
	mfum -= bytes;
	asize -= bytes;

	/* Evict MRU data. */
	wt -= m - total_evicted;
	w = wt * (int64_t)(arc_pd >> 16) >> 16;
	e = MIN((int64_t)(asize - arc_c), (int64_t)(mrud - w));
	bytes = arc_evict_impl(arc_mru, ARC_BUFC_DATA, e);
	total_evicted += bytes;
	mrud -= bytes;
	asize -= bytes;

	/* Evict MFU data. */
	e = asize - arc_c;
	bytes = arc_evict_impl(arc_mfu, ARC_BUFC_DATA, e);
	mfud -= bytes;
	total_evicted += bytes;

	/*
	* Evict ghost lists
	*
	* Size of each state's ghost list represents how much that state
	* may grow by shrinking the other states. Would it need to shrink
	* other states to zero (that is unlikely), its ghost size would be
	* equal to sum of other three state sizes. But excessive ghost
	* size may result in false ghost hits (too far back), that may
	* never result in real cache hits if several states are competing.
	* So choose some arbitraty point of 1/2 of other state sizes.
	*/
	gsrd = (mrum + mfud + mfum) / 2;
	e = zfs_refcount_count(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]) -
	gsrd;
	(void) arc_evict_impl(arc_mru_ghost, ARC_BUFC_DATA, e);

	gsrm = (mrud + mfud + mfum) / 2;
	e = zfs_refcount_count(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]) -
	gsrm;
	(void) arc_evict_impl(arc_mru_ghost, ARC_BUFC_METADATA, e);

	gsfd = (mrud + mrum + mfum) / 2;
	e = zfs_refcount_count(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]) -
	gsfd;
	(void) arc_evict_impl(arc_mfu_ghost, ARC_BUFC_DATA, e);

	gsfm = (mrud + mrum + mfud) / 2;
	e = zfs_refcount_count(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]) -
	gsfm;
	(void) arc_evict_impl(arc_mfu_ghost, ARC_BUFC_METADATA, e);

	return (total_evicted);
	}

	void
	arc_flush(spa_t *spa, boolean_t retry)
	{
	uint64_t guid = 0;

	/*
	* If retry is B_TRUE, a spa must not be specified since we have
	* no good way to determine if all of a spa's buffers have been
	* evicted from an arc state.
	*/
	ASSERT(!retry \|\| spa == NULL);

	if (spa != NULL)
	guid = spa_load_guid(spa);

	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry);
	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry);

	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry);
	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry);

	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry);
	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry);

	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry);
	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry);

	(void) arc_flush_state(arc_uncached, guid, ARC_BUFC_DATA, retry);
	(void) arc_flush_state(arc_uncached, guid, ARC_BUFC_METADATA, retry);
	}

	void
	arc_reduce_target_size(int64_t to_free)
	{
	uint64_t c = arc_c;

	if (c <= arc_c_min)
	return;

	/*
	* All callers want the ARC to actually evict (at least) this much
	* memory. Therefore we reduce from the lower of the current size and
	* the target size. This way, even if arc_c is much higher than
	* arc_size (as can be the case after many calls to arc_freed(), we will
	* immediately have arc_c < arc_size and therefore the arc_evict_zthr
	* will evict.
	*/
	uint64_t asize = aggsum_value(&arc_sums.arcstat_size);
	if (asize < c)
	to_free += c - asize;
	arc_c = MAX((int64_t)c - to_free, (int64_t)arc_c_min);

	/* See comment in arc_evict_cb_check() on why lock+flag */
	mutex_enter(&arc_evict_lock);
	arc_evict_needed = B_TRUE;
	mutex_exit(&arc_evict_lock);
	zthr_wakeup(arc_evict_zthr);
	}

	/*
	* Determine if the system is under memory pressure and is asking
	* to reclaim memory. A return value of B_TRUE indicates that the system
	* is under memory pressure and that the arc should adjust accordingly.
	*/
	boolean_t
	arc_reclaim_needed(void)
	{
	return (arc_available_memory() < 0);
	}

	void
	arc_kmem_reap_soon(void)
	{
	size_t i;
	kmem_cache_t *prev_cache = NULL;
	kmem_cache_t *prev_data_cache = NULL;

	#ifdef _KERNEL
	#if defined(_ILP32)
	/*
	* Reclaim unused memory from all kmem caches.
	*/
	kmem_reap();
	#endif
	#endif

	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
	#if defined(_ILP32)
	/* reach upper limit of cache size on 32-bit */
	if (zio_buf_cache[i] == NULL)
	break;
	#endif
	if (zio_buf_cache[i] != prev_cache) {
	prev_cache = zio_buf_cache[i];
	kmem_cache_reap_now(zio_buf_cache[i]);
	}
	if (zio_data_buf_cache[i] != prev_data_cache) {
	prev_data_cache = zio_data_buf_cache[i];
	kmem_cache_reap_now(zio_data_buf_cache[i]);
	}
	}
	kmem_cache_reap_now(buf_cache);
	kmem_cache_reap_now(hdr_full_cache);
	kmem_cache_reap_now(hdr_l2only_cache);
	kmem_cache_reap_now(zfs_btree_leaf_cache);
	abd_cache_reap_now();
	}

	static boolean_t
	arc_evict_cb_check(void arg, zthr_t zthr)
	{
	(void) arg, (void) zthr;

	#ifdef ZFS_DEBUG
	/*
	* This is necessary in order to keep the kstat information
	* up to date for tools that display kstat data such as the
	* mdb ::arc dcmd and the Linux crash utility. These tools
	* typically do not call kstat's update function, but simply
	* dump out stats from the most recent update. Without
	* this call, these commands may show stale stats for the
	* anon, mru, mru_ghost, mfu, and mfu_ghost lists. Even
	* with this call, the data might be out of date if the
	* evict thread hasn't been woken recently; but that should
	* suffice. The arc_state_t structures can be queried
	* directly if more accurate information is needed.
	*/
	if (arc_ksp != NULL)
	arc_ksp->ks_update(arc_ksp, KSTAT_READ);
	#endif

	/*
	* We have to rely on arc_wait_for_eviction() to tell us when to
	* evict, rather than checking if we are overflowing here, so that we
	* are sure to not leave arc_wait_for_eviction() waiting on aew_cv.
	* If we have become "not overflowing" since arc_wait_for_eviction()
	* checked, we need to wake it up. We could broadcast the CV here,
	* but arc_wait_for_eviction() may have not yet gone to sleep. We
	* would need to use a mutex to ensure that this function doesn't
	* broadcast until arc_wait_for_eviction() has gone to sleep (e.g.
	* the arc_evict_lock). However, the lock ordering of such a lock
	* would necessarily be incorrect with respect to the zthr_lock,
	* which is held before this function is called, and is held by
	* arc_wait_for_eviction() when it calls zthr_wakeup().
	*/
	if (arc_evict_needed)
	return (B_TRUE);

	/*
	* If we have buffers in uncached state, evict them periodically.
	*/
	return ((zfs_refcount_count(&arc_uncached->arcs_esize[ARC_BUFC_DATA]) +
	zfs_refcount_count(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]) &&
	ddi_get_lbolt() - arc_last_uncached_flush >
	MSEC_TO_TICK(arc_min_prefetch_ms / 2)));
	}

	/*
	* Keep arc_size under arc_c by running arc_evict which evicts data
	* from the ARC.
	*/
	static void
	arc_evict_cb(void arg, zthr_t zthr)
	{
	(void) arg, (void) zthr;

	uint64_t evicted = 0;
	fstrans_cookie_t cookie = spl_fstrans_mark();

	/* Always try to evict from uncached state. */
	arc_last_uncached_flush = ddi_get_lbolt();
	evicted += arc_flush_state(arc_uncached, 0, ARC_BUFC_DATA, B_FALSE);
	evicted += arc_flush_state(arc_uncached, 0, ARC_BUFC_METADATA, B_FALSE);

	/* Evict from other states only if told to. */
	if (arc_evict_needed)
	evicted += arc_evict();

	/*
	* If evicted is zero, we couldn't evict anything
	* via arc_evict(). This could be due to hash lock
	* collisions, but more likely due to the majority of
	* arc buffers being unevictable. Therefore, even if
	* arc_size is above arc_c, another pass is unlikely to
	* be helpful and could potentially cause us to enter an
	* infinite loop. Additionally, zthr_iscancelled() is
	* checked here so that if the arc is shutting down, the
	* broadcast will wake any remaining arc evict waiters.
	*/
	mutex_enter(&arc_evict_lock);
	arc_evict_needed = !zthr_iscancelled(arc_evict_zthr) &&
	evicted > 0 && aggsum_compare(&arc_sums.arcstat_size, arc_c) > 0;
	if (!arc_evict_needed) {
	/*
	* We're either no longer overflowing, or we
	* can't evict anything more, so we should wake
	* arc_get_data_impl() sooner.
	*/
	arc_evict_waiter_t *aw;
	while ((aw = list_remove_head(&arc_evict_waiters)) != NULL) {
	cv_broadcast(&aw->aew_cv);
	}
	arc_set_need_free();
	}
	mutex_exit(&arc_evict_lock);
	spl_fstrans_unmark(cookie);
	}

	static boolean_t
	arc_reap_cb_check(void arg, zthr_t zthr)
	{
	(void) arg, (void) zthr;

	int64_t free_memory = arc_available_memory();
	static int reap_cb_check_counter = 0;

	/*
	* If a kmem reap is already active, don't schedule more. We must
	* check for this because kmem_cache_reap_soon() won't actually
	* block on the cache being reaped (this is to prevent callers from
	* becoming implicitly blocked by a system-wide kmem reap -- which,
	* on a system with many, many full magazines, can take minutes).
	*/
	if (!kmem_cache_reap_active() && free_memory < 0) {

	arc_no_grow = B_TRUE;
	arc_warm = B_TRUE;
	/*
	* Wait at least zfs_grow_retry (default 5) seconds
	* before considering growing.
	*/
	arc_growtime = gethrtime() + SEC2NSEC(arc_grow_retry);
	return (B_TRUE);
	} else if (free_memory < arc_c >> arc_no_grow_shift) {
	arc_no_grow = B_TRUE;
	} else if (gethrtime() >= arc_growtime) {
	arc_no_grow = B_FALSE;
	}

	/*
	* Called unconditionally every 60 seconds to reclaim unused
	* zstd compression and decompression context. This is done
	* here to avoid the need for an independent thread.
	*/
	if (!((reap_cb_check_counter++) % 60))
	zfs_zstd_cache_reap_now();

	return (B_FALSE);
	}

	/*
	* Keep enough free memory in the system by reaping the ARC's kmem
	* caches. To cause more slabs to be reapable, we may reduce the
	* target size of the cache (arc_c), causing the arc_evict_cb()
	* to free more buffers.
	*/
	static void
	arc_reap_cb(void arg, zthr_t zthr)
	{
	(void) arg, (void) zthr;

	int64_t free_memory;
	fstrans_cookie_t cookie = spl_fstrans_mark();

	/*
	* Kick off asynchronous kmem_reap()'s of all our caches.
	*/
	arc_kmem_reap_soon();

	/*
	* Wait at least arc_kmem_cache_reap_retry_ms between
	* arc_kmem_reap_soon() calls. Without this check it is possible to
	* end up in a situation where we spend lots of time reaping
	* caches, while we're near arc_c_min. Waiting here also gives the
	* subsequent free memory check a chance of finding that the
	* asynchronous reap has already freed enough memory, and we don't
	* need to call arc_reduce_target_size().
	*/
	delay((hz * arc_kmem_cache_reap_retry_ms + 999) / 1000);

	/*
	* Reduce the target size as needed to maintain the amount of free
	* memory in the system at a fraction of the arc_size (1/128th by
	* default). If oversubscribed (free_memory < 0) then reduce the
	* target arc_size by the deficit amount plus the fractional
	* amount. If free memory is positive but less than the fractional
	* amount, reduce by what is needed to hit the fractional amount.
	*/
	free_memory = arc_available_memory();

	int64_t can_free = arc_c - arc_c_min;
	if (can_free > 0) {
	int64_t to_free = (can_free >> arc_shrink_shift) - free_memory;
	if (to_free > 0)
	arc_reduce_target_size(to_free);
	}
	spl_fstrans_unmark(cookie);
	}

	#ifdef _KERNEL
	/*
	* Determine the amount of memory eligible for eviction contained in the
	* ARC. All clean data reported by the ghost lists can always be safely
	* evicted. Due to arc_c_min, the same does not hold for all clean data
	* contained by the regular mru and mfu lists.
	*
	* In the case of the regular mru and mfu lists, we need to report as
	* much clean data as possible, such that evicting that same reported
	* data will not bring arc_size below arc_c_min. Thus, in certain
	* circumstances, the total amount of clean data in the mru and mfu
	* lists might not actually be evictable.
	*
	* The following two distinct cases are accounted for:
	*
	* 1. The sum of the amount of dirty data contained by both the mru and
	* mfu lists, plus the ARC's other accounting (e.g. the anon list),
	* is greater than or equal to arc_c_min.
	* (i.e. amount of dirty data >= arc_c_min)
	*
	* This is the easy case; all clean data contained by the mru and mfu
	* lists is evictable. Evicting all clean data can only drop arc_size
	* to the amount of dirty data, which is greater than arc_c_min.
	*
	* 2. The sum of the amount of dirty data contained by both the mru and
	* mfu lists, plus the ARC's other accounting (e.g. the anon list),
	* is less than arc_c_min.
	* (i.e. arc_c_min > amount of dirty data)
	*
	* 2.1. arc_size is greater than or equal arc_c_min.
	* (i.e. arc_size >= arc_c_min > amount of dirty data)
	*
	* In this case, not all clean data from the regular mru and mfu
	* lists is actually evictable; we must leave enough clean data
	* to keep arc_size above arc_c_min. Thus, the maximum amount of
	* evictable data from the two lists combined, is exactly the
	* difference between arc_size and arc_c_min.
	*
	* 2.2. arc_size is less than arc_c_min
	* (i.e. arc_c_min > arc_size > amount of dirty data)
	*
	* In this case, none of the data contained in the mru and mfu
	* lists is evictable, even if it's clean. Since arc_size is
	* already below arc_c_min, evicting any more would only
	* increase this negative difference.
	*/

	#endif /* _KERNEL */

	/*
	* Adapt arc info given the number of bytes we are trying to add and
	* the state that we are coming from. This function is only called
	* when we are adding new content to the cache.
	*/
	static void
	arc_adapt(uint64_t bytes)
	{
	/*
	* Wake reap thread if we do not have any available memory
	*/
	if (arc_reclaim_needed()) {
	zthr_wakeup(arc_reap_zthr);
	return;
	}

	if (arc_no_grow)
	return;

	if (arc_c >= arc_c_max)
	return;

	/*
	* If we're within (2 * maxblocksize) bytes of the target
	* cache size, increment the target cache size
	*/
	if (aggsum_upper_bound(&arc_sums.arcstat_size) +
	2 * SPA_MAXBLOCKSIZE >= arc_c) {
	uint64_t dc = MAX(bytes, SPA_OLD_MAXBLOCKSIZE);
	if (atomic_add_64_nv(&arc_c, dc) > arc_c_max)
	arc_c = arc_c_max;
	}
	}

	/*
	* Check if arc_size has grown past our upper threshold, determined by
	* zfs_arc_overflow_shift.
	*/
	static arc_ovf_level_t
	arc_is_overflowing(boolean_t use_reserve)
	{
	/* Always allow at least one block of overflow */
	int64_t overflow = MAX(SPA_MAXBLOCKSIZE,
	arc_c >> zfs_arc_overflow_shift);

	/*
	* We just compare the lower bound here for performance reasons. Our
	* primary goals are to make sure that the arc never grows without
	* bound, and that it can reach its maximum size. This check
	* accomplishes both goals. The maximum amount we could run over by is
	* 2 * aggsum_borrow_multiplier * NUM_CPUS * the average size of a block
	* in the ARC. In practice, that's in the tens of MB, which is low
	* enough to be safe.
	*/
	int64_t over = aggsum_lower_bound(&arc_sums.arcstat_size) -
	arc_c - overflow / 2;
	if (!use_reserve)
	overflow /= 2;
	return (over < 0 ? ARC_OVF_NONE :
	over < overflow ? ARC_OVF_SOME : ARC_OVF_SEVERE);
	}

	static abd_t *
	arc_get_data_abd(arc_buf_hdr_t hdr, uint64_t size, const void tag,
	int alloc_flags)
	{
	arc_buf_contents_t type = arc_buf_type(hdr);

	arc_get_data_impl(hdr, size, tag, alloc_flags);
	if (alloc_flags & ARC_HDR_ALLOC_LINEAR)
	return (abd_alloc_linear(size, type == ARC_BUFC_METADATA));
	else
	return (abd_alloc(size, type == ARC_BUFC_METADATA));
	}

	static void *
	arc_get_data_buf(arc_buf_hdr_t hdr, uint64_t size, const void tag)
	{
	arc_buf_contents_t type = arc_buf_type(hdr);

	arc_get_data_impl(hdr, size, tag, 0);
	if (type == ARC_BUFC_METADATA) {
	return (zio_buf_alloc(size));
	} else {
	ASSERT(type == ARC_BUFC_DATA);
	return (zio_data_buf_alloc(size));
	}
	}

	/*
	* Wait for the specified amount of data (in bytes) to be evicted from the
	* ARC, and for there to be sufficient free memory in the system. Waiting for
	* eviction ensures that the memory used by the ARC decreases. Waiting for
	* free memory ensures that the system won't run out of free pages, regardless
	* of ARC behavior and settings. See arc_lowmem_init().
	*/
	void
	arc_wait_for_eviction(uint64_t amount, boolean_t use_reserve)
	{
	switch (arc_is_overflowing(use_reserve)) {
	case ARC_OVF_NONE:
	return;
	case ARC_OVF_SOME:
	/*
	* This is a bit racy without taking arc_evict_lock, but the
	* worst that can happen is we either call zthr_wakeup() extra
	* time due to race with other thread here, or the set flag
	* get cleared by arc_evict_cb(), which is unlikely due to
	* big hysteresis, but also not important since at this level
	* of overflow the eviction is purely advisory. Same time
	* taking the global lock here every time without waiting for
	* the actual eviction creates a significant lock contention.
	*/
	if (!arc_evict_needed) {
	arc_evict_needed = B_TRUE;
	zthr_wakeup(arc_evict_zthr);
	}
	return;
	case ARC_OVF_SEVERE:
	default:
	{
	arc_evict_waiter_t aw;
	list_link_init(&aw.aew_node);
	cv_init(&aw.aew_cv, NULL, CV_DEFAULT, NULL);

	uint64_t last_count = 0;
	mutex_enter(&arc_evict_lock);
	if (!list_is_empty(&arc_evict_waiters)) {
	arc_evict_waiter_t *last =
	list_tail(&arc_evict_waiters);
	last_count = last->aew_count;
	} else if (!arc_evict_needed) {
	arc_evict_needed = B_TRUE;
	zthr_wakeup(arc_evict_zthr);
	}
	/*
	* Note, the last waiter's count may be less than
	* arc_evict_count if we are low on memory in which
	* case arc_evict_state_impl() may have deferred
	* wakeups (but still incremented arc_evict_count).
	*/
	aw.aew_count = MAX(last_count, arc_evict_count) + amount;

	list_insert_tail(&arc_evict_waiters, &aw);

	arc_set_need_free();

	DTRACE_PROBE3(arc__wait__for__eviction,
	uint64_t, amount,
	uint64_t, arc_evict_count,
	uint64_t, aw.aew_count);

	/*
	* We will be woken up either when arc_evict_count reaches
	* aew_count, or when the ARC is no longer overflowing and
	* eviction completes.
	* In case of "false" wakeup, we will still be on the list.
	*/
	do {
	cv_wait(&aw.aew_cv, &arc_evict_lock);
	} while (list_link_active(&aw.aew_node));
	mutex_exit(&arc_evict_lock);

	cv_destroy(&aw.aew_cv);
	}
	}
	}

	/*
	* Allocate a block and return it to the caller. If we are hitting the
	* hard limit for the cache size, we must sleep, waiting for the eviction
	* thread to catch up. If we're past the target size but below the hard
	* limit, we'll only signal the reclaim thread and continue on.
	*/
	static void
	arc_get_data_impl(arc_buf_hdr_t hdr, uint64_t size, const void tag,
	int alloc_flags)
	{
	arc_adapt(size);

	/*
	* If arc_size is currently overflowing, we must be adding data
	* faster than we are evicting. To ensure we don't compound the
	* problem by adding more data and forcing arc_size to grow even
	* further past it's target size, we wait for the eviction thread to
	* make some progress. We also wait for there to be sufficient free
	* memory in the system, as measured by arc_free_memory().
	*
	* Specifically, we wait for zfs_arc_eviction_pct percent of the
	* requested size to be evicted. This should be more than 100%, to
	* ensure that that progress is also made towards getting arc_size
	* under arc_c. See the comment above zfs_arc_eviction_pct.
	*/
	arc_wait_for_eviction(size * zfs_arc_eviction_pct / 100,
	alloc_flags & ARC_HDR_USE_RESERVE);

	arc_buf_contents_t type = arc_buf_type(hdr);
	if (type == ARC_BUFC_METADATA) {
	arc_space_consume(size, ARC_SPACE_META);
	} else {
	arc_space_consume(size, ARC_SPACE_DATA);
	}

	/*
	* Update the state size. Note that ghost states have a
	* "ghost size" and so don't need to be updated.
	*/
	arc_state_t *state = hdr->b_l1hdr.b_state;
	if (!GHOST_STATE(state)) {

	(void) zfs_refcount_add_many(&state->arcs_size[type], size,
	tag);

	/*
	* If this is reached via arc_read, the link is
	* protected by the hash lock. If reached via
	* arc_buf_alloc, the header should not be accessed by
	* any other thread. And, if reached via arc_read_done,
	* the hash lock will protect it if it's found in the
	* hash table; otherwise no other thread should be
	* trying to [add\|remove]_reference it.
	*/
	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
	ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
	(void) zfs_refcount_add_many(&state->arcs_esize[type],
	size, tag);
	}
	}
	}

	static void
	arc_free_data_abd(arc_buf_hdr_t hdr, abd_t abd, uint64_t size,
	const void *tag)
	{
	arc_free_data_impl(hdr, size, tag);
	abd_free(abd);
	}

	static void
	arc_free_data_buf(arc_buf_hdr_t hdr, void buf, uint64_t size, const void *tag)
	{
	arc_buf_contents_t type = arc_buf_type(hdr);

	arc_free_data_impl(hdr, size, tag);
	if (type == ARC_BUFC_METADATA) {
	zio_buf_free(buf, size);
	} else {
	ASSERT(type == ARC_BUFC_DATA);
	zio_data_buf_free(buf, size);
	}
	}

	/*
	* Free the arc data buffer.
	*/
	static void
	arc_free_data_impl(arc_buf_hdr_t hdr, uint64_t size, const void tag)
	{
	arc_state_t *state = hdr->b_l1hdr.b_state;
	arc_buf_contents_t type = arc_buf_type(hdr);

	/* protected by hash lock, if in the hash table */
	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
	ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
	ASSERT(state != arc_anon && state != arc_l2c_only);

	(void) zfs_refcount_remove_many(&state->arcs_esize[type],
	size, tag);
	}
	(void) zfs_refcount_remove_many(&state->arcs_size[type], size, tag);

	VERIFY3U(hdr->b_type, ==, type);
	if (type == ARC_BUFC_METADATA) {
	arc_space_return(size, ARC_SPACE_META);
	} else {
	ASSERT(type == ARC_BUFC_DATA);
	arc_space_return(size, ARC_SPACE_DATA);
	}
	}

	/*
	* This routine is called whenever a buffer is accessed.
	*/
	static void
	arc_access(arc_buf_hdr_t *hdr, arc_flags_t arc_flags, boolean_t hit)
	{
	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
	ASSERT(HDR_HAS_L1HDR(hdr));

	/*
	* Update buffer prefetch status.
	*/
	boolean_t was_prefetch = HDR_PREFETCH(hdr);
	boolean_t now_prefetch = arc_flags & ARC_FLAG_PREFETCH;
	if (was_prefetch != now_prefetch) {
	if (was_prefetch) {
	ARCSTAT_CONDSTAT(hit, demand_hit, demand_iohit,
	HDR_PRESCIENT_PREFETCH(hdr), prescient, predictive,
	prefetch);
	}
	if (HDR_HAS_L2HDR(hdr))
	l2arc_hdr_arcstats_decrement_state(hdr);
	if (was_prefetch) {
	arc_hdr_clear_flags(hdr,
	ARC_FLAG_PREFETCH \| ARC_FLAG_PRESCIENT_PREFETCH);
	} else {
	arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
	}
	if (HDR_HAS_L2HDR(hdr))
	l2arc_hdr_arcstats_increment_state(hdr);
	}
	if (now_prefetch) {
	if (arc_flags & ARC_FLAG_PRESCIENT_PREFETCH) {
	arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH);
	ARCSTAT_BUMP(arcstat_prescient_prefetch);
	} else {
	ARCSTAT_BUMP(arcstat_predictive_prefetch);
	}
	}
	if (arc_flags & ARC_FLAG_L2CACHE)
	arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);

	clock_t now = ddi_get_lbolt();
	if (hdr->b_l1hdr.b_state == arc_anon) {
	arc_state_t *new_state;
	/*
	* This buffer is not in the cache, and does not appear in
	* our "ghost" lists. Add it to the MRU or uncached state.
	*/
	ASSERT0(hdr->b_l1hdr.b_arc_access);
	hdr->b_l1hdr.b_arc_access = now;
	if (HDR_UNCACHED(hdr)) {
	new_state = arc_uncached;
	DTRACE_PROBE1(new_state__uncached, arc_buf_hdr_t *,
	hdr);
	} else {
	new_state = arc_mru;
	DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
	}
	arc_change_state(new_state, hdr);
	} else if (hdr->b_l1hdr.b_state == arc_mru) {
	/*
	* This buffer has been accessed once recently and either
	* its read is still in progress or it is in the cache.
	*/
	if (HDR_IO_IN_PROGRESS(hdr)) {
	hdr->b_l1hdr.b_arc_access = now;
	return;
	}
	hdr->b_l1hdr.b_mru_hits++;
	ARCSTAT_BUMP(arcstat_mru_hits);

	/*
	* If the previous access was a prefetch, then it already
	* handled possible promotion, so nothing more to do for now.
	*/
	if (was_prefetch) {
	hdr->b_l1hdr.b_arc_access = now;
	return;
	}

	/*
	* If more than ARC_MINTIME have passed from the previous
	* hit, promote the buffer to the MFU state.
	*/
	if (ddi_time_after(now, hdr->b_l1hdr.b_arc_access +
	ARC_MINTIME)) {
	hdr->b_l1hdr.b_arc_access = now;
	DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
	arc_change_state(arc_mfu, hdr);
	}
	} else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
	arc_state_t *new_state;
	/*
	* This buffer has been accessed once recently, but was
	* evicted from the cache. Would we have bigger MRU, it
	* would be an MRU hit, so handle it the same way, except
	* we don't need to check the previous access time.
	*/
	hdr->b_l1hdr.b_mru_ghost_hits++;
	ARCSTAT_BUMP(arcstat_mru_ghost_hits);
	hdr->b_l1hdr.b_arc_access = now;
	wmsum_add(&arc_mru_ghost->arcs_hits[arc_buf_type(hdr)],
	arc_hdr_size(hdr));
	if (was_prefetch) {
	new_state = arc_mru;
	DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
	} else {
	new_state = arc_mfu;
	DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
	}
	arc_change_state(new_state, hdr);
	} else if (hdr->b_l1hdr.b_state == arc_mfu) {
	/*
	* This buffer has been accessed more than once and either
	* still in the cache or being restored from one of ghosts.
	*/
	if (!HDR_IO_IN_PROGRESS(hdr)) {
	hdr->b_l1hdr.b_mfu_hits++;
	ARCSTAT_BUMP(arcstat_mfu_hits);
	}
	hdr->b_l1hdr.b_arc_access = now;
	} else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
	/*
	* This buffer has been accessed more than once recently, but
	* has been evicted from the cache. Would we have bigger MFU
	* it would stay in cache, so move it back to MFU state.
	*/
	hdr->b_l1hdr.b_mfu_ghost_hits++;
	ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
	hdr->b_l1hdr.b_arc_access = now;
	wmsum_add(&arc_mfu_ghost->arcs_hits[arc_buf_type(hdr)],
	arc_hdr_size(hdr));
	DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
	arc_change_state(arc_mfu, hdr);
	} else if (hdr->b_l1hdr.b_state == arc_uncached) {
	/*
	* This buffer is uncacheable, but we got a hit. Probably
	* a demand read after prefetch. Nothing more to do here.
	*/
	if (!HDR_IO_IN_PROGRESS(hdr))
	ARCSTAT_BUMP(arcstat_uncached_hits);
	hdr->b_l1hdr.b_arc_access = now;
	} else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
	/*
	* This buffer is on the 2nd Level ARC and was not accessed
	* for a long time, so treat it as new and put into MRU.
	*/
	hdr->b_l1hdr.b_arc_access = now;
	DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
	arc_change_state(arc_mru, hdr);
	} else {
	cmn_err(CE_PANIC, "invalid arc state 0x%p",
	hdr->b_l1hdr.b_state);
	}
	}

	/*
	* This routine is called by dbuf_hold() to update the arc_access() state
	* which otherwise would be skipped for entries in the dbuf cache.
	*/
	void
	arc_buf_access(arc_buf_t *buf)
	{
	arc_buf_hdr_t *hdr = buf->b_hdr;

	/*
	* Avoid taking the hash_lock when possible as an optimization.
	* The header must be checked again under the hash_lock in order
	* to handle the case where it is concurrently being released.
	*/
	if (hdr->b_l1hdr.b_state == arc_anon \|\| HDR_EMPTY(hdr))
	return;

	kmutex_t *hash_lock = HDR_LOCK(hdr);
	mutex_enter(hash_lock);

	if (hdr->b_l1hdr.b_state == arc_anon \|\| HDR_EMPTY(hdr)) {
	mutex_exit(hash_lock);
	ARCSTAT_BUMP(arcstat_access_skip);
	return;
	}

	ASSERT(hdr->b_l1hdr.b_state == arc_mru \|\|
	hdr->b_l1hdr.b_state == arc_mfu \|\|
	hdr->b_l1hdr.b_state == arc_uncached);

	DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
	arc_access(hdr, 0, B_TRUE);
	mutex_exit(hash_lock);

	ARCSTAT_BUMP(arcstat_hits);
	ARCSTAT_CONDSTAT(B_TRUE /* demand */, demand, prefetch,
	!HDR_ISTYPE_METADATA(hdr), data, metadata, hits);
	}

	/* a generic arc_read_done_func_t which you can use */
	void
	arc_bcopy_func(zio_t zio, const zbookmark_phys_t zb, const blkptr_t *bp,
	arc_buf_t buf, void arg)
	{
	(void) zio, (void) zb, (void) bp;

	if (buf == NULL)
	return;

	memcpy(arg, buf->b_data, arc_buf_size(buf));
	arc_buf_destroy(buf, arg);
	}

	/* a generic arc_read_done_func_t */
	void
	arc_getbuf_func(zio_t zio, const zbookmark_phys_t zb, const blkptr_t *bp,
	arc_buf_t buf, void arg)
	{
	(void) zb, (void) bp;
	arc_buf_t **bufp = arg;

	if (buf == NULL) {
	ASSERT(zio == NULL \|\| zio->io_error != 0);
	*bufp = NULL;
	} else {
	ASSERT(zio == NULL \|\| zio->io_error == 0);
	*bufp = buf;
	ASSERT(buf->b_data != NULL);
	}
	}

	static void
	arc_hdr_verify(arc_buf_hdr_t hdr, blkptr_t bp)
	{
	if (BP_IS_HOLE(bp) \|\| BP_IS_EMBEDDED(bp)) {
	ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0);
	ASSERT3U(arc_hdr_get_compress(hdr), ==, ZIO_COMPRESS_OFF);
	} else {
	if (HDR_COMPRESSION_ENABLED(hdr)) {
	ASSERT3U(arc_hdr_get_compress(hdr), ==,
	BP_GET_COMPRESS(bp));
	}
	ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
	ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp));
	ASSERT3U(!!HDR_PROTECTED(hdr), ==, BP_IS_PROTECTED(bp));
	}
	}

	static void
	arc_read_done(zio_t *zio)
	{
	blkptr_t *bp = zio->io_bp;
	arc_buf_hdr_t *hdr = zio->io_private;
	kmutex_t *hash_lock = NULL;
	arc_callback_t *callback_list;
	arc_callback_t *acb;

	/*
	* The hdr was inserted into hash-table and removed from lists
	* prior to starting I/O. We should find this header, since
	* it's in the hash table, and it should be legit since it's
	* not possible to evict it during the I/O. The only possible
	* reason for it not to be found is if we were freed during the
	* read.
	*/
	if (HDR_IN_HASH_TABLE(hdr)) {
	arc_buf_hdr_t *found;

	ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp));
	ASSERT3U(hdr->b_dva.dva_word[0], ==,
	BP_IDENTITY(zio->io_bp)->dva_word[0]);
	ASSERT3U(hdr->b_dva.dva_word[1], ==,
	BP_IDENTITY(zio->io_bp)->dva_word[1]);

	found = buf_hash_find(hdr->b_spa, zio->io_bp, &hash_lock);

	ASSERT((found == hdr &&
	DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) \|\|
	(found == hdr && HDR_L2_READING(hdr)));
	ASSERT3P(hash_lock, !=, NULL);
	}

	if (BP_IS_PROTECTED(bp)) {
	hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp);
	hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;
	zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt,
	hdr->b_crypt_hdr.b_iv);

	if (zio->io_error == 0) {
	if (BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG) {
	void *tmpbuf;

	tmpbuf = abd_borrow_buf_copy(zio->io_abd,
	sizeof (zil_chain_t));
	zio_crypt_decode_mac_zil(tmpbuf,
	hdr->b_crypt_hdr.b_mac);
	abd_return_buf(zio->io_abd, tmpbuf,
	sizeof (zil_chain_t));
	} else {
	zio_crypt_decode_mac_bp(bp,
	hdr->b_crypt_hdr.b_mac);
	}
	}
	}

	if (zio->io_error == 0) {
	/* byteswap if necessary */
	if (BP_SHOULD_BYTESWAP(zio->io_bp)) {
	if (BP_GET_LEVEL(zio->io_bp) > 0) {
	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
	} else {
	hdr->b_l1hdr.b_byteswap =
	DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
	}
	} else {
	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
	}
	if (!HDR_L2_READING(hdr)) {
	hdr->b_complevel = zio->io_prop.zp_complevel;
	}
	}

	arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED);
	if (l2arc_noprefetch && HDR_PREFETCH(hdr))
	arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE);

	callback_list = hdr->b_l1hdr.b_acb;
	ASSERT3P(callback_list, !=, NULL);
	hdr->b_l1hdr.b_acb = NULL;

	/*
	* If a read request has a callback (i.e. acb_done is not NULL), then we
	* make a buf containing the data according to the parameters which were
	* passed in. The implementation of arc_buf_alloc_impl() ensures that we
	* aren't needlessly decompressing the data multiple times.
	*/
	int callback_cnt = 0;
	for (acb = callback_list; acb != NULL; acb = acb->acb_next) {

	/* We need the last one to call below in original order. */
	callback_list = acb;

	if (!acb->acb_done \|\| acb->acb_nobuf)
	continue;

	callback_cnt++;

	if (zio->io_error != 0)
	continue;

	int error = arc_buf_alloc_impl(hdr, zio->io_spa,
	&acb->acb_zb, acb->acb_private, acb->acb_encrypted,
	acb->acb_compressed, acb->acb_noauth, B_TRUE,
	&acb->acb_buf);

	/*
	* Assert non-speculative zios didn't fail because an
	* encryption key wasn't loaded
	*/
	ASSERT((zio->io_flags & ZIO_FLAG_SPECULATIVE) \|\|
	error != EACCES);

	/*
	* If we failed to decrypt, report an error now (as the zio
	* layer would have done if it had done the transforms).
	*/
	if (error == ECKSUM) {
	ASSERT(BP_IS_PROTECTED(bp));
	error = SET_ERROR(EIO);
	if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) {
	spa_log_error(zio->io_spa, &acb->acb_zb,
	&zio->io_bp->blk_birth);
	(void) zfs_ereport_post(
	FM_EREPORT_ZFS_AUTHENTICATION,
	zio->io_spa, NULL, &acb->acb_zb, zio, 0);
	}
	}

	if (error != 0) {
	/*
	* Decompression or decryption failed. Set
	* io_error so that when we call acb_done
	* (below), we will indicate that the read
	* failed. Note that in the unusual case
	* where one callback is compressed and another
	* uncompressed, we will mark all of them
	* as failed, even though the uncompressed
	* one can't actually fail. In this case,
	* the hdr will not be anonymous, because
	* if there are multiple callbacks, it's
	* because multiple threads found the same
	* arc buf in the hash table.
	*/
	zio->io_error = error;
	}
	}

	/*
	* If there are multiple callbacks, we must have the hash lock,
	* because the only way for multiple threads to find this hdr is
	* in the hash table. This ensures that if there are multiple
	* callbacks, the hdr is not anonymous. If it were anonymous,
	* we couldn't use arc_buf_destroy() in the error case below.
	*/
	ASSERT(callback_cnt < 2 \|\| hash_lock != NULL);

	if (zio->io_error == 0) {
	arc_hdr_verify(hdr, zio->io_bp);
	} else {
	arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
	if (hdr->b_l1hdr.b_state != arc_anon)
	arc_change_state(arc_anon, hdr);
	if (HDR_IN_HASH_TABLE(hdr))
	buf_hash_remove(hdr);
	}

	arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
	(void) remove_reference(hdr, hdr);

	if (hash_lock != NULL)
	mutex_exit(hash_lock);

	/* execute each callback and free its structure */
	while ((acb = callback_list) != NULL) {
	if (acb->acb_done != NULL) {
	if (zio->io_error != 0 && acb->acb_buf != NULL) {
	/*
	* If arc_buf_alloc_impl() fails during
	* decompression, the buf will still be
	* allocated, and needs to be freed here.
	*/
	arc_buf_destroy(acb->acb_buf,
	acb->acb_private);
	acb->acb_buf = NULL;
	}
	acb->acb_done(zio, &zio->io_bookmark, zio->io_bp,
	acb->acb_buf, acb->acb_private);
	}

	if (acb->acb_zio_dummy != NULL) {
	acb->acb_zio_dummy->io_error = zio->io_error;
	zio_nowait(acb->acb_zio_dummy);
	}

	callback_list = acb->acb_prev;
	if (acb->acb_wait) {
	mutex_enter(&acb->acb_wait_lock);
	acb->acb_wait_error = zio->io_error;
	acb->acb_wait = B_FALSE;
	cv_signal(&acb->acb_wait_cv);
	mutex_exit(&acb->acb_wait_lock);
	/* acb will be freed by the waiting thread. */
	} else {
	kmem_free(acb, sizeof (arc_callback_t));
	}
	}
	}

	/*
	* "Read" the block at the specified DVA (in bp) via the
	* cache. If the block is found in the cache, invoke the provided
	* callback immediately and return. Note that the `zio' parameter
	* in the callback will be NULL in this case, since no IO was
	* required. If the block is not in the cache pass the read request
	* on to the spa with a substitute callback function, so that the
	* requested block will be added to the cache.
	*
	* If a read request arrives for a block that has a read in-progress,
	* either wait for the in-progress read to complete (and return the
	* results); or, if this is a read with a "done" func, add a record
	* to the read to invoke the "done" func when the read completes,
	* and return; or just return.
	*
	* arc_read_done() will invoke all the requested "done" functions
	* for readers of this block.
	*/
	int
	arc_read(zio_t pio, spa_t spa, const blkptr_t *bp,
	arc_read_done_func_t done, void private, zio_priority_t priority,
	int zio_flags, arc_flags_t arc_flags, const zbookmark_phys_t zb)
	{
	arc_buf_hdr_t *hdr = NULL;
	kmutex_t *hash_lock = NULL;
	zio_t *rzio;
	uint64_t guid = spa_load_guid(spa);
	boolean_t compressed_read = (zio_flags & ZIO_FLAG_RAW_COMPRESS) != 0;
	boolean_t encrypted_read = BP_IS_ENCRYPTED(bp) &&
	(zio_flags & ZIO_FLAG_RAW_ENCRYPT) != 0;
	boolean_t noauth_read = BP_IS_AUTHENTICATED(bp) &&
	(zio_flags & ZIO_FLAG_RAW_ENCRYPT) != 0;
	boolean_t embedded_bp = !!BP_IS_EMBEDDED(bp);
	boolean_t no_buf = *arc_flags & ARC_FLAG_NO_BUF;
	arc_buf_t *buf = NULL;
	int rc = 0;

	ASSERT(!embedded_bp \|\|
	BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA);
	ASSERT(!BP_IS_HOLE(bp));
	ASSERT(!BP_IS_REDACTED(bp));

	/*
	* Normally SPL_FSTRANS will already be set since kernel threads which
	* expect to call the DMU interfaces will set it when created. System
	* calls are similarly handled by setting/cleaning the bit in the
	* registered callback (module/os/.../zfs/zpl_*).
	*
	* External consumers such as Lustre which call the exported DMU
	* interfaces may not have set SPL_FSTRANS. To avoid a deadlock
	* on the hash_lock always set and clear the bit.
	*/
	fstrans_cookie_t cookie = spl_fstrans_mark();
	top:
	/*
	* Verify the block pointer contents are reasonable. This should
	* always be the case since the blkptr is protected by a checksum.
	* However, if there is damage it's desirable to detect this early
	* and treat it as a checksum error. This allows an alternate blkptr
	* to be tried when one is available (e.g. ditto blocks).
	*/
	if (!zfs_blkptr_verify(spa, bp, (zio_flags & ZIO_FLAG_CONFIG_WRITER) ?
	BLK_CONFIG_HELD : BLK_CONFIG_NEEDED, BLK_VERIFY_LOG)) {
	rc = SET_ERROR(ECKSUM);
	goto done;
	}

	if (!embedded_bp) {
	/*
	* Embedded BP's have no DVA and require no I/O to "read".
	* Create an anonymous arc buf to back it.
	*/
	hdr = buf_hash_find(guid, bp, &hash_lock);
	}

	/*
	* Determine if we have an L1 cache hit or a cache miss. For simplicity
	* we maintain encrypted data separately from compressed / uncompressed
	* data. If the user is requesting raw encrypted data and we don't have
	* that in the header we will read from disk to guarantee that we can
	* get it even if the encryption keys aren't loaded.
	*/
	if (hdr != NULL && HDR_HAS_L1HDR(hdr) && (HDR_HAS_RABD(hdr) \|\|
	(hdr->b_l1hdr.b_pabd != NULL && !encrypted_read))) {
	boolean_t is_data = !HDR_ISTYPE_METADATA(hdr);

	if (HDR_IO_IN_PROGRESS(hdr)) {
	if (*arc_flags & ARC_FLAG_CACHED_ONLY) {
	mutex_exit(hash_lock);
	ARCSTAT_BUMP(arcstat_cached_only_in_progress);
	rc = SET_ERROR(ENOENT);
	goto done;
	}

	zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head;
	ASSERT3P(head_zio, !=, NULL);
	if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) &&
	priority == ZIO_PRIORITY_SYNC_READ) {
	/*
	* This is a sync read that needs to wait for
	* an in-flight async read. Request that the
	* zio have its priority upgraded.
	*/
	zio_change_priority(head_zio, priority);
	DTRACE_PROBE1(arc__async__upgrade__sync,
	arc_buf_hdr_t *, hdr);
	ARCSTAT_BUMP(arcstat_async_upgrade_sync);
	}

	DTRACE_PROBE1(arc__iohit, arc_buf_hdr_t *, hdr);
	arc_access(hdr, *arc_flags, B_FALSE);

	/*
	* If there are multiple threads reading the same block
	* and that block is not yet in the ARC, then only one
	* thread will do the physical I/O and all other
	* threads will wait until that I/O completes.
	* Synchronous reads use the acb_wait_cv whereas nowait
	* reads register a callback. Both are signalled/called
	* in arc_read_done.
	*
	* Errors of the physical I/O may need to be propagated.
	* Synchronous read errors are returned here from
	* arc_read_done via acb_wait_error. Nowait reads
	* attach the acb_zio_dummy zio to pio and
	* arc_read_done propagates the physical I/O's io_error
	* to acb_zio_dummy, and thereby to pio.
	*/
	arc_callback_t *acb = NULL;
	if (done \|\| pio \|\| *arc_flags & ARC_FLAG_WAIT) {
	acb = kmem_zalloc(sizeof (arc_callback_t),
	KM_SLEEP);
	acb->acb_done = done;
	acb->acb_private = private;
	acb->acb_compressed = compressed_read;
	acb->acb_encrypted = encrypted_read;
	acb->acb_noauth = noauth_read;
	acb->acb_nobuf = no_buf;
	if (*arc_flags & ARC_FLAG_WAIT) {
	acb->acb_wait = B_TRUE;
	mutex_init(&acb->acb_wait_lock, NULL,
	MUTEX_DEFAULT, NULL);
	cv_init(&acb->acb_wait_cv, NULL,
	CV_DEFAULT, NULL);
	}
	acb->acb_zb = *zb;
	if (pio != NULL) {
	acb->acb_zio_dummy = zio_null(pio,
	spa, NULL, NULL, NULL, zio_flags);
	}
	acb->acb_zio_head = head_zio;
	acb->acb_next = hdr->b_l1hdr.b_acb;
	hdr->b_l1hdr.b_acb->acb_prev = acb;
	hdr->b_l1hdr.b_acb = acb;
	}
	mutex_exit(hash_lock);

	ARCSTAT_BUMP(arcstat_iohits);
	ARCSTAT_CONDSTAT(!(*arc_flags & ARC_FLAG_PREFETCH),
	demand, prefetch, is_data, data, metadata, iohits);

	if (*arc_flags & ARC_FLAG_WAIT) {
	mutex_enter(&acb->acb_wait_lock);
	while (acb->acb_wait) {
	cv_wait(&acb->acb_wait_cv,
	&acb->acb_wait_lock);
	}
	rc = acb->acb_wait_error;
	mutex_exit(&acb->acb_wait_lock);
	mutex_destroy(&acb->acb_wait_lock);
	cv_destroy(&acb->acb_wait_cv);
	kmem_free(acb, sizeof (arc_callback_t));
	}
	goto out;
	}

	ASSERT(hdr->b_l1hdr.b_state == arc_mru \|\|
	hdr->b_l1hdr.b_state == arc_mfu \|\|
	hdr->b_l1hdr.b_state == arc_uncached);

	DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
	arc_access(hdr, *arc_flags, B_TRUE);

	if (done && !no_buf) {
	ASSERT(!embedded_bp \|\| !BP_IS_HOLE(bp));

	/* Get a buf with the desired data in it. */
	rc = arc_buf_alloc_impl(hdr, spa, zb, private,
	encrypted_read, compressed_read, noauth_read,
	B_TRUE, &buf);
	if (rc == ECKSUM) {
	/*
	* Convert authentication and decryption errors
	* to EIO (and generate an ereport if needed)
	* before leaving the ARC.
	*/
	rc = SET_ERROR(EIO);
	if ((zio_flags & ZIO_FLAG_SPECULATIVE) == 0) {
	spa_log_error(spa, zb, &hdr->b_birth);
	(void) zfs_ereport_post(
	FM_EREPORT_ZFS_AUTHENTICATION,
	spa, NULL, zb, NULL, 0);
	}
	}
	if (rc != 0) {
	arc_buf_destroy_impl(buf);
	buf = NULL;
	(void) remove_reference(hdr, private);
	}

	/* assert any errors weren't due to unloaded keys */
	ASSERT((zio_flags & ZIO_FLAG_SPECULATIVE) \|\|
	rc != EACCES);
	}
	mutex_exit(hash_lock);
	ARCSTAT_BUMP(arcstat_hits);
	ARCSTAT_CONDSTAT(!(*arc_flags & ARC_FLAG_PREFETCH),
	demand, prefetch, is_data, data, metadata, hits);
	*arc_flags \|= ARC_FLAG_CACHED;
	goto done;
	} else {
	uint64_t lsize = BP_GET_LSIZE(bp);
	uint64_t psize = BP_GET_PSIZE(bp);
	arc_callback_t *acb;
	vdev_t *vd = NULL;
	uint64_t addr = 0;
	boolean_t devw = B_FALSE;
	uint64_t size;
	abd_t *hdr_abd;
	int alloc_flags = encrypted_read ? ARC_HDR_ALLOC_RDATA : 0;
	arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);

	if (*arc_flags & ARC_FLAG_CACHED_ONLY) {
	if (hash_lock != NULL)
	mutex_exit(hash_lock);
	rc = SET_ERROR(ENOENT);
	goto done;
	}

	if (hdr == NULL) {
	/*
	* This block is not in the cache or it has
	* embedded data.
	*/
	arc_buf_hdr_t *exists = NULL;
	hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
	BP_IS_PROTECTED(bp), BP_GET_COMPRESS(bp), 0, type);

	if (!embedded_bp) {
	hdr->b_dva = *BP_IDENTITY(bp);
	hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
	exists = buf_hash_insert(hdr, &hash_lock);
	}
	if (exists != NULL) {
	/* somebody beat us to the hash insert */
	mutex_exit(hash_lock);
	buf_discard_identity(hdr);
	arc_hdr_destroy(hdr);
	goto top; /* restart the IO request */
	}
	} else {
	/*
	* This block is in the ghost cache or encrypted data
	* was requested and we didn't have it. If it was
	* L2-only (and thus didn't have an L1 hdr),
	* we realloc the header to add an L1 hdr.
	*/
	if (!HDR_HAS_L1HDR(hdr)) {
	hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
	hdr_full_cache);
	}

	if (GHOST_STATE(hdr->b_l1hdr.b_state)) {
	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
	ASSERT(!HDR_HAS_RABD(hdr));
	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
	ASSERT0(zfs_refcount_count(
	&hdr->b_l1hdr.b_refcnt));
	ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
	#ifdef ZFS_DEBUG
	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
	#endif
	} else if (HDR_IO_IN_PROGRESS(hdr)) {
	/*
	* If this header already had an IO in progress
	* and we are performing another IO to fetch
	* encrypted data we must wait until the first
	* IO completes so as not to confuse
	* arc_read_done(). This should be very rare
	* and so the performance impact shouldn't
	* matter.
	*/
	arc_callback_t *acb = kmem_zalloc(
	sizeof (arc_callback_t), KM_SLEEP);
	acb->acb_wait = B_TRUE;
	mutex_init(&acb->acb_wait_lock, NULL,
	MUTEX_DEFAULT, NULL);
	cv_init(&acb->acb_wait_cv, NULL, CV_DEFAULT,
	NULL);
	acb->acb_zio_head =
	hdr->b_l1hdr.b_acb->acb_zio_head;
	acb->acb_next = hdr->b_l1hdr.b_acb;
	hdr->b_l1hdr.b_acb->acb_prev = acb;
	hdr->b_l1hdr.b_acb = acb;
	mutex_exit(hash_lock);
	mutex_enter(&acb->acb_wait_lock);
	while (acb->acb_wait) {
	cv_wait(&acb->acb_wait_cv,
	&acb->acb_wait_lock);
	}
	mutex_exit(&acb->acb_wait_lock);
	mutex_destroy(&acb->acb_wait_lock);
	cv_destroy(&acb->acb_wait_cv);
	kmem_free(acb, sizeof (arc_callback_t));
	goto top;
	}
	}
	if (*arc_flags & ARC_FLAG_UNCACHED) {
	arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED);
	if (!encrypted_read)
	alloc_flags \|= ARC_HDR_ALLOC_LINEAR;
	}

	/*
	* Take additional reference for IO_IN_PROGRESS. It stops
	* arc_access() from putting this header without any buffers
	* and so other references but obviously nonevictable onto
	* the evictable list of MRU or MFU state.
	*/
	add_reference(hdr, hdr);
	if (!embedded_bp)
	arc_access(hdr, *arc_flags, B_FALSE);
	arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
	arc_hdr_alloc_abd(hdr, alloc_flags);
	if (encrypted_read) {
	ASSERT(HDR_HAS_RABD(hdr));
	size = HDR_GET_PSIZE(hdr);
	hdr_abd = hdr->b_crypt_hdr.b_rabd;
	zio_flags \|= ZIO_FLAG_RAW;
	} else {
	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
	size = arc_hdr_size(hdr);
	hdr_abd = hdr->b_l1hdr.b_pabd;

	if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) {
	zio_flags \|= ZIO_FLAG_RAW_COMPRESS;
	}

	/*
	* For authenticated bp's, we do not ask the ZIO layer
	* to authenticate them since this will cause the entire
	* IO to fail if the key isn't loaded. Instead, we
	* defer authentication until arc_buf_fill(), which will
	* verify the data when the key is available.
	*/
	if (BP_IS_AUTHENTICATED(bp))
	zio_flags \|= ZIO_FLAG_RAW_ENCRYPT;
	}

	if (BP_IS_AUTHENTICATED(bp))
	arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH);
	if (BP_GET_LEVEL(bp) > 0)
	arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT);
	ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));

	acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
	acb->acb_done = done;
	acb->acb_private = private;
	acb->acb_compressed = compressed_read;
	acb->acb_encrypted = encrypted_read;
	acb->acb_noauth = noauth_read;
	acb->acb_zb = *zb;

	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
	hdr->b_l1hdr.b_acb = acb;

	if (HDR_HAS_L2HDR(hdr) &&
	(vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
	devw = hdr->b_l2hdr.b_dev->l2ad_writing;
	addr = hdr->b_l2hdr.b_daddr;
	/*
	* Lock out L2ARC device removal.
	*/
	if (vdev_is_dead(vd) \|\|
	!spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
	vd = NULL;
	}

	/*
	* We count both async reads and scrub IOs as asynchronous so
	* that both can be upgraded in the event of a cache hit while
	* the read IO is still in-flight.
	*/
	if (priority == ZIO_PRIORITY_ASYNC_READ \|\|
	priority == ZIO_PRIORITY_SCRUB)
	arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
	else
	arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);

	/*
	* At this point, we have a level 1 cache miss or a blkptr
	* with embedded data. Try again in L2ARC if possible.
	*/
	ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize);

	/*
	* Skip ARC stat bump for block pointers with embedded
	* data. The data are read from the blkptr itself via
	* decode_embedded_bp_compressed().
	*/
	if (!embedded_bp) {
	DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr,
	blkptr_t *, bp, uint64_t, lsize,
	zbookmark_phys_t *, zb);
	ARCSTAT_BUMP(arcstat_misses);
	ARCSTAT_CONDSTAT(!(*arc_flags & ARC_FLAG_PREFETCH),
	demand, prefetch, !HDR_ISTYPE_METADATA(hdr), data,
	metadata, misses);
	zfs_racct_read(size, 1);
	}

	/* Check if the spa even has l2 configured */
	const boolean_t spa_has_l2 = l2arc_ndev != 0 &&
	spa->spa_l2cache.sav_count > 0;

	if (vd != NULL && spa_has_l2 && !(l2arc_norw && devw)) {
	/*
	* Read from the L2ARC if the following are true:
	* 1. The L2ARC vdev was previously cached.
	* 2. This buffer still has L2ARC metadata.
	* 3. This buffer isn't currently writing to the L2ARC.
	* 4. The L2ARC entry wasn't evicted, which may
	* also have invalidated the vdev.
	*/
	if (HDR_HAS_L2HDR(hdr) &&
	!HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr)) {
	l2arc_read_callback_t *cb;
	abd_t *abd;
	uint64_t asize;

	DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
	ARCSTAT_BUMP(arcstat_l2_hits);
	hdr->b_l2hdr.b_hits++;

	cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
	KM_SLEEP);
	cb->l2rcb_hdr = hdr;
	cb->l2rcb_bp = *bp;
	cb->l2rcb_zb = *zb;
	cb->l2rcb_flags = zio_flags;

	/*
	* When Compressed ARC is disabled, but the
	* L2ARC block is compressed, arc_hdr_size()
	* will have returned LSIZE rather than PSIZE.
	*/
	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
	!HDR_COMPRESSION_ENABLED(hdr) &&
	HDR_GET_PSIZE(hdr) != 0) {
	size = HDR_GET_PSIZE(hdr);
	}

	asize = vdev_psize_to_asize(vd, size);
	if (asize != size) {
	abd = abd_alloc_for_io(asize,
	HDR_ISTYPE_METADATA(hdr));
	cb->l2rcb_abd = abd;
	} else {
	abd = hdr_abd;
	}

	ASSERT(addr >= VDEV_LABEL_START_SIZE &&
	addr + asize <= vd->vdev_psize -
	VDEV_LABEL_END_SIZE);

	/*
	* l2arc read. The SCL_L2ARC lock will be
	* released by l2arc_read_done().
	* Issue a null zio if the underlying buffer
	* was squashed to zero size by compression.
	*/
	ASSERT3U(arc_hdr_get_compress(hdr), !=,
	ZIO_COMPRESS_EMPTY);
	rzio = zio_read_phys(pio, vd, addr,
	asize, abd,
	ZIO_CHECKSUM_OFF,
	l2arc_read_done, cb, priority,
	zio_flags \| ZIO_FLAG_CANFAIL \|
	ZIO_FLAG_DONT_PROPAGATE \|
	ZIO_FLAG_DONT_RETRY, B_FALSE);
	acb->acb_zio_head = rzio;

	if (hash_lock != NULL)
	mutex_exit(hash_lock);

	DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
	zio_t *, rzio);
	ARCSTAT_INCR(arcstat_l2_read_bytes,
	HDR_GET_PSIZE(hdr));

	if (*arc_flags & ARC_FLAG_NOWAIT) {
	zio_nowait(rzio);
	goto out;
	}

	ASSERT(*arc_flags & ARC_FLAG_WAIT);
	if (zio_wait(rzio) == 0)
	goto out;

	/* l2arc read error; goto zio_read() */
	if (hash_lock != NULL)
	mutex_enter(hash_lock);
	} else {
	DTRACE_PROBE1(l2arc__miss,
	arc_buf_hdr_t *, hdr);
	ARCSTAT_BUMP(arcstat_l2_misses);
	if (HDR_L2_WRITING(hdr))
	ARCSTAT_BUMP(arcstat_l2_rw_clash);
	spa_config_exit(spa, SCL_L2ARC, vd);
	}
	} else {
	if (vd != NULL)
	spa_config_exit(spa, SCL_L2ARC, vd);

	/*
	* Only a spa with l2 should contribute to l2
	* miss stats. (Including the case of having a
	* faulted cache device - that's also a miss.)
	*/
	if (spa_has_l2) {
	/*
	* Skip ARC stat bump for block pointers with
	* embedded data. The data are read from the
	* blkptr itself via
	* decode_embedded_bp_compressed().
	*/
	if (!embedded_bp) {
	DTRACE_PROBE1(l2arc__miss,
	arc_buf_hdr_t *, hdr);
	ARCSTAT_BUMP(arcstat_l2_misses);
	}
	}
	}

	rzio = zio_read(pio, spa, bp, hdr_abd, size,
	arc_read_done, hdr, priority, zio_flags, zb);
	acb->acb_zio_head = rzio;

	if (hash_lock != NULL)
	mutex_exit(hash_lock);

	if (*arc_flags & ARC_FLAG_WAIT) {
	rc = zio_wait(rzio);
	goto out;
	}

	ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
	zio_nowait(rzio);
	}

	out:
	/* embedded bps don't actually go to disk */
	if (!embedded_bp)
	spa_read_history_add(spa, zb, *arc_flags);
	spl_fstrans_unmark(cookie);
	return (rc);

	done:
	if (done)
	done(NULL, zb, bp, buf, private);
	if (pio && rc != 0) {
	zio_t *zio = zio_null(pio, spa, NULL, NULL, NULL, zio_flags);
	zio->io_error = rc;
	zio_nowait(zio);
	}
	goto out;
	}

	arc_prune_t *
	arc_add_prune_callback(arc_prune_func_t func, void private)
	{
	arc_prune_t *p;

	p = kmem_alloc(sizeof (*p), KM_SLEEP);
	p->p_pfunc = func;
	p->p_private = private;
	list_link_init(&p->p_node);
	zfs_refcount_create(&p->p_refcnt);

	mutex_enter(&arc_prune_mtx);
	zfs_refcount_add(&p->p_refcnt, &arc_prune_list);
	list_insert_head(&arc_prune_list, p);
	mutex_exit(&arc_prune_mtx);

	return (p);
	}

	void
	arc_remove_prune_callback(arc_prune_t *p)
	{
	boolean_t wait = B_FALSE;
	mutex_enter(&arc_prune_mtx);
	list_remove(&arc_prune_list, p);
	if (zfs_refcount_remove(&p->p_refcnt, &arc_prune_list) > 0)
	wait = B_TRUE;
	mutex_exit(&arc_prune_mtx);

	/* wait for arc_prune_task to finish */
	if (wait)
	taskq_wait_outstanding(arc_prune_taskq, 0);
	ASSERT0(zfs_refcount_count(&p->p_refcnt));
	zfs_refcount_destroy(&p->p_refcnt);
	kmem_free(p, sizeof (*p));
	}

	/*
	* Helper function for arc_prune_async() it is responsible for safely
	* handling the execution of a registered arc_prune_func_t.
	*/
	static void
	arc_prune_task(void *ptr)
	{
	arc_prune_t ap = (arc_prune_t )ptr;
	arc_prune_func_t *func = ap->p_pfunc;

	if (func != NULL)
	func(ap->p_adjust, ap->p_private);

	zfs_refcount_remove(&ap->p_refcnt, func);
	}

	/*
	* Notify registered consumers they must drop holds on a portion of the ARC
	* buffers they reference. This provides a mechanism to ensure the ARC can
	* honor the metadata limit and reclaim otherwise pinned ARC buffers.
	*
	* This operation is performed asynchronously so it may be safely called
	* in the context of the arc_reclaim_thread(). A reference is taken here
	* for each registered arc_prune_t and the arc_prune_task() is responsible
	* for releasing it once the registered arc_prune_func_t has completed.
	*/
	static void
	arc_prune_async(uint64_t adjust)
	{
	arc_prune_t *ap;

	mutex_enter(&arc_prune_mtx);
	for (ap = list_head(&arc_prune_list); ap != NULL;
	ap = list_next(&arc_prune_list, ap)) {

	if (zfs_refcount_count(&ap->p_refcnt) >= 2)
	continue;

	zfs_refcount_add(&ap->p_refcnt, ap->p_pfunc);
	ap->p_adjust = adjust;
	if (taskq_dispatch(arc_prune_taskq, arc_prune_task,
	ap, TQ_SLEEP) == TASKQID_INVALID) {
	zfs_refcount_remove(&ap->p_refcnt, ap->p_pfunc);
	continue;
	}
	ARCSTAT_BUMP(arcstat_prune);
	}
	mutex_exit(&arc_prune_mtx);
	}

	/*
	* Notify the arc that a block was freed, and thus will never be used again.
	*/
	void
	arc_freed(spa_t spa, const blkptr_t bp)
	{
	arc_buf_hdr_t *hdr;
	kmutex_t *hash_lock;
	uint64_t guid = spa_load_guid(spa);

	ASSERT(!BP_IS_EMBEDDED(bp));

	hdr = buf_hash_find(guid, bp, &hash_lock);
	if (hdr == NULL)
	return;

	/*
	* We might be trying to free a block that is still doing I/O
	* (i.e. prefetch) or has some other reference (i.e. a dedup-ed,
	* dmu_sync-ed block). A block may also have a reference if it is
	* part of a dedup-ed, dmu_synced write. The dmu_sync() function would
	* have written the new block to its final resting place on disk but
	* without the dedup flag set. This would have left the hdr in the MRU
	* state and discoverable. When the txg finally syncs it detects that
	* the block was overridden in open context and issues an override I/O.
	* Since this is a dedup block, the override I/O will determine if the
	* block is already in the DDT. If so, then it will replace the io_bp
	* with the bp from the DDT and allow the I/O to finish. When the I/O
	* reaches the done callback, dbuf_write_override_done, it will
	* check to see if the io_bp and io_bp_override are identical.
	* If they are not, then it indicates that the bp was replaced with
	* the bp in the DDT and the override bp is freed. This allows
	* us to arrive here with a reference on a block that is being
	* freed. So if we have an I/O in progress, or a reference to
	* this hdr, then we don't destroy the hdr.
	*/
	if (!HDR_HAS_L1HDR(hdr) \|\|
	zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
	arc_change_state(arc_anon, hdr);
	arc_hdr_destroy(hdr);
	mutex_exit(hash_lock);
	} else {
	mutex_exit(hash_lock);
	}

	}

	/*
	* Release this buffer from the cache, making it an anonymous buffer. This
	* must be done after a read and prior to modifying the buffer contents.
	* If the buffer has more than one reference, we must make
	* a new hdr for the buffer.
	*/
	void
	arc_release(arc_buf_t buf, const void tag)
	{
	arc_buf_hdr_t *hdr = buf->b_hdr;

	/*
	* It would be nice to assert that if its DMU metadata (level >
	* 0 \|\| it's the dnode file), then it must be syncing context.
	* But we don't know that information at this level.
	*/

	ASSERT(HDR_HAS_L1HDR(hdr));

	/*
	* We don't grab the hash lock prior to this check, because if
	* the buffer's header is in the arc_anon state, it won't be
	* linked into the hash table.
	*/
	if (hdr->b_l1hdr.b_state == arc_anon) {
	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
	ASSERT(!HDR_IN_HASH_TABLE(hdr));
	ASSERT(!HDR_HAS_L2HDR(hdr));

	ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
	ASSERT(ARC_BUF_LAST(buf));
	ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));

	hdr->b_l1hdr.b_arc_access = 0;

	/*
	* If the buf is being overridden then it may already
	* have a hdr that is not empty.
	*/
	buf_discard_identity(hdr);
	arc_buf_thaw(buf);

	return;
	}

	kmutex_t *hash_lock = HDR_LOCK(hdr);
	mutex_enter(hash_lock);

	/*
	* This assignment is only valid as long as the hash_lock is
	* held, we must be careful not to reference state or the
	* b_state field after dropping the lock.
	*/
	arc_state_t *state = hdr->b_l1hdr.b_state;
	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
	ASSERT3P(state, !=, arc_anon);

	/* this buffer is not on any list */
	ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0);

	if (HDR_HAS_L2HDR(hdr)) {
	mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);

	/*
	* We have to recheck this conditional again now that
	* we're holding the l2ad_mtx to prevent a race with
	* another thread which might be concurrently calling
	* l2arc_evict(). In that case, l2arc_evict() might have
	* destroyed the header's L2 portion as we were waiting
	* to acquire the l2ad_mtx.
	*/
	if (HDR_HAS_L2HDR(hdr))
	arc_hdr_l2hdr_destroy(hdr);

	mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
	}

	/*
	* Do we have more than one buf?
	*/
	if (hdr->b_l1hdr.b_buf != buf \|\| !ARC_BUF_LAST(buf)) {
	arc_buf_hdr_t *nhdr;
	uint64_t spa = hdr->b_spa;
	uint64_t psize = HDR_GET_PSIZE(hdr);
	uint64_t lsize = HDR_GET_LSIZE(hdr);
	boolean_t protected = HDR_PROTECTED(hdr);
	enum zio_compress compress = arc_hdr_get_compress(hdr);
	arc_buf_contents_t type = arc_buf_type(hdr);
	VERIFY3U(hdr->b_type, ==, type);

	ASSERT(hdr->b_l1hdr.b_buf != buf \|\| buf->b_next != NULL);
	VERIFY3S(remove_reference(hdr, tag), >, 0);

	if (ARC_BUF_SHARED(buf) && !ARC_BUF_COMPRESSED(buf)) {
	ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
	ASSERT(ARC_BUF_LAST(buf));
	}

	/*
	* Pull the data off of this hdr and attach it to
	* a new anonymous hdr. Also find the last buffer
	* in the hdr's buffer list.
	*/
	arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
	ASSERT3P(lastbuf, !=, NULL);

	/*
	* If the current arc_buf_t and the hdr are sharing their data
	* buffer, then we must stop sharing that block.
	*/
	if (ARC_BUF_SHARED(buf)) {
	ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
	ASSERT(!arc_buf_is_shared(lastbuf));

	/*
	* First, sever the block sharing relationship between
	* buf and the arc_buf_hdr_t.
	*/
	arc_unshare_buf(hdr, buf);

	/*
	* Now we need to recreate the hdr's b_pabd. Since we
	* have lastbuf handy, we try to share with it, but if
	* we can't then we allocate a new b_pabd and copy the
	* data from buf into it.
	*/
	if (arc_can_share(hdr, lastbuf)) {
	arc_share_buf(hdr, lastbuf);
	} else {
	arc_hdr_alloc_abd(hdr, 0);
	abd_copy_from_buf(hdr->b_l1hdr.b_pabd,
	buf->b_data, psize);
	}
	VERIFY3P(lastbuf->b_data, !=, NULL);
	} else if (HDR_SHARED_DATA(hdr)) {
	/*
	* Uncompressed shared buffers are always at the end
	* of the list. Compressed buffers don't have the
	* same requirements. This makes it hard to
	* simply assert that the lastbuf is shared so
	* we rely on the hdr's compression flags to determine
	* if we have a compressed, shared buffer.
	*/
	ASSERT(arc_buf_is_shared(lastbuf) \|\|
	arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
	ASSERT(!arc_buf_is_shared(buf));
	}

	ASSERT(hdr->b_l1hdr.b_pabd != NULL \|\| HDR_HAS_RABD(hdr));
	ASSERT3P(state, !=, arc_l2c_only);

	(void) zfs_refcount_remove_many(&state->arcs_size[type],
	arc_buf_size(buf), buf);

	if (zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
	ASSERT3P(state, !=, arc_l2c_only);
	(void) zfs_refcount_remove_many(
	&state->arcs_esize[type],
	arc_buf_size(buf), buf);
	}

	arc_cksum_verify(buf);
	arc_buf_unwatch(buf);

	/* if this is the last uncompressed buf free the checksum */
	if (!arc_hdr_has_uncompressed_buf(hdr))
	arc_cksum_free(hdr);

	mutex_exit(hash_lock);

	nhdr = arc_hdr_alloc(spa, psize, lsize, protected,
	compress, hdr->b_complevel, type);
	ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL);
	ASSERT0(zfs_refcount_count(&nhdr->b_l1hdr.b_refcnt));
	VERIFY3U(nhdr->b_type, ==, type);
	ASSERT(!HDR_SHARED_DATA(nhdr));

	nhdr->b_l1hdr.b_buf = buf;
	(void) zfs_refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
	buf->b_hdr = nhdr;

	(void) zfs_refcount_add_many(&arc_anon->arcs_size[type],
	arc_buf_size(buf), buf);
	} else {
	ASSERT(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
	/* protected by hash lock, or hdr is on arc_anon */
	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
	hdr->b_l1hdr.b_mru_hits = 0;
	hdr->b_l1hdr.b_mru_ghost_hits = 0;
	hdr->b_l1hdr.b_mfu_hits = 0;
	hdr->b_l1hdr.b_mfu_ghost_hits = 0;
	arc_change_state(arc_anon, hdr);
	hdr->b_l1hdr.b_arc_access = 0;

	mutex_exit(hash_lock);
	buf_discard_identity(hdr);
	arc_buf_thaw(buf);
	}
	}

	int
	arc_released(arc_buf_t *buf)
	{
	return (buf->b_data != NULL &&
	buf->b_hdr->b_l1hdr.b_state == arc_anon);
	}

	#ifdef ZFS_DEBUG
	int
	arc_referenced(arc_buf_t *buf)
	{
	return (zfs_refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
	}
	#endif

	static void
	arc_write_ready(zio_t *zio)
	{
	arc_write_callback_t *callback = zio->io_private;
	arc_buf_t *buf = callback->awcb_buf;
	arc_buf_hdr_t *hdr = buf->b_hdr;
	blkptr_t *bp = zio->io_bp;
	uint64_t psize = BP_IS_HOLE(bp) ? 0 : BP_GET_PSIZE(bp);
	fstrans_cookie_t cookie = spl_fstrans_mark();

	ASSERT(HDR_HAS_L1HDR(hdr));
	ASSERT(!zfs_refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);

	/*
	* If we're reexecuting this zio because the pool suspended, then
	* cleanup any state that was previously set the first time the
	* callback was invoked.
	*/
	if (zio->io_flags & ZIO_FLAG_REEXECUTED) {
	arc_cksum_free(hdr);
	arc_buf_unwatch(buf);
	if (hdr->b_l1hdr.b_pabd != NULL) {
	if (ARC_BUF_SHARED(buf)) {
	arc_unshare_buf(hdr, buf);
	} else {
	ASSERT(!arc_buf_is_shared(buf));
	arc_hdr_free_abd(hdr, B_FALSE);
	}
	}

	if (HDR_HAS_RABD(hdr))
	arc_hdr_free_abd(hdr, B_TRUE);
	}
	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
	ASSERT(!HDR_HAS_RABD(hdr));
	ASSERT(!HDR_SHARED_DATA(hdr));
	ASSERT(!arc_buf_is_shared(buf));

	callback->awcb_ready(zio, buf, callback->awcb_private);

	if (HDR_IO_IN_PROGRESS(hdr)) {
	ASSERT(zio->io_flags & ZIO_FLAG_REEXECUTED);
	} else {
	arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
	add_reference(hdr, hdr); /* For IO_IN_PROGRESS. */
	}

	if (BP_IS_PROTECTED(bp)) {
	/* ZIL blocks are written through zio_rewrite */
	ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_INTENT_LOG);

	if (BP_SHOULD_BYTESWAP(bp)) {
	if (BP_GET_LEVEL(bp) > 0) {
	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
	} else {
	hdr->b_l1hdr.b_byteswap =
	DMU_OT_BYTESWAP(BP_GET_TYPE(bp));
	}
	} else {
	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
	}

	arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
	hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp);
	hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;
	zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt,
	hdr->b_crypt_hdr.b_iv);
	zio_crypt_decode_mac_bp(bp, hdr->b_crypt_hdr.b_mac);
	} else {
	arc_hdr_clear_flags(hdr, ARC_FLAG_PROTECTED);
	}

	/*
	* If this block was written for raw encryption but the zio layer
	* ended up only authenticating it, adjust the buffer flags now.
	*/
	if (BP_IS_AUTHENTICATED(bp) && ARC_BUF_ENCRYPTED(buf)) {
	arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH);
	buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
	if (BP_GET_COMPRESS(bp) == ZIO_COMPRESS_OFF)
	buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
	} else if (BP_IS_HOLE(bp) && ARC_BUF_ENCRYPTED(buf)) {
	buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
	buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
	}

	/* this must be done after the buffer flags are adjusted */
	arc_cksum_compute(buf);

	enum zio_compress compress;
	if (BP_IS_HOLE(bp) \|\| BP_IS_EMBEDDED(bp)) {
	compress = ZIO_COMPRESS_OFF;
	} else {
	ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
	compress = BP_GET_COMPRESS(bp);
	}
	HDR_SET_PSIZE(hdr, psize);
	arc_hdr_set_compress(hdr, compress);
	hdr->b_complevel = zio->io_prop.zp_complevel;

	if (zio->io_error != 0 \|\| psize == 0)
	goto out;

	/*
	* Fill the hdr with data. If the buffer is encrypted we have no choice
	* but to copy the data into b_radb. If the hdr is compressed, the data
	* we want is available from the zio, otherwise we can take it from
	* the buf.
	*
	* We might be able to share the buf's data with the hdr here. However,
	* doing so would cause the ARC to be full of linear ABDs if we write a
	* lot of shareable data. As a compromise, we check whether scattered
	* ABDs are allowed, and assume that if they are then the user wants
	* the ARC to be primarily filled with them regardless of the data being
	* written. Therefore, if they're allowed then we allocate one and copy
	* the data into it; otherwise, we share the data directly if we can.
	*/
	if (ARC_BUF_ENCRYPTED(buf)) {
	ASSERT3U(psize, >, 0);
	ASSERT(ARC_BUF_COMPRESSED(buf));
	arc_hdr_alloc_abd(hdr, ARC_HDR_ALLOC_RDATA \|
	ARC_HDR_USE_RESERVE);
	abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);
	} else if (!(HDR_UNCACHED(hdr) \|\|
	abd_size_alloc_linear(arc_buf_size(buf))) \|\|
	!arc_can_share(hdr, buf)) {
	/*
	* Ideally, we would always copy the io_abd into b_pabd, but the
	* user may have disabled compressed ARC, thus we must check the
	* hdr's compression setting rather than the io_bp's.
	*/
	if (BP_IS_ENCRYPTED(bp)) {
	ASSERT3U(psize, >, 0);
	arc_hdr_alloc_abd(hdr, ARC_HDR_ALLOC_RDATA \|
	ARC_HDR_USE_RESERVE);
	abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);
	} else if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF &&
	!ARC_BUF_COMPRESSED(buf)) {
	ASSERT3U(psize, >, 0);
	arc_hdr_alloc_abd(hdr, ARC_HDR_USE_RESERVE);
	abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize);
	} else {
	ASSERT3U(zio->io_orig_size, ==, arc_hdr_size(hdr));
	arc_hdr_alloc_abd(hdr, ARC_HDR_USE_RESERVE);
	abd_copy_from_buf(hdr->b_l1hdr.b_pabd, buf->b_data,
	arc_buf_size(buf));
	}
	} else {
	ASSERT3P(buf->b_data, ==, abd_to_buf(zio->io_orig_abd));
	ASSERT3U(zio->io_orig_size, ==, arc_buf_size(buf));
	ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
	ASSERT(ARC_BUF_LAST(buf));

	arc_share_buf(hdr, buf);
	}

	out:
	arc_hdr_verify(hdr, bp);
	spl_fstrans_unmark(cookie);
	}

	static void
	arc_write_children_ready(zio_t *zio)
	{
	arc_write_callback_t *callback = zio->io_private;
	arc_buf_t *buf = callback->awcb_buf;

	callback->awcb_children_ready(zio, buf, callback->awcb_private);
	}

	static void
	arc_write_done(zio_t *zio)
	{
	arc_write_callback_t *callback = zio->io_private;
	arc_buf_t *buf = callback->awcb_buf;
	arc_buf_hdr_t *hdr = buf->b_hdr;

	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);

	if (zio->io_error == 0) {
	arc_hdr_verify(hdr, zio->io_bp);

	if (BP_IS_HOLE(zio->io_bp) \|\| BP_IS_EMBEDDED(zio->io_bp)) {
	buf_discard_identity(hdr);
	} else {
	hdr->b_dva = *BP_IDENTITY(zio->io_bp);
	hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
	}
	} else {
	ASSERT(HDR_EMPTY(hdr));
	}

	/*
	* If the block to be written was all-zero or compressed enough to be
	* embedded in the BP, no write was performed so there will be no
	* dva/birth/checksum. The buffer must therefore remain anonymous
	* (and uncached).
	*/
	if (!HDR_EMPTY(hdr)) {
	arc_buf_hdr_t *exists;
	kmutex_t *hash_lock;

	ASSERT3U(zio->io_error, ==, 0);

	arc_cksum_verify(buf);

	exists = buf_hash_insert(hdr, &hash_lock);
	if (exists != NULL) {
	/*
	* This can only happen if we overwrite for
	* sync-to-convergence, because we remove
	* buffers from the hash table when we arc_free().
	*/
	if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
	if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
	panic("bad overwrite, hdr=%p exists=%p",
	(void )hdr, (void )exists);
	ASSERT(zfs_refcount_is_zero(
	&exists->b_l1hdr.b_refcnt));
	arc_change_state(arc_anon, exists);
	arc_hdr_destroy(exists);
	mutex_exit(hash_lock);
	exists = buf_hash_insert(hdr, &hash_lock);
	ASSERT3P(exists, ==, NULL);
	} else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
	/* nopwrite */
	ASSERT(zio->io_prop.zp_nopwrite);
	if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
	panic("bad nopwrite, hdr=%p exists=%p",
	(void )hdr, (void )exists);
	} else {
	/* Dedup */
	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
	ASSERT(ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
	ASSERT(hdr->b_l1hdr.b_state == arc_anon);
	ASSERT(BP_GET_DEDUP(zio->io_bp));
	ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
	}
	}
	arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
	VERIFY3S(remove_reference(hdr, hdr), >, 0);
	/* if it's not anon, we are doing a scrub */
	if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
	arc_access(hdr, 0, B_FALSE);
	mutex_exit(hash_lock);
	} else {
	arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
	VERIFY3S(remove_reference(hdr, hdr), >, 0);
	}

	callback->awcb_done(zio, buf, callback->awcb_private);

	abd_free(zio->io_abd);
	kmem_free(callback, sizeof (arc_write_callback_t));
	}

	zio_t *
	arc_write(zio_t pio, spa_t spa, uint64_t txg,
	blkptr_t bp, arc_buf_t buf, boolean_t uncached, boolean_t l2arc,
	const zio_prop_t zp, arc_write_done_func_t ready,
	arc_write_done_func_t children_ready, arc_write_done_func_t done,
	void *private, zio_priority_t priority, int zio_flags,
	const zbookmark_phys_t *zb)
	{
	arc_buf_hdr_t *hdr = buf->b_hdr;
	arc_write_callback_t *callback;
	zio_t *zio;
	zio_prop_t localprop = *zp;

	ASSERT3P(ready, !=, NULL);
	ASSERT3P(done, !=, NULL);
	ASSERT(!HDR_IO_ERROR(hdr));
	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
	if (uncached)
	arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED);
	else if (l2arc)
	arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);

	if (ARC_BUF_ENCRYPTED(buf)) {
	ASSERT(ARC_BUF_COMPRESSED(buf));
	localprop.zp_encrypt = B_TRUE;
	localprop.zp_compress = HDR_GET_COMPRESS(hdr);
	localprop.zp_complevel = hdr->b_complevel;
	localprop.zp_byteorder =
	(hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?
	ZFS_HOST_BYTEORDER : !ZFS_HOST_BYTEORDER;
	memcpy(localprop.zp_salt, hdr->b_crypt_hdr.b_salt,
	ZIO_DATA_SALT_LEN);
	memcpy(localprop.zp_iv, hdr->b_crypt_hdr.b_iv,
	ZIO_DATA_IV_LEN);
	memcpy(localprop.zp_mac, hdr->b_crypt_hdr.b_mac,
	ZIO_DATA_MAC_LEN);
	if (DMU_OT_IS_ENCRYPTED(localprop.zp_type)) {
	localprop.zp_nopwrite = B_FALSE;
	localprop.zp_copies =
	MIN(localprop.zp_copies, SPA_DVAS_PER_BP - 1);
	}
	zio_flags \|= ZIO_FLAG_RAW;
	} else if (ARC_BUF_COMPRESSED(buf)) {
	ASSERT3U(HDR_GET_LSIZE(hdr), !=, arc_buf_size(buf));
	localprop.zp_compress = HDR_GET_COMPRESS(hdr);
	localprop.zp_complevel = hdr->b_complevel;
	zio_flags \|= ZIO_FLAG_RAW_COMPRESS;
	}
	callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
	callback->awcb_ready = ready;
	callback->awcb_children_ready = children_ready;
	callback->awcb_done = done;
	callback->awcb_private = private;
	callback->awcb_buf = buf;

	/*
	* The hdr's b_pabd is now stale, free it now. A new data block
	* will be allocated when the zio pipeline calls arc_write_ready().
	*/
	if (hdr->b_l1hdr.b_pabd != NULL) {
	/*
	* If the buf is currently sharing the data block with
	* the hdr then we need to break that relationship here.
	* The hdr will remain with a NULL data pointer and the
	* buf will take sole ownership of the block.
	*/
	if (ARC_BUF_SHARED(buf)) {
	arc_unshare_buf(hdr, buf);
	} else {
	ASSERT(!arc_buf_is_shared(buf));
	arc_hdr_free_abd(hdr, B_FALSE);
	}
	VERIFY3P(buf->b_data, !=, NULL);
	}

	if (HDR_HAS_RABD(hdr))
	arc_hdr_free_abd(hdr, B_TRUE);

	if (!(zio_flags & ZIO_FLAG_RAW))
	arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF);

	ASSERT(!arc_buf_is_shared(buf));
	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);

	zio = zio_write(pio, spa, txg, bp,
	abd_get_from_buf(buf->b_data, HDR_GET_LSIZE(hdr)),
	HDR_GET_LSIZE(hdr), arc_buf_size(buf), &localprop, arc_write_ready,
	(children_ready != NULL) ? arc_write_children_ready : NULL,
	arc_write_done, callback, priority, zio_flags, zb);

	return (zio);
	}

	void
	arc_tempreserve_clear(uint64_t reserve)
	{
	atomic_add_64(&arc_tempreserve, -reserve);
	ASSERT((int64_t)arc_tempreserve >= 0);
	}

	int
	arc_tempreserve_space(spa_t *spa, uint64_t reserve, uint64_t txg)
	{
	int error;
	uint64_t anon_size;

	if (!arc_no_grow &&
	reserve > arc_c/4 &&
	reserve * 4 > (2ULL << SPA_MAXBLOCKSHIFT))
	arc_c = MIN(arc_c_max, reserve * 4);

	/*
	* Throttle when the calculated memory footprint for the TXG
	* exceeds the target ARC size.
	*/
	if (reserve > arc_c) {
	DMU_TX_STAT_BUMP(dmu_tx_memory_reserve);
	return (SET_ERROR(ERESTART));
	}

	/*
	* Don't count loaned bufs as in flight dirty data to prevent long
	* network delays from blocking transactions that are ready to be
	* assigned to a txg.
	*/

	/* assert that it has not wrapped around */
	ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0);

	anon_size = MAX((int64_t)
	(zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_DATA]) +
	zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_METADATA]) -
	arc_loaned_bytes), 0);

	/*
	* Writes will, almost always, require additional memory allocations
	* in order to compress/encrypt/etc the data. We therefore need to
	* make sure that there is sufficient available memory for this.
	*/
	error = arc_memory_throttle(spa, reserve, txg);
	if (error != 0)
	return (error);

	/*
	* Throttle writes when the amount of dirty data in the cache
	* gets too large. We try to keep the cache less than half full
	* of dirty blocks so that our sync times don't grow too large.
	*
	* In the case of one pool being built on another pool, we want
	* to make sure we don't end up throttling the lower (backing)
	* pool when the upper pool is the majority contributor to dirty
	* data. To insure we make forward progress during throttling, we
	* also check the current pool's net dirty data and only throttle
	* if it exceeds zfs_arc_pool_dirty_percent of the anonymous dirty
	* data in the cache.
	*
	* Note: if two requests come in concurrently, we might let them
	* both succeed, when one of them should fail. Not a huge deal.
	*/
	uint64_t total_dirty = reserve + arc_tempreserve + anon_size;
	uint64_t spa_dirty_anon = spa_dirty_data(spa);
	uint64_t rarc_c = arc_warm ? arc_c : arc_c_max;
	if (total_dirty > rarc_c * zfs_arc_dirty_limit_percent / 100 &&
	anon_size > rarc_c * zfs_arc_anon_limit_percent / 100 &&
	spa_dirty_anon > anon_size * zfs_arc_pool_dirty_percent / 100) {
	#ifdef ZFS_DEBUG
	uint64_t meta_esize = zfs_refcount_count(
	&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
	uint64_t data_esize =
	zfs_refcount_count(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
	dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
	"anon_data=%lluK tempreserve=%lluK rarc_c=%lluK\n",
	(u_longlong_t)arc_tempreserve >> 10,
	(u_longlong_t)meta_esize >> 10,
	(u_longlong_t)data_esize >> 10,
	(u_longlong_t)reserve >> 10,
	(u_longlong_t)rarc_c >> 10);
	#endif
	DMU_TX_STAT_BUMP(dmu_tx_dirty_throttle);
	return (SET_ERROR(ERESTART));
	}
	atomic_add_64(&arc_tempreserve, reserve);
	return (0);
	}

	static void
	arc_kstat_update_state(arc_state_t state, kstat_named_t size,
	kstat_named_t data, kstat_named_t metadata,
	kstat_named_t evict_data, kstat_named_t evict_metadata)
	{
	data->value.ui64 =
	zfs_refcount_count(&state->arcs_size[ARC_BUFC_DATA]);
	metadata->value.ui64 =
	zfs_refcount_count(&state->arcs_size[ARC_BUFC_METADATA]);
	size->value.ui64 = data->value.ui64 + metadata->value.ui64;
	evict_data->value.ui64 =
	zfs_refcount_count(&state->arcs_esize[ARC_BUFC_DATA]);
	evict_metadata->value.ui64 =
	zfs_refcount_count(&state->arcs_esize[ARC_BUFC_METADATA]);
	}

	static int
	arc_kstat_update(kstat_t *ksp, int rw)
	{
	arc_stats_t *as = ksp->ks_data;

	if (rw == KSTAT_WRITE)
	return (SET_ERROR(EACCES));

	as->arcstat_hits.value.ui64 =
	wmsum_value(&arc_sums.arcstat_hits);
	as->arcstat_iohits.value.ui64 =
	wmsum_value(&arc_sums.arcstat_iohits);
	as->arcstat_misses.value.ui64 =
	wmsum_value(&arc_sums.arcstat_misses);
	as->arcstat_demand_data_hits.value.ui64 =
	wmsum_value(&arc_sums.arcstat_demand_data_hits);
	as->arcstat_demand_data_iohits.value.ui64 =
	wmsum_value(&arc_sums.arcstat_demand_data_iohits);
	as->arcstat_demand_data_misses.value.ui64 =
	wmsum_value(&arc_sums.arcstat_demand_data_misses);
	as->arcstat_demand_metadata_hits.value.ui64 =
	wmsum_value(&arc_sums.arcstat_demand_metadata_hits);
	as->arcstat_demand_metadata_iohits.value.ui64 =
	wmsum_value(&arc_sums.arcstat_demand_metadata_iohits);
	as->arcstat_demand_metadata_misses.value.ui64 =
	wmsum_value(&arc_sums.arcstat_demand_metadata_misses);
	as->arcstat_prefetch_data_hits.value.ui64 =
	wmsum_value(&arc_sums.arcstat_prefetch_data_hits);
	as->arcstat_prefetch_data_iohits.value.ui64 =
	wmsum_value(&arc_sums.arcstat_prefetch_data_iohits);
	as->arcstat_prefetch_data_misses.value.ui64 =
	wmsum_value(&arc_sums.arcstat_prefetch_data_misses);
	as->arcstat_prefetch_metadata_hits.value.ui64 =
	wmsum_value(&arc_sums.arcstat_prefetch_metadata_hits);
	as->arcstat_prefetch_metadata_iohits.value.ui64 =
	wmsum_value(&arc_sums.arcstat_prefetch_metadata_iohits);
	as->arcstat_prefetch_metadata_misses.value.ui64 =
	wmsum_value(&arc_sums.arcstat_prefetch_metadata_misses);
	as->arcstat_mru_hits.value.ui64 =
	wmsum_value(&arc_sums.arcstat_mru_hits);
	as->arcstat_mru_ghost_hits.value.ui64 =
	wmsum_value(&arc_sums.arcstat_mru_ghost_hits);
	as->arcstat_mfu_hits.value.ui64 =
	wmsum_value(&arc_sums.arcstat_mfu_hits);
	as->arcstat_mfu_ghost_hits.value.ui64 =
	wmsum_value(&arc_sums.arcstat_mfu_ghost_hits);
	as->arcstat_uncached_hits.value.ui64 =
	wmsum_value(&arc_sums.arcstat_uncached_hits);
	as->arcstat_deleted.value.ui64 =
	wmsum_value(&arc_sums.arcstat_deleted);
	as->arcstat_mutex_miss.value.ui64 =
	wmsum_value(&arc_sums.arcstat_mutex_miss);
	as->arcstat_access_skip.value.ui64 =
	wmsum_value(&arc_sums.arcstat_access_skip);
	as->arcstat_evict_skip.value.ui64 =
	wmsum_value(&arc_sums.arcstat_evict_skip);
	as->arcstat_evict_not_enough.value.ui64 =
	wmsum_value(&arc_sums.arcstat_evict_not_enough);
	as->arcstat_evict_l2_cached.value.ui64 =
	wmsum_value(&arc_sums.arcstat_evict_l2_cached);
	as->arcstat_evict_l2_eligible.value.ui64 =
	wmsum_value(&arc_sums.arcstat_evict_l2_eligible);
	as->arcstat_evict_l2_eligible_mfu.value.ui64 =
	wmsum_value(&arc_sums.arcstat_evict_l2_eligible_mfu);
	as->arcstat_evict_l2_eligible_mru.value.ui64 =
	wmsum_value(&arc_sums.arcstat_evict_l2_eligible_mru);
	as->arcstat_evict_l2_ineligible.value.ui64 =
	wmsum_value(&arc_sums.arcstat_evict_l2_ineligible);
	as->arcstat_evict_l2_skip.value.ui64 =
	wmsum_value(&arc_sums.arcstat_evict_l2_skip);
	as->arcstat_hash_collisions.value.ui64 =
	wmsum_value(&arc_sums.arcstat_hash_collisions);
	as->arcstat_hash_chains.value.ui64 =
	wmsum_value(&arc_sums.arcstat_hash_chains);
	as->arcstat_size.value.ui64 =
	aggsum_value(&arc_sums.arcstat_size);
	as->arcstat_compressed_size.value.ui64 =
	wmsum_value(&arc_sums.arcstat_compressed_size);
	as->arcstat_uncompressed_size.value.ui64 =
	wmsum_value(&arc_sums.arcstat_uncompressed_size);
	as->arcstat_overhead_size.value.ui64 =
	wmsum_value(&arc_sums.arcstat_overhead_size);
	as->arcstat_hdr_size.value.ui64 =
	wmsum_value(&arc_sums.arcstat_hdr_size);
	as->arcstat_data_size.value.ui64 =
	wmsum_value(&arc_sums.arcstat_data_size);
	as->arcstat_metadata_size.value.ui64 =
	wmsum_value(&arc_sums.arcstat_metadata_size);
	as->arcstat_dbuf_size.value.ui64 =
	wmsum_value(&arc_sums.arcstat_dbuf_size);
	#if defined(COMPAT_FREEBSD11)
	as->arcstat_other_size.value.ui64 =
	wmsum_value(&arc_sums.arcstat_bonus_size) +
	wmsum_value(&arc_sums.arcstat_dnode_size) +
	wmsum_value(&arc_sums.arcstat_dbuf_size);
	#endif

	arc_kstat_update_state(arc_anon,
	&as->arcstat_anon_size,
	&as->arcstat_anon_data,
	&as->arcstat_anon_metadata,
	&as->arcstat_anon_evictable_data,
	&as->arcstat_anon_evictable_metadata);
	arc_kstat_update_state(arc_mru,
	&as->arcstat_mru_size,
	&as->arcstat_mru_data,
	&as->arcstat_mru_metadata,
	&as->arcstat_mru_evictable_data,
	&as->arcstat_mru_evictable_metadata);
	arc_kstat_update_state(arc_mru_ghost,
	&as->arcstat_mru_ghost_size,
	&as->arcstat_mru_ghost_data,
	&as->arcstat_mru_ghost_metadata,
	&as->arcstat_mru_ghost_evictable_data,
	&as->arcstat_mru_ghost_evictable_metadata);
	arc_kstat_update_state(arc_mfu,
	&as->arcstat_mfu_size,
	&as->arcstat_mfu_data,
	&as->arcstat_mfu_metadata,
	&as->arcstat_mfu_evictable_data,
	&as->arcstat_mfu_evictable_metadata);
	arc_kstat_update_state(arc_mfu_ghost,
	&as->arcstat_mfu_ghost_size,
	&as->arcstat_mfu_ghost_data,
	&as->arcstat_mfu_ghost_metadata,
	&as->arcstat_mfu_ghost_evictable_data,
	&as->arcstat_mfu_ghost_evictable_metadata);
	arc_kstat_update_state(arc_uncached,
	&as->arcstat_uncached_size,
	&as->arcstat_uncached_data,
	&as->arcstat_uncached_metadata,
	&as->arcstat_uncached_evictable_data,
	&as->arcstat_uncached_evictable_metadata);

	as->arcstat_dnode_size.value.ui64 =
	wmsum_value(&arc_sums.arcstat_dnode_size);
	as->arcstat_bonus_size.value.ui64 =
	wmsum_value(&arc_sums.arcstat_bonus_size);
	as->arcstat_l2_hits.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_hits);
	as->arcstat_l2_misses.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_misses);
	as->arcstat_l2_prefetch_asize.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_prefetch_asize);
	as->arcstat_l2_mru_asize.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_mru_asize);
	as->arcstat_l2_mfu_asize.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_mfu_asize);
	as->arcstat_l2_bufc_data_asize.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_bufc_data_asize);
	as->arcstat_l2_bufc_metadata_asize.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_bufc_metadata_asize);
	as->arcstat_l2_feeds.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_feeds);
	as->arcstat_l2_rw_clash.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_rw_clash);
	as->arcstat_l2_read_bytes.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_read_bytes);
	as->arcstat_l2_write_bytes.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_write_bytes);
	as->arcstat_l2_writes_sent.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_writes_sent);
	as->arcstat_l2_writes_done.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_writes_done);
	as->arcstat_l2_writes_error.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_writes_error);
	as->arcstat_l2_writes_lock_retry.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_writes_lock_retry);
	as->arcstat_l2_evict_lock_retry.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_evict_lock_retry);
	as->arcstat_l2_evict_reading.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_evict_reading);
	as->arcstat_l2_evict_l1cached.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_evict_l1cached);
	as->arcstat_l2_free_on_write.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_free_on_write);
	as->arcstat_l2_abort_lowmem.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_abort_lowmem);
	as->arcstat_l2_cksum_bad.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_cksum_bad);
	as->arcstat_l2_io_error.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_io_error);
	as->arcstat_l2_lsize.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_lsize);
	as->arcstat_l2_psize.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_psize);
	as->arcstat_l2_hdr_size.value.ui64 =
	aggsum_value(&arc_sums.arcstat_l2_hdr_size);
	as->arcstat_l2_log_blk_writes.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_log_blk_writes);
	as->arcstat_l2_log_blk_asize.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_log_blk_asize);
	as->arcstat_l2_log_blk_count.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_log_blk_count);
	as->arcstat_l2_rebuild_success.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_rebuild_success);
	as->arcstat_l2_rebuild_abort_unsupported.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_unsupported);
	as->arcstat_l2_rebuild_abort_io_errors.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_io_errors);
	as->arcstat_l2_rebuild_abort_dh_errors.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_dh_errors);
	as->arcstat_l2_rebuild_abort_cksum_lb_errors.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors);
	as->arcstat_l2_rebuild_abort_lowmem.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_lowmem);
	as->arcstat_l2_rebuild_size.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_rebuild_size);
	as->arcstat_l2_rebuild_asize.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_rebuild_asize);
	as->arcstat_l2_rebuild_bufs.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_rebuild_bufs);
	as->arcstat_l2_rebuild_bufs_precached.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_rebuild_bufs_precached);
	as->arcstat_l2_rebuild_log_blks.value.ui64 =
	wmsum_value(&arc_sums.arcstat_l2_rebuild_log_blks);
	as->arcstat_memory_throttle_count.value.ui64 =
	wmsum_value(&arc_sums.arcstat_memory_throttle_count);
	as->arcstat_memory_direct_count.value.ui64 =
	wmsum_value(&arc_sums.arcstat_memory_direct_count);
	as->arcstat_memory_indirect_count.value.ui64 =
	wmsum_value(&arc_sums.arcstat_memory_indirect_count);

	as->arcstat_memory_all_bytes.value.ui64 =
	arc_all_memory();
	as->arcstat_memory_free_bytes.value.ui64 =
	arc_free_memory();
	as->arcstat_memory_available_bytes.value.i64 =
	arc_available_memory();

	as->arcstat_prune.value.ui64 =
	wmsum_value(&arc_sums.arcstat_prune);
	as->arcstat_meta_used.value.ui64 =
	wmsum_value(&arc_sums.arcstat_meta_used);
	as->arcstat_async_upgrade_sync.value.ui64 =
	wmsum_value(&arc_sums.arcstat_async_upgrade_sync);
	as->arcstat_predictive_prefetch.value.ui64 =
	wmsum_value(&arc_sums.arcstat_predictive_prefetch);
	as->arcstat_demand_hit_predictive_prefetch.value.ui64 =
	wmsum_value(&arc_sums.arcstat_demand_hit_predictive_prefetch);
	as->arcstat_demand_iohit_predictive_prefetch.value.ui64 =
	wmsum_value(&arc_sums.arcstat_demand_iohit_predictive_prefetch);
	as->arcstat_prescient_prefetch.value.ui64 =
	wmsum_value(&arc_sums.arcstat_prescient_prefetch);
	as->arcstat_demand_hit_prescient_prefetch.value.ui64 =
	wmsum_value(&arc_sums.arcstat_demand_hit_prescient_prefetch);
	as->arcstat_demand_iohit_prescient_prefetch.value.ui64 =
	wmsum_value(&arc_sums.arcstat_demand_iohit_prescient_prefetch);
	as->arcstat_raw_size.value.ui64 =
	wmsum_value(&arc_sums.arcstat_raw_size);
	as->arcstat_cached_only_in_progress.value.ui64 =
	wmsum_value(&arc_sums.arcstat_cached_only_in_progress);
	as->arcstat_abd_chunk_waste_size.value.ui64 =
	wmsum_value(&arc_sums.arcstat_abd_chunk_waste_size);

	return (0);
	}

	/*
	* This function must return indices evenly distributed between all
	* sublists of the multilist. This is needed due to how the ARC eviction
	* code is laid out; arc_evict_state() assumes ARC buffers are evenly
	* distributed between all sublists and uses this assumption when
	* deciding which sublist to evict from and how much to evict from it.
	*/
	static unsigned int
	arc_state_multilist_index_func(multilist_t ml, void obj)
	{
	arc_buf_hdr_t *hdr = obj;

	/*
	* We rely on b_dva to generate evenly distributed index
	* numbers using buf_hash below. So, as an added precaution,
	* let's make sure we never add empty buffers to the arc lists.
	*/
	ASSERT(!HDR_EMPTY(hdr));

	/*
	* The assumption here, is the hash value for a given
	* arc_buf_hdr_t will remain constant throughout its lifetime
	* (i.e. its b_spa, b_dva, and b_birth fields don't change).
	* Thus, we don't need to store the header's sublist index
	* on insertion, as this index can be recalculated on removal.
	*
	* Also, the low order bits of the hash value are thought to be
	* distributed evenly. Otherwise, in the case that the multilist
	* has a power of two number of sublists, each sublists' usage
	* would not be evenly distributed. In this context full 64bit
	* division would be a waste of time, so limit it to 32 bits.
	*/
	return ((unsigned int)buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
	multilist_get_num_sublists(ml));
	}

	static unsigned int
	arc_state_l2c_multilist_index_func(multilist_t ml, void obj)
	{
	panic("Header %p insert into arc_l2c_only %p", obj, ml);
	}

	#define WARN_IF_TUNING_IGNORED(tuning, value, do_warn) do { \
	if ((do_warn) && (tuning) && ((tuning) != (value))) { \
	cmn_err(CE_WARN, \
	"ignoring tunable %s (using %llu instead)", \
	(#tuning), (u_longlong_t)(value)); \
	} \
	} while (0)

	/*
	* Called during module initialization and periodically thereafter to
	* apply reasonable changes to the exposed performance tunings. Can also be
	* called explicitly by param_set_arc_*() functions when ARC tunables are
	* updated manually. Non-zero zfs_* values which differ from the currently set
	* values will be applied.
	*/
	void
	arc_tuning_update(boolean_t verbose)
	{
	uint64_t allmem = arc_all_memory();

	/* Valid range: 32M - <arc_c_max> */
	if ((zfs_arc_min) && (zfs_arc_min != arc_c_min) &&
	(zfs_arc_min >= 2ULL << SPA_MAXBLOCKSHIFT) &&
	(zfs_arc_min <= arc_c_max)) {
	arc_c_min = zfs_arc_min;
	arc_c = MAX(arc_c, arc_c_min);
	}
	WARN_IF_TUNING_IGNORED(zfs_arc_min, arc_c_min, verbose);

	/* Valid range: 64M - <all physical memory> */
	if ((zfs_arc_max) && (zfs_arc_max != arc_c_max) &&
	(zfs_arc_max >= MIN_ARC_MAX) && (zfs_arc_max < allmem) &&
	(zfs_arc_max > arc_c_min)) {
	arc_c_max = zfs_arc_max;
	arc_c = MIN(arc_c, arc_c_max);
	if (arc_dnode_limit > arc_c_max)
	arc_dnode_limit = arc_c_max;
	}
	WARN_IF_TUNING_IGNORED(zfs_arc_max, arc_c_max, verbose);

	/* Valid range: 0 - <all physical memory> */
	arc_dnode_limit = zfs_arc_dnode_limit ? zfs_arc_dnode_limit :
	MIN(zfs_arc_dnode_limit_percent, 100) * arc_c_max / 100;
	WARN_IF_TUNING_IGNORED(zfs_arc_dnode_limit, arc_dnode_limit, verbose);

	/* Valid range: 1 - N */
	if (zfs_arc_grow_retry)
	arc_grow_retry = zfs_arc_grow_retry;

	/* Valid range: 1 - N */
	if (zfs_arc_shrink_shift) {
	arc_shrink_shift = zfs_arc_shrink_shift;
	arc_no_grow_shift = MIN(arc_no_grow_shift, arc_shrink_shift -1);
	}

	/* Valid range: 1 - N ms */
	if (zfs_arc_min_prefetch_ms)
	arc_min_prefetch_ms = zfs_arc_min_prefetch_ms;

	/* Valid range: 1 - N ms */
	if (zfs_arc_min_prescient_prefetch_ms) {
	arc_min_prescient_prefetch_ms =
	zfs_arc_min_prescient_prefetch_ms;
	}

	/* Valid range: 0 - 100 */
	if (zfs_arc_lotsfree_percent <= 100)
	arc_lotsfree_percent = zfs_arc_lotsfree_percent;
	WARN_IF_TUNING_IGNORED(zfs_arc_lotsfree_percent, arc_lotsfree_percent,
	verbose);

	/* Valid range: 0 - <all physical memory> */
	if ((zfs_arc_sys_free) && (zfs_arc_sys_free != arc_sys_free))
	arc_sys_free = MIN(zfs_arc_sys_free, allmem);
	WARN_IF_TUNING_IGNORED(zfs_arc_sys_free, arc_sys_free, verbose);
	}

	static void
	arc_state_multilist_init(multilist_t *ml,
	multilist_sublist_index_func_t index_func, int maxcountp)
	{
	multilist_create(ml, sizeof (arc_buf_hdr_t),
	offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), index_func);
	maxcountp = MAX(maxcountp, multilist_get_num_sublists(ml));
	}

	static void
	arc_state_init(void)
	{
	int num_sublists = 0;

	arc_state_multilist_init(&arc_mru->arcs_list[ARC_BUFC_METADATA],
	arc_state_multilist_index_func, &num_sublists);
	arc_state_multilist_init(&arc_mru->arcs_list[ARC_BUFC_DATA],
	arc_state_multilist_index_func, &num_sublists);
	arc_state_multilist_init(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
	arc_state_multilist_index_func, &num_sublists);
	arc_state_multilist_init(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
	arc_state_multilist_index_func, &num_sublists);
	arc_state_multilist_init(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
	arc_state_multilist_index_func, &num_sublists);
	arc_state_multilist_init(&arc_mfu->arcs_list[ARC_BUFC_DATA],
	arc_state_multilist_index_func, &num_sublists);
	arc_state_multilist_init(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
	arc_state_multilist_index_func, &num_sublists);
	arc_state_multilist_init(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
	arc_state_multilist_index_func, &num_sublists);
	arc_state_multilist_init(&arc_uncached->arcs_list[ARC_BUFC_METADATA],
	arc_state_multilist_index_func, &num_sublists);
	arc_state_multilist_init(&arc_uncached->arcs_list[ARC_BUFC_DATA],
	arc_state_multilist_index_func, &num_sublists);

	/*
	* L2 headers should never be on the L2 state list since they don't
	* have L1 headers allocated. Special index function asserts that.
	*/
	arc_state_multilist_init(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
	arc_state_l2c_multilist_index_func, &num_sublists);
	arc_state_multilist_init(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
	arc_state_l2c_multilist_index_func, &num_sublists);

	/*
	* Keep track of the number of markers needed to reclaim buffers from
	* any ARC state. The markers will be pre-allocated so as to minimize
	* the number of memory allocations performed by the eviction thread.
	*/
	arc_state_evict_marker_count = num_sublists;

	zfs_refcount_create(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
	zfs_refcount_create(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
	zfs_refcount_create(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
	zfs_refcount_create(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
	zfs_refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);
	zfs_refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]);
	zfs_refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
	zfs_refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_DATA]);
	zfs_refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]);
	zfs_refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]);
	zfs_refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]);
	zfs_refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]);
	zfs_refcount_create(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]);
	zfs_refcount_create(&arc_uncached->arcs_esize[ARC_BUFC_DATA]);

	zfs_refcount_create(&arc_anon->arcs_size[ARC_BUFC_DATA]);
	zfs_refcount_create(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
	zfs_refcount_create(&arc_mru->arcs_size[ARC_BUFC_DATA]);
	zfs_refcount_create(&arc_mru->arcs_size[ARC_BUFC_METADATA]);
	zfs_refcount_create(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]);
	zfs_refcount_create(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]);
	zfs_refcount_create(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
	zfs_refcount_create(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
	zfs_refcount_create(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]);
	zfs_refcount_create(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]);
	zfs_refcount_create(&arc_l2c_only->arcs_size[ARC_BUFC_DATA]);
	zfs_refcount_create(&arc_l2c_only->arcs_size[ARC_BUFC_METADATA]);
	zfs_refcount_create(&arc_uncached->arcs_size[ARC_BUFC_DATA]);
	zfs_refcount_create(&arc_uncached->arcs_size[ARC_BUFC_METADATA]);

	wmsum_init(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA], 0);
	wmsum_init(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA], 0);
	wmsum_init(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA], 0);
	wmsum_init(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA], 0);

	wmsum_init(&arc_sums.arcstat_hits, 0);
	wmsum_init(&arc_sums.arcstat_iohits, 0);
	wmsum_init(&arc_sums.arcstat_misses, 0);
	wmsum_init(&arc_sums.arcstat_demand_data_hits, 0);
	wmsum_init(&arc_sums.arcstat_demand_data_iohits, 0);
	wmsum_init(&arc_sums.arcstat_demand_data_misses, 0);
	wmsum_init(&arc_sums.arcstat_demand_metadata_hits, 0);
	wmsum_init(&arc_sums.arcstat_demand_metadata_iohits, 0);
	wmsum_init(&arc_sums.arcstat_demand_metadata_misses, 0);
	wmsum_init(&arc_sums.arcstat_prefetch_data_hits, 0);
	wmsum_init(&arc_sums.arcstat_prefetch_data_iohits, 0);
	wmsum_init(&arc_sums.arcstat_prefetch_data_misses, 0);
	wmsum_init(&arc_sums.arcstat_prefetch_metadata_hits, 0);
	wmsum_init(&arc_sums.arcstat_prefetch_metadata_iohits, 0);
	wmsum_init(&arc_sums.arcstat_prefetch_metadata_misses, 0);
	wmsum_init(&arc_sums.arcstat_mru_hits, 0);
	wmsum_init(&arc_sums.arcstat_mru_ghost_hits, 0);
	wmsum_init(&arc_sums.arcstat_mfu_hits, 0);
	wmsum_init(&arc_sums.arcstat_mfu_ghost_hits, 0);
	wmsum_init(&arc_sums.arcstat_uncached_hits, 0);
	wmsum_init(&arc_sums.arcstat_deleted, 0);
	wmsum_init(&arc_sums.arcstat_mutex_miss, 0);
	wmsum_init(&arc_sums.arcstat_access_skip, 0);
	wmsum_init(&arc_sums.arcstat_evict_skip, 0);
	wmsum_init(&arc_sums.arcstat_evict_not_enough, 0);
	wmsum_init(&arc_sums.arcstat_evict_l2_cached, 0);
	wmsum_init(&arc_sums.arcstat_evict_l2_eligible, 0);
	wmsum_init(&arc_sums.arcstat_evict_l2_eligible_mfu, 0);
	wmsum_init(&arc_sums.arcstat_evict_l2_eligible_mru, 0);
	wmsum_init(&arc_sums.arcstat_evict_l2_ineligible, 0);
	wmsum_init(&arc_sums.arcstat_evict_l2_skip, 0);
	wmsum_init(&arc_sums.arcstat_hash_collisions, 0);
	wmsum_init(&arc_sums.arcstat_hash_chains, 0);
	aggsum_init(&arc_sums.arcstat_size, 0);
	wmsum_init(&arc_sums.arcstat_compressed_size, 0);
	wmsum_init(&arc_sums.arcstat_uncompressed_size, 0);
	wmsum_init(&arc_sums.arcstat_overhead_size, 0);
	wmsum_init(&arc_sums.arcstat_hdr_size, 0);
	wmsum_init(&arc_sums.arcstat_data_size, 0);
	wmsum_init(&arc_sums.arcstat_metadata_size, 0);
	wmsum_init(&arc_sums.arcstat_dbuf_size, 0);
	wmsum_init(&arc_sums.arcstat_dnode_size, 0);
	wmsum_init(&arc_sums.arcstat_bonus_size, 0);
	wmsum_init(&arc_sums.arcstat_l2_hits, 0);
	wmsum_init(&arc_sums.arcstat_l2_misses, 0);
	wmsum_init(&arc_sums.arcstat_l2_prefetch_asize, 0);
	wmsum_init(&arc_sums.arcstat_l2_mru_asize, 0);
	wmsum_init(&arc_sums.arcstat_l2_mfu_asize, 0);
	wmsum_init(&arc_sums.arcstat_l2_bufc_data_asize, 0);
	wmsum_init(&arc_sums.arcstat_l2_bufc_metadata_asize, 0);
	wmsum_init(&arc_sums.arcstat_l2_feeds, 0);
	wmsum_init(&arc_sums.arcstat_l2_rw_clash, 0);
	wmsum_init(&arc_sums.arcstat_l2_read_bytes, 0);
	wmsum_init(&arc_sums.arcstat_l2_write_bytes, 0);
	wmsum_init(&arc_sums.arcstat_l2_writes_sent, 0);
	wmsum_init(&arc_sums.arcstat_l2_writes_done, 0);
	wmsum_init(&arc_sums.arcstat_l2_writes_error, 0);
	wmsum_init(&arc_sums.arcstat_l2_writes_lock_retry, 0);
	wmsum_init(&arc_sums.arcstat_l2_evict_lock_retry, 0);
	wmsum_init(&arc_sums.arcstat_l2_evict_reading, 0);
	wmsum_init(&arc_sums.arcstat_l2_evict_l1cached, 0);
	wmsum_init(&arc_sums.arcstat_l2_free_on_write, 0);
	wmsum_init(&arc_sums.arcstat_l2_abort_lowmem, 0);
	wmsum_init(&arc_sums.arcstat_l2_cksum_bad, 0);
	wmsum_init(&arc_sums.arcstat_l2_io_error, 0);
	wmsum_init(&arc_sums.arcstat_l2_lsize, 0);
	wmsum_init(&arc_sums.arcstat_l2_psize, 0);
	aggsum_init(&arc_sums.arcstat_l2_hdr_size, 0);
	wmsum_init(&arc_sums.arcstat_l2_log_blk_writes, 0);
	wmsum_init(&arc_sums.arcstat_l2_log_blk_asize, 0);
	wmsum_init(&arc_sums.arcstat_l2_log_blk_count, 0);
	wmsum_init(&arc_sums.arcstat_l2_rebuild_success, 0);
	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_unsupported, 0);
	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_io_errors, 0);
	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_dh_errors, 0);
	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors, 0);
	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_lowmem, 0);
	wmsum_init(&arc_sums.arcstat_l2_rebuild_size, 0);
	wmsum_init(&arc_sums.arcstat_l2_rebuild_asize, 0);
	wmsum_init(&arc_sums.arcstat_l2_rebuild_bufs, 0);
	wmsum_init(&arc_sums.arcstat_l2_rebuild_bufs_precached, 0);
	wmsum_init(&arc_sums.arcstat_l2_rebuild_log_blks, 0);
	wmsum_init(&arc_sums.arcstat_memory_throttle_count, 0);
	wmsum_init(&arc_sums.arcstat_memory_direct_count, 0);
	wmsum_init(&arc_sums.arcstat_memory_indirect_count, 0);
	wmsum_init(&arc_sums.arcstat_prune, 0);
	wmsum_init(&arc_sums.arcstat_meta_used, 0);
	wmsum_init(&arc_sums.arcstat_async_upgrade_sync, 0);
	wmsum_init(&arc_sums.arcstat_predictive_prefetch, 0);
	wmsum_init(&arc_sums.arcstat_demand_hit_predictive_prefetch, 0);
	wmsum_init(&arc_sums.arcstat_demand_iohit_predictive_prefetch, 0);
	wmsum_init(&arc_sums.arcstat_prescient_prefetch, 0);
	wmsum_init(&arc_sums.arcstat_demand_hit_prescient_prefetch, 0);
	wmsum_init(&arc_sums.arcstat_demand_iohit_prescient_prefetch, 0);
	wmsum_init(&arc_sums.arcstat_raw_size, 0);
	wmsum_init(&arc_sums.arcstat_cached_only_in_progress, 0);
	wmsum_init(&arc_sums.arcstat_abd_chunk_waste_size, 0);

	arc_anon->arcs_state = ARC_STATE_ANON;
	arc_mru->arcs_state = ARC_STATE_MRU;
	arc_mru_ghost->arcs_state = ARC_STATE_MRU_GHOST;
	arc_mfu->arcs_state = ARC_STATE_MFU;
	arc_mfu_ghost->arcs_state = ARC_STATE_MFU_GHOST;
	arc_l2c_only->arcs_state = ARC_STATE_L2C_ONLY;
	arc_uncached->arcs_state = ARC_STATE_UNCACHED;
	}

	static void
	arc_state_fini(void)
	{
	zfs_refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
	zfs_refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
	zfs_refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
	zfs_refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
	zfs_refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);
	zfs_refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]);
	zfs_refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
	zfs_refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_DATA]);
	zfs_refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]);
	zfs_refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]);
	zfs_refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]);
	zfs_refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]);
	zfs_refcount_destroy(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]);
	zfs_refcount_destroy(&arc_uncached->arcs_esize[ARC_BUFC_DATA]);

	zfs_refcount_destroy(&arc_anon->arcs_size[ARC_BUFC_DATA]);
	zfs_refcount_destroy(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
	zfs_refcount_destroy(&arc_mru->arcs_size[ARC_BUFC_DATA]);
	zfs_refcount_destroy(&arc_mru->arcs_size[ARC_BUFC_METADATA]);
	zfs_refcount_destroy(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]);
	zfs_refcount_destroy(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]);
	zfs_refcount_destroy(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
	zfs_refcount_destroy(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
	zfs_refcount_destroy(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]);
	zfs_refcount_destroy(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]);
	zfs_refcount_destroy(&arc_l2c_only->arcs_size[ARC_BUFC_DATA]);
	zfs_refcount_destroy(&arc_l2c_only->arcs_size[ARC_BUFC_METADATA]);
	zfs_refcount_destroy(&arc_uncached->arcs_size[ARC_BUFC_DATA]);
	zfs_refcount_destroy(&arc_uncached->arcs_size[ARC_BUFC_METADATA]);

	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
	multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
	multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
	multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]);
	multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]);
	multilist_destroy(&arc_uncached->arcs_list[ARC_BUFC_METADATA]);
	multilist_destroy(&arc_uncached->arcs_list[ARC_BUFC_DATA]);

	wmsum_fini(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA]);
	wmsum_fini(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA]);
	wmsum_fini(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA]);
	wmsum_fini(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA]);

	wmsum_fini(&arc_sums.arcstat_hits);
	wmsum_fini(&arc_sums.arcstat_iohits);
	wmsum_fini(&arc_sums.arcstat_misses);
	wmsum_fini(&arc_sums.arcstat_demand_data_hits);
	wmsum_fini(&arc_sums.arcstat_demand_data_iohits);
	wmsum_fini(&arc_sums.arcstat_demand_data_misses);
	wmsum_fini(&arc_sums.arcstat_demand_metadata_hits);
	wmsum_fini(&arc_sums.arcstat_demand_metadata_iohits);
	wmsum_fini(&arc_sums.arcstat_demand_metadata_misses);
	wmsum_fini(&arc_sums.arcstat_prefetch_data_hits);
	wmsum_fini(&arc_sums.arcstat_prefetch_data_iohits);
	wmsum_fini(&arc_sums.arcstat_prefetch_data_misses);
	wmsum_fini(&arc_sums.arcstat_prefetch_metadata_hits);
	wmsum_fini(&arc_sums.arcstat_prefetch_metadata_iohits);
	wmsum_fini(&arc_sums.arcstat_prefetch_metadata_misses);
	wmsum_fini(&arc_sums.arcstat_mru_hits);
	wmsum_fini(&arc_sums.arcstat_mru_ghost_hits);
	wmsum_fini(&arc_sums.arcstat_mfu_hits);
	wmsum_fini(&arc_sums.arcstat_mfu_ghost_hits);
	wmsum_fini(&arc_sums.arcstat_uncached_hits);
	wmsum_fini(&arc_sums.arcstat_deleted);
	wmsum_fini(&arc_sums.arcstat_mutex_miss);
	wmsum_fini(&arc_sums.arcstat_access_skip);
	wmsum_fini(&arc_sums.arcstat_evict_skip);
	wmsum_fini(&arc_sums.arcstat_evict_not_enough);
	wmsum_fini(&arc_sums.arcstat_evict_l2_cached);
	wmsum_fini(&arc_sums.arcstat_evict_l2_eligible);
	wmsum_fini(&arc_sums.arcstat_evict_l2_eligible_mfu);
	wmsum_fini(&arc_sums.arcstat_evict_l2_eligible_mru);
	wmsum_fini(&arc_sums.arcstat_evict_l2_ineligible);
	wmsum_fini(&arc_sums.arcstat_evict_l2_skip);
	wmsum_fini(&arc_sums.arcstat_hash_collisions);
	wmsum_fini(&arc_sums.arcstat_hash_chains);
	aggsum_fini(&arc_sums.arcstat_size);
	wmsum_fini(&arc_sums.arcstat_compressed_size);
	wmsum_fini(&arc_sums.arcstat_uncompressed_size);
	wmsum_fini(&arc_sums.arcstat_overhead_size);
	wmsum_fini(&arc_sums.arcstat_hdr_size);
	wmsum_fini(&arc_sums.arcstat_data_size);
	wmsum_fini(&arc_sums.arcstat_metadata_size);
	wmsum_fini(&arc_sums.arcstat_dbuf_size);
	wmsum_fini(&arc_sums.arcstat_dnode_size);
	wmsum_fini(&arc_sums.arcstat_bonus_size);
	wmsum_fini(&arc_sums.arcstat_l2_hits);
	wmsum_fini(&arc_sums.arcstat_l2_misses);
	wmsum_fini(&arc_sums.arcstat_l2_prefetch_asize);
	wmsum_fini(&arc_sums.arcstat_l2_mru_asize);
	wmsum_fini(&arc_sums.arcstat_l2_mfu_asize);
	wmsum_fini(&arc_sums.arcstat_l2_bufc_data_asize);
	wmsum_fini(&arc_sums.arcstat_l2_bufc_metadata_asize);
	wmsum_fini(&arc_sums.arcstat_l2_feeds);
	wmsum_fini(&arc_sums.arcstat_l2_rw_clash);
	wmsum_fini(&arc_sums.arcstat_l2_read_bytes);
	wmsum_fini(&arc_sums.arcstat_l2_write_bytes);
	wmsum_fini(&arc_sums.arcstat_l2_writes_sent);
	wmsum_fini(&arc_sums.arcstat_l2_writes_done);
	wmsum_fini(&arc_sums.arcstat_l2_writes_error);
	wmsum_fini(&arc_sums.arcstat_l2_writes_lock_retry);
	wmsum_fini(&arc_sums.arcstat_l2_evict_lock_retry);
	wmsum_fini(&arc_sums.arcstat_l2_evict_reading);
	wmsum_fini(&arc_sums.arcstat_l2_evict_l1cached);
	wmsum_fini(&arc_sums.arcstat_l2_free_on_write);
	wmsum_fini(&arc_sums.arcstat_l2_abort_lowmem);
	wmsum_fini(&arc_sums.arcstat_l2_cksum_bad);
	wmsum_fini(&arc_sums.arcstat_l2_io_error);
	wmsum_fini(&arc_sums.arcstat_l2_lsize);
	wmsum_fini(&arc_sums.arcstat_l2_psize);
	aggsum_fini(&arc_sums.arcstat_l2_hdr_size);
	wmsum_fini(&arc_sums.arcstat_l2_log_blk_writes);
	wmsum_fini(&arc_sums.arcstat_l2_log_blk_asize);
	wmsum_fini(&arc_sums.arcstat_l2_log_blk_count);
	wmsum_fini(&arc_sums.arcstat_l2_rebuild_success);
	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_unsupported);
	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_io_errors);
	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_dh_errors);
	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors);
	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_lowmem);
	wmsum_fini(&arc_sums.arcstat_l2_rebuild_size);
	wmsum_fini(&arc_sums.arcstat_l2_rebuild_asize);
	wmsum_fini(&arc_sums.arcstat_l2_rebuild_bufs);
	wmsum_fini(&arc_sums.arcstat_l2_rebuild_bufs_precached);
	wmsum_fini(&arc_sums.arcstat_l2_rebuild_log_blks);
	wmsum_fini(&arc_sums.arcstat_memory_throttle_count);
	wmsum_fini(&arc_sums.arcstat_memory_direct_count);
	wmsum_fini(&arc_sums.arcstat_memory_indirect_count);
	wmsum_fini(&arc_sums.arcstat_prune);
	wmsum_fini(&arc_sums.arcstat_meta_used);
	wmsum_fini(&arc_sums.arcstat_async_upgrade_sync);
	wmsum_fini(&arc_sums.arcstat_predictive_prefetch);
	wmsum_fini(&arc_sums.arcstat_demand_hit_predictive_prefetch);
	wmsum_fini(&arc_sums.arcstat_demand_iohit_predictive_prefetch);
	wmsum_fini(&arc_sums.arcstat_prescient_prefetch);
	wmsum_fini(&arc_sums.arcstat_demand_hit_prescient_prefetch);
	wmsum_fini(&arc_sums.arcstat_demand_iohit_prescient_prefetch);
	wmsum_fini(&arc_sums.arcstat_raw_size);
	wmsum_fini(&arc_sums.arcstat_cached_only_in_progress);
	wmsum_fini(&arc_sums.arcstat_abd_chunk_waste_size);
	}

	uint64_t
	arc_target_bytes(void)
	{
	return (arc_c);
	}

	void
	arc_set_limits(uint64_t allmem)
	{
	/* Set min cache to 1/32 of all memory, or 32MB, whichever is more. */
	arc_c_min = MAX(allmem / 32, 2ULL << SPA_MAXBLOCKSHIFT);

	/* How to set default max varies by platform. */
	arc_c_max = arc_default_max(arc_c_min, allmem);
	}
	void
	arc_init(void)
	{
	uint64_t percent, allmem = arc_all_memory();
	mutex_init(&arc_evict_lock, NULL, MUTEX_DEFAULT, NULL);
	list_create(&arc_evict_waiters, sizeof (arc_evict_waiter_t),
	offsetof(arc_evict_waiter_t, aew_node));

	arc_min_prefetch_ms = 1000;
	arc_min_prescient_prefetch_ms = 6000;

	#if defined(_KERNEL)
	arc_lowmem_init();
	#endif

	arc_set_limits(allmem);

	#ifdef _KERNEL
	/*
	* If zfs_arc_max is non-zero at init, meaning it was set in the kernel
	* environment before the module was loaded, don't block setting the
	* maximum because it is less than arc_c_min, instead, reset arc_c_min
	* to a lower value.
	* zfs_arc_min will be handled by arc_tuning_update().
	*/
	if (zfs_arc_max != 0 && zfs_arc_max >= MIN_ARC_MAX &&
	zfs_arc_max < allmem) {
	arc_c_max = zfs_arc_max;
	if (arc_c_min >= arc_c_max) {
	arc_c_min = MAX(zfs_arc_max / 2,
	2ULL << SPA_MAXBLOCKSHIFT);
	}
	}
	#else
	/*
	* In userland, there's only the memory pressure that we artificially
	* create (see arc_available_memory()). Don't let arc_c get too
	* small, because it can cause transactions to be larger than
	* arc_c, causing arc_tempreserve_space() to fail.
	*/
	arc_c_min = MAX(arc_c_max / 2, 2ULL << SPA_MAXBLOCKSHIFT);
	#endif

	arc_c = arc_c_min;
	/*
	* 32-bit fixed point fractions of metadata from total ARC size,
	* MRU data from all data and MRU metadata from all metadata.
	*/
	arc_meta = (1ULL << 32) / 4; /* Metadata is 25% of arc_c. */
	arc_pd = (1ULL << 32) / 2; /* Data MRU is 50% of data. */
	arc_pm = (1ULL << 32) / 2; /* Metadata MRU is 50% of metadata. */

	percent = MIN(zfs_arc_dnode_limit_percent, 100);
	arc_dnode_limit = arc_c_max * percent / 100;

	/* Apply user specified tunings */
	arc_tuning_update(B_TRUE);

	/* if kmem_flags are set, lets try to use less memory */
	if (kmem_debugging())
	arc_c = arc_c / 2;
	if (arc_c < arc_c_min)
	arc_c = arc_c_min;

	arc_register_hotplug();

	arc_state_init();

	buf_init();

	list_create(&arc_prune_list, sizeof (arc_prune_t),
	offsetof(arc_prune_t, p_node));
	mutex_init(&arc_prune_mtx, NULL, MUTEX_DEFAULT, NULL);

	arc_prune_taskq = taskq_create("arc_prune", zfs_arc_prune_task_threads,
	defclsyspri, 100, INT_MAX, TASKQ_PREPOPULATE \| TASKQ_DYNAMIC);

	arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
	sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);

	if (arc_ksp != NULL) {
	arc_ksp->ks_data = &arc_stats;
	arc_ksp->ks_update = arc_kstat_update;
	kstat_install(arc_ksp);
	}

	arc_state_evict_markers =
	arc_state_alloc_markers(arc_state_evict_marker_count);
	arc_evict_zthr = zthr_create_timer("arc_evict",
	arc_evict_cb_check, arc_evict_cb, NULL, SEC2NSEC(1), defclsyspri);
	arc_reap_zthr = zthr_create_timer("arc_reap",
	arc_reap_cb_check, arc_reap_cb, NULL, SEC2NSEC(1), minclsyspri);

	arc_warm = B_FALSE;

	/*
	* Calculate maximum amount of dirty data per pool.
	*
	* If it has been set by a module parameter, take that.
	* Otherwise, use a percentage of physical memory defined by
	* zfs_dirty_data_max_percent (default 10%) with a cap at
	* zfs_dirty_data_max_max (default 4G or 25% of physical memory).
	*/
	#ifdef __LP64__
	if (zfs_dirty_data_max_max == 0)
	zfs_dirty_data_max_max = MIN(4ULL * 1024 * 1024 * 1024,
	allmem * zfs_dirty_data_max_max_percent / 100);
	#else
	if (zfs_dirty_data_max_max == 0)
	zfs_dirty_data_max_max = MIN(1ULL * 1024 * 1024 * 1024,
	allmem * zfs_dirty_data_max_max_percent / 100);
	#endif

	if (zfs_dirty_data_max == 0) {
	zfs_dirty_data_max = allmem *
	zfs_dirty_data_max_percent / 100;
	zfs_dirty_data_max = MIN(zfs_dirty_data_max,
	zfs_dirty_data_max_max);
	}

	if (zfs_wrlog_data_max == 0) {

	/*
	* dp_wrlog_total is reduced for each txg at the end of
	* spa_sync(). However, dp_dirty_total is reduced every time
	* a block is written out. Thus under normal operation,
	* dp_wrlog_total could grow 2 times as big as
	* zfs_dirty_data_max.
	*/
	zfs_wrlog_data_max = zfs_dirty_data_max * 2;
	}
	}

	void
	arc_fini(void)
	{
	arc_prune_t *p;

	#ifdef _KERNEL
	arc_lowmem_fini();
	#endif /* _KERNEL */

	/* Use B_TRUE to ensure all buffers are evicted */
	arc_flush(NULL, B_TRUE);

	if (arc_ksp != NULL) {
	kstat_delete(arc_ksp);
	arc_ksp = NULL;
	}

	taskq_wait(arc_prune_taskq);
	taskq_destroy(arc_prune_taskq);

	mutex_enter(&arc_prune_mtx);
	while ((p = list_remove_head(&arc_prune_list)) != NULL) {
	zfs_refcount_remove(&p->p_refcnt, &arc_prune_list);
	zfs_refcount_destroy(&p->p_refcnt);
	kmem_free(p, sizeof (*p));
	}
	mutex_exit(&arc_prune_mtx);

	list_destroy(&arc_prune_list);
	mutex_destroy(&arc_prune_mtx);

	(void) zthr_cancel(arc_evict_zthr);
	(void) zthr_cancel(arc_reap_zthr);
	arc_state_free_markers(arc_state_evict_markers,
	arc_state_evict_marker_count);

	mutex_destroy(&arc_evict_lock);
	list_destroy(&arc_evict_waiters);

	/*
	* Free any buffers that were tagged for destruction. This needs
	* to occur before arc_state_fini() runs and destroys the aggsum
	* values which are updated when freeing scatter ABDs.
	*/
	l2arc_do_free_on_write();

	/*
	* buf_fini() must proceed arc_state_fini() because buf_fin() may
	* trigger the release of kmem magazines, which can callback to
	* arc_space_return() which accesses aggsums freed in act_state_fini().
	*/
	buf_fini();
	arc_state_fini();

	arc_unregister_hotplug();

	/*
	* We destroy the zthrs after all the ARC state has been
	* torn down to avoid the case of them receiving any
	* wakeup() signals after they are destroyed.
	*/
	zthr_destroy(arc_evict_zthr);
	zthr_destroy(arc_reap_zthr);

	ASSERT0(arc_loaned_bytes);
	}

	/*
	* Level 2 ARC
	*
	* The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk.
	* It uses dedicated storage devices to hold cached data, which are populated
	* using large infrequent writes. The main role of this cache is to boost
	* the performance of random read workloads. The intended L2ARC devices
	* include short-stroked disks, solid state disks, and other media with
	* substantially faster read latency than disk.
	*
	* +-----------------------+
	* \| ARC \|
	* +-----------------------+
	* \| ^ ^
	* \| \| \|
	* l2arc_feed_thread() arc_read()
	* \| \| \|
	* \| l2arc read \|
	* V \| \|
	* +---------------+ \|
	* \| L2ARC \| \|
	* +---------------+ \|
	* \| ^ \|
	* l2arc_write() \| \|
	* \| \| \|
	* V \| \|
	* +-------+ +-------+
	* \| vdev \| \| vdev \|
	* \| cache \| \| cache \|
	* +-------+ +-------+
	* +=========+ .-----.
	* : L2ARC : \|-_____-\|
	* : devices : \| Disks \|
	* +=========+ `-_____-'
	*
	* Read requests are satisfied from the following sources, in order:
	*
	* 1) ARC
	* 2) vdev cache of L2ARC devices
	* 3) L2ARC devices
	* 4) vdev cache of disks
	* 5) disks
	*
	* Some L2ARC device types exhibit extremely slow write performance.
	* To accommodate for this there are some significant differences between
	* the L2ARC and traditional cache design:
	*
	* 1. There is no eviction path from the ARC to the L2ARC. Evictions from
	* the ARC behave as usual, freeing buffers and placing headers on ghost
	* lists. The ARC does not send buffers to the L2ARC during eviction as
	* this would add inflated write latencies for all ARC memory pressure.
	*
	* 2. The L2ARC attempts to cache data from the ARC before it is evicted.
	* It does this by periodically scanning buffers from the eviction-end of
	* the MFU and MRU ARC lists, copying them to the L2ARC devices if they are
	* not already there. It scans until a headroom of buffers is satisfied,
	* which itself is a buffer for ARC eviction. If a compressible buffer is
	* found during scanning and selected for writing to an L2ARC device, we
	* temporarily boost scanning headroom during the next scan cycle to make
	* sure we adapt to compression effects (which might significantly reduce
	* the data volume we write to L2ARC). The thread that does this is
	* l2arc_feed_thread(), illustrated below; example sizes are included to
	* provide a better sense of ratio than this diagram:
	*
	* head --> tail
	* +---------------------+----------+
	* ARC_mfu \|:::::#:::::::::::::::\|o#o###o###\|-->. # already on L2ARC
	* +---------------------+----------+ \| o L2ARC eligible
	* ARC_mru \|:#:::::::::::::::::::\|#o#ooo####\|-->\| : ARC buffer
	* +---------------------+----------+ \|
	* 15.9 Gbytes ^ 32 Mbytes \|
	* headroom \|
	* l2arc_feed_thread()
	* \|
	* l2arc write hand <--[oooo]--'
	* \| 8 Mbyte
	* \| write max
	* V
	* +==============================+
	* L2ARC dev \|####\|#\|###\|###\| \|####\| ... \|
	* +==============================+
	* 32 Gbytes
	*
	* 3. If an ARC buffer is copied to the L2ARC but then hit instead of
	* evicted, then the L2ARC has cached a buffer much sooner than it probably
	* needed to, potentially wasting L2ARC device bandwidth and storage. It is
	* safe to say that this is an uncommon case, since buffers at the end of
	* the ARC lists have moved there due to inactivity.
	*
	* 4. If the ARC evicts faster than the L2ARC can maintain a headroom,
	* then the L2ARC simply misses copying some buffers. This serves as a
	* pressure valve to prevent heavy read workloads from both stalling the ARC
	* with waits and clogging the L2ARC with writes. This also helps prevent
	* the potential for the L2ARC to churn if it attempts to cache content too
	* quickly, such as during backups of the entire pool.
	*
	* 5. After system boot and before the ARC has filled main memory, there are
	* no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru
	* lists can remain mostly static. Instead of searching from tail of these
	* lists as pictured, the l2arc_feed_thread() will search from the list heads
	* for eligible buffers, greatly increasing its chance of finding them.
	*
	* The L2ARC device write speed is also boosted during this time so that
	* the L2ARC warms up faster. Since there have been no ARC evictions yet,
	* there are no L2ARC reads, and no fear of degrading read performance
	* through increased writes.
	*
	* 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
	* the vdev queue can aggregate them into larger and fewer writes. Each
	* device is written to in a rotor fashion, sweeping writes through
	* available space then repeating.
	*
	* 7. The L2ARC does not store dirty content. It never needs to flush
	* write buffers back to disk based storage.
	*
	* 8. If an ARC buffer is written (and dirtied) which also exists in the
	* L2ARC, the now stale L2ARC buffer is immediately dropped.
	*
	* The performance of the L2ARC can be tweaked by a number of tunables, which
	* may be necessary for different workloads:
	*
	* l2arc_write_max max write bytes per interval
	* l2arc_write_boost extra write bytes during device warmup
	* l2arc_noprefetch skip caching prefetched buffers
	* l2arc_headroom number of max device writes to precache
	* l2arc_headroom_boost when we find compressed buffers during ARC
	* scanning, we multiply headroom by this
	* percentage factor for the next scan cycle,
	* since more compressed buffers are likely to
	* be present
	* l2arc_feed_secs seconds between L2ARC writing
	*
	* Tunables may be removed or added as future performance improvements are
	* integrated, and also may become zpool properties.
	*
	* There are three key functions that control how the L2ARC warms up:
	*
	* l2arc_write_eligible() check if a buffer is eligible to cache
	* l2arc_write_size() calculate how much to write
	* l2arc_write_interval() calculate sleep delay between writes
	*
	* These three functions determine what to write, how much, and how quickly
	* to send writes.
	*
	* L2ARC persistence:
	*
	* When writing buffers to L2ARC, we periodically add some metadata to
	* make sure we can pick them up after reboot, thus dramatically reducing
	* the impact that any downtime has on the performance of storage systems
	* with large caches.
	*
	* The implementation works fairly simply by integrating the following two
	* modifications:
	*
	* *) When writing to the L2ARC, we occasionally write a "l2arc log block",
	* which is an additional piece of metadata which describes what's been
	* written. This allows us to rebuild the arc_buf_hdr_t structures of the
	* main ARC buffers. There are 2 linked-lists of log blocks headed by
	* dh_start_lbps[2]. We alternate which chain we append to, so they are
	* time-wise and offset-wise interleaved, but that is an optimization rather
	* than for correctness. The log block also includes a pointer to the
	* previous block in its chain.
	*
	* *) We reserve SPA_MINBLOCKSIZE of space at the start of each L2ARC device
	* for our header bookkeeping purposes. This contains a device header,
	* which contains our top-level reference structures. We update it each
	* time we write a new log block, so that we're able to locate it in the
	* L2ARC device. If this write results in an inconsistent device header
	* (e.g. due to power failure), we detect this by verifying the header's
	* checksum and simply fail to reconstruct the L2ARC after reboot.
	*
	* Implementation diagram:
	*
	* +=== L2ARC device (not to scale) ======================================+
	* \| ___two newest log block pointers__.__________ \|
	* \| / \dh_start_lbps[1] \|
	* \| / \ \dh_start_lbps[0]\|
	* \|.___/__. V V \|
	* \|\|L2 dev\|....\|lb \|bufs \|lb \|bufs \|lb \|bufs \|lb \|bufs \|lb \|---(empty)---\|
	* \|\| hdr\| ^ /^ /^ / / \|
	* \|+------+ ...--\-------/ \-----/--\------/ / \|
	* \| \--------------/ \--------------/ \|
	* +======================================================================+
	*
	* As can be seen on the diagram, rather than using a simple linked list,
	* we use a pair of linked lists with alternating elements. This is a
	* performance enhancement due to the fact that we only find out the
	* address of the next log block access once the current block has been
	* completely read in. Obviously, this hurts performance, because we'd be
	* keeping the device's I/O queue at only a 1 operation deep, thus
	* incurring a large amount of I/O round-trip latency. Having two lists
	* allows us to fetch two log blocks ahead of where we are currently
	* rebuilding L2ARC buffers.
	*
	* On-device data structures:
	*
	* L2ARC device header: l2arc_dev_hdr_phys_t
	* L2ARC log block: l2arc_log_blk_phys_t
	*
	* L2ARC reconstruction:
	*
	* When writing data, we simply write in the standard rotary fashion,
	* evicting buffers as we go and simply writing new data over them (writing
	* a new log block every now and then). This obviously means that once we
	* loop around the end of the device, we will start cutting into an already
	* committed log block (and its referenced data buffers), like so:
	*
	* current write head__ __old tail
	* \ /
	* V V
	* <--\|bufs \|lb \|bufs \|lb \| \|bufs \|lb \|bufs \|lb \|-->
	* ^ ^^^^^^^^^___________________________________
	* \| \
	* <<nextwrite>> may overwrite this blk and/or its bufs --'
	*
	* When importing the pool, we detect this situation and use it to stop
	* our scanning process (see l2arc_rebuild).
	*
	* There is one significant caveat to consider when rebuilding ARC contents
	* from an L2ARC device: what about invalidated buffers? Given the above
	* construction, we cannot update blocks which we've already written to amend
	* them to remove buffers which were invalidated. Thus, during reconstruction,
	* we might be populating the cache with buffers for data that's not on the
	* main pool anymore, or may have been overwritten!
	*
	* As it turns out, this isn't a problem. Every arc_read request includes
	* both the DVA and, crucially, the birth TXG of the BP the caller is
	* looking for. So even if the cache were populated by completely rotten
	* blocks for data that had been long deleted and/or overwritten, we'll
	* never actually return bad data from the cache, since the DVA with the
	* birth TXG uniquely identify a block in space and time - once created,
	* a block is immutable on disk. The worst thing we have done is wasted
	* some time and memory at l2arc rebuild to reconstruct outdated ARC
	* entries that will get dropped from the l2arc as it is being updated
	* with new blocks.
	*
	* L2ARC buffers that have been evicted by l2arc_evict() ahead of the write
	* hand are not restored. This is done by saving the offset (in bytes)
	* l2arc_evict() has evicted to in the L2ARC device header and taking it
	* into account when restoring buffers.
	*/

	static boolean_t
	l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr)
	{
	/*
	* A buffer is not eligible for the L2ARC if it:
	* 1. belongs to a different spa.
	* 2. is already cached on the L2ARC.
	* 3. has an I/O in progress (it may be an incomplete read).
	* 4. is flagged not eligible (zfs property).
	*/
	if (hdr->b_spa != spa_guid \|\| HDR_HAS_L2HDR(hdr) \|\|
	HDR_IO_IN_PROGRESS(hdr) \|\| !HDR_L2CACHE(hdr))
	return (B_FALSE);

	return (B_TRUE);
	}

	static uint64_t
	l2arc_write_size(l2arc_dev_t *dev)
	{
	uint64_t size;

	/*
	* Make sure our globals have meaningful values in case the user
	* altered them.
	*/
	size = l2arc_write_max;
	if (size == 0) {
	cmn_err(CE_NOTE, "l2arc_write_max must be greater than zero, "
	"resetting it to the default (%d)", L2ARC_WRITE_SIZE);
	size = l2arc_write_max = L2ARC_WRITE_SIZE;
	}

	if (arc_warm == B_FALSE)
	size += l2arc_write_boost;

	/* We need to add in the worst case scenario of log block overhead. */
	size += l2arc_log_blk_overhead(size, dev);
	if (dev->l2ad_vdev->vdev_has_trim && l2arc_trim_ahead > 0) {
	/*
	* Trim ahead of the write size 64MB or (l2arc_trim_ahead/100)
	* times the writesize, whichever is greater.
	*/
	size += MAX(64 * 1024 * 1024,
	(size * l2arc_trim_ahead) / 100);
	}

	/*
	* Make sure the write size does not exceed the size of the cache
	* device. This is important in l2arc_evict(), otherwise infinite
	* iteration can occur.
	*/
	size = MIN(size, (dev->l2ad_end - dev->l2ad_start) / 4);

	size = P2ROUNDUP(size, 1ULL << dev->l2ad_vdev->vdev_ashift);

	return (size);

	}

	static clock_t
	l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
	{
	clock_t interval, next, now;

	/*
	* If the ARC lists are busy, increase our write rate; if the
	* lists are stale, idle back. This is achieved by checking
	* how much we previously wrote - if it was more than half of
	* what we wanted, schedule the next write much sooner.
	*/
	if (l2arc_feed_again && wrote > (wanted / 2))
	interval = (hz * l2arc_feed_min_ms) / 1000;
	else
	interval = hz * l2arc_feed_secs;

	now = ddi_get_lbolt();
	next = MAX(now, MIN(now + interval, began + interval));

	return (next);
	}

	/*
	* Cycle through L2ARC devices. This is how L2ARC load balances.
	* If a device is returned, this also returns holding the spa config lock.
	*/
	static l2arc_dev_t *
	l2arc_dev_get_next(void)
	{
	l2arc_dev_t first, next = NULL;

	/*
	* Lock out the removal of spas (spa_namespace_lock), then removal
	* of cache devices (l2arc_dev_mtx). Once a device has been selected,
	* both locks will be dropped and a spa config lock held instead.
	*/
	mutex_enter(&spa_namespace_lock);
	mutex_enter(&l2arc_dev_mtx);

	/* if there are no vdevs, there is nothing to do */
	if (l2arc_ndev == 0)
	goto out;

	first = NULL;
	next = l2arc_dev_last;
	do {
	/* loop around the list looking for a non-faulted vdev */
	if (next == NULL) {
	next = list_head(l2arc_dev_list);
	} else {
	next = list_next(l2arc_dev_list, next);
	if (next == NULL)
	next = list_head(l2arc_dev_list);
	}

	/* if we have come back to the start, bail out */
	if (first == NULL)
	first = next;
	else if (next == first)
	break;

	ASSERT3P(next, !=, NULL);
	} while (vdev_is_dead(next->l2ad_vdev) \|\| next->l2ad_rebuild \|\|
	next->l2ad_trim_all);

	/* if we were unable to find any usable vdevs, return NULL */
	if (vdev_is_dead(next->l2ad_vdev) \|\| next->l2ad_rebuild \|\|
	next->l2ad_trim_all)
	next = NULL;

	l2arc_dev_last = next;

	out:
	mutex_exit(&l2arc_dev_mtx);

	/*
	* Grab the config lock to prevent the 'next' device from being
	* removed while we are writing to it.
	*/
	if (next != NULL)
	spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER);
	mutex_exit(&spa_namespace_lock);

	return (next);
	}

	/*
	* Free buffers that were tagged for destruction.
	*/
	static void
	l2arc_do_free_on_write(void)
	{
	l2arc_data_free_t *df;

	mutex_enter(&l2arc_free_on_write_mtx);
	while ((df = list_remove_head(l2arc_free_on_write)) != NULL) {
	ASSERT3P(df->l2df_abd, !=, NULL);
	abd_free(df->l2df_abd);
	kmem_free(df, sizeof (l2arc_data_free_t));
	}
	mutex_exit(&l2arc_free_on_write_mtx);
	}

	/*
	* A write to a cache device has completed. Update all headers to allow
	* reads from these buffers to begin.
	*/
	static void
	l2arc_write_done(zio_t *zio)
	{
	l2arc_write_callback_t *cb;
	l2arc_lb_abd_buf_t *abd_buf;
	l2arc_lb_ptr_buf_t *lb_ptr_buf;
	l2arc_dev_t *dev;
	l2arc_dev_hdr_phys_t *l2dhdr;
	list_t *buflist;
	arc_buf_hdr_t head, hdr, *hdr_prev;
	kmutex_t *hash_lock;
	int64_t bytes_dropped = 0;

	cb = zio->io_private;
	ASSERT3P(cb, !=, NULL);
	dev = cb->l2wcb_dev;
	l2dhdr = dev->l2ad_dev_hdr;
	ASSERT3P(dev, !=, NULL);
	head = cb->l2wcb_head;
	ASSERT3P(head, !=, NULL);
	buflist = &dev->l2ad_buflist;
	ASSERT3P(buflist, !=, NULL);
	DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
	l2arc_write_callback_t *, cb);

	/*
	* All writes completed, or an error was hit.
	*/
	top:
	mutex_enter(&dev->l2ad_mtx);
	for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
	hdr_prev = list_prev(buflist, hdr);

	hash_lock = HDR_LOCK(hdr);

	/*
	* We cannot use mutex_enter or else we can deadlock
	* with l2arc_write_buffers (due to swapping the order
	* the hash lock and l2ad_mtx are taken).
	*/
	if (!mutex_tryenter(hash_lock)) {
	/*
	* Missed the hash lock. We must retry so we
	* don't leave the ARC_FLAG_L2_WRITING bit set.
	*/
	ARCSTAT_BUMP(arcstat_l2_writes_lock_retry);

	/*
	* We don't want to rescan the headers we've
	* already marked as having been written out, so
	* we reinsert the head node so we can pick up
	* where we left off.
	*/
	list_remove(buflist, head);
	list_insert_after(buflist, hdr, head);

	mutex_exit(&dev->l2ad_mtx);

	/*
	* We wait for the hash lock to become available
	* to try and prevent busy waiting, and increase
	* the chance we'll be able to acquire the lock
	* the next time around.
	*/
	mutex_enter(hash_lock);
	mutex_exit(hash_lock);
	goto top;
	}

	/*
	* We could not have been moved into the arc_l2c_only
	* state while in-flight due to our ARC_FLAG_L2_WRITING
	* bit being set. Let's just ensure that's being enforced.
	*/
	ASSERT(HDR_HAS_L1HDR(hdr));

	/*
	* Skipped - drop L2ARC entry and mark the header as no
	* longer L2 eligibile.
	*/
	if (zio->io_error != 0) {
	/*
	* Error - drop L2ARC entry.
	*/
	list_remove(buflist, hdr);
	arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);

	uint64_t psize = HDR_GET_PSIZE(hdr);
	l2arc_hdr_arcstats_decrement(hdr);

	bytes_dropped +=
	vdev_psize_to_asize(dev->l2ad_vdev, psize);
	(void) zfs_refcount_remove_many(&dev->l2ad_alloc,
	arc_hdr_size(hdr), hdr);
	}

	/*
	* Allow ARC to begin reads and ghost list evictions to
	* this L2ARC entry.
	*/
	arc_hdr_clear_flags(hdr, ARC_FLAG_L2_WRITING);

	mutex_exit(hash_lock);
	}

	/*
	* Free the allocated abd buffers for writing the log blocks.
	* If the zio failed reclaim the allocated space and remove the
	* pointers to these log blocks from the log block pointer list
	* of the L2ARC device.
	*/
	while ((abd_buf = list_remove_tail(&cb->l2wcb_abd_list)) != NULL) {
	abd_free(abd_buf->abd);
	zio_buf_free(abd_buf, sizeof (*abd_buf));
	if (zio->io_error != 0) {
	lb_ptr_buf = list_remove_head(&dev->l2ad_lbptr_list);
	/*
	* L2BLK_GET_PSIZE returns aligned size for log
	* blocks.
	*/
	uint64_t asize =
	L2BLK_GET_PSIZE((lb_ptr_buf->lb_ptr)->lbp_prop);
	bytes_dropped += asize;
	ARCSTAT_INCR(arcstat_l2_log_blk_asize, -asize);
	ARCSTAT_BUMPDOWN(arcstat_l2_log_blk_count);
	zfs_refcount_remove_many(&dev->l2ad_lb_asize, asize,
	lb_ptr_buf);
	zfs_refcount_remove(&dev->l2ad_lb_count, lb_ptr_buf);
	kmem_free(lb_ptr_buf->lb_ptr,
	sizeof (l2arc_log_blkptr_t));
	kmem_free(lb_ptr_buf, sizeof (l2arc_lb_ptr_buf_t));
	}
	}
	list_destroy(&cb->l2wcb_abd_list);

	if (zio->io_error != 0) {
	ARCSTAT_BUMP(arcstat_l2_writes_error);

	/*
	* Restore the lbps array in the header to its previous state.
	* If the list of log block pointers is empty, zero out the
	* log block pointers in the device header.
	*/
	lb_ptr_buf = list_head(&dev->l2ad_lbptr_list);
	for (int i = 0; i < 2; i++) {
	if (lb_ptr_buf == NULL) {
	/*
	* If the list is empty zero out the device
	* header. Otherwise zero out the second log
	* block pointer in the header.
	*/
	if (i == 0) {
	memset(l2dhdr, 0,
	dev->l2ad_dev_hdr_asize);
	} else {
	memset(&l2dhdr->dh_start_lbps[i], 0,
	sizeof (l2arc_log_blkptr_t));
	}
	break;
	}
	memcpy(&l2dhdr->dh_start_lbps[i], lb_ptr_buf->lb_ptr,
	sizeof (l2arc_log_blkptr_t));
	lb_ptr_buf = list_next(&dev->l2ad_lbptr_list,
	lb_ptr_buf);
	}
	}

	ARCSTAT_BUMP(arcstat_l2_writes_done);
	list_remove(buflist, head);
	ASSERT(!HDR_HAS_L1HDR(head));
	kmem_cache_free(hdr_l2only_cache, head);
	mutex_exit(&dev->l2ad_mtx);

	ASSERT(dev->l2ad_vdev != NULL);
	vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0);

	l2arc_do_free_on_write();

	kmem_free(cb, sizeof (l2arc_write_callback_t));
	}

	static int
	l2arc_untransform(zio_t zio, l2arc_read_callback_t cb)
	{
	int ret;
	spa_t *spa = zio->io_spa;
	arc_buf_hdr_t *hdr = cb->l2rcb_hdr;
	blkptr_t *bp = zio->io_bp;
	uint8_t salt[ZIO_DATA_SALT_LEN];
	uint8_t iv[ZIO_DATA_IV_LEN];
	uint8_t mac[ZIO_DATA_MAC_LEN];
	boolean_t no_crypt = B_FALSE;

	/*
	* ZIL data is never be written to the L2ARC, so we don't need
	* special handling for its unique MAC storage.
	*/
	ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_INTENT_LOG);
	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);

	/*
	* If the data was encrypted, decrypt it now. Note that
	* we must check the bp here and not the hdr, since the
	* hdr does not have its encryption parameters updated
	* until arc_read_done().
	*/
	if (BP_IS_ENCRYPTED(bp)) {
	abd_t *eabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr,
	ARC_HDR_USE_RESERVE);

	zio_crypt_decode_params_bp(bp, salt, iv);
	zio_crypt_decode_mac_bp(bp, mac);

	ret = spa_do_crypt_abd(B_FALSE, spa, &cb->l2rcb_zb,
	BP_GET_TYPE(bp), BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp),
	salt, iv, mac, HDR_GET_PSIZE(hdr), eabd,
	hdr->b_l1hdr.b_pabd, &no_crypt);
	if (ret != 0) {
	arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr);
	goto error;
	}

	/*
	* If we actually performed decryption, replace b_pabd
	* with the decrypted data. Otherwise we can just throw
	* our decryption buffer away.
	*/
	if (!no_crypt) {
	arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
	arc_hdr_size(hdr), hdr);
	hdr->b_l1hdr.b_pabd = eabd;
	zio->io_abd = eabd;
	} else {
	arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr);
	}
	}

	/*
	* If the L2ARC block was compressed, but ARC compression
	* is disabled we decompress the data into a new buffer and
	* replace the existing data.
	*/
	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
	!HDR_COMPRESSION_ENABLED(hdr)) {
	abd_t *cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr,
	ARC_HDR_USE_RESERVE);
	void *tmp = abd_borrow_buf(cabd, arc_hdr_size(hdr));

	ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
	hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr),
	HDR_GET_LSIZE(hdr), &hdr->b_complevel);
	if (ret != 0) {
	abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr));
	arc_free_data_abd(hdr, cabd, arc_hdr_size(hdr), hdr);
	goto error;
	}

	abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr));
	arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
	arc_hdr_size(hdr), hdr);
	hdr->b_l1hdr.b_pabd = cabd;
	zio->io_abd = cabd;
	zio->io_size = HDR_GET_LSIZE(hdr);
	}

	return (0);

	error:
	return (ret);
	}


	/*
	* A read to a cache device completed. Validate buffer contents before
	* handing over to the regular ARC routines.
	*/
	static void
	l2arc_read_done(zio_t *zio)
	{
	int tfm_error = 0;
	l2arc_read_callback_t *cb = zio->io_private;
	arc_buf_hdr_t *hdr;
	kmutex_t *hash_lock;
	boolean_t valid_cksum;
	boolean_t using_rdata = (BP_IS_ENCRYPTED(&cb->l2rcb_bp) &&
	(cb->l2rcb_flags & ZIO_FLAG_RAW_ENCRYPT));

	ASSERT3P(zio->io_vd, !=, NULL);
	ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);

	spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);

	ASSERT3P(cb, !=, NULL);
	hdr = cb->l2rcb_hdr;
	ASSERT3P(hdr, !=, NULL);

	hash_lock = HDR_LOCK(hdr);
	mutex_enter(hash_lock);
	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));

	/*
	* If the data was read into a temporary buffer,
	* move it and free the buffer.
	*/
	if (cb->l2rcb_abd != NULL) {
	ASSERT3U(arc_hdr_size(hdr), <, zio->io_size);
	if (zio->io_error == 0) {
	if (using_rdata) {
	abd_copy(hdr->b_crypt_hdr.b_rabd,
	cb->l2rcb_abd, arc_hdr_size(hdr));
	} else {
	abd_copy(hdr->b_l1hdr.b_pabd,
	cb->l2rcb_abd, arc_hdr_size(hdr));
	}
	}

	/*
	* The following must be done regardless of whether
	* there was an error:
	* - free the temporary buffer
	* - point zio to the real ARC buffer
	* - set zio size accordingly
	* These are required because zio is either re-used for
	* an I/O of the block in the case of the error
	* or the zio is passed to arc_read_done() and it
	* needs real data.
	*/
	abd_free(cb->l2rcb_abd);
	zio->io_size = zio->io_orig_size = arc_hdr_size(hdr);

	if (using_rdata) {
	ASSERT(HDR_HAS_RABD(hdr));
	zio->io_abd = zio->io_orig_abd =
	hdr->b_crypt_hdr.b_rabd;
	} else {
	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
	zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd;
	}
	}

	ASSERT3P(zio->io_abd, !=, NULL);

	/*
	* Check this survived the L2ARC journey.
	*/
	ASSERT(zio->io_abd == hdr->b_l1hdr.b_pabd \|\|
	(HDR_HAS_RABD(hdr) && zio->io_abd == hdr->b_crypt_hdr.b_rabd));
	zio->io_bp_copy = cb->l2rcb_bp; /* XXX fix in L2ARC 2.0 */
	zio->io_bp = &zio->io_bp_copy; /* XXX fix in L2ARC 2.0 */
	zio->io_prop.zp_complevel = hdr->b_complevel;

	valid_cksum = arc_cksum_is_equal(hdr, zio);

	/*
	* b_rabd will always match the data as it exists on disk if it is
	* being used. Therefore if we are reading into b_rabd we do not
	* attempt to untransform the data.
	*/
	if (valid_cksum && !using_rdata)
	tfm_error = l2arc_untransform(zio, cb);

	if (valid_cksum && tfm_error == 0 && zio->io_error == 0 &&
	!HDR_L2_EVICTED(hdr)) {
	mutex_exit(hash_lock);
	zio->io_private = hdr;
	arc_read_done(zio);
	} else {
	/*
	* Buffer didn't survive caching. Increment stats and
	* reissue to the original storage device.
	*/
	if (zio->io_error != 0) {
	ARCSTAT_BUMP(arcstat_l2_io_error);
	} else {
	zio->io_error = SET_ERROR(EIO);
	}
	if (!valid_cksum \|\| tfm_error != 0)
	ARCSTAT_BUMP(arcstat_l2_cksum_bad);

	/*
	* If there's no waiter, issue an async i/o to the primary
	* storage now. If there is a waiter, the caller must
	* issue the i/o in a context where it's OK to block.
	*/
	if (zio->io_waiter == NULL) {
	zio_t *pio = zio_unique_parent(zio);
	void *abd = (using_rdata) ?
	hdr->b_crypt_hdr.b_rabd : hdr->b_l1hdr.b_pabd;

	ASSERT(!pio \|\| pio->io_child_type == ZIO_CHILD_LOGICAL);

	zio = zio_read(pio, zio->io_spa, zio->io_bp,
	abd, zio->io_size, arc_read_done,
	hdr, zio->io_priority, cb->l2rcb_flags,
	&cb->l2rcb_zb);

	/*
	* Original ZIO will be freed, so we need to update
	* ARC header with the new ZIO pointer to be used
	* by zio_change_priority() in arc_read().
	*/
	for (struct arc_callback *acb = hdr->b_l1hdr.b_acb;
	acb != NULL; acb = acb->acb_next)
	acb->acb_zio_head = zio;

	mutex_exit(hash_lock);
	zio_nowait(zio);
	} else {
	mutex_exit(hash_lock);
	}
	}

	kmem_free(cb, sizeof (l2arc_read_callback_t));
	}

	/*
	* This is the list priority from which the L2ARC will search for pages to
	* cache. This is used within loops (0..3) to cycle through lists in the
	* desired order. This order can have a significant effect on cache
	* performance.
	*
	* Currently the metadata lists are hit first, MFU then MRU, followed by
	* the data lists. This function returns a locked list, and also returns
	* the lock pointer.
	*/
	static multilist_sublist_t *
	l2arc_sublist_lock(int list_num)
	{
	multilist_t *ml = NULL;
	unsigned int idx;

	ASSERT(list_num >= 0 && list_num < L2ARC_FEED_TYPES);

	switch (list_num) {
	case 0:
	ml = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
	break;
	case 1:
	ml = &arc_mru->arcs_list[ARC_BUFC_METADATA];
	break;
	case 2:
	ml = &arc_mfu->arcs_list[ARC_BUFC_DATA];
	break;
	case 3:
	ml = &arc_mru->arcs_list[ARC_BUFC_DATA];
	break;
	default:
	return (NULL);
	}

	/*
	* Return a randomly-selected sublist. This is acceptable
	* because the caller feeds only a little bit of data for each
	* call (8MB). Subsequent calls will result in different
	* sublists being selected.
	*/
	idx = multilist_get_random_index(ml);
	- return (multilist_sublist_lock(ml, idx));
	+ return (multilist_sublist_lock_idx(ml, idx));
	}

	/*
	* Calculates the maximum overhead of L2ARC metadata log blocks for a given
	* L2ARC write size. l2arc_evict and l2arc_write_size need to include this
	* overhead in processing to make sure there is enough headroom available
	* when writing buffers.
	*/
	static inline uint64_t
	l2arc_log_blk_overhead(uint64_t write_sz, l2arc_dev_t *dev)
	{
	if (dev->l2ad_log_entries == 0) {
	return (0);
	} else {
	uint64_t log_entries = write_sz >> SPA_MINBLOCKSHIFT;

	uint64_t log_blocks = (log_entries +
	dev->l2ad_log_entries - 1) /
	dev->l2ad_log_entries;

	return (vdev_psize_to_asize(dev->l2ad_vdev,
	sizeof (l2arc_log_blk_phys_t)) * log_blocks);
	}
	}

	/*
	* Evict buffers from the device write hand to the distance specified in
	* bytes. This distance may span populated buffers, it may span nothing.
	* This is clearing a region on the L2ARC device ready for writing.
	* If the 'all' boolean is set, every buffer is evicted.
	*/
	static void
	l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
	{
	list_t *buflist;
	arc_buf_hdr_t hdr, hdr_prev;
	kmutex_t *hash_lock;
	uint64_t taddr;
	l2arc_lb_ptr_buf_t lb_ptr_buf, lb_ptr_buf_prev;
	vdev_t *vd = dev->l2ad_vdev;
	boolean_t rerun;

	buflist = &dev->l2ad_buflist;

	top:
	rerun = B_FALSE;
	if (dev->l2ad_hand + distance > dev->l2ad_end) {
	/*
	* When there is no space to accommodate upcoming writes,
	* evict to the end. Then bump the write and evict hands
	* to the start and iterate. This iteration does not
	* happen indefinitely as we make sure in
	* l2arc_write_size() that when the write hand is reset,
	* the write size does not exceed the end of the device.
	*/
	rerun = B_TRUE;
	taddr = dev->l2ad_end;
	} else {
	taddr = dev->l2ad_hand + distance;
	}
	DTRACE_PROBE4(l2arc__evict, l2arc_dev_t , dev, list_t , buflist,
	uint64_t, taddr, boolean_t, all);

	if (!all) {
	/*
	* This check has to be placed after deciding whether to
	* iterate (rerun).
	*/
	if (dev->l2ad_first) {
	/*
	* This is the first sweep through the device. There is
	* nothing to evict. We have already trimmmed the
	* whole device.
	*/
	goto out;
	} else {
	/*
	* Trim the space to be evicted.
	*/
	if (vd->vdev_has_trim && dev->l2ad_evict < taddr &&
	l2arc_trim_ahead > 0) {
	/*
	* We have to drop the spa_config lock because
	* vdev_trim_range() will acquire it.
	* l2ad_evict already accounts for the label
	* size. To prevent vdev_trim_ranges() from
	* adding it again, we subtract it from
	* l2ad_evict.
	*/
	spa_config_exit(dev->l2ad_spa, SCL_L2ARC, dev);
	vdev_trim_simple(vd,
	dev->l2ad_evict - VDEV_LABEL_START_SIZE,
	taddr - dev->l2ad_evict);
	spa_config_enter(dev->l2ad_spa, SCL_L2ARC, dev,
	RW_READER);
	}

	/*
	* When rebuilding L2ARC we retrieve the evict hand
	* from the header of the device. Of note, l2arc_evict()
	* does not actually delete buffers from the cache
	* device, but trimming may do so depending on the
	* hardware implementation. Thus keeping track of the
	* evict hand is useful.
	*/
	dev->l2ad_evict = MAX(dev->l2ad_evict, taddr);
	}
	}

	retry:
	mutex_enter(&dev->l2ad_mtx);
	/*
	* We have to account for evicted log blocks. Run vdev_space_update()
	* on log blocks whose offset (in bytes) is before the evicted offset
	* (in bytes) by searching in the list of pointers to log blocks
	* present in the L2ARC device.
	*/
	for (lb_ptr_buf = list_tail(&dev->l2ad_lbptr_list); lb_ptr_buf;
	lb_ptr_buf = lb_ptr_buf_prev) {

	lb_ptr_buf_prev = list_prev(&dev->l2ad_lbptr_list, lb_ptr_buf);

	/* L2BLK_GET_PSIZE returns aligned size for log blocks */
	uint64_t asize = L2BLK_GET_PSIZE(
	(lb_ptr_buf->lb_ptr)->lbp_prop);

	/*
	* We don't worry about log blocks left behind (ie
	* lbp_payload_start < l2ad_hand) because l2arc_write_buffers()
	* will never write more than l2arc_evict() evicts.
	*/
	if (!all && l2arc_log_blkptr_valid(dev, lb_ptr_buf->lb_ptr)) {
	break;
	} else {
	vdev_space_update(vd, -asize, 0, 0);
	ARCSTAT_INCR(arcstat_l2_log_blk_asize, -asize);
	ARCSTAT_BUMPDOWN(arcstat_l2_log_blk_count);
	zfs_refcount_remove_many(&dev->l2ad_lb_asize, asize,
	lb_ptr_buf);
	zfs_refcount_remove(&dev->l2ad_lb_count, lb_ptr_buf);
	list_remove(&dev->l2ad_lbptr_list, lb_ptr_buf);
	kmem_free(lb_ptr_buf->lb_ptr,
	sizeof (l2arc_log_blkptr_t));
	kmem_free(lb_ptr_buf, sizeof (l2arc_lb_ptr_buf_t));
	}
	}

	for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) {
	hdr_prev = list_prev(buflist, hdr);

	ASSERT(!HDR_EMPTY(hdr));
	hash_lock = HDR_LOCK(hdr);

	/*
	* We cannot use mutex_enter or else we can deadlock
	* with l2arc_write_buffers (due to swapping the order
	* the hash lock and l2ad_mtx are taken).
	*/
	if (!mutex_tryenter(hash_lock)) {
	/*
	* Missed the hash lock. Retry.
	*/
	ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
	mutex_exit(&dev->l2ad_mtx);
	mutex_enter(hash_lock);
	mutex_exit(hash_lock);
	goto retry;
	}

	/*
	* A header can't be on this list if it doesn't have L2 header.
	*/
	ASSERT(HDR_HAS_L2HDR(hdr));

	/* Ensure this header has finished being written. */
	ASSERT(!HDR_L2_WRITING(hdr));
	ASSERT(!HDR_L2_WRITE_HEAD(hdr));

	if (!all && (hdr->b_l2hdr.b_daddr >= dev->l2ad_evict \|\|
	hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) {
	/*
	* We've evicted to the target address,
	* or the end of the device.
	*/
	mutex_exit(hash_lock);
	break;
	}

	if (!HDR_HAS_L1HDR(hdr)) {
	ASSERT(!HDR_L2_READING(hdr));
	/*
	* This doesn't exist in the ARC. Destroy.
	* arc_hdr_destroy() will call list_remove()
	* and decrement arcstat_l2_lsize.
	*/
	arc_change_state(arc_anon, hdr);
	arc_hdr_destroy(hdr);
	} else {
	ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
	ARCSTAT_BUMP(arcstat_l2_evict_l1cached);
	/*
	* Invalidate issued or about to be issued
	* reads, since we may be about to write
	* over this location.
	*/
	if (HDR_L2_READING(hdr)) {
	ARCSTAT_BUMP(arcstat_l2_evict_reading);
	arc_hdr_set_flags(hdr, ARC_FLAG_L2_EVICTED);
	}

	arc_hdr_l2hdr_destroy(hdr);
	}
	mutex_exit(hash_lock);
	}
	mutex_exit(&dev->l2ad_mtx);

	out:
	/*
	* We need to check if we evict all buffers, otherwise we may iterate
	* unnecessarily.
	*/
	if (!all && rerun) {
	/*
	* Bump device hand to the device start if it is approaching the
	* end. l2arc_evict() has already evicted ahead for this case.
	*/
	dev->l2ad_hand = dev->l2ad_start;
	dev->l2ad_evict = dev->l2ad_start;
	dev->l2ad_first = B_FALSE;
	goto top;
	}

	if (!all) {
	/*
	* In case of cache device removal (all) the following
	* assertions may be violated without functional consequences
	* as the device is about to be removed.
	*/
	ASSERT3U(dev->l2ad_hand + distance, <, dev->l2ad_end);
	if (!dev->l2ad_first)
	ASSERT3U(dev->l2ad_hand, <=, dev->l2ad_evict);
	}
	}

	/*
	* Handle any abd transforms that might be required for writing to the L2ARC.
	* If successful, this function will always return an abd with the data
	* transformed as it is on disk in a new abd of asize bytes.
	*/
	static int
	l2arc_apply_transforms(spa_t spa, arc_buf_hdr_t hdr, uint64_t asize,
	abd_t **abd_out)
	{
	int ret;
	void *tmp = NULL;
	abd_t cabd = NULL, eabd = NULL, *to_write = hdr->b_l1hdr.b_pabd;
	enum zio_compress compress = HDR_GET_COMPRESS(hdr);
	uint64_t psize = HDR_GET_PSIZE(hdr);
	uint64_t size = arc_hdr_size(hdr);
	boolean_t ismd = HDR_ISTYPE_METADATA(hdr);
	boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
	dsl_crypto_key_t *dck = NULL;
	uint8_t mac[ZIO_DATA_MAC_LEN] = { 0 };
	boolean_t no_crypt = B_FALSE;

	ASSERT((HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
	!HDR_COMPRESSION_ENABLED(hdr)) \|\|
	HDR_ENCRYPTED(hdr) \|\| HDR_SHARED_DATA(hdr) \|\| psize != asize);
	ASSERT3U(psize, <=, asize);

	/*
	* If this data simply needs its own buffer, we simply allocate it
	* and copy the data. This may be done to eliminate a dependency on a
	* shared buffer or to reallocate the buffer to match asize.
	*/
	if (HDR_HAS_RABD(hdr) && asize != psize) {
	ASSERT3U(asize, >=, psize);
	to_write = abd_alloc_for_io(asize, ismd);
	abd_copy(to_write, hdr->b_crypt_hdr.b_rabd, psize);
	if (psize != asize)
	abd_zero_off(to_write, psize, asize - psize);
	goto out;
	}

	if ((compress == ZIO_COMPRESS_OFF \|\| HDR_COMPRESSION_ENABLED(hdr)) &&
	!HDR_ENCRYPTED(hdr)) {
	ASSERT3U(size, ==, psize);
	to_write = abd_alloc_for_io(asize, ismd);
	abd_copy(to_write, hdr->b_l1hdr.b_pabd, size);
	if (size != asize)
	abd_zero_off(to_write, size, asize - size);
	goto out;
	}

	if (compress != ZIO_COMPRESS_OFF && !HDR_COMPRESSION_ENABLED(hdr)) {
	/*
	* In some cases, we can wind up with size > asize, so
	* we need to opt for the larger allocation option here.
	*
	* (We also need abd_return_buf_copy in all cases because
	* it's an ASSERT() to modify the buffer before returning it
	* with arc_return_buf(), and all the compressors
	* write things before deciding to fail compression in nearly
	* every case.)
	*/
	uint64_t bufsize = MAX(size, asize);
	cabd = abd_alloc_for_io(bufsize, ismd);
	tmp = abd_borrow_buf(cabd, bufsize);

	psize = zio_compress_data(compress, to_write, &tmp, size,
	hdr->b_complevel);

	if (psize >= asize) {
	psize = HDR_GET_PSIZE(hdr);
	abd_return_buf_copy(cabd, tmp, bufsize);
	HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF);
	to_write = cabd;
	abd_copy(to_write, hdr->b_l1hdr.b_pabd, psize);
	if (psize != asize)
	abd_zero_off(to_write, psize, asize - psize);
	goto encrypt;
	}
	ASSERT3U(psize, <=, HDR_GET_PSIZE(hdr));
	if (psize < asize)
	memset((char *)tmp + psize, 0, bufsize - psize);
	psize = HDR_GET_PSIZE(hdr);
	abd_return_buf_copy(cabd, tmp, bufsize);
	to_write = cabd;
	}

	encrypt:
	if (HDR_ENCRYPTED(hdr)) {
	eabd = abd_alloc_for_io(asize, ismd);

	/*
	* If the dataset was disowned before the buffer
	* made it to this point, the key to re-encrypt
	* it won't be available. In this case we simply
	* won't write the buffer to the L2ARC.
	*/
	ret = spa_keystore_lookup_key(spa, hdr->b_crypt_hdr.b_dsobj,
	FTAG, &dck);
	if (ret != 0)
	goto error;

	ret = zio_do_crypt_abd(B_TRUE, &dck->dck_key,
	hdr->b_crypt_hdr.b_ot, bswap, hdr->b_crypt_hdr.b_salt,
	hdr->b_crypt_hdr.b_iv, mac, psize, to_write, eabd,
	&no_crypt);
	if (ret != 0)
	goto error;

	if (no_crypt)
	abd_copy(eabd, to_write, psize);

	if (psize != asize)
	abd_zero_off(eabd, psize, asize - psize);

	/* assert that the MAC we got here matches the one we saved */
	ASSERT0(memcmp(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN));
	spa_keystore_dsl_key_rele(spa, dck, FTAG);

	if (to_write == cabd)
	abd_free(cabd);

	to_write = eabd;
	}

	out:
	ASSERT3P(to_write, !=, hdr->b_l1hdr.b_pabd);
	*abd_out = to_write;
	return (0);

	error:
	if (dck != NULL)
	spa_keystore_dsl_key_rele(spa, dck, FTAG);
	if (cabd != NULL)
	abd_free(cabd);
	if (eabd != NULL)
	abd_free(eabd);

	*abd_out = NULL;
	return (ret);
	}

	static void
	l2arc_blk_fetch_done(zio_t *zio)
	{
	l2arc_read_callback_t *cb;

	cb = zio->io_private;
	if (cb->l2rcb_abd != NULL)
	abd_free(cb->l2rcb_abd);
	kmem_free(cb, sizeof (l2arc_read_callback_t));
	}

	/*
	* Find and write ARC buffers to the L2ARC device.
	*
	* An ARC_FLAG_L2_WRITING flag is set so that the L2ARC buffers are not valid
	* for reading until they have completed writing.
	* The headroom_boost is an in-out parameter used to maintain headroom boost
	* state between calls to this function.
	*
	* Returns the number of bytes actually written (which may be smaller than
	* the delta by which the device hand has changed due to alignment and the
	* writing of log blocks).
	*/
	static uint64_t
	l2arc_write_buffers(spa_t spa, l2arc_dev_t dev, uint64_t target_sz)
	{
	- arc_buf_hdr_t hdr, hdr_prev, *head;
	- uint64_t write_asize, write_psize, write_lsize, headroom;
	- boolean_t full;
	+ arc_buf_hdr_t hdr, head, *marker;
	+ uint64_t write_asize, write_psize, headroom;
	+ boolean_t full, from_head = !arc_warm;
	l2arc_write_callback_t *cb = NULL;
	zio_t pio, wzio;
	uint64_t guid = spa_load_guid(spa);
	l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;

	ASSERT3P(dev->l2ad_vdev, !=, NULL);

	pio = NULL;
	- write_lsize = write_asize = write_psize = 0;
	+ write_asize = write_psize = 0;
	full = B_FALSE;
	head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE);
	arc_hdr_set_flags(head, ARC_FLAG_L2_WRITE_HEAD \| ARC_FLAG_HAS_L2HDR);
	+ marker = arc_state_alloc_marker();

	/*
	* Copy buffers for L2ARC writing.
	*/
	for (int pass = 0; pass < L2ARC_FEED_TYPES; pass++) {
	/*
	* If pass == 1 or 3, we cache MRU metadata and data
	* respectively.
	*/
	if (l2arc_mfuonly) {
	if (pass == 1 \|\| pass == 3)
	continue;
	}

	- multilist_sublist_t *mls = l2arc_sublist_lock(pass);
	uint64_t passed_sz = 0;
	-
	- VERIFY3P(mls, !=, NULL);
	+ headroom = target_sz * l2arc_headroom;
	+ if (zfs_compressed_arc_enabled)
	+ headroom = (headroom * l2arc_headroom_boost) / 100;

	/*
	- * L2ARC fast warmup.
	- *
	* Until the ARC is warm and starts to evict, read from the
	* head of the ARC lists rather than the tail.
	*/
	- if (arc_warm == B_FALSE)
	+ multilist_sublist_t *mls = l2arc_sublist_lock(pass);
	+ ASSERT3P(mls, !=, NULL);
	+ if (from_head)
	hdr = multilist_sublist_head(mls);
	else
	hdr = multilist_sublist_tail(mls);

	- headroom = target_sz * l2arc_headroom;
	- if (zfs_compressed_arc_enabled)
	- headroom = (headroom * l2arc_headroom_boost) / 100;
	-
	- for (; hdr; hdr = hdr_prev) {
	+ while (hdr != NULL) {
	kmutex_t *hash_lock;
	abd_t *to_write = NULL;

	- if (arc_warm == B_FALSE)
	- hdr_prev = multilist_sublist_next(mls, hdr);
	- else
	- hdr_prev = multilist_sublist_prev(mls, hdr);
	-
	hash_lock = HDR_LOCK(hdr);
	if (!mutex_tryenter(hash_lock)) {
	- /*
	- * Skip this buffer rather than waiting.
	- */
	+skip:
	+ /* Skip this buffer rather than waiting. */
	+ if (from_head)
	+ hdr = multilist_sublist_next(mls, hdr);
	+ else
	+ hdr = multilist_sublist_prev(mls, hdr);
	continue;
	}

	passed_sz += HDR_GET_LSIZE(hdr);
	if (l2arc_headroom != 0 && passed_sz > headroom) {
	/*
	* Searched too far.
	*/
	mutex_exit(hash_lock);
	break;
	}

	if (!l2arc_write_eligible(guid, hdr)) {
	mutex_exit(hash_lock);
	- continue;
	+ goto skip;
	}

	ASSERT(HDR_HAS_L1HDR(hdr));
	-
	ASSERT3U(HDR_GET_PSIZE(hdr), >, 0);
	ASSERT3U(arc_hdr_size(hdr), >, 0);
	ASSERT(hdr->b_l1hdr.b_pabd != NULL \|\|
	HDR_HAS_RABD(hdr));
	uint64_t psize = HDR_GET_PSIZE(hdr);
	uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev,
	psize);

	/*
	* If the allocated size of this buffer plus the max
	* size for the pending log block exceeds the evicted
	* target size, terminate writing buffers for this run.
	*/
	if (write_asize + asize +
	sizeof (l2arc_log_blk_phys_t) > target_sz) {
	full = B_TRUE;
	mutex_exit(hash_lock);
	break;
	}

	/*
	- * We rely on the L1 portion of the header below, so
	- * it's invalid for this header to have been evicted out
	- * of the ghost cache, prior to being written out. The
	- * ARC_FLAG_L2_WRITING bit ensures this won't happen.
	+ * We should not sleep with sublist lock held or it
	+ * may block ARC eviction. Insert a marker to save
	+ * the position and drop the lock.
	*/
	- arc_hdr_set_flags(hdr, ARC_FLAG_L2_WRITING);
	+ if (from_head) {
	+ multilist_sublist_insert_after(mls, hdr,
	+ marker);
	+ } else {
	+ multilist_sublist_insert_before(mls, hdr,
	+ marker);
	+ }
	+ multilist_sublist_unlock(mls);

	/*
	* If this header has b_rabd, we can use this since it
	* must always match the data exactly as it exists on
	* disk. Otherwise, the L2ARC can normally use the
	* hdr's data, but if we're sharing data between the
	* hdr and one of its bufs, L2ARC needs its own copy of
	* the data so that the ZIO below can't race with the
	* buf consumer. To ensure that this copy will be
	* available for the lifetime of the ZIO and be cleaned
	* up afterwards, we add it to the l2arc_free_on_write
	* queue. If we need to apply any transforms to the
	* data (compression, encryption) we will also need the
	* extra buffer.
	*/
	if (HDR_HAS_RABD(hdr) && psize == asize) {
	to_write = hdr->b_crypt_hdr.b_rabd;
	} else if ((HDR_COMPRESSION_ENABLED(hdr) \|\|
	HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) &&
	!HDR_ENCRYPTED(hdr) && !HDR_SHARED_DATA(hdr) &&
	psize == asize) {
	to_write = hdr->b_l1hdr.b_pabd;
	} else {
	int ret;
	arc_buf_contents_t type = arc_buf_type(hdr);

	ret = l2arc_apply_transforms(spa, hdr, asize,
	&to_write);
	if (ret != 0) {
	arc_hdr_clear_flags(hdr,
	- ARC_FLAG_L2_WRITING);
	+ ARC_FLAG_L2CACHE);
	mutex_exit(hash_lock);
	- continue;
	+ goto next;
	}

	l2arc_free_abd_on_write(to_write, asize, type);
	}

	+ hdr->b_l2hdr.b_dev = dev;
	+ hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
	+ hdr->b_l2hdr.b_hits = 0;
	+ hdr->b_l2hdr.b_arcs_state =
	+ hdr->b_l1hdr.b_state->arcs_state;
	+ mutex_enter(&dev->l2ad_mtx);
	if (pio == NULL) {
	/*
	* Insert a dummy header on the buflist so
	* l2arc_write_done() can find where the
	* write buffers begin without searching.
	*/
	- mutex_enter(&dev->l2ad_mtx);
	list_insert_head(&dev->l2ad_buflist, head);
	- mutex_exit(&dev->l2ad_mtx);
	+ }
	+ list_insert_head(&dev->l2ad_buflist, hdr);
	+ mutex_exit(&dev->l2ad_mtx);
	+ arc_hdr_set_flags(hdr, ARC_FLAG_HAS_L2HDR \|
	+ ARC_FLAG_L2_WRITING);
	+
	+ (void) zfs_refcount_add_many(&dev->l2ad_alloc,
	+ arc_hdr_size(hdr), hdr);
	+ l2arc_hdr_arcstats_increment(hdr);

	+ boolean_t commit = l2arc_log_blk_insert(dev, hdr);
	+ mutex_exit(hash_lock);
	+
	+ if (pio == NULL) {
	cb = kmem_alloc(
	sizeof (l2arc_write_callback_t), KM_SLEEP);
	cb->l2wcb_dev = dev;
	cb->l2wcb_head = head;
	- /*
	- * Create a list to save allocated abd buffers
	- * for l2arc_log_blk_commit().
	- */
	list_create(&cb->l2wcb_abd_list,
	sizeof (l2arc_lb_abd_buf_t),
	offsetof(l2arc_lb_abd_buf_t, node));
	pio = zio_root(spa, l2arc_write_done, cb,
	ZIO_FLAG_CANFAIL);
	}

	- hdr->b_l2hdr.b_dev = dev;
	- hdr->b_l2hdr.b_hits = 0;
	-
	- hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
	- hdr->b_l2hdr.b_arcs_state =
	- hdr->b_l1hdr.b_state->arcs_state;
	- arc_hdr_set_flags(hdr, ARC_FLAG_HAS_L2HDR);
	-
	- mutex_enter(&dev->l2ad_mtx);
	- list_insert_head(&dev->l2ad_buflist, hdr);
	- mutex_exit(&dev->l2ad_mtx);
	-
	- (void) zfs_refcount_add_many(&dev->l2ad_alloc,
	- arc_hdr_size(hdr), hdr);
	-
	wzio = zio_write_phys(pio, dev->l2ad_vdev,
	- hdr->b_l2hdr.b_daddr, asize, to_write,
	+ dev->l2ad_hand, asize, to_write,
	ZIO_CHECKSUM_OFF, NULL, hdr,
	ZIO_PRIORITY_ASYNC_WRITE,
	ZIO_FLAG_CANFAIL, B_FALSE);

	- write_lsize += HDR_GET_LSIZE(hdr);
	DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
	zio_t *, wzio);
	+ zio_nowait(wzio);

	write_psize += psize;
	write_asize += asize;
	dev->l2ad_hand += asize;
	- l2arc_hdr_arcstats_increment(hdr);
	vdev_space_update(dev->l2ad_vdev, asize, 0, 0);

	- mutex_exit(hash_lock);
	-
	- /*
	- * Append buf info to current log and commit if full.
	- * arcstat_l2_{size,asize} kstats are updated
	- * internally.
	- */
	- if (l2arc_log_blk_insert(dev, hdr)) {
	- /*
	- * l2ad_hand will be adjusted in
	- * l2arc_log_blk_commit().
	- */
	+ if (commit) {
	+ /* l2ad_hand will be adjusted inside. */
	write_asize +=
	l2arc_log_blk_commit(dev, pio, cb);
	}

	- zio_nowait(wzio);
	+next:
	+ multilist_sublist_lock(mls);
	+ if (from_head)
	+ hdr = multilist_sublist_next(mls, marker);
	+ else
	+ hdr = multilist_sublist_prev(mls, marker);
	+ multilist_sublist_remove(mls, marker);
	}

	multilist_sublist_unlock(mls);

	if (full == B_TRUE)
	break;
	}

	+ arc_state_free_marker(marker);
	+
	/* No buffers selected for writing? */
	if (pio == NULL) {
	- ASSERT0(write_lsize);
	+ ASSERT0(write_psize);
	ASSERT(!HDR_HAS_L1HDR(head));
	kmem_cache_free(hdr_l2only_cache, head);

	/*
	* Although we did not write any buffers l2ad_evict may
	* have advanced.
	*/
	if (dev->l2ad_evict != l2dhdr->dh_evict)
	l2arc_dev_hdr_update(dev);

	return (0);
	}

	if (!dev->l2ad_first)
	ASSERT3U(dev->l2ad_hand, <=, dev->l2ad_evict);

	ASSERT3U(write_asize, <=, target_sz);
	ARCSTAT_BUMP(arcstat_l2_writes_sent);
	ARCSTAT_INCR(arcstat_l2_write_bytes, write_psize);

	dev->l2ad_writing = B_TRUE;
	(void) zio_wait(pio);
	dev->l2ad_writing = B_FALSE;

	/*
	* Update the device header after the zio completes as
	* l2arc_write_done() may have updated the memory holding the log block
	* pointers in the device header.
	*/
	l2arc_dev_hdr_update(dev);

	return (write_asize);
	}

	static boolean_t
	l2arc_hdr_limit_reached(void)
	{
	int64_t s = aggsum_upper_bound(&arc_sums.arcstat_l2_hdr_size);

	return (arc_reclaim_needed() \|\|
	(s > (arc_warm ? arc_c : arc_c_max) * l2arc_meta_percent / 100));
	}

	/*
	* This thread feeds the L2ARC at regular intervals. This is the beating
	* heart of the L2ARC.
	*/
	static __attribute__((noreturn)) void
	l2arc_feed_thread(void *unused)
	{
	(void) unused;
	callb_cpr_t cpr;
	l2arc_dev_t *dev;
	spa_t *spa;
	uint64_t size, wrote;
	clock_t begin, next = ddi_get_lbolt();
	fstrans_cookie_t cookie;

	CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG);

	mutex_enter(&l2arc_feed_thr_lock);

	cookie = spl_fstrans_mark();
	while (l2arc_thread_exit == 0) {
	CALLB_CPR_SAFE_BEGIN(&cpr);
	(void) cv_timedwait_idle(&l2arc_feed_thr_cv,
	&l2arc_feed_thr_lock, next);
	CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock);
	next = ddi_get_lbolt() + hz;

	/*
	* Quick check for L2ARC devices.
	*/
	mutex_enter(&l2arc_dev_mtx);
	if (l2arc_ndev == 0) {
	mutex_exit(&l2arc_dev_mtx);
	continue;
	}
	mutex_exit(&l2arc_dev_mtx);
	begin = ddi_get_lbolt();

	/*
	* This selects the next l2arc device to write to, and in
	* doing so the next spa to feed from: dev->l2ad_spa. This
	* will return NULL if there are now no l2arc devices or if
	* they are all faulted.
	*
	* If a device is returned, its spa's config lock is also
	* held to prevent device removal. l2arc_dev_get_next()
	* will grab and release l2arc_dev_mtx.
	*/
	if ((dev = l2arc_dev_get_next()) == NULL)
	continue;

	spa = dev->l2ad_spa;
	ASSERT3P(spa, !=, NULL);

	/*
	* If the pool is read-only then force the feed thread to
	* sleep a little longer.
	*/
	if (!spa_writeable(spa)) {
	next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz;
	spa_config_exit(spa, SCL_L2ARC, dev);
	continue;
	}

	/*
	* Avoid contributing to memory pressure.
	*/
	if (l2arc_hdr_limit_reached()) {
	ARCSTAT_BUMP(arcstat_l2_abort_lowmem);
	spa_config_exit(spa, SCL_L2ARC, dev);
	continue;
	}

	ARCSTAT_BUMP(arcstat_l2_feeds);

	size = l2arc_write_size(dev);

	/*
	* Evict L2ARC buffers that will be overwritten.
	*/
	l2arc_evict(dev, size, B_FALSE);

	/*
	* Write ARC buffers.
	*/
	wrote = l2arc_write_buffers(spa, dev, size);

	/*
	* Calculate interval between writes.
	*/
	next = l2arc_write_interval(begin, size, wrote);
	spa_config_exit(spa, SCL_L2ARC, dev);
	}
	spl_fstrans_unmark(cookie);

	l2arc_thread_exit = 0;
	cv_broadcast(&l2arc_feed_thr_cv);
	CALLB_CPR_EXIT(&cpr); /* drops l2arc_feed_thr_lock */
	thread_exit();
	}

	boolean_t
	l2arc_vdev_present(vdev_t *vd)
	{
	return (l2arc_vdev_get(vd) != NULL);
	}

	/*
	* Returns the l2arc_dev_t associated with a particular vdev_t or NULL if
	* the vdev_t isn't an L2ARC device.
	*/
	l2arc_dev_t *
	l2arc_vdev_get(vdev_t *vd)
	{
	l2arc_dev_t *dev;

	mutex_enter(&l2arc_dev_mtx);
	for (dev = list_head(l2arc_dev_list); dev != NULL;
	dev = list_next(l2arc_dev_list, dev)) {
	if (dev->l2ad_vdev == vd)
	break;
	}
	mutex_exit(&l2arc_dev_mtx);

	return (dev);
	}

	static void
	l2arc_rebuild_dev(l2arc_dev_t *dev, boolean_t reopen)
	{
	l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
	uint64_t l2dhdr_asize = dev->l2ad_dev_hdr_asize;
	spa_t *spa = dev->l2ad_spa;

	/*
	* The L2ARC has to hold at least the payload of one log block for
	* them to be restored (persistent L2ARC). The payload of a log block
	* depends on the amount of its log entries. We always write log blocks
	* with 1022 entries. How many of them are committed or restored depends
	* on the size of the L2ARC device. Thus the maximum payload of
	* one log block is 1022 * SPA_MAXBLOCKSIZE = 16GB. If the L2ARC device
	* is less than that, we reduce the amount of committed and restored
	* log entries per block so as to enable persistence.
	*/
	if (dev->l2ad_end < l2arc_rebuild_blocks_min_l2size) {
	dev->l2ad_log_entries = 0;
	} else {
	dev->l2ad_log_entries = MIN((dev->l2ad_end -
	dev->l2ad_start) >> SPA_MAXBLOCKSHIFT,
	L2ARC_LOG_BLK_MAX_ENTRIES);
	}

	/*
	* Read the device header, if an error is returned do not rebuild L2ARC.
	*/
	if (l2arc_dev_hdr_read(dev) == 0 && dev->l2ad_log_entries > 0) {
	/*
	* If we are onlining a cache device (vdev_reopen) that was
	* still present (l2arc_vdev_present()) and rebuild is enabled,
	* we should evict all ARC buffers and pointers to log blocks
	* and reclaim their space before restoring its contents to
	* L2ARC.
	*/
	if (reopen) {
	if (!l2arc_rebuild_enabled) {
	return;
	} else {
	l2arc_evict(dev, 0, B_TRUE);
	/* start a new log block */
	dev->l2ad_log_ent_idx = 0;
	dev->l2ad_log_blk_payload_asize = 0;
	dev->l2ad_log_blk_payload_start = 0;
	}
	}
	/*
	* Just mark the device as pending for a rebuild. We won't
	* be starting a rebuild in line here as it would block pool
	* import. Instead spa_load_impl will hand that off to an
	* async task which will call l2arc_spa_rebuild_start.
	*/
	dev->l2ad_rebuild = B_TRUE;
	} else if (spa_writeable(spa)) {
	/*
	* In this case TRIM the whole device if l2arc_trim_ahead > 0,
	* otherwise create a new header. We zero out the memory holding
	* the header to reset dh_start_lbps. If we TRIM the whole
	* device the new header will be written by
	* vdev_trim_l2arc_thread() at the end of the TRIM to update the
	* trim_state in the header too. When reading the header, if
	* trim_state is not VDEV_TRIM_COMPLETE and l2arc_trim_ahead > 0
	* we opt to TRIM the whole device again.
	*/
	if (l2arc_trim_ahead > 0) {
	dev->l2ad_trim_all = B_TRUE;
	} else {
	memset(l2dhdr, 0, l2dhdr_asize);
	l2arc_dev_hdr_update(dev);
	}
	}
	}

	/*
	* Add a vdev for use by the L2ARC. By this point the spa has already
	* validated the vdev and opened it.
	*/
	void
	l2arc_add_vdev(spa_t spa, vdev_t vd)
	{
	l2arc_dev_t *adddev;
	uint64_t l2dhdr_asize;

	ASSERT(!l2arc_vdev_present(vd));

	/*
	* Create a new l2arc device entry.
	*/
	adddev = vmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP);
	adddev->l2ad_spa = spa;
	adddev->l2ad_vdev = vd;
	/* leave extra size for an l2arc device header */
	l2dhdr_asize = adddev->l2ad_dev_hdr_asize =
	MAX(sizeof (*adddev->l2ad_dev_hdr), 1 << vd->vdev_ashift);
	adddev->l2ad_start = VDEV_LABEL_START_SIZE + l2dhdr_asize;
	adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
	ASSERT3U(adddev->l2ad_start, <, adddev->l2ad_end);
	adddev->l2ad_hand = adddev->l2ad_start;
	adddev->l2ad_evict = adddev->l2ad_start;
	adddev->l2ad_first = B_TRUE;
	adddev->l2ad_writing = B_FALSE;
	adddev->l2ad_trim_all = B_FALSE;
	list_link_init(&adddev->l2ad_node);
	adddev->l2ad_dev_hdr = kmem_zalloc(l2dhdr_asize, KM_SLEEP);

	mutex_init(&adddev->l2ad_mtx, NULL, MUTEX_DEFAULT, NULL);
	/*
	* This is a list of all ARC buffers that are still valid on the
	* device.
	*/
	list_create(&adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
	offsetof(arc_buf_hdr_t, b_l2hdr.b_l2node));

	/*
	* This is a list of pointers to log blocks that are still present
	* on the device.
	*/
	list_create(&adddev->l2ad_lbptr_list, sizeof (l2arc_lb_ptr_buf_t),
	offsetof(l2arc_lb_ptr_buf_t, node));

	vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
	zfs_refcount_create(&adddev->l2ad_alloc);
	zfs_refcount_create(&adddev->l2ad_lb_asize);
	zfs_refcount_create(&adddev->l2ad_lb_count);

	/*
	* Decide if dev is eligible for L2ARC rebuild or whole device
	* trimming. This has to happen before the device is added in the
	* cache device list and l2arc_dev_mtx is released. Otherwise
	* l2arc_feed_thread() might already start writing on the
	* device.
	*/
	l2arc_rebuild_dev(adddev, B_FALSE);

	/*
	* Add device to global list
	*/
	mutex_enter(&l2arc_dev_mtx);
	list_insert_head(l2arc_dev_list, adddev);
	atomic_inc_64(&l2arc_ndev);
	mutex_exit(&l2arc_dev_mtx);
	}

	/*
	* Decide if a vdev is eligible for L2ARC rebuild, called from vdev_reopen()
	* in case of onlining a cache device.
	*/
	void
	l2arc_rebuild_vdev(vdev_t *vd, boolean_t reopen)
	{
	l2arc_dev_t *dev = NULL;

	dev = l2arc_vdev_get(vd);
	ASSERT3P(dev, !=, NULL);

	/*
	* In contrast to l2arc_add_vdev() we do not have to worry about
	* l2arc_feed_thread() invalidating previous content when onlining a
	* cache device. The device parameters (l2ad*) are not cleared when
	* offlining the device and writing new buffers will not invalidate
	* all previous content. In worst case only buffers that have not had
	* their log block written to the device will be lost.
	* When onlining the cache device (ie offline->online without exporting
	* the pool in between) this happens:
	* vdev_reopen() -> vdev_open() -> l2arc_rebuild_vdev()
	* \| \|
	* vdev_is_dead() = B_FALSE l2ad_rebuild = B_TRUE
	* During the time where vdev_is_dead = B_FALSE and until l2ad_rebuild
	* is set to B_TRUE we might write additional buffers to the device.
	*/
	l2arc_rebuild_dev(dev, reopen);
	}

	/*
	* Remove a vdev from the L2ARC.
	*/
	void
	l2arc_remove_vdev(vdev_t *vd)
	{
	l2arc_dev_t *remdev = NULL;

	/*
	* Find the device by vdev
	*/
	remdev = l2arc_vdev_get(vd);
	ASSERT3P(remdev, !=, NULL);

	/*
	* Cancel any ongoing or scheduled rebuild.
	*/
	mutex_enter(&l2arc_rebuild_thr_lock);
	if (remdev->l2ad_rebuild_began == B_TRUE) {
	remdev->l2ad_rebuild_cancel = B_TRUE;
	while (remdev->l2ad_rebuild == B_TRUE)
	cv_wait(&l2arc_rebuild_thr_cv, &l2arc_rebuild_thr_lock);
	}
	mutex_exit(&l2arc_rebuild_thr_lock);

	/*
	* Remove device from global list
	*/
	mutex_enter(&l2arc_dev_mtx);
	list_remove(l2arc_dev_list, remdev);
	l2arc_dev_last = NULL; /* may have been invalidated */
	atomic_dec_64(&l2arc_ndev);
	mutex_exit(&l2arc_dev_mtx);

	/*
	* Clear all buflists and ARC references. L2ARC device flush.
	*/
	l2arc_evict(remdev, 0, B_TRUE);
	list_destroy(&remdev->l2ad_buflist);
	ASSERT(list_is_empty(&remdev->l2ad_lbptr_list));
	list_destroy(&remdev->l2ad_lbptr_list);
	mutex_destroy(&remdev->l2ad_mtx);
	zfs_refcount_destroy(&remdev->l2ad_alloc);
	zfs_refcount_destroy(&remdev->l2ad_lb_asize);
	zfs_refcount_destroy(&remdev->l2ad_lb_count);
	kmem_free(remdev->l2ad_dev_hdr, remdev->l2ad_dev_hdr_asize);
	vmem_free(remdev, sizeof (l2arc_dev_t));
	}

	void
	l2arc_init(void)
	{
	l2arc_thread_exit = 0;
	l2arc_ndev = 0;

	mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
	cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
	mutex_init(&l2arc_rebuild_thr_lock, NULL, MUTEX_DEFAULT, NULL);
	cv_init(&l2arc_rebuild_thr_cv, NULL, CV_DEFAULT, NULL);
	mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);

	l2arc_dev_list = &L2ARC_dev_list;
	l2arc_free_on_write = &L2ARC_free_on_write;
	list_create(l2arc_dev_list, sizeof (l2arc_dev_t),
	offsetof(l2arc_dev_t, l2ad_node));
	list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t),
	offsetof(l2arc_data_free_t, l2df_list_node));
	}

	void
	l2arc_fini(void)
	{
	mutex_destroy(&l2arc_feed_thr_lock);
	cv_destroy(&l2arc_feed_thr_cv);
	mutex_destroy(&l2arc_rebuild_thr_lock);
	cv_destroy(&l2arc_rebuild_thr_cv);
	mutex_destroy(&l2arc_dev_mtx);
	mutex_destroy(&l2arc_free_on_write_mtx);

	list_destroy(l2arc_dev_list);
	list_destroy(l2arc_free_on_write);
	}

	void
	l2arc_start(void)
	{
	if (!(spa_mode_global & SPA_MODE_WRITE))
	return;

	(void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
	TS_RUN, defclsyspri);
	}

	void
	l2arc_stop(void)
	{
	if (!(spa_mode_global & SPA_MODE_WRITE))
	return;

	mutex_enter(&l2arc_feed_thr_lock);
	cv_signal(&l2arc_feed_thr_cv); /* kick thread out of startup */
	l2arc_thread_exit = 1;
	while (l2arc_thread_exit != 0)
	cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
	mutex_exit(&l2arc_feed_thr_lock);
	}

	/*
	* Punches out rebuild threads for the L2ARC devices in a spa. This should
	* be called after pool import from the spa async thread, since starting
	* these threads directly from spa_import() will make them part of the
	* "zpool import" context and delay process exit (and thus pool import).
	*/
	void
	l2arc_spa_rebuild_start(spa_t *spa)
	{
	ASSERT(MUTEX_HELD(&spa_namespace_lock));

	/*
	* Locate the spa's l2arc devices and kick off rebuild threads.
	*/
	for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
	l2arc_dev_t *dev =
	l2arc_vdev_get(spa->spa_l2cache.sav_vdevs[i]);
	if (dev == NULL) {
	/* Don't attempt a rebuild if the vdev is UNAVAIL */
	continue;
	}
	mutex_enter(&l2arc_rebuild_thr_lock);
	if (dev->l2ad_rebuild && !dev->l2ad_rebuild_cancel) {
	dev->l2ad_rebuild_began = B_TRUE;
	(void) thread_create(NULL, 0, l2arc_dev_rebuild_thread,
	dev, 0, &p0, TS_RUN, minclsyspri);
	}
	mutex_exit(&l2arc_rebuild_thr_lock);
	}
	}

	/*
	* Main entry point for L2ARC rebuilding.
	*/
	static __attribute__((noreturn)) void
	l2arc_dev_rebuild_thread(void *arg)
	{
	l2arc_dev_t *dev = arg;

	VERIFY(!dev->l2ad_rebuild_cancel);
	VERIFY(dev->l2ad_rebuild);
	(void) l2arc_rebuild(dev);
	mutex_enter(&l2arc_rebuild_thr_lock);
	dev->l2ad_rebuild_began = B_FALSE;
	dev->l2ad_rebuild = B_FALSE;
	mutex_exit(&l2arc_rebuild_thr_lock);

	thread_exit();
	}

	/*
	* This function implements the actual L2ARC metadata rebuild. It:
	* starts reading the log block chain and restores each block's contents
	* to memory (reconstructing arc_buf_hdr_t's).
	*
	* Operation stops under any of the following conditions:
	*
	* 1) We reach the end of the log block chain.
	* 2) We encounter any error condition (cksum errors, io errors)
	*/
	static int
	l2arc_rebuild(l2arc_dev_t *dev)
	{
	vdev_t *vd = dev->l2ad_vdev;
	spa_t *spa = vd->vdev_spa;
	int err = 0;
	l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
	l2arc_log_blk_phys_t this_lb, next_lb;
	zio_t this_io = NULL, next_io = NULL;
	l2arc_log_blkptr_t lbps[2];
	l2arc_lb_ptr_buf_t *lb_ptr_buf;
	boolean_t lock_held;

	this_lb = vmem_zalloc(sizeof (*this_lb), KM_SLEEP);
	next_lb = vmem_zalloc(sizeof (*next_lb), KM_SLEEP);

	/*
	* We prevent device removal while issuing reads to the device,
	* then during the rebuilding phases we drop this lock again so
	* that a spa_unload or device remove can be initiated - this is
	* safe, because the spa will signal us to stop before removing
	* our device and wait for us to stop.
	*/
	spa_config_enter(spa, SCL_L2ARC, vd, RW_READER);
	lock_held = B_TRUE;

	/*
	* Retrieve the persistent L2ARC device state.
	* L2BLK_GET_PSIZE returns aligned size for log blocks.
	*/
	dev->l2ad_evict = MAX(l2dhdr->dh_evict, dev->l2ad_start);
	dev->l2ad_hand = MAX(l2dhdr->dh_start_lbps[0].lbp_daddr +
	L2BLK_GET_PSIZE((&l2dhdr->dh_start_lbps[0])->lbp_prop),
	dev->l2ad_start);
	dev->l2ad_first = !!(l2dhdr->dh_flags & L2ARC_DEV_HDR_EVICT_FIRST);

	vd->vdev_trim_action_time = l2dhdr->dh_trim_action_time;
	vd->vdev_trim_state = l2dhdr->dh_trim_state;

	/*
	* In case the zfs module parameter l2arc_rebuild_enabled is false
	* we do not start the rebuild process.
	*/
	if (!l2arc_rebuild_enabled)
	goto out;

	/* Prepare the rebuild process */
	memcpy(lbps, l2dhdr->dh_start_lbps, sizeof (lbps));

	/* Start the rebuild process */
	for (;;) {
	if (!l2arc_log_blkptr_valid(dev, &lbps[0]))
	break;

	if ((err = l2arc_log_blk_read(dev, &lbps[0], &lbps[1],
	this_lb, next_lb, this_io, &next_io)) != 0)
	goto out;

	/*
	* Our memory pressure valve. If the system is running low
	* on memory, rather than swamping memory with new ARC buf
	* hdrs, we opt not to rebuild the L2ARC. At this point,
	* however, we have already set up our L2ARC dev to chain in
	* new metadata log blocks, so the user may choose to offline/
	* online the L2ARC dev at a later time (or re-import the pool)
	* to reconstruct it (when there's less memory pressure).
	*/
	if (l2arc_hdr_limit_reached()) {
	ARCSTAT_BUMP(arcstat_l2_rebuild_abort_lowmem);
	cmn_err(CE_NOTE, "System running low on memory, "
	"aborting L2ARC rebuild.");
	err = SET_ERROR(ENOMEM);
	goto out;
	}

	spa_config_exit(spa, SCL_L2ARC, vd);
	lock_held = B_FALSE;

	/*
	* Now that we know that the next_lb checks out alright, we
	* can start reconstruction from this log block.
	* L2BLK_GET_PSIZE returns aligned size for log blocks.
	*/
	uint64_t asize = L2BLK_GET_PSIZE((&lbps[0])->lbp_prop);
	l2arc_log_blk_restore(dev, this_lb, asize);

	/*
	* log block restored, include its pointer in the list of
	* pointers to log blocks present in the L2ARC device.
	*/
	lb_ptr_buf = kmem_zalloc(sizeof (l2arc_lb_ptr_buf_t), KM_SLEEP);
	lb_ptr_buf->lb_ptr = kmem_zalloc(sizeof (l2arc_log_blkptr_t),
	KM_SLEEP);
	memcpy(lb_ptr_buf->lb_ptr, &lbps[0],
	sizeof (l2arc_log_blkptr_t));
	mutex_enter(&dev->l2ad_mtx);
	list_insert_tail(&dev->l2ad_lbptr_list, lb_ptr_buf);
	ARCSTAT_INCR(arcstat_l2_log_blk_asize, asize);
	ARCSTAT_BUMP(arcstat_l2_log_blk_count);
	zfs_refcount_add_many(&dev->l2ad_lb_asize, asize, lb_ptr_buf);
	zfs_refcount_add(&dev->l2ad_lb_count, lb_ptr_buf);
	mutex_exit(&dev->l2ad_mtx);
	vdev_space_update(vd, asize, 0, 0);

	/*
	* Protection against loops of log blocks:
	*
	* l2ad_hand l2ad_evict
	* V V
	* l2ad_start \|=======================================\| l2ad_end
	* -----\|\|\|----\|\|\|---\|\|\|----\|\|\|
	* (3) (2) (1) (0)
	* ---\|\|\|---\|\|\|----\|\|\|---\|\|\|
	* (7) (6) (5) (4)
	*
	* In this situation the pointer of log block (4) passes
	* l2arc_log_blkptr_valid() but the log block should not be
	* restored as it is overwritten by the payload of log block
	* (0). Only log blocks (0)-(3) should be restored. We check
	* whether l2ad_evict lies in between the payload starting
	* offset of the next log block (lbps[1].lbp_payload_start)
	* and the payload starting offset of the present log block
	* (lbps[0].lbp_payload_start). If true and this isn't the
	* first pass, we are looping from the beginning and we should
	* stop.
	*/
	if (l2arc_range_check_overlap(lbps[1].lbp_payload_start,
	lbps[0].lbp_payload_start, dev->l2ad_evict) &&
	!dev->l2ad_first)
	goto out;

	kpreempt(KPREEMPT_SYNC);
	for (;;) {
	mutex_enter(&l2arc_rebuild_thr_lock);
	if (dev->l2ad_rebuild_cancel) {
	dev->l2ad_rebuild = B_FALSE;
	cv_signal(&l2arc_rebuild_thr_cv);
	mutex_exit(&l2arc_rebuild_thr_lock);
	err = SET_ERROR(ECANCELED);
	goto out;
	}
	mutex_exit(&l2arc_rebuild_thr_lock);
	if (spa_config_tryenter(spa, SCL_L2ARC, vd,
	RW_READER)) {
	lock_held = B_TRUE;
	break;
	}
	/*
	* L2ARC config lock held by somebody in writer,
	* possibly due to them trying to remove us. They'll
	* likely to want us to shut down, so after a little
	* delay, we check l2ad_rebuild_cancel and retry
	* the lock again.
	*/
	delay(1);
	}

	/*
	* Continue with the next log block.
	*/
	lbps[0] = lbps[1];
	lbps[1] = this_lb->lb_prev_lbp;
	PTR_SWAP(this_lb, next_lb);
	this_io = next_io;
	next_io = NULL;
	}

	if (this_io != NULL)
	l2arc_log_blk_fetch_abort(this_io);
	out:
	if (next_io != NULL)
	l2arc_log_blk_fetch_abort(next_io);
	vmem_free(this_lb, sizeof (*this_lb));
	vmem_free(next_lb, sizeof (*next_lb));

	if (!l2arc_rebuild_enabled) {
	spa_history_log_internal(spa, "L2ARC rebuild", NULL,
	"disabled");
	} else if (err == 0 && zfs_refcount_count(&dev->l2ad_lb_count) > 0) {
	ARCSTAT_BUMP(arcstat_l2_rebuild_success);
	spa_history_log_internal(spa, "L2ARC rebuild", NULL,
	"successful, restored %llu blocks",
	(u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
	} else if (err == 0 && zfs_refcount_count(&dev->l2ad_lb_count) == 0) {
	/*
	* No error but also nothing restored, meaning the lbps array
	* in the device header points to invalid/non-present log
	* blocks. Reset the header.
	*/
	spa_history_log_internal(spa, "L2ARC rebuild", NULL,
	"no valid log blocks");
	memset(l2dhdr, 0, dev->l2ad_dev_hdr_asize);
	l2arc_dev_hdr_update(dev);
	} else if (err == ECANCELED) {
	/*
	* In case the rebuild was canceled do not log to spa history
	* log as the pool may be in the process of being removed.
	*/
	zfs_dbgmsg("L2ARC rebuild aborted, restored %llu blocks",
	(u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
	} else if (err != 0) {
	spa_history_log_internal(spa, "L2ARC rebuild", NULL,
	"aborted, restored %llu blocks",
	(u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
	}

	if (lock_held)
	spa_config_exit(spa, SCL_L2ARC, vd);

	return (err);
	}

	/*
	* Attempts to read the device header on the provided L2ARC device and writes
	* it to `hdr'. On success, this function returns 0, otherwise the appropriate
	* error code is returned.
	*/
	static int
	l2arc_dev_hdr_read(l2arc_dev_t *dev)
	{
	int err;
	uint64_t guid;
	l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
	const uint64_t l2dhdr_asize = dev->l2ad_dev_hdr_asize;
	abd_t *abd;

	guid = spa_guid(dev->l2ad_vdev->vdev_spa);

	abd = abd_get_from_buf(l2dhdr, l2dhdr_asize);

	err = zio_wait(zio_read_phys(NULL, dev->l2ad_vdev,
	VDEV_LABEL_START_SIZE, l2dhdr_asize, abd,
	ZIO_CHECKSUM_LABEL, NULL, NULL, ZIO_PRIORITY_SYNC_READ,
	ZIO_FLAG_CANFAIL \| ZIO_FLAG_DONT_PROPAGATE \| ZIO_FLAG_DONT_RETRY \|
	ZIO_FLAG_SPECULATIVE, B_FALSE));

	abd_free(abd);

	if (err != 0) {
	ARCSTAT_BUMP(arcstat_l2_rebuild_abort_dh_errors);
	zfs_dbgmsg("L2ARC IO error (%d) while reading device header, "
	"vdev guid: %llu", err,
	(u_longlong_t)dev->l2ad_vdev->vdev_guid);
	return (err);
	}

	if (l2dhdr->dh_magic == BSWAP_64(L2ARC_DEV_HDR_MAGIC))
	byteswap_uint64_array(l2dhdr, sizeof (*l2dhdr));

	if (l2dhdr->dh_magic != L2ARC_DEV_HDR_MAGIC \|\|
	l2dhdr->dh_spa_guid != guid \|\|
	l2dhdr->dh_vdev_guid != dev->l2ad_vdev->vdev_guid \|\|
	l2dhdr->dh_version != L2ARC_PERSISTENT_VERSION \|\|
	l2dhdr->dh_log_entries != dev->l2ad_log_entries \|\|
	l2dhdr->dh_end != dev->l2ad_end \|\|
	!l2arc_range_check_overlap(dev->l2ad_start, dev->l2ad_end,
	l2dhdr->dh_evict) \|\|
	(l2dhdr->dh_trim_state != VDEV_TRIM_COMPLETE &&
	l2arc_trim_ahead > 0)) {
	/*
	* Attempt to rebuild a device containing no actual dev hdr
	* or containing a header from some other pool or from another
	* version of persistent L2ARC.
	*/
	ARCSTAT_BUMP(arcstat_l2_rebuild_abort_unsupported);
	return (SET_ERROR(ENOTSUP));
	}

	return (0);
	}

	/*
	* Reads L2ARC log blocks from storage and validates their contents.
	*
	* This function implements a simple fetcher to make sure that while
	* we're processing one buffer the L2ARC is already fetching the next
	* one in the chain.
	*
	* The arguments this_lp and next_lp point to the current and next log block
	* address in the block chain. Similarly, this_lb and next_lb hold the
	* l2arc_log_blk_phys_t's of the current and next L2ARC blk.
	*
	* The `this_io' and `next_io' arguments are used for block fetching.
	* When issuing the first blk IO during rebuild, you should pass NULL for
	* `this_io'. This function will then issue a sync IO to read the block and
	* also issue an async IO to fetch the next block in the block chain. The
	* fetched IO is returned in `next_io'. On subsequent calls to this
	* function, pass the value returned in `next_io' from the previous call
	* as `this_io' and a fresh `next_io' pointer to hold the next fetch IO.
	* Prior to the call, you should initialize your `next_io' pointer to be
	* NULL. If no fetch IO was issued, the pointer is left set at NULL.
	*
	* On success, this function returns 0, otherwise it returns an appropriate
	* error code. On error the fetching IO is aborted and cleared before
	* returning from this function. Therefore, if we return `success', the
	* caller can assume that we have taken care of cleanup of fetch IOs.
	*/
	static int
	l2arc_log_blk_read(l2arc_dev_t *dev,
	const l2arc_log_blkptr_t this_lbp, const l2arc_log_blkptr_t next_lbp,
	l2arc_log_blk_phys_t this_lb, l2arc_log_blk_phys_t next_lb,
	zio_t this_io, zio_t *next_io)
	{
	int err = 0;
	zio_cksum_t cksum;
	abd_t *abd = NULL;
	uint64_t asize;

	ASSERT(this_lbp != NULL && next_lbp != NULL);
	ASSERT(this_lb != NULL && next_lb != NULL);
	ASSERT(next_io != NULL && *next_io == NULL);
	ASSERT(l2arc_log_blkptr_valid(dev, this_lbp));

	/*
	* Check to see if we have issued the IO for this log block in a
	* previous run. If not, this is the first call, so issue it now.
	*/
	if (this_io == NULL) {
	this_io = l2arc_log_blk_fetch(dev->l2ad_vdev, this_lbp,
	this_lb);
	}

	/*
	* Peek to see if we can start issuing the next IO immediately.
	*/
	if (l2arc_log_blkptr_valid(dev, next_lbp)) {
	/*
	* Start issuing IO for the next log block early - this
	* should help keep the L2ARC device busy while we
	* decompress and restore this log block.
	*/
	*next_io = l2arc_log_blk_fetch(dev->l2ad_vdev, next_lbp,
	next_lb);
	}

	/* Wait for the IO to read this log block to complete */
	if ((err = zio_wait(this_io)) != 0) {
	ARCSTAT_BUMP(arcstat_l2_rebuild_abort_io_errors);
	zfs_dbgmsg("L2ARC IO error (%d) while reading log block, "
	"offset: %llu, vdev guid: %llu", err,
	(u_longlong_t)this_lbp->lbp_daddr,
	(u_longlong_t)dev->l2ad_vdev->vdev_guid);
	goto cleanup;
	}

	/*
	* Make sure the buffer checks out.
	* L2BLK_GET_PSIZE returns aligned size for log blocks.
	*/
	asize = L2BLK_GET_PSIZE((this_lbp)->lbp_prop);
	fletcher_4_native(this_lb, asize, NULL, &cksum);
	if (!ZIO_CHECKSUM_EQUAL(cksum, this_lbp->lbp_cksum)) {
	ARCSTAT_BUMP(arcstat_l2_rebuild_abort_cksum_lb_errors);
	zfs_dbgmsg("L2ARC log block cksum failed, offset: %llu, "
	"vdev guid: %llu, l2ad_hand: %llu, l2ad_evict: %llu",
	(u_longlong_t)this_lbp->lbp_daddr,
	(u_longlong_t)dev->l2ad_vdev->vdev_guid,
	(u_longlong_t)dev->l2ad_hand,
	(u_longlong_t)dev->l2ad_evict);
	err = SET_ERROR(ECKSUM);
	goto cleanup;
	}

	/* Now we can take our time decoding this buffer */
	switch (L2BLK_GET_COMPRESS((this_lbp)->lbp_prop)) {
	case ZIO_COMPRESS_OFF:
	break;
	case ZIO_COMPRESS_LZ4:
	abd = abd_alloc_for_io(asize, B_TRUE);
	abd_copy_from_buf_off(abd, this_lb, 0, asize);
	if ((err = zio_decompress_data(
	L2BLK_GET_COMPRESS((this_lbp)->lbp_prop),
	abd, this_lb, asize, sizeof (*this_lb), NULL)) != 0) {
	err = SET_ERROR(EINVAL);
	goto cleanup;
	}
	break;
	default:
	err = SET_ERROR(EINVAL);
	goto cleanup;
	}
	if (this_lb->lb_magic == BSWAP_64(L2ARC_LOG_BLK_MAGIC))
	byteswap_uint64_array(this_lb, sizeof (*this_lb));
	if (this_lb->lb_magic != L2ARC_LOG_BLK_MAGIC) {
	err = SET_ERROR(EINVAL);
	goto cleanup;
	}
	cleanup:
	/* Abort an in-flight fetch I/O in case of error */
	if (err != 0 && *next_io != NULL) {
	l2arc_log_blk_fetch_abort(*next_io);
	*next_io = NULL;
	}
	if (abd != NULL)
	abd_free(abd);
	return (err);
	}

	/*
	* Restores the payload of a log block to ARC. This creates empty ARC hdr
	* entries which only contain an l2arc hdr, essentially restoring the
	* buffers to their L2ARC evicted state. This function also updates space
	* usage on the L2ARC vdev to make sure it tracks restored buffers.
	*/
	static void
	l2arc_log_blk_restore(l2arc_dev_t dev, const l2arc_log_blk_phys_t lb,
	uint64_t lb_asize)
	{
	uint64_t size = 0, asize = 0;
	uint64_t log_entries = dev->l2ad_log_entries;

	/*
	* Usually arc_adapt() is called only for data, not headers, but
	* since we may allocate significant amount of memory here, let ARC
	* grow its arc_c.
	*/
	arc_adapt(log_entries * HDR_L2ONLY_SIZE);

	for (int i = log_entries - 1; i >= 0; i--) {
	/*
	* Restore goes in the reverse temporal direction to preserve
	* correct temporal ordering of buffers in the l2ad_buflist.
	* l2arc_hdr_restore also does a list_insert_tail instead of
	* list_insert_head on the l2ad_buflist:
	*
	* LIST l2ad_buflist LIST
	* HEAD <------ (time) ------ TAIL
	* direction +-----+-----+-----+-----+-----+ direction
	* of l2arc <== \| buf \| buf \| buf \| buf \| buf \| ===> of rebuild
	* fill +-----+-----+-----+-----+-----+
	* ^ ^
	* \| \|
	* \| \|
	* l2arc_feed_thread l2arc_rebuild
	* will place new bufs here restores bufs here
	*
	* During l2arc_rebuild() the device is not used by
	* l2arc_feed_thread() as dev->l2ad_rebuild is set to true.
	*/
	size += L2BLK_GET_LSIZE((&lb->lb_entries[i])->le_prop);
	asize += vdev_psize_to_asize(dev->l2ad_vdev,
	L2BLK_GET_PSIZE((&lb->lb_entries[i])->le_prop));
	l2arc_hdr_restore(&lb->lb_entries[i], dev);
	}

	/*
	* Record rebuild stats:
	* size Logical size of restored buffers in the L2ARC
	* asize Aligned size of restored buffers in the L2ARC
	*/
	ARCSTAT_INCR(arcstat_l2_rebuild_size, size);
	ARCSTAT_INCR(arcstat_l2_rebuild_asize, asize);
	ARCSTAT_INCR(arcstat_l2_rebuild_bufs, log_entries);
	ARCSTAT_F_AVG(arcstat_l2_log_blk_avg_asize, lb_asize);
	ARCSTAT_F_AVG(arcstat_l2_data_to_meta_ratio, asize / lb_asize);
	ARCSTAT_BUMP(arcstat_l2_rebuild_log_blks);
	}

	/*
	* Restores a single ARC buf hdr from a log entry. The ARC buffer is put
	* into a state indicating that it has been evicted to L2ARC.
	*/
	static void
	l2arc_hdr_restore(const l2arc_log_ent_phys_t le, l2arc_dev_t dev)
	{
	arc_buf_hdr_t hdr, exists;
	kmutex_t *hash_lock;
	arc_buf_contents_t type = L2BLK_GET_TYPE((le)->le_prop);
	uint64_t asize;

	/*
	* Do all the allocation before grabbing any locks, this lets us
	* sleep if memory is full and we don't have to deal with failed
	* allocations.
	*/
	hdr = arc_buf_alloc_l2only(L2BLK_GET_LSIZE((le)->le_prop), type,
	dev, le->le_dva, le->le_daddr,
	L2BLK_GET_PSIZE((le)->le_prop), le->le_birth,
	L2BLK_GET_COMPRESS((le)->le_prop), le->le_complevel,
	L2BLK_GET_PROTECTED((le)->le_prop),
	L2BLK_GET_PREFETCH((le)->le_prop),
	L2BLK_GET_STATE((le)->le_prop));
	asize = vdev_psize_to_asize(dev->l2ad_vdev,
	L2BLK_GET_PSIZE((le)->le_prop));

	/*
	* vdev_space_update() has to be called before arc_hdr_destroy() to
	* avoid underflow since the latter also calls vdev_space_update().
	*/
	l2arc_hdr_arcstats_increment(hdr);
	vdev_space_update(dev->l2ad_vdev, asize, 0, 0);

	mutex_enter(&dev->l2ad_mtx);
	list_insert_tail(&dev->l2ad_buflist, hdr);
	(void) zfs_refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr);
	mutex_exit(&dev->l2ad_mtx);

	exists = buf_hash_insert(hdr, &hash_lock);
	if (exists) {
	/* Buffer was already cached, no need to restore it. */
	arc_hdr_destroy(hdr);
	/*
	* If the buffer is already cached, check whether it has
	* L2ARC metadata. If not, enter them and update the flag.
	* This is important is case of onlining a cache device, since
	* we previously evicted all L2ARC metadata from ARC.
	*/
	if (!HDR_HAS_L2HDR(exists)) {
	arc_hdr_set_flags(exists, ARC_FLAG_HAS_L2HDR);
	exists->b_l2hdr.b_dev = dev;
	exists->b_l2hdr.b_daddr = le->le_daddr;
	exists->b_l2hdr.b_arcs_state =
	L2BLK_GET_STATE((le)->le_prop);
	mutex_enter(&dev->l2ad_mtx);
	list_insert_tail(&dev->l2ad_buflist, exists);
	(void) zfs_refcount_add_many(&dev->l2ad_alloc,
	arc_hdr_size(exists), exists);
	mutex_exit(&dev->l2ad_mtx);
	l2arc_hdr_arcstats_increment(exists);
	vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
	}
	ARCSTAT_BUMP(arcstat_l2_rebuild_bufs_precached);
	}

	mutex_exit(hash_lock);
	}

	/*
	* Starts an asynchronous read IO to read a log block. This is used in log
	* block reconstruction to start reading the next block before we are done
	* decoding and reconstructing the current block, to keep the l2arc device
	* nice and hot with read IO to process.
	* The returned zio will contain a newly allocated memory buffers for the IO
	* data which should then be freed by the caller once the zio is no longer
	* needed (i.e. due to it having completed). If you wish to abort this
	* zio, you should do so using l2arc_log_blk_fetch_abort, which takes
	* care of disposing of the allocated buffers correctly.
	*/
	static zio_t *
	l2arc_log_blk_fetch(vdev_t vd, const l2arc_log_blkptr_t lbp,
	l2arc_log_blk_phys_t *lb)
	{
	uint32_t asize;
	zio_t *pio;
	l2arc_read_callback_t *cb;

	/* L2BLK_GET_PSIZE returns aligned size for log blocks */
	asize = L2BLK_GET_PSIZE((lbp)->lbp_prop);
	ASSERT(asize <= sizeof (l2arc_log_blk_phys_t));

	cb = kmem_zalloc(sizeof (l2arc_read_callback_t), KM_SLEEP);
	cb->l2rcb_abd = abd_get_from_buf(lb, asize);
	pio = zio_root(vd->vdev_spa, l2arc_blk_fetch_done, cb,
	ZIO_FLAG_CANFAIL \| ZIO_FLAG_DONT_PROPAGATE \| ZIO_FLAG_DONT_RETRY);
	(void) zio_nowait(zio_read_phys(pio, vd, lbp->lbp_daddr, asize,
	cb->l2rcb_abd, ZIO_CHECKSUM_OFF, NULL, NULL,
	ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL \|
	ZIO_FLAG_DONT_PROPAGATE \| ZIO_FLAG_DONT_RETRY, B_FALSE));

	return (pio);
	}

	/*
	* Aborts a zio returned from l2arc_log_blk_fetch and frees the data
	* buffers allocated for it.
	*/
	static void
	l2arc_log_blk_fetch_abort(zio_t *zio)
	{
	(void) zio_wait(zio);
	}

	/*
	* Creates a zio to update the device header on an l2arc device.
	*/
	void
	l2arc_dev_hdr_update(l2arc_dev_t *dev)
	{
	l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
	const uint64_t l2dhdr_asize = dev->l2ad_dev_hdr_asize;
	abd_t *abd;
	int err;

	VERIFY(spa_config_held(dev->l2ad_spa, SCL_STATE_ALL, RW_READER));

	l2dhdr->dh_magic = L2ARC_DEV_HDR_MAGIC;
	l2dhdr->dh_version = L2ARC_PERSISTENT_VERSION;
	l2dhdr->dh_spa_guid = spa_guid(dev->l2ad_vdev->vdev_spa);
	l2dhdr->dh_vdev_guid = dev->l2ad_vdev->vdev_guid;
	l2dhdr->dh_log_entries = dev->l2ad_log_entries;
	l2dhdr->dh_evict = dev->l2ad_evict;
	l2dhdr->dh_start = dev->l2ad_start;
	l2dhdr->dh_end = dev->l2ad_end;
	l2dhdr->dh_lb_asize = zfs_refcount_count(&dev->l2ad_lb_asize);
	l2dhdr->dh_lb_count = zfs_refcount_count(&dev->l2ad_lb_count);
	l2dhdr->dh_flags = 0;
	l2dhdr->dh_trim_action_time = dev->l2ad_vdev->vdev_trim_action_time;
	l2dhdr->dh_trim_state = dev->l2ad_vdev->vdev_trim_state;
	if (dev->l2ad_first)
	l2dhdr->dh_flags \|= L2ARC_DEV_HDR_EVICT_FIRST;

	abd = abd_get_from_buf(l2dhdr, l2dhdr_asize);

	err = zio_wait(zio_write_phys(NULL, dev->l2ad_vdev,
	VDEV_LABEL_START_SIZE, l2dhdr_asize, abd, ZIO_CHECKSUM_LABEL, NULL,
	NULL, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE));

	abd_free(abd);

	if (err != 0) {
	zfs_dbgmsg("L2ARC IO error (%d) while writing device header, "
	"vdev guid: %llu", err,
	(u_longlong_t)dev->l2ad_vdev->vdev_guid);
	}
	}

	/*
	* Commits a log block to the L2ARC device. This routine is invoked from
	* l2arc_write_buffers when the log block fills up.
	* This function allocates some memory to temporarily hold the serialized
	* buffer to be written. This is then released in l2arc_write_done.
	*/
	static uint64_t
	l2arc_log_blk_commit(l2arc_dev_t dev, zio_t pio, l2arc_write_callback_t *cb)
	{
	l2arc_log_blk_phys_t *lb = &dev->l2ad_log_blk;
	l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
	uint64_t psize, asize;
	zio_t *wzio;
	l2arc_lb_abd_buf_t *abd_buf;
	uint8_t *tmpbuf = NULL;
	l2arc_lb_ptr_buf_t *lb_ptr_buf;

	VERIFY3S(dev->l2ad_log_ent_idx, ==, dev->l2ad_log_entries);

	abd_buf = zio_buf_alloc(sizeof (*abd_buf));
	abd_buf->abd = abd_get_from_buf(lb, sizeof (*lb));
	lb_ptr_buf = kmem_zalloc(sizeof (l2arc_lb_ptr_buf_t), KM_SLEEP);
	lb_ptr_buf->lb_ptr = kmem_zalloc(sizeof (l2arc_log_blkptr_t), KM_SLEEP);

	/* link the buffer into the block chain */
	lb->lb_prev_lbp = l2dhdr->dh_start_lbps[1];
	lb->lb_magic = L2ARC_LOG_BLK_MAGIC;

	/*
	* l2arc_log_blk_commit() may be called multiple times during a single
	* l2arc_write_buffers() call. Save the allocated abd buffers in a list
	* so we can free them in l2arc_write_done() later on.
	*/
	list_insert_tail(&cb->l2wcb_abd_list, abd_buf);

	/* try to compress the buffer */
	psize = zio_compress_data(ZIO_COMPRESS_LZ4,
	abd_buf->abd, (void *) &tmpbuf, sizeof (lb), 0);

	/* a log block is never entirely zero */
	ASSERT(psize != 0);
	asize = vdev_psize_to_asize(dev->l2ad_vdev, psize);
	ASSERT(asize <= sizeof (*lb));

	/*
	* Update the start log block pointer in the device header to point
	* to the log block we're about to write.
	*/
	l2dhdr->dh_start_lbps[1] = l2dhdr->dh_start_lbps[0];
	l2dhdr->dh_start_lbps[0].lbp_daddr = dev->l2ad_hand;
	l2dhdr->dh_start_lbps[0].lbp_payload_asize =
	dev->l2ad_log_blk_payload_asize;
	l2dhdr->dh_start_lbps[0].lbp_payload_start =
	dev->l2ad_log_blk_payload_start;
	L2BLK_SET_LSIZE(
	(&l2dhdr->dh_start_lbps[0])->lbp_prop, sizeof (*lb));
	L2BLK_SET_PSIZE(
	(&l2dhdr->dh_start_lbps[0])->lbp_prop, asize);
	L2BLK_SET_CHECKSUM(
	(&l2dhdr->dh_start_lbps[0])->lbp_prop,
	ZIO_CHECKSUM_FLETCHER_4);
	if (asize < sizeof (*lb)) {
	/* compression succeeded */
	memset(tmpbuf + psize, 0, asize - psize);
	L2BLK_SET_COMPRESS(
	(&l2dhdr->dh_start_lbps[0])->lbp_prop,
	ZIO_COMPRESS_LZ4);
	} else {
	/* compression failed */
	memcpy(tmpbuf, lb, sizeof (*lb));
	L2BLK_SET_COMPRESS(
	(&l2dhdr->dh_start_lbps[0])->lbp_prop,
	ZIO_COMPRESS_OFF);
	}

	/* checksum what we're about to write */
	fletcher_4_native(tmpbuf, asize, NULL,
	&l2dhdr->dh_start_lbps[0].lbp_cksum);

	abd_free(abd_buf->abd);

	/* perform the write itself */
	abd_buf->abd = abd_get_from_buf(tmpbuf, sizeof (*lb));
	abd_take_ownership_of_buf(abd_buf->abd, B_TRUE);
	wzio = zio_write_phys(pio, dev->l2ad_vdev, dev->l2ad_hand,
	asize, abd_buf->abd, ZIO_CHECKSUM_OFF, NULL, NULL,
	ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE);
	DTRACE_PROBE2(l2arc__write, vdev_t , dev->l2ad_vdev, zio_t , wzio);
	(void) zio_nowait(wzio);

	dev->l2ad_hand += asize;
	/*
	* Include the committed log block's pointer in the list of pointers
	* to log blocks present in the L2ARC device.
	*/
	memcpy(lb_ptr_buf->lb_ptr, &l2dhdr->dh_start_lbps[0],
	sizeof (l2arc_log_blkptr_t));
	mutex_enter(&dev->l2ad_mtx);
	list_insert_head(&dev->l2ad_lbptr_list, lb_ptr_buf);
	ARCSTAT_INCR(arcstat_l2_log_blk_asize, asize);
	ARCSTAT_BUMP(arcstat_l2_log_blk_count);
	zfs_refcount_add_many(&dev->l2ad_lb_asize, asize, lb_ptr_buf);
	zfs_refcount_add(&dev->l2ad_lb_count, lb_ptr_buf);
	mutex_exit(&dev->l2ad_mtx);
	vdev_space_update(dev->l2ad_vdev, asize, 0, 0);

	/* bump the kstats */
	ARCSTAT_INCR(arcstat_l2_write_bytes, asize);
	ARCSTAT_BUMP(arcstat_l2_log_blk_writes);
	ARCSTAT_F_AVG(arcstat_l2_log_blk_avg_asize, asize);
	ARCSTAT_F_AVG(arcstat_l2_data_to_meta_ratio,
	dev->l2ad_log_blk_payload_asize / asize);

	/* start a new log block */
	dev->l2ad_log_ent_idx = 0;
	dev->l2ad_log_blk_payload_asize = 0;
	dev->l2ad_log_blk_payload_start = 0;

	return (asize);
	}

	/*
	* Validates an L2ARC log block address to make sure that it can be read
	* from the provided L2ARC device.
	*/
	boolean_t
	l2arc_log_blkptr_valid(l2arc_dev_t dev, const l2arc_log_blkptr_t lbp)
	{
	/* L2BLK_GET_PSIZE returns aligned size for log blocks */
	uint64_t asize = L2BLK_GET_PSIZE((lbp)->lbp_prop);
	uint64_t end = lbp->lbp_daddr + asize - 1;
	uint64_t start = lbp->lbp_payload_start;
	boolean_t evicted = B_FALSE;

	/*
	* A log block is valid if all of the following conditions are true:
	* - it fits entirely (including its payload) between l2ad_start and
	* l2ad_end
	* - it has a valid size
	* - neither the log block itself nor part of its payload was evicted
	* by l2arc_evict():
	*
	* l2ad_hand l2ad_evict
	* \| \| lbp_daddr
	* \| start \| \| end
	* \| \| \| \| \|
	* V V V V V
	* l2ad_start ============================================ l2ad_end
	* --------------------------\|\|\|\|
	* ^ ^
	* \| log block
	* payload
	*/

	evicted =
	l2arc_range_check_overlap(start, end, dev->l2ad_hand) \|\|
	l2arc_range_check_overlap(start, end, dev->l2ad_evict) \|\|
	l2arc_range_check_overlap(dev->l2ad_hand, dev->l2ad_evict, start) \|\|
	l2arc_range_check_overlap(dev->l2ad_hand, dev->l2ad_evict, end);

	return (start >= dev->l2ad_start && end <= dev->l2ad_end &&
	asize > 0 && asize <= sizeof (l2arc_log_blk_phys_t) &&
	(!evicted \|\| dev->l2ad_first));
	}

	/*
	* Inserts ARC buffer header `hdr' into the current L2ARC log block on
	* the device. The buffer being inserted must be present in L2ARC.
	* Returns B_TRUE if the L2ARC log block is full and needs to be committed
	* to L2ARC, or B_FALSE if it still has room for more ARC buffers.
	*/
	static boolean_t
	l2arc_log_blk_insert(l2arc_dev_t dev, const arc_buf_hdr_t hdr)
	{
	l2arc_log_blk_phys_t *lb = &dev->l2ad_log_blk;
	l2arc_log_ent_phys_t *le;

	if (dev->l2ad_log_entries == 0)
	return (B_FALSE);

	int index = dev->l2ad_log_ent_idx++;

	ASSERT3S(index, <, dev->l2ad_log_entries);
	ASSERT(HDR_HAS_L2HDR(hdr));

	le = &lb->lb_entries[index];
	memset(le, 0, sizeof (*le));
	le->le_dva = hdr->b_dva;
	le->le_birth = hdr->b_birth;
	le->le_daddr = hdr->b_l2hdr.b_daddr;
	if (index == 0)
	dev->l2ad_log_blk_payload_start = le->le_daddr;
	L2BLK_SET_LSIZE((le)->le_prop, HDR_GET_LSIZE(hdr));
	L2BLK_SET_PSIZE((le)->le_prop, HDR_GET_PSIZE(hdr));
	L2BLK_SET_COMPRESS((le)->le_prop, HDR_GET_COMPRESS(hdr));
	le->le_complevel = hdr->b_complevel;
	L2BLK_SET_TYPE((le)->le_prop, hdr->b_type);
	L2BLK_SET_PROTECTED((le)->le_prop, !!(HDR_PROTECTED(hdr)));
	L2BLK_SET_PREFETCH((le)->le_prop, !!(HDR_PREFETCH(hdr)));
	- L2BLK_SET_STATE((le)->le_prop, hdr->b_l1hdr.b_state->arcs_state);
	+ L2BLK_SET_STATE((le)->le_prop, hdr->b_l2hdr.b_arcs_state);

	dev->l2ad_log_blk_payload_asize += vdev_psize_to_asize(dev->l2ad_vdev,
	HDR_GET_PSIZE(hdr));

	return (dev->l2ad_log_ent_idx == dev->l2ad_log_entries);
	}

	/*
	* Checks whether a given L2ARC device address sits in a time-sequential
	* range. The trick here is that the L2ARC is a rotary buffer, so we can't
	* just do a range comparison, we need to handle the situation in which the
	* range wraps around the end of the L2ARC device. Arguments:
	* bottom -- Lower end of the range to check (written to earlier).
	* top -- Upper end of the range to check (written to later).
	* check -- The address for which we want to determine if it sits in
	* between the top and bottom.
	*
	* The 3-way conditional below represents the following cases:
	*
	* bottom < top : Sequentially ordered case:
	* <check>--------+-------------------+
	* \| (overlap here?) \|
	* L2ARC dev V V
	* \|---------------<bottom>============<top>--------------\|
	*
	* bottom > top: Looped-around case:
	* <check>--------+------------------+
	* \| (overlap here?) \|
	* L2ARC dev V V
	* \|===============<top>---------------<bottom>===========\|
	* ^ ^
	* \| (or here?) \|
	* +---------------+---------<check>
	*
	* top == bottom : Just a single address comparison.
	*/
	boolean_t
	l2arc_range_check_overlap(uint64_t bottom, uint64_t top, uint64_t check)
	{
	if (bottom < top)
	return (bottom <= check && check <= top);
	else if (bottom > top)
	return (check <= top \|\| bottom <= check);
	else
	return (check == top);
	}

	EXPORT_SYMBOL(arc_buf_size);
	EXPORT_SYMBOL(arc_write);
	EXPORT_SYMBOL(arc_read);
	EXPORT_SYMBOL(arc_buf_info);
	EXPORT_SYMBOL(arc_getbuf_func);
	EXPORT_SYMBOL(arc_add_prune_callback);
	EXPORT_SYMBOL(arc_remove_prune_callback);

	ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min, param_set_arc_min,
	spl_param_get_u64, ZMOD_RW, "Minimum ARC size in bytes");

	ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, max, param_set_arc_max,
	spl_param_get_u64, ZMOD_RW, "Maximum ARC size in bytes");

	ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, meta_balance, UINT, ZMOD_RW,
	"Balance between metadata and data on ghost hits.");

	ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, grow_retry, param_set_arc_int,
	param_get_uint, ZMOD_RW, "Seconds before growing ARC size");

	ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, shrink_shift, param_set_arc_int,
	param_get_uint, ZMOD_RW, "log2(fraction of ARC to reclaim)");

	ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, pc_percent, UINT, ZMOD_RW,
	"Percent of pagecache to reclaim ARC to");

	ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, average_blocksize, UINT, ZMOD_RD,
	"Target average block size");

	ZFS_MODULE_PARAM(zfs, zfs_, compressed_arc_enabled, INT, ZMOD_RW,
	"Disable compressed ARC buffers");

	ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min_prefetch_ms, param_set_arc_int,
	param_get_uint, ZMOD_RW, "Min life of prefetch block in ms");

	ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min_prescient_prefetch_ms,
	param_set_arc_int, param_get_uint, ZMOD_RW,
	"Min life of prescient prefetched block in ms");

	ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, write_max, U64, ZMOD_RW,
	"Max write bytes per interval");

	ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, write_boost, U64, ZMOD_RW,
	"Extra write bytes during device warmup");

	ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, headroom, U64, ZMOD_RW,
	"Number of max device writes to precache");

	ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, headroom_boost, U64, ZMOD_RW,
	"Compressed l2arc_headroom multiplier");

	ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, trim_ahead, U64, ZMOD_RW,
	"TRIM ahead L2ARC write size multiplier");

	ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, feed_secs, U64, ZMOD_RW,
	"Seconds between L2ARC writing");

	ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, feed_min_ms, U64, ZMOD_RW,
	"Min feed interval in milliseconds");

	ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, noprefetch, INT, ZMOD_RW,
	"Skip caching prefetched buffers");

	ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, feed_again, INT, ZMOD_RW,
	"Turbo L2ARC warmup");

	ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, norw, INT, ZMOD_RW,
	"No reads during writes");

	ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, meta_percent, UINT, ZMOD_RW,
	"Percent of ARC size allowed for L2ARC-only headers");

	ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_enabled, INT, ZMOD_RW,
	"Rebuild the L2ARC when importing a pool");

	ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_blocks_min_l2size, U64, ZMOD_RW,
	"Min size in bytes to write rebuild log blocks in L2ARC");

	ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, mfuonly, INT, ZMOD_RW,
	"Cache only MFU data from ARC into L2ARC");

	ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, exclude_special, INT, ZMOD_RW,
	"Exclude dbufs on special vdevs from being cached to L2ARC if set.");

	ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, lotsfree_percent, param_set_arc_int,
	param_get_uint, ZMOD_RW, "System free memory I/O throttle in bytes");

	ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, sys_free, param_set_arc_u64,
	spl_param_get_u64, ZMOD_RW, "System free memory target size in bytes");

	ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, dnode_limit, param_set_arc_u64,
	spl_param_get_u64, ZMOD_RW, "Minimum bytes of dnodes in ARC");

	ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, dnode_limit_percent,
	param_set_arc_int, param_get_uint, ZMOD_RW,
	"Percent of ARC meta buffers for dnodes");

	ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, dnode_reduce_percent, UINT, ZMOD_RW,
	"Percentage of excess dnodes to try to unpin");

	ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, eviction_pct, UINT, ZMOD_RW,
	"When full, ARC allocation waits for eviction of this % of alloc size");

	ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, evict_batch_limit, UINT, ZMOD_RW,
	"The number of headers to evict per sublist before moving to the next");

	ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, prune_task_threads, INT, ZMOD_RW,
	"Number of arc_prune threads");
	diff --git a/sys/contrib/openzfs/module/zfs/brt.c b/sys/contrib/openzfs/module/zfs/brt.c
	index 225ddaca1e54..bf8fdf6ea4b4 100644
	--- a/sys/contrib/openzfs/module/zfs/brt.c
	+++ b/sys/contrib/openzfs/module/zfs/brt.c
	@@ -1,1737 +1,1674 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2020, 2021, 2022 by Pawel Jakub Dawidek
	*/

	#include <sys/zfs_context.h>
	#include <sys/spa.h>
	#include <sys/spa_impl.h>
	#include <sys/zio.h>
	#include <sys/brt.h>
	#include <sys/brt_impl.h>
	#include <sys/ddt.h>
	#include <sys/bitmap.h>
	#include <sys/zap.h>
	#include <sys/dmu_tx.h>
	#include <sys/arc.h>
	#include <sys/dsl_pool.h>
	#include <sys/dsl_scan.h>
	#include <sys/vdev_impl.h>
	#include <sys/kstat.h>
	#include <sys/wmsum.h>

	/*
	* Block Cloning design.
	*
	* Block Cloning allows to manually clone a file (or a subset of its blocks)
	* into another (or the same) file by just creating additional references to
	* the data blocks without copying the data itself. Those references are kept
	* in the Block Reference Tables (BRTs).
	*
	* In many ways this is similar to the existing deduplication, but there are
	* some important differences:
	*
	* - Deduplication is automatic and Block Cloning is not - one has to use a
	* dedicated system call(s) to clone the given file/blocks.
	* - Deduplication keeps all data blocks in its table, even those referenced
	* just once. Block Cloning creates an entry in its tables only when there
	* are at least two references to the given data block. If the block was
	* never explicitly cloned or the second to last reference was dropped,
	* there will be neither space nor performance overhead.
	* - Deduplication needs data to work - one needs to pass real data to the
	* write(2) syscall, so hash can be calculated. Block Cloning doesn't require
	* data, just block pointers to the data, so it is extremely fast, as we pay
	* neither the cost of reading the data, nor the cost of writing the data -
	* we operate exclusively on metadata.
	* - If the D (dedup) bit is not set in the block pointer, it means that
	* the block is not in the dedup table (DDT) and we won't consult the DDT
	* when we need to free the block. Block Cloning must be consulted on every
	* free, because we cannot modify the source BP (eg. by setting something
	* similar to the D bit), thus we have no hint if the block is in the
	* Block Reference Table (BRT), so we need to look into the BRT. There is
	* an optimization in place that allows us to eliminate the majority of BRT
	* lookups which is described below in the "Minimizing free penalty" section.
	* - The BRT entry is much smaller than the DDT entry - for BRT we only store
	* 64bit offset and 64bit reference counter.
	* - Dedup keys are cryptographic hashes, so two blocks that are close to each
	* other on disk are most likely in totally different parts of the DDT.
	* The BRT entry keys are offsets into a single top-level VDEV, so data blocks
	* from one file should have BRT entries close to each other.
	* - Scrub will only do a single pass over a block that is referenced multiple
	* times in the DDT. Unfortunately it is not currently (if at all) possible
	* with Block Cloning and block referenced multiple times will be scrubbed
	* multiple times. The new, sorted scrub should be able to eliminate
	* duplicated reads given enough memory.
	* - Deduplication requires cryptographically strong hash as a checksum or
	* additional data verification. Block Cloning works with any checksum
	* algorithm or even with checksumming disabled.
	*
	* As mentioned above, the BRT entries are much smaller than the DDT entries.
	* To uniquely identify a block we just need its vdev id and offset. We also
	* need to maintain a reference counter. The vdev id will often repeat, as there
	* is a small number of top-level VDEVs and a large number of blocks stored in
	* each VDEV. We take advantage of that to reduce the BRT entry size further by
	* maintaining one BRT for each top-level VDEV, so we can then have only offset
	* and counter as the BRT entry.
	*
	* Minimizing free penalty.
	*
	* Block Cloning allows creating additional references to any existing block.
	* When we free a block there is no hint in the block pointer whether the block
	* was cloned or not, so on each free we have to check if there is a
	* corresponding entry in the BRT or not. If there is, we need to decrease
	* the reference counter. Doing BRT lookup on every free can potentially be
	* expensive by requiring additional I/Os if the BRT doesn't fit into memory.
	* This is the main problem with deduplication, so we've learned our lesson and
	* try not to repeat the same mistake here. How do we do that? We divide each
	* top-level VDEV into 16MB regions. For each region we maintain a counter that
	* is a sum of all the BRT entries that have offsets within the region. This
	* creates the entries count array of 16bit numbers for each top-level VDEV.
	* The entries count array is always kept in memory and updated on disk in the
	* same transaction group as the BRT updates to keep everything in-sync. We can
	* keep the array in memory, because it is very small. With 16MB regions and
	* 1TB VDEV the array requires only 128kB of memory (we may decide to decrease
	* the region size even further in the future). Now, when we want to free
	* a block, we first consult the array. If the counter for the whole region is
	* zero, there is no need to look for the BRT entry, as there isn't one for
	* sure. If the counter for the region is greater than zero, only then we will
	* do a BRT lookup and if an entry is found we will decrease the reference
	* counter in the BRT entry and in the entry counters array.
	*
	* The entry counters array is small, but can potentially be larger for very
	* large VDEVs or smaller regions. In this case we don't want to rewrite entire
	* array on every change. We then divide the array into 32kB block and keep
	* a bitmap of dirty blocks within a transaction group. When we sync the
	* transaction group we can only update the parts of the entry counters array
	* that were modified. Note: Keeping track of the dirty parts of the entry
	* counters array is implemented, but updating only parts of the array on disk
	* is not yet implemented - for now we will update entire array if there was
	* any change.
	*
	* The implementation tries to be economic: if BRT is not used, or no longer
	* used, there will be no entries in the MOS and no additional memory used (eg.
	* the entry counters array is only allocated if needed).
	*
	* Interaction between Deduplication and Block Cloning.
	*
	* If both functionalities are in use, we could end up with a block that is
	* referenced multiple times in both DDT and BRT. When we free one of the
	* references we couldn't tell where it belongs, so we would have to decide
	* what table takes the precedence: do we first clear DDT references or BRT
	* references? To avoid this dilemma BRT cooperates with DDT - if a given block
	* is being cloned using BRT and the BP has the D (dedup) bit set, BRT will
	* lookup DDT entry instead and increase the counter there. No BRT entry
	* will be created for a block which has the D (dedup) bit set.
	* BRT may be more efficient for manual deduplication, but if the block is
	* already in the DDT, then creating additional BRT entry would be less
	* efficient. This clever idea was proposed by Allan Jude.
	*
	* Block Cloning across datasets.
	*
	* Block Cloning is not limited to cloning blocks within the same dataset.
	* It is possible (and very useful) to clone blocks between different datasets.
	* One use case is recovering files from snapshots. By cloning the files into
	* dataset we need no additional storage. Without Block Cloning we would need
	* additional space for those files.
	* Another interesting use case is moving the files between datasets
	* (copying the file content to the new dataset and removing the source file).
	* In that case Block Cloning will only be used briefly, because the BRT entries
	* will be removed when the source is removed.
	* Block Cloning across encrypted datasets is supported as long as both
	* datasets share the same master key (e.g. snapshots and clones)
	*
	* Block Cloning flow through ZFS layers.
	*
	* Note: Block Cloning can be used both for cloning file system blocks and ZVOL
	* blocks. As of this writing no interface is implemented that allows for block
	* cloning within a ZVOL.
	* FreeBSD and Linux provides copy_file_range(2) system call and we will use it
	* for blocking cloning.
	*
	* ssize_t
	* copy_file_range(int infd, off_t inoffp, int outfd, off_t outoffp,
	* size_t len, unsigned int flags);
	*
	* Even though offsets and length represent bytes, they have to be
	* block-aligned or we will return an error so the upper layer can
	* fallback to the generic mechanism that will just copy the data.
	* Using copy_file_range(2) will call OS-independent zfs_clone_range() function.
	* This function was implemented based on zfs_write(), but instead of writing
	* the given data we first read block pointers using the new dmu_read_l0_bps()
	* function from the source file. Once we have BPs from the source file we call
	* the dmu_brt_clone() function on the destination file. This function
	* allocates BPs for us. We iterate over all source BPs. If the given BP is
	* a hole or an embedded block, we just copy BP as-is. If it points to a real
	* data we place this BP on a BRT pending list using the brt_pending_add()
	* function.
	*
	* We use this pending list to keep track of all BPs that got new references
	* within this transaction group.
	*
	* Some special cases to consider and how we address them:
	* - The block we want to clone may have been created within the same
	* transaction group that we are trying to clone. Such block has no BP
	* allocated yet, so cannot be immediately cloned. We return EAGAIN.
	* - The block we want to clone may have been modified within the same
	* transaction group. We return EAGAIN.
	* - A block may be cloned multiple times during one transaction group (that's
	* why pending list is actually a tree and not an append-only list - this
	* way we can figure out faster if this block is cloned for the first time
	* in this txg or consecutive time).
	* - A block may be cloned and freed within the same transaction group
	* (see dbuf_undirty()).
	* - A block may be cloned and within the same transaction group the clone
	* can be cloned again (see dmu_read_l0_bps()).
	* - A file might have been deleted, but the caller still has a file descriptor
	* open to this file and clones it.
	*
	* When we free a block we have an additional step in the ZIO pipeline where we
	* call the zio_brt_free() function. We then call the brt_entry_decref()
	* that loads the corresponding BRT entry (if one exists) and decreases
	* reference counter. If this is not the last reference we will stop ZIO
	* pipeline here. If this is the last reference or the block is not in the
	* BRT, we continue the pipeline and free the block as usual.
	*
	* At the beginning of spa_sync() where there can be no more block cloning,
	* but before issuing frees we call brt_pending_apply(). This function applies
	* all the new clones to the BRT table - we load BRT entries and update
	* reference counters. To sync new BRT entries to disk, we use brt_sync()
	* function. This function will sync all dirty per-top-level-vdev BRTs,
	* the entry counters arrays, etc.
	*
	* Block Cloning and ZIL.
	*
	* Every clone operation is divided into chunks (similar to write) and each
	* chunk is cloned in a separate transaction. The chunk size is determined by
	* how many BPs we can fit into a single ZIL entry.
	* Replaying clone operation is different from the regular clone operation,
	* as when we log clone operations we cannot use the source object - it may
	* reside on a different dataset, so we log BPs we want to clone.
	* The ZIL is replayed when we mount the given dataset, not when the pool is
	* imported. Taking this into account it is possible that the pool is imported
	* without mounting datasets and the source dataset is destroyed before the
	* destination dataset is mounted and its ZIL replayed.
	* To address this situation we leverage zil_claim() mechanism where ZFS will
	* parse all the ZILs on pool import. When we come across TX_CLONE_RANGE
	* entries, we will bump reference counters for their BPs in the BRT. Then
	* on mount and ZIL replay we bump the reference counters once more, while the
	* first references are dropped during ZIL destroy by zil_free_clone_range().
	* It is possible that after zil_claim() we never mount the destination, so
	* we never replay its ZIL and just destroy it. In this case the only taken
	* references will be dropped by zil_free_clone_range(), since the cloning is
	* not going to ever take place.
	*/

	static kmem_cache_t *brt_entry_cache;
	static kmem_cache_t *brt_pending_entry_cache;

	/*
	* Enable/disable prefetching of BRT entries that we are going to modify.
	*/
	-int zfs_brt_prefetch = 1;
	+static int brt_zap_prefetch = 1;

	#ifdef ZFS_DEBUG
	#define BRT_DEBUG(...) do { \
	if ((zfs_flags & ZFS_DEBUG_BRT) != 0) { \
	__dprintf(B_TRUE, __FILE__, __func__, __LINE__, __VA_ARGS__); \
	} \
	} while (0)
	#else
	#define BRT_DEBUG(...) do { } while (0)
	#endif

	-int brt_zap_leaf_blockshift = 12;
	-int brt_zap_indirect_blockshift = 12;
	+static int brt_zap_default_bs = 12;
	+static int brt_zap_default_ibs = 12;

	static kstat_t *brt_ksp;

	typedef struct brt_stats {
	kstat_named_t brt_addref_entry_in_memory;
	kstat_named_t brt_addref_entry_not_on_disk;
	kstat_named_t brt_addref_entry_on_disk;
	kstat_named_t brt_addref_entry_read_lost_race;
	kstat_named_t brt_decref_entry_in_memory;
	kstat_named_t brt_decref_entry_loaded_from_disk;
	kstat_named_t brt_decref_entry_not_in_memory;
	kstat_named_t brt_decref_entry_not_on_disk;
	kstat_named_t brt_decref_entry_read_lost_race;
	kstat_named_t brt_decref_entry_still_referenced;
	kstat_named_t brt_decref_free_data_later;
	kstat_named_t brt_decref_free_data_now;
	kstat_named_t brt_decref_no_entry;
	} brt_stats_t;

	static brt_stats_t brt_stats = {
	{ "addref_entry_in_memory", KSTAT_DATA_UINT64 },
	{ "addref_entry_not_on_disk", KSTAT_DATA_UINT64 },
	{ "addref_entry_on_disk", KSTAT_DATA_UINT64 },
	{ "addref_entry_read_lost_race", KSTAT_DATA_UINT64 },
	{ "decref_entry_in_memory", KSTAT_DATA_UINT64 },
	{ "decref_entry_loaded_from_disk", KSTAT_DATA_UINT64 },
	{ "decref_entry_not_in_memory", KSTAT_DATA_UINT64 },
	{ "decref_entry_not_on_disk", KSTAT_DATA_UINT64 },
	{ "decref_entry_read_lost_race", KSTAT_DATA_UINT64 },
	{ "decref_entry_still_referenced", KSTAT_DATA_UINT64 },
	{ "decref_free_data_later", KSTAT_DATA_UINT64 },
	{ "decref_free_data_now", KSTAT_DATA_UINT64 },
	{ "decref_no_entry", KSTAT_DATA_UINT64 }
	};

	struct {
	wmsum_t brt_addref_entry_in_memory;
	wmsum_t brt_addref_entry_not_on_disk;
	wmsum_t brt_addref_entry_on_disk;
	wmsum_t brt_addref_entry_read_lost_race;
	wmsum_t brt_decref_entry_in_memory;
	wmsum_t brt_decref_entry_loaded_from_disk;
	wmsum_t brt_decref_entry_not_in_memory;
	wmsum_t brt_decref_entry_not_on_disk;
	wmsum_t brt_decref_entry_read_lost_race;
	wmsum_t brt_decref_entry_still_referenced;
	wmsum_t brt_decref_free_data_later;
	wmsum_t brt_decref_free_data_now;
	wmsum_t brt_decref_no_entry;
	} brt_sums;

	#define BRTSTAT_BUMP(stat) wmsum_add(&brt_sums.stat, 1)

	static int brt_entry_compare(const void x1, const void x2);
	static int brt_pending_entry_compare(const void x1, const void x2);

	static void
	brt_rlock(brt_t *brt)
	{
	rw_enter(&brt->brt_lock, RW_READER);
	}

	static void
	brt_wlock(brt_t *brt)
	{
	rw_enter(&brt->brt_lock, RW_WRITER);
	}

	static void
	brt_unlock(brt_t *brt)
	{
	rw_exit(&brt->brt_lock);
	}

	static uint16_t
	brt_vdev_entcount_get(const brt_vdev_t *brtvd, uint64_t idx)
	{

	ASSERT3U(idx, <, brtvd->bv_size);

	if (unlikely(brtvd->bv_need_byteswap)) {
	return (BSWAP_16(brtvd->bv_entcount[idx]));
	} else {
	return (brtvd->bv_entcount[idx]);
	}
	}

	static void
	brt_vdev_entcount_set(brt_vdev_t *brtvd, uint64_t idx, uint16_t entcnt)
	{

	ASSERT3U(idx, <, brtvd->bv_size);

	if (unlikely(brtvd->bv_need_byteswap)) {
	brtvd->bv_entcount[idx] = BSWAP_16(entcnt);
	} else {
	brtvd->bv_entcount[idx] = entcnt;
	}
	}

	static void
	brt_vdev_entcount_inc(brt_vdev_t *brtvd, uint64_t idx)
	{
	uint16_t entcnt;

	ASSERT3U(idx, <, brtvd->bv_size);

	entcnt = brt_vdev_entcount_get(brtvd, idx);
	ASSERT(entcnt < UINT16_MAX);

	brt_vdev_entcount_set(brtvd, idx, entcnt + 1);
	}

	static void
	brt_vdev_entcount_dec(brt_vdev_t *brtvd, uint64_t idx)
	{
	uint16_t entcnt;

	ASSERT3U(idx, <, brtvd->bv_size);

	entcnt = brt_vdev_entcount_get(brtvd, idx);
	ASSERT(entcnt > 0);

	brt_vdev_entcount_set(brtvd, idx, entcnt - 1);
	}

	#ifdef ZFS_DEBUG
	static void
	brt_vdev_dump(brt_vdev_t *brtvd)
	{
	uint64_t idx;

	zfs_dbgmsg(" BRT vdevid=%llu meta_dirty=%d entcount_dirty=%d "
	"size=%llu totalcount=%llu nblocks=%llu bitmapsize=%zu\n",
	(u_longlong_t)brtvd->bv_vdevid,
	brtvd->bv_meta_dirty, brtvd->bv_entcount_dirty,
	(u_longlong_t)brtvd->bv_size,
	(u_longlong_t)brtvd->bv_totalcount,
	(u_longlong_t)brtvd->bv_nblocks,
	(size_t)BT_SIZEOFMAP(brtvd->bv_nblocks));
	if (brtvd->bv_totalcount > 0) {
	zfs_dbgmsg(" entcounts:");
	for (idx = 0; idx < brtvd->bv_size; idx++) {
	uint16_t entcnt = brt_vdev_entcount_get(brtvd, idx);
	if (entcnt > 0) {
	zfs_dbgmsg(" [%04llu] %hu",
	(u_longlong_t)idx, entcnt);
	}
	}
	}
	if (brtvd->bv_entcount_dirty) {
	char *bitmap;

	bitmap = kmem_alloc(brtvd->bv_nblocks + 1, KM_SLEEP);
	for (idx = 0; idx < brtvd->bv_nblocks; idx++) {
	bitmap[idx] =
	BT_TEST(brtvd->bv_bitmap, idx) ? 'x' : '.';
	}
	bitmap[idx] = '\0';
	zfs_dbgmsg(" dirty: %s", bitmap);
	kmem_free(bitmap, brtvd->bv_nblocks + 1);
	}
	}
	#endif

	static brt_vdev_t *
	brt_vdev(brt_t *brt, uint64_t vdevid)
	{
	brt_vdev_t *brtvd;

	ASSERT(RW_LOCK_HELD(&brt->brt_lock));

	if (vdevid < brt->brt_nvdevs) {
	brtvd = &brt->brt_vdevs[vdevid];
	} else {
	brtvd = NULL;
	}

	return (brtvd);
	}

	static void
	brt_vdev_create(brt_t brt, brt_vdev_t brtvd, dmu_tx_t *tx)
	{
	char name[64];

	ASSERT(RW_WRITE_HELD(&brt->brt_lock));
	ASSERT0(brtvd->bv_mos_brtvdev);
	ASSERT0(brtvd->bv_mos_entries);
	ASSERT(brtvd->bv_entcount != NULL);
	ASSERT(brtvd->bv_size > 0);
	ASSERT(brtvd->bv_bitmap != NULL);
	ASSERT(brtvd->bv_nblocks > 0);

	brtvd->bv_mos_entries = zap_create_flags(brt->brt_mos, 0,
	ZAP_FLAG_HASH64 \| ZAP_FLAG_UINT64_KEY, DMU_OTN_ZAP_METADATA,
	- brt_zap_leaf_blockshift, brt_zap_indirect_blockshift, DMU_OT_NONE,
	- 0, tx);
	+ brt_zap_default_bs, brt_zap_default_ibs, DMU_OT_NONE, 0, tx);
	VERIFY(brtvd->bv_mos_entries != 0);
	BRT_DEBUG("MOS entries created, object=%llu",
	(u_longlong_t)brtvd->bv_mos_entries);

	/*
	* We allocate DMU buffer to store the bv_entcount[] array.
	* We will keep array size (bv_size) and cummulative count for all
	* bv_entcount[]s (bv_totalcount) in the bonus buffer.
	*/
	brtvd->bv_mos_brtvdev = dmu_object_alloc(brt->brt_mos,
	DMU_OTN_UINT64_METADATA, BRT_BLOCKSIZE,
	DMU_OTN_UINT64_METADATA, sizeof (brt_vdev_phys_t), tx);
	VERIFY(brtvd->bv_mos_brtvdev != 0);
	BRT_DEBUG("MOS BRT VDEV created, object=%llu",
	(u_longlong_t)brtvd->bv_mos_brtvdev);

	snprintf(name, sizeof (name), "%s%llu", BRT_OBJECT_VDEV_PREFIX,
	(u_longlong_t)brtvd->bv_vdevid);
	VERIFY0(zap_add(brt->brt_mos, DMU_POOL_DIRECTORY_OBJECT, name,
	sizeof (uint64_t), 1, &brtvd->bv_mos_brtvdev, tx));
	BRT_DEBUG("Pool directory object created, object=%s", name);

	spa_feature_incr(brt->brt_spa, SPA_FEATURE_BLOCK_CLONING, tx);
	}

	static void
	brt_vdev_realloc(brt_t brt, brt_vdev_t brtvd)
	{
	vdev_t *vd;
	uint16_t *entcount;
	ulong_t *bitmap;
	uint64_t nblocks, size;

	ASSERT(RW_WRITE_HELD(&brt->brt_lock));

	spa_config_enter(brt->brt_spa, SCL_VDEV, FTAG, RW_READER);
	vd = vdev_lookup_top(brt->brt_spa, brtvd->bv_vdevid);
	size = (vdev_get_min_asize(vd) - 1) / brt->brt_rangesize + 1;
	spa_config_exit(brt->brt_spa, SCL_VDEV, FTAG);

	entcount = vmem_zalloc(sizeof (entcount[0]) * size, KM_SLEEP);
	nblocks = BRT_RANGESIZE_TO_NBLOCKS(size);
	bitmap = kmem_zalloc(BT_SIZEOFMAP(nblocks), KM_SLEEP);

	if (!brtvd->bv_initiated) {
	ASSERT0(brtvd->bv_size);
	ASSERT(brtvd->bv_entcount == NULL);
	ASSERT(brtvd->bv_bitmap == NULL);
	ASSERT0(brtvd->bv_nblocks);

	avl_create(&brtvd->bv_tree, brt_entry_compare,
	sizeof (brt_entry_t), offsetof(brt_entry_t, bre_node));
	} else {
	ASSERT(brtvd->bv_size > 0);
	ASSERT(brtvd->bv_entcount != NULL);
	ASSERT(brtvd->bv_bitmap != NULL);
	ASSERT(brtvd->bv_nblocks > 0);
	/*
	* TODO: Allow vdev shrinking. We only need to implement
	* shrinking the on-disk BRT VDEV object.
	* dmu_free_range(brt->brt_mos, brtvd->bv_mos_brtvdev, offset,
	* size, tx);
	*/
	ASSERT3U(brtvd->bv_size, <=, size);

	memcpy(entcount, brtvd->bv_entcount,
	sizeof (entcount[0]) * MIN(size, brtvd->bv_size));
	memcpy(bitmap, brtvd->bv_bitmap, MIN(BT_SIZEOFMAP(nblocks),
	BT_SIZEOFMAP(brtvd->bv_nblocks)));
	vmem_free(brtvd->bv_entcount,
	sizeof (entcount[0]) * brtvd->bv_size);
	kmem_free(brtvd->bv_bitmap, BT_SIZEOFMAP(brtvd->bv_nblocks));
	}

	brtvd->bv_size = size;
	brtvd->bv_entcount = entcount;
	brtvd->bv_bitmap = bitmap;
	brtvd->bv_nblocks = nblocks;
	if (!brtvd->bv_initiated) {
	brtvd->bv_need_byteswap = FALSE;
	brtvd->bv_initiated = TRUE;
	BRT_DEBUG("BRT VDEV %llu initiated.",
	(u_longlong_t)brtvd->bv_vdevid);
	}
	}

	static void
	brt_vdev_load(brt_t brt, brt_vdev_t brtvd)
	{
	char name[64];
	dmu_buf_t *db;
	brt_vdev_phys_t *bvphys;
	int error;

	snprintf(name, sizeof (name), "%s%llu", BRT_OBJECT_VDEV_PREFIX,
	(u_longlong_t)brtvd->bv_vdevid);
	error = zap_lookup(brt->brt_mos, DMU_POOL_DIRECTORY_OBJECT, name,
	sizeof (uint64_t), 1, &brtvd->bv_mos_brtvdev);
	if (error != 0)
	return;
	ASSERT(brtvd->bv_mos_brtvdev != 0);

	error = dmu_bonus_hold(brt->brt_mos, brtvd->bv_mos_brtvdev, FTAG, &db);
	ASSERT0(error);
	if (error != 0)
	return;

	bvphys = db->db_data;
	if (brt->brt_rangesize == 0) {
	brt->brt_rangesize = bvphys->bvp_rangesize;
	} else {
	ASSERT3U(brt->brt_rangesize, ==, bvphys->bvp_rangesize);
	}

	ASSERT(!brtvd->bv_initiated);
	brt_vdev_realloc(brt, brtvd);

	/* TODO: We don't support VDEV shrinking. */
	ASSERT3U(bvphys->bvp_size, <=, brtvd->bv_size);

	/*
	* If VDEV grew, we will leave new bv_entcount[] entries zeroed out.
	*/
	error = dmu_read(brt->brt_mos, brtvd->bv_mos_brtvdev, 0,
	MIN(brtvd->bv_size, bvphys->bvp_size) * sizeof (uint16_t),
	brtvd->bv_entcount, DMU_READ_NO_PREFETCH);
	ASSERT0(error);

	brtvd->bv_mos_entries = bvphys->bvp_mos_entries;
	ASSERT(brtvd->bv_mos_entries != 0);
	brtvd->bv_need_byteswap =
	(bvphys->bvp_byteorder != BRT_NATIVE_BYTEORDER);
	brtvd->bv_totalcount = bvphys->bvp_totalcount;
	brtvd->bv_usedspace = bvphys->bvp_usedspace;
	brtvd->bv_savedspace = bvphys->bvp_savedspace;
	brt->brt_usedspace += brtvd->bv_usedspace;
	brt->brt_savedspace += brtvd->bv_savedspace;

	dmu_buf_rele(db, FTAG);

	BRT_DEBUG("MOS BRT VDEV %s loaded: mos_brtvdev=%llu, mos_entries=%llu",
	name, (u_longlong_t)brtvd->bv_mos_brtvdev,
	(u_longlong_t)brtvd->bv_mos_entries);
	}

	static void
	brt_vdev_dealloc(brt_t brt, brt_vdev_t brtvd)
	{

	ASSERT(RW_WRITE_HELD(&brt->brt_lock));
	ASSERT(brtvd->bv_initiated);

	vmem_free(brtvd->bv_entcount, sizeof (uint16_t) * brtvd->bv_size);
	brtvd->bv_entcount = NULL;
	kmem_free(brtvd->bv_bitmap, BT_SIZEOFMAP(brtvd->bv_nblocks));
	brtvd->bv_bitmap = NULL;
	ASSERT0(avl_numnodes(&brtvd->bv_tree));
	avl_destroy(&brtvd->bv_tree);

	brtvd->bv_size = 0;
	brtvd->bv_nblocks = 0;

	brtvd->bv_initiated = FALSE;
	BRT_DEBUG("BRT VDEV %llu deallocated.", (u_longlong_t)brtvd->bv_vdevid);
	}

	static void
	brt_vdev_destroy(brt_t brt, brt_vdev_t brtvd, dmu_tx_t *tx)
	{
	char name[64];
	uint64_t count;
	dmu_buf_t *db;
	brt_vdev_phys_t *bvphys;

	ASSERT(RW_WRITE_HELD(&brt->brt_lock));
	ASSERT(brtvd->bv_mos_brtvdev != 0);
	ASSERT(brtvd->bv_mos_entries != 0);

	VERIFY0(zap_count(brt->brt_mos, brtvd->bv_mos_entries, &count));
	VERIFY0(count);
	VERIFY0(zap_destroy(brt->brt_mos, brtvd->bv_mos_entries, tx));
	BRT_DEBUG("MOS entries destroyed, object=%llu",
	(u_longlong_t)brtvd->bv_mos_entries);
	brtvd->bv_mos_entries = 0;

	VERIFY0(dmu_bonus_hold(brt->brt_mos, brtvd->bv_mos_brtvdev, FTAG, &db));
	bvphys = db->db_data;
	ASSERT0(bvphys->bvp_totalcount);
	ASSERT0(bvphys->bvp_usedspace);
	ASSERT0(bvphys->bvp_savedspace);
	dmu_buf_rele(db, FTAG);

	VERIFY0(dmu_object_free(brt->brt_mos, brtvd->bv_mos_brtvdev, tx));
	BRT_DEBUG("MOS BRT VDEV destroyed, object=%llu",
	(u_longlong_t)brtvd->bv_mos_brtvdev);
	brtvd->bv_mos_brtvdev = 0;

	snprintf(name, sizeof (name), "%s%llu", BRT_OBJECT_VDEV_PREFIX,
	(u_longlong_t)brtvd->bv_vdevid);
	VERIFY0(zap_remove(brt->brt_mos, DMU_POOL_DIRECTORY_OBJECT, name, tx));
	BRT_DEBUG("Pool directory object removed, object=%s", name);

	brt_vdev_dealloc(brt, brtvd);

	spa_feature_decr(brt->brt_spa, SPA_FEATURE_BLOCK_CLONING, tx);
	}

	static void
	brt_vdevs_expand(brt_t *brt, uint64_t nvdevs)
	{
	brt_vdev_t brtvd, vdevs;
	uint64_t vdevid;

	ASSERT(RW_WRITE_HELD(&brt->brt_lock));
	ASSERT3U(nvdevs, >, brt->brt_nvdevs);

	vdevs = kmem_zalloc(sizeof (vdevs[0]) * nvdevs, KM_SLEEP);
	if (brt->brt_nvdevs > 0) {
	ASSERT(brt->brt_vdevs != NULL);

	memcpy(vdevs, brt->brt_vdevs,
	sizeof (brt_vdev_t) * brt->brt_nvdevs);
	kmem_free(brt->brt_vdevs,
	sizeof (brt_vdev_t) * brt->brt_nvdevs);
	}
	for (vdevid = brt->brt_nvdevs; vdevid < nvdevs; vdevid++) {
	brtvd = &vdevs[vdevid];

	brtvd->bv_vdevid = vdevid;
	brtvd->bv_initiated = FALSE;
	}

	BRT_DEBUG("BRT VDEVs expanded from %llu to %llu.",
	(u_longlong_t)brt->brt_nvdevs, (u_longlong_t)nvdevs);

	brt->brt_vdevs = vdevs;
	brt->brt_nvdevs = nvdevs;
	}

	static boolean_t
	brt_vdev_lookup(brt_t brt, brt_vdev_t brtvd, const brt_entry_t *bre)
	{
	uint64_t idx;

	ASSERT(RW_LOCK_HELD(&brt->brt_lock));

	idx = bre->bre_offset / brt->brt_rangesize;
	if (brtvd->bv_entcount != NULL && idx < brtvd->bv_size) {
	/* VDEV wasn't expanded. */
	return (brt_vdev_entcount_get(brtvd, idx) > 0);
	}

	return (FALSE);
	}

	static void
	brt_vdev_addref(brt_t brt, brt_vdev_t brtvd, const brt_entry_t *bre,
	uint64_t dsize)
	{
	uint64_t idx;

	ASSERT(RW_LOCK_HELD(&brt->brt_lock));
	ASSERT(brtvd != NULL);
	ASSERT(brtvd->bv_entcount != NULL);

	brt->brt_savedspace += dsize;
	brtvd->bv_savedspace += dsize;
	brtvd->bv_meta_dirty = TRUE;

	if (bre->bre_refcount > 1) {
	return;
	}

	brt->brt_usedspace += dsize;
	brtvd->bv_usedspace += dsize;

	idx = bre->bre_offset / brt->brt_rangesize;
	if (idx >= brtvd->bv_size) {
	/* VDEV has been expanded. */
	brt_vdev_realloc(brt, brtvd);
	}

	ASSERT3U(idx, <, brtvd->bv_size);

	brtvd->bv_totalcount++;
	brt_vdev_entcount_inc(brtvd, idx);
	brtvd->bv_entcount_dirty = TRUE;
	idx = idx / BRT_BLOCKSIZE / 8;
	BT_SET(brtvd->bv_bitmap, idx);

	#ifdef ZFS_DEBUG
	if (zfs_flags & ZFS_DEBUG_BRT)
	brt_vdev_dump(brtvd);
	#endif
	}

	static void
	brt_vdev_decref(brt_t brt, brt_vdev_t brtvd, const brt_entry_t *bre,
	uint64_t dsize)
	{
	uint64_t idx;

	ASSERT(RW_WRITE_HELD(&brt->brt_lock));
	ASSERT(brtvd != NULL);
	ASSERT(brtvd->bv_entcount != NULL);

	brt->brt_savedspace -= dsize;
	brtvd->bv_savedspace -= dsize;
	brtvd->bv_meta_dirty = TRUE;

	if (bre->bre_refcount > 0) {
	return;
	}

	brt->brt_usedspace -= dsize;
	brtvd->bv_usedspace -= dsize;

	idx = bre->bre_offset / brt->brt_rangesize;
	ASSERT3U(idx, <, brtvd->bv_size);

	ASSERT(brtvd->bv_totalcount > 0);
	brtvd->bv_totalcount--;
	brt_vdev_entcount_dec(brtvd, idx);
	brtvd->bv_entcount_dirty = TRUE;
	idx = idx / BRT_BLOCKSIZE / 8;
	BT_SET(brtvd->bv_bitmap, idx);

	#ifdef ZFS_DEBUG
	if (zfs_flags & ZFS_DEBUG_BRT)
	brt_vdev_dump(brtvd);
	#endif
	}

	static void
	brt_vdev_sync(brt_t brt, brt_vdev_t brtvd, dmu_tx_t *tx)
	{
	dmu_buf_t *db;
	brt_vdev_phys_t *bvphys;

	ASSERT(brtvd->bv_meta_dirty);
	ASSERT(brtvd->bv_mos_brtvdev != 0);
	ASSERT(dmu_tx_is_syncing(tx));

	VERIFY0(dmu_bonus_hold(brt->brt_mos, brtvd->bv_mos_brtvdev, FTAG, &db));

	if (brtvd->bv_entcount_dirty) {
	/*
	* TODO: Walk brtvd->bv_bitmap and write only the dirty blocks.
	*/
	dmu_write(brt->brt_mos, brtvd->bv_mos_brtvdev, 0,
	brtvd->bv_size * sizeof (brtvd->bv_entcount[0]),
	brtvd->bv_entcount, tx);
	memset(brtvd->bv_bitmap, 0, BT_SIZEOFMAP(brtvd->bv_nblocks));
	brtvd->bv_entcount_dirty = FALSE;
	}

	dmu_buf_will_dirty(db, tx);
	bvphys = db->db_data;
	bvphys->bvp_mos_entries = brtvd->bv_mos_entries;
	bvphys->bvp_size = brtvd->bv_size;
	if (brtvd->bv_need_byteswap) {
	bvphys->bvp_byteorder = BRT_NON_NATIVE_BYTEORDER;
	} else {
	bvphys->bvp_byteorder = BRT_NATIVE_BYTEORDER;
	}
	bvphys->bvp_totalcount = brtvd->bv_totalcount;
	bvphys->bvp_rangesize = brt->brt_rangesize;
	bvphys->bvp_usedspace = brtvd->bv_usedspace;
	bvphys->bvp_savedspace = brtvd->bv_savedspace;
	dmu_buf_rele(db, FTAG);

	brtvd->bv_meta_dirty = FALSE;
	}

	static void
	brt_vdevs_alloc(brt_t *brt, boolean_t load)
	{
	brt_vdev_t *brtvd;
	uint64_t vdevid;

	brt_wlock(brt);

	brt_vdevs_expand(brt, brt->brt_spa->spa_root_vdev->vdev_children);

	if (load) {
	for (vdevid = 0; vdevid < brt->brt_nvdevs; vdevid++) {
	brtvd = &brt->brt_vdevs[vdevid];
	ASSERT(brtvd->bv_entcount == NULL);

	brt_vdev_load(brt, brtvd);
	}
	}

	if (brt->brt_rangesize == 0) {
	brt->brt_rangesize = BRT_RANGESIZE;
	}

	brt_unlock(brt);
	}

	static void
	brt_vdevs_free(brt_t *brt)
	{
	brt_vdev_t *brtvd;
	uint64_t vdevid;

	brt_wlock(brt);

	for (vdevid = 0; vdevid < brt->brt_nvdevs; vdevid++) {
	brtvd = &brt->brt_vdevs[vdevid];
	if (brtvd->bv_initiated)
	brt_vdev_dealloc(brt, brtvd);
	}
	kmem_free(brt->brt_vdevs, sizeof (brt_vdev_t) * brt->brt_nvdevs);

	brt_unlock(brt);
	}

	static void
	brt_entry_fill(const blkptr_t bp, brt_entry_t bre, uint64_t *vdevidp)
	{

	bre->bre_offset = DVA_GET_OFFSET(&bp->blk_dva[0]);
	bre->bre_refcount = 0;

	*vdevidp = DVA_GET_VDEV(&bp->blk_dva[0]);
	}

	static int
	brt_entry_compare(const void x1, const void x2)
	{
	const brt_entry_t *bre1 = x1;
	const brt_entry_t *bre2 = x2;

	return (TREE_CMP(bre1->bre_offset, bre2->bre_offset));
	}

	static int
	brt_entry_lookup(brt_t brt, brt_vdev_t brtvd, brt_entry_t *bre)
	{
	uint64_t mos_entries;
	- uint64_t one, physsize;
	int error;

	ASSERT(RW_LOCK_HELD(&brt->brt_lock));

	if (!brt_vdev_lookup(brt, brtvd, bre))
	return (SET_ERROR(ENOENT));

	/*
	* Remember mos_entries object number. After we reacquire the BRT lock,
	* the brtvd pointer may be invalid.
	*/
	mos_entries = brtvd->bv_mos_entries;
	if (mos_entries == 0)
	return (SET_ERROR(ENOENT));

	brt_unlock(brt);

	- error = zap_length_uint64(brt->brt_mos, mos_entries, &bre->bre_offset,
	- BRT_KEY_WORDS, &one, &physsize);
	- if (error == 0) {
	- ASSERT3U(one, ==, 1);
	- ASSERT3U(physsize, ==, sizeof (bre->bre_refcount));
	-
	- error = zap_lookup_uint64(brt->brt_mos, mos_entries,
	- &bre->bre_offset, BRT_KEY_WORDS, 1,
	- sizeof (bre->bre_refcount), &bre->bre_refcount);
	- BRT_DEBUG("ZAP lookup: object=%llu vdev=%llu offset=%llu "
	- "count=%llu error=%d", (u_longlong_t)mos_entries,
	- (u_longlong_t)brtvd->bv_vdevid,
	- (u_longlong_t)bre->bre_offset,
	- error == 0 ? (u_longlong_t)bre->bre_refcount : 0, error);
	- }
	+ error = zap_lookup_uint64(brt->brt_mos, mos_entries, &bre->bre_offset,
	+ BRT_KEY_WORDS, 1, sizeof (bre->bre_refcount), &bre->bre_refcount);

	brt_wlock(brt);

	return (error);
	}

	static void
	brt_entry_prefetch(brt_t brt, uint64_t vdevid, brt_entry_t bre)
	{
	brt_vdev_t *brtvd;
	uint64_t mos_entries = 0;

	brt_rlock(brt);
	brtvd = brt_vdev(brt, vdevid);
	if (brtvd != NULL)
	mos_entries = brtvd->bv_mos_entries;
	brt_unlock(brt);

	if (mos_entries == 0)
	return;

	- BRT_DEBUG("ZAP prefetch: object=%llu vdev=%llu offset=%llu",
	- (u_longlong_t)mos_entries, (u_longlong_t)vdevid,
	- (u_longlong_t)bre->bre_offset);
	(void) zap_prefetch_uint64(brt->brt_mos, mos_entries,
	(uint64_t *)&bre->bre_offset, BRT_KEY_WORDS);
	}

	-static int
	-brt_entry_update(brt_t brt, brt_vdev_t brtvd, brt_entry_t bre, dmu_tx_t tx)
	-{
	- int error;
	-
	- ASSERT(RW_LOCK_HELD(&brt->brt_lock));
	- ASSERT(brtvd->bv_mos_entries != 0);
	- ASSERT(bre->bre_refcount > 0);
	-
	- error = zap_update_uint64(brt->brt_mos, brtvd->bv_mos_entries,
	- (uint64_t *)&bre->bre_offset, BRT_KEY_WORDS, 1,
	- sizeof (bre->bre_refcount), &bre->bre_refcount, tx);
	- BRT_DEBUG("ZAP update: object=%llu vdev=%llu offset=%llu count=%llu "
	- "error=%d", (u_longlong_t)brtvd->bv_mos_entries,
	- (u_longlong_t)brtvd->bv_vdevid, (u_longlong_t)bre->bre_offset,
	- (u_longlong_t)bre->bre_refcount, error);
	-
	- return (error);
	-}
	-
	-static int
	-brt_entry_remove(brt_t brt, brt_vdev_t brtvd, brt_entry_t bre, dmu_tx_t tx)
	-{
	- int error;
	-
	- ASSERT(RW_LOCK_HELD(&brt->brt_lock));
	- ASSERT(brtvd->bv_mos_entries != 0);
	- ASSERT0(bre->bre_refcount);
	-
	- error = zap_remove_uint64(brt->brt_mos, brtvd->bv_mos_entries,
	- (uint64_t *)&bre->bre_offset, BRT_KEY_WORDS, tx);
	- BRT_DEBUG("ZAP remove: object=%llu vdev=%llu offset=%llu count=%llu "
	- "error=%d", (u_longlong_t)brtvd->bv_mos_entries,
	- (u_longlong_t)brtvd->bv_vdevid, (u_longlong_t)bre->bre_offset,
	- (u_longlong_t)bre->bre_refcount, error);
	-
	- return (error);
	-}
	-
	/*
	* Return TRUE if we _can_ have BRT entry for this bp. It might be false
	* positive, but gives us quick answer if we should look into BRT, which
	* may require reads and thus will be more expensive.
	*/
	boolean_t
	brt_maybe_exists(spa_t spa, const blkptr_t bp)
	{
	brt_t *brt = spa->spa_brt;
	brt_vdev_t *brtvd;
	brt_entry_t bre_search;
	boolean_t mayexists = FALSE;
	uint64_t vdevid;

	brt_entry_fill(bp, &bre_search, &vdevid);

	brt_rlock(brt);

	brtvd = brt_vdev(brt, vdevid);
	if (brtvd != NULL && brtvd->bv_initiated) {
	if (!avl_is_empty(&brtvd->bv_tree) \|\|
	brt_vdev_lookup(brt, brtvd, &bre_search)) {
	mayexists = TRUE;
	}
	}

	brt_unlock(brt);

	return (mayexists);
	}

	uint64_t
	brt_get_dspace(spa_t *spa)
	{
	brt_t *brt = spa->spa_brt;

	if (brt == NULL)
	return (0);

	return (brt->brt_savedspace);
	}

	uint64_t
	brt_get_used(spa_t *spa)
	{
	brt_t *brt = spa->spa_brt;

	if (brt == NULL)
	return (0);

	return (brt->brt_usedspace);
	}

	uint64_t
	brt_get_saved(spa_t *spa)
	{
	brt_t *brt = spa->spa_brt;

	if (brt == NULL)
	return (0);

	return (brt->brt_savedspace);
	}

	uint64_t
	brt_get_ratio(spa_t *spa)
	{
	brt_t *brt = spa->spa_brt;

	if (brt->brt_usedspace == 0)
	return (100);

	return ((brt->brt_usedspace + brt->brt_savedspace) * 100 /
	brt->brt_usedspace);
	}

	static int
	brt_kstats_update(kstat_t *ksp, int rw)
	{
	brt_stats_t *bs = ksp->ks_data;

	if (rw == KSTAT_WRITE)
	return (EACCES);

	bs->brt_addref_entry_in_memory.value.ui64 =
	wmsum_value(&brt_sums.brt_addref_entry_in_memory);
	bs->brt_addref_entry_not_on_disk.value.ui64 =
	wmsum_value(&brt_sums.brt_addref_entry_not_on_disk);
	bs->brt_addref_entry_on_disk.value.ui64 =
	wmsum_value(&brt_sums.brt_addref_entry_on_disk);
	bs->brt_addref_entry_read_lost_race.value.ui64 =
	wmsum_value(&brt_sums.brt_addref_entry_read_lost_race);
	bs->brt_decref_entry_in_memory.value.ui64 =
	wmsum_value(&brt_sums.brt_decref_entry_in_memory);
	bs->brt_decref_entry_loaded_from_disk.value.ui64 =
	wmsum_value(&brt_sums.brt_decref_entry_loaded_from_disk);
	bs->brt_decref_entry_not_in_memory.value.ui64 =
	wmsum_value(&brt_sums.brt_decref_entry_not_in_memory);
	bs->brt_decref_entry_not_on_disk.value.ui64 =
	wmsum_value(&brt_sums.brt_decref_entry_not_on_disk);
	bs->brt_decref_entry_read_lost_race.value.ui64 =
	wmsum_value(&brt_sums.brt_decref_entry_read_lost_race);
	bs->brt_decref_entry_still_referenced.value.ui64 =
	wmsum_value(&brt_sums.brt_decref_entry_still_referenced);
	bs->brt_decref_free_data_later.value.ui64 =
	wmsum_value(&brt_sums.brt_decref_free_data_later);
	bs->brt_decref_free_data_now.value.ui64 =
	wmsum_value(&brt_sums.brt_decref_free_data_now);
	bs->brt_decref_no_entry.value.ui64 =
	wmsum_value(&brt_sums.brt_decref_no_entry);

	return (0);
	}

	static void
	brt_stat_init(void)
	{

	wmsum_init(&brt_sums.brt_addref_entry_in_memory, 0);
	wmsum_init(&brt_sums.brt_addref_entry_not_on_disk, 0);
	wmsum_init(&brt_sums.brt_addref_entry_on_disk, 0);
	wmsum_init(&brt_sums.brt_addref_entry_read_lost_race, 0);
	wmsum_init(&brt_sums.brt_decref_entry_in_memory, 0);
	wmsum_init(&brt_sums.brt_decref_entry_loaded_from_disk, 0);
	wmsum_init(&brt_sums.brt_decref_entry_not_in_memory, 0);
	wmsum_init(&brt_sums.brt_decref_entry_not_on_disk, 0);
	wmsum_init(&brt_sums.brt_decref_entry_read_lost_race, 0);
	wmsum_init(&brt_sums.brt_decref_entry_still_referenced, 0);
	wmsum_init(&brt_sums.brt_decref_free_data_later, 0);
	wmsum_init(&brt_sums.brt_decref_free_data_now, 0);
	wmsum_init(&brt_sums.brt_decref_no_entry, 0);

	brt_ksp = kstat_create("zfs", 0, "brtstats", "misc", KSTAT_TYPE_NAMED,
	sizeof (brt_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
	if (brt_ksp != NULL) {
	brt_ksp->ks_data = &brt_stats;
	brt_ksp->ks_update = brt_kstats_update;
	kstat_install(brt_ksp);
	}
	}

	static void
	brt_stat_fini(void)
	{
	if (brt_ksp != NULL) {
	kstat_delete(brt_ksp);
	brt_ksp = NULL;
	}

	wmsum_fini(&brt_sums.brt_addref_entry_in_memory);
	wmsum_fini(&brt_sums.brt_addref_entry_not_on_disk);
	wmsum_fini(&brt_sums.brt_addref_entry_on_disk);
	wmsum_fini(&brt_sums.brt_addref_entry_read_lost_race);
	wmsum_fini(&brt_sums.brt_decref_entry_in_memory);
	wmsum_fini(&brt_sums.brt_decref_entry_loaded_from_disk);
	wmsum_fini(&brt_sums.brt_decref_entry_not_in_memory);
	wmsum_fini(&brt_sums.brt_decref_entry_not_on_disk);
	wmsum_fini(&brt_sums.brt_decref_entry_read_lost_race);
	wmsum_fini(&brt_sums.brt_decref_entry_still_referenced);
	wmsum_fini(&brt_sums.brt_decref_free_data_later);
	wmsum_fini(&brt_sums.brt_decref_free_data_now);
	wmsum_fini(&brt_sums.brt_decref_no_entry);
	}

	void
	brt_init(void)
	{
	brt_entry_cache = kmem_cache_create("brt_entry_cache",
	sizeof (brt_entry_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
	brt_pending_entry_cache = kmem_cache_create("brt_pending_entry_cache",
	sizeof (brt_pending_entry_t), 0, NULL, NULL, NULL, NULL, NULL, 0);

	brt_stat_init();
	}

	void
	brt_fini(void)
	{
	brt_stat_fini();

	kmem_cache_destroy(brt_entry_cache);
	kmem_cache_destroy(brt_pending_entry_cache);
	}

	static brt_entry_t *
	brt_entry_alloc(const brt_entry_t *bre_init)
	{
	brt_entry_t *bre;

	bre = kmem_cache_alloc(brt_entry_cache, KM_SLEEP);
	bre->bre_offset = bre_init->bre_offset;
	bre->bre_refcount = bre_init->bre_refcount;

	return (bre);
	}

	static void
	brt_entry_free(brt_entry_t *bre)
	{

	kmem_cache_free(brt_entry_cache, bre);
	}

	static void
	brt_entry_addref(brt_t brt, const blkptr_t bp)
	{
	brt_vdev_t *brtvd;
	brt_entry_t bre, racebre;
	brt_entry_t bre_search;
	avl_index_t where;
	uint64_t vdevid;
	int error;

	ASSERT(!RW_WRITE_HELD(&brt->brt_lock));

	brt_entry_fill(bp, &bre_search, &vdevid);

	brt_wlock(brt);

	brtvd = brt_vdev(brt, vdevid);
	if (brtvd == NULL) {
	ASSERT3U(vdevid, >=, brt->brt_nvdevs);

	/* New VDEV was added. */
	brt_vdevs_expand(brt, vdevid + 1);
	brtvd = brt_vdev(brt, vdevid);
	}
	ASSERT(brtvd != NULL);
	if (!brtvd->bv_initiated)
	brt_vdev_realloc(brt, brtvd);

	bre = avl_find(&brtvd->bv_tree, &bre_search, NULL);
	if (bre != NULL) {
	BRTSTAT_BUMP(brt_addref_entry_in_memory);
	} else {
	/*
	* brt_entry_lookup() may drop the BRT (read) lock and
	* reacquire it (write).
	*/
	error = brt_entry_lookup(brt, brtvd, &bre_search);
	/* bre_search now contains correct bre_refcount */
	ASSERT(error == 0 \|\| error == ENOENT);
	if (error == 0)
	BRTSTAT_BUMP(brt_addref_entry_on_disk);
	else
	BRTSTAT_BUMP(brt_addref_entry_not_on_disk);
	/*
	* When the BRT lock was dropped, brt_vdevs[] may have been
	* expanded and reallocated, we need to update brtvd's pointer.
	*/
	brtvd = brt_vdev(brt, vdevid);
	ASSERT(brtvd != NULL);

	racebre = avl_find(&brtvd->bv_tree, &bre_search, &where);
	if (racebre == NULL) {
	bre = brt_entry_alloc(&bre_search);
	ASSERT(RW_WRITE_HELD(&brt->brt_lock));
	avl_insert(&brtvd->bv_tree, bre, where);
	brt->brt_nentries++;
	} else {
	/*
	* The entry was added when the BRT lock was dropped in
	* brt_entry_lookup().
	*/
	BRTSTAT_BUMP(brt_addref_entry_read_lost_race);
	bre = racebre;
	}
	}
	bre->bre_refcount++;
	brt_vdev_addref(brt, brtvd, bre, bp_get_dsize(brt->brt_spa, bp));

	brt_unlock(brt);
	}

	/* Return TRUE if block should be freed immediately. */
	boolean_t
	brt_entry_decref(spa_t spa, const blkptr_t bp)
	{
	brt_t *brt = spa->spa_brt;
	brt_vdev_t *brtvd;
	brt_entry_t bre, racebre;
	brt_entry_t bre_search;
	avl_index_t where;
	uint64_t vdevid;
	int error;

	brt_entry_fill(bp, &bre_search, &vdevid);

	brt_wlock(brt);

	brtvd = brt_vdev(brt, vdevid);
	ASSERT(brtvd != NULL);

	bre = avl_find(&brtvd->bv_tree, &bre_search, NULL);
	if (bre != NULL) {
	BRTSTAT_BUMP(brt_decref_entry_in_memory);
	goto out;
	} else {
	BRTSTAT_BUMP(brt_decref_entry_not_in_memory);
	}

	/*
	* brt_entry_lookup() may drop the BRT lock and reacquire it.
	*/
	error = brt_entry_lookup(brt, brtvd, &bre_search);
	/* bre_search now contains correct bre_refcount */
	ASSERT(error == 0 \|\| error == ENOENT);
	/*
	* When the BRT lock was dropped, brt_vdevs[] may have been expanded
	* and reallocated, we need to update brtvd's pointer.
	*/
	brtvd = brt_vdev(brt, vdevid);
	ASSERT(brtvd != NULL);

	if (error == ENOENT) {
	BRTSTAT_BUMP(brt_decref_entry_not_on_disk);
	bre = NULL;
	goto out;
	}

	racebre = avl_find(&brtvd->bv_tree, &bre_search, &where);
	if (racebre != NULL) {
	/*
	* The entry was added when the BRT lock was dropped in
	* brt_entry_lookup().
	*/
	BRTSTAT_BUMP(brt_decref_entry_read_lost_race);
	bre = racebre;
	goto out;
	}

	BRTSTAT_BUMP(brt_decref_entry_loaded_from_disk);
	bre = brt_entry_alloc(&bre_search);
	ASSERT(RW_WRITE_HELD(&brt->brt_lock));
	avl_insert(&brtvd->bv_tree, bre, where);
	brt->brt_nentries++;

	out:
	if (bre == NULL) {
	/*
	* This is a free of a regular (not cloned) block.
	*/
	brt_unlock(brt);
	BRTSTAT_BUMP(brt_decref_no_entry);
	return (B_TRUE);
	}
	if (bre->bre_refcount == 0) {
	brt_unlock(brt);
	BRTSTAT_BUMP(brt_decref_free_data_now);
	return (B_TRUE);
	}

	ASSERT(bre->bre_refcount > 0);
	bre->bre_refcount--;
	if (bre->bre_refcount == 0)
	BRTSTAT_BUMP(brt_decref_free_data_later);
	else
	BRTSTAT_BUMP(brt_decref_entry_still_referenced);
	brt_vdev_decref(brt, brtvd, bre, bp_get_dsize(brt->brt_spa, bp));

	brt_unlock(brt);

	return (B_FALSE);
	}

	uint64_t
	brt_entry_get_refcount(spa_t spa, const blkptr_t bp)
	{
	brt_t *brt = spa->spa_brt;
	brt_vdev_t *brtvd;
	brt_entry_t bre_search, *bre;
	uint64_t vdevid, refcnt;
	int error;

	brt_entry_fill(bp, &bre_search, &vdevid);

	brt_rlock(brt);

	brtvd = brt_vdev(brt, vdevid);
	ASSERT(brtvd != NULL);

	bre = avl_find(&brtvd->bv_tree, &bre_search, NULL);
	if (bre == NULL) {
	error = brt_entry_lookup(brt, brtvd, &bre_search);
	ASSERT(error == 0 \|\| error == ENOENT);
	if (error == ENOENT)
	refcnt = 0;
	else
	refcnt = bre_search.bre_refcount;
	} else
	refcnt = bre->bre_refcount;

	brt_unlock(brt);
	return (refcnt);
	}

	static void
	brt_prefetch(brt_t brt, const blkptr_t bp)
	{
	brt_entry_t bre;
	uint64_t vdevid;

	ASSERT(bp != NULL);

	- if (!zfs_brt_prefetch)
	+ if (!brt_zap_prefetch)
	return;

	brt_entry_fill(bp, &bre, &vdevid);

	brt_entry_prefetch(brt, vdevid, &bre);
	}

	static int
	brt_pending_entry_compare(const void x1, const void x2)
	{
	const brt_pending_entry_t bpe1 = x1, bpe2 = x2;
	const blkptr_t bp1 = &bpe1->bpe_bp, bp2 = &bpe2->bpe_bp;
	int cmp;

	- cmp = TREE_CMP(BP_PHYSICAL_BIRTH(bp1), BP_PHYSICAL_BIRTH(bp2));
	+ cmp = TREE_CMP(DVA_GET_VDEV(&bp1->blk_dva[0]),
	+ DVA_GET_VDEV(&bp2->blk_dva[0]));
	if (cmp == 0) {
	- cmp = TREE_CMP(DVA_GET_VDEV(&bp1->blk_dva[0]),
	- DVA_GET_VDEV(&bp2->blk_dva[0]));
	- if (cmp == 0) {
	- cmp = TREE_CMP(DVA_GET_OFFSET(&bp1->blk_dva[0]),
	- DVA_GET_OFFSET(&bp2->blk_dva[0]));
	+ cmp = TREE_CMP(DVA_GET_OFFSET(&bp1->blk_dva[0]),
	+ DVA_GET_OFFSET(&bp2->blk_dva[0]));
	+ if (unlikely(cmp == 0)) {
	+ cmp = TREE_CMP(BP_PHYSICAL_BIRTH(bp1),
	+ BP_PHYSICAL_BIRTH(bp2));
	}
	}

	return (cmp);
	}

	void
	brt_pending_add(spa_t spa, const blkptr_t bp, dmu_tx_t *tx)
	{
	brt_t *brt;
	avl_tree_t *pending_tree;
	kmutex_t *pending_lock;
	brt_pending_entry_t bpe, newbpe;
	avl_index_t where;
	uint64_t txg;

	brt = spa->spa_brt;
	txg = dmu_tx_get_txg(tx);
	ASSERT3U(txg, !=, 0);
	pending_tree = &brt->brt_pending_tree[txg & TXG_MASK];
	pending_lock = &brt->brt_pending_lock[txg & TXG_MASK];

	newbpe = kmem_cache_alloc(brt_pending_entry_cache, KM_SLEEP);
	newbpe->bpe_bp = *bp;
	newbpe->bpe_count = 1;

	mutex_enter(pending_lock);

	bpe = avl_find(pending_tree, newbpe, &where);
	if (bpe == NULL) {
	avl_insert(pending_tree, newbpe, where);
	newbpe = NULL;
	} else {
	bpe->bpe_count++;
	}

	mutex_exit(pending_lock);

	if (newbpe != NULL) {
	ASSERT(bpe != NULL);
	ASSERT(bpe != newbpe);
	kmem_cache_free(brt_pending_entry_cache, newbpe);
	} else {
	ASSERT(bpe == NULL);
	- }

	- /* Prefetch BRT entry, as we will need it in the syncing context. */
	- brt_prefetch(brt, bp);
	+ /* Prefetch BRT entry for the syncing context. */
	+ brt_prefetch(brt, bp);
	+ }
	}

	void
	brt_pending_remove(spa_t spa, const blkptr_t bp, dmu_tx_t *tx)
	{
	brt_t *brt;
	avl_tree_t *pending_tree;
	kmutex_t *pending_lock;
	brt_pending_entry_t *bpe, bpe_search;
	uint64_t txg;

	brt = spa->spa_brt;
	txg = dmu_tx_get_txg(tx);
	ASSERT3U(txg, !=, 0);
	pending_tree = &brt->brt_pending_tree[txg & TXG_MASK];
	pending_lock = &brt->brt_pending_lock[txg & TXG_MASK];

	bpe_search.bpe_bp = *bp;

	mutex_enter(pending_lock);

	bpe = avl_find(pending_tree, &bpe_search, NULL);
	/* I believe we should always find bpe when this function is called. */
	if (bpe != NULL) {
	ASSERT(bpe->bpe_count > 0);

	bpe->bpe_count--;
	if (bpe->bpe_count == 0) {
	avl_remove(pending_tree, bpe);
	kmem_cache_free(brt_pending_entry_cache, bpe);
	}
	}

	mutex_exit(pending_lock);
	}

	void
	brt_pending_apply(spa_t *spa, uint64_t txg)
	{
	- brt_t *brt;
	+ brt_t *brt = spa->spa_brt;
	brt_pending_entry_t *bpe;
	avl_tree_t *pending_tree;
	- kmutex_t *pending_lock;
	void *c;

	ASSERT3U(txg, !=, 0);

	- brt = spa->spa_brt;
	+ /*
	+ * We are in syncing context, so no other brt_pending_tree accesses
	+ * are possible for the TXG. Don't need to acquire brt_pending_lock.
	+ */
	pending_tree = &brt->brt_pending_tree[txg & TXG_MASK];
	- pending_lock = &brt->brt_pending_lock[txg & TXG_MASK];
	-
	- mutex_enter(pending_lock);

	c = NULL;
	while ((bpe = avl_destroy_nodes(pending_tree, &c)) != NULL) {
	boolean_t added_to_ddt;

	- mutex_exit(pending_lock);
	-
	for (int i = 0; i < bpe->bpe_count; i++) {
	/*
	* If the block has DEDUP bit set, it means that it
	* already exists in the DEDUP table, so we can just
	* use that instead of creating new entry in
	* the BRT table.
	*/
	if (BP_GET_DEDUP(&bpe->bpe_bp)) {
	added_to_ddt = ddt_addref(spa, &bpe->bpe_bp);
	} else {
	added_to_ddt = B_FALSE;
	}
	if (!added_to_ddt)
	brt_entry_addref(brt, &bpe->bpe_bp);
	}

	kmem_cache_free(brt_pending_entry_cache, bpe);
	- mutex_enter(pending_lock);
	}
	-
	- mutex_exit(pending_lock);
	}

	static void
	-brt_sync_entry(brt_t brt, brt_vdev_t brtvd, brt_entry_t bre, dmu_tx_t tx)
	+brt_sync_entry(dnode_t dn, brt_entry_t bre, dmu_tx_t *tx)
	{
	-
	- ASSERT(RW_WRITE_HELD(&brt->brt_lock));
	- ASSERT(brtvd->bv_mos_entries != 0);
	-
	if (bre->bre_refcount == 0) {
	- int error;
	-
	- error = brt_entry_remove(brt, brtvd, bre, tx);
	- ASSERT(error == 0 \|\| error == ENOENT);
	- /*
	- * If error == ENOENT then zfs_clone_range() was done from a
	- * removed (but opened) file (open(), unlink()).
	- */
	- ASSERT(brt_entry_lookup(brt, brtvd, bre) == ENOENT);
	+ int error = zap_remove_uint64_by_dnode(dn, &bre->bre_offset,
	+ BRT_KEY_WORDS, tx);
	+ VERIFY(error == 0 \|\| error == ENOENT);
	} else {
	- VERIFY0(brt_entry_update(brt, brtvd, bre, tx));
	+ VERIFY0(zap_update_uint64_by_dnode(dn, &bre->bre_offset,
	+ BRT_KEY_WORDS, 1, sizeof (bre->bre_refcount),
	+ &bre->bre_refcount, tx));
	}
	}

	static void
	brt_sync_table(brt_t brt, dmu_tx_t tx)
	{
	brt_vdev_t *brtvd;
	brt_entry_t *bre;
	+ dnode_t *dn;
	uint64_t vdevid;
	void *c;

	brt_wlock(brt);

	for (vdevid = 0; vdevid < brt->brt_nvdevs; vdevid++) {
	brtvd = &brt->brt_vdevs[vdevid];

	if (!brtvd->bv_initiated)
	continue;

	if (!brtvd->bv_meta_dirty) {
	ASSERT(!brtvd->bv_entcount_dirty);
	ASSERT0(avl_numnodes(&brtvd->bv_tree));
	continue;
	}

	ASSERT(!brtvd->bv_entcount_dirty \|\|
	avl_numnodes(&brtvd->bv_tree) != 0);

	if (brtvd->bv_mos_brtvdev == 0)
	brt_vdev_create(brt, brtvd, tx);

	+ VERIFY0(dnode_hold(brt->brt_mos, brtvd->bv_mos_entries,
	+ FTAG, &dn));
	+
	c = NULL;
	while ((bre = avl_destroy_nodes(&brtvd->bv_tree, &c)) != NULL) {
	- brt_sync_entry(brt, brtvd, bre, tx);
	+ brt_sync_entry(dn, bre, tx);
	brt_entry_free(bre);
	ASSERT(brt->brt_nentries > 0);
	brt->brt_nentries--;
	}

	+ dnode_rele(dn, FTAG);
	+
	brt_vdev_sync(brt, brtvd, tx);

	if (brtvd->bv_totalcount == 0)
	brt_vdev_destroy(brt, brtvd, tx);
	}

	ASSERT0(brt->brt_nentries);

	brt_unlock(brt);
	}

	void
	brt_sync(spa_t *spa, uint64_t txg)
	{
	dmu_tx_t *tx;
	brt_t *brt;

	ASSERT(spa_syncing_txg(spa) == txg);

	brt = spa->spa_brt;
	brt_rlock(brt);
	if (brt->brt_nentries == 0) {
	/* No changes. */
	brt_unlock(brt);
	return;
	}
	brt_unlock(brt);

	tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);

	brt_sync_table(brt, tx);

	dmu_tx_commit(tx);
	}

	static void
	brt_table_alloc(brt_t *brt)
	{

	for (int i = 0; i < TXG_SIZE; i++) {
	avl_create(&brt->brt_pending_tree[i],
	brt_pending_entry_compare,
	sizeof (brt_pending_entry_t),
	offsetof(brt_pending_entry_t, bpe_node));
	mutex_init(&brt->brt_pending_lock[i], NULL, MUTEX_DEFAULT,
	NULL);
	}
	}

	static void
	brt_table_free(brt_t *brt)
	{

	for (int i = 0; i < TXG_SIZE; i++) {
	ASSERT(avl_is_empty(&brt->brt_pending_tree[i]));

	avl_destroy(&brt->brt_pending_tree[i]);
	mutex_destroy(&brt->brt_pending_lock[i]);
	}
	}

	static void
	brt_alloc(spa_t *spa)
	{
	brt_t *brt;

	ASSERT(spa->spa_brt == NULL);

	brt = kmem_zalloc(sizeof (*brt), KM_SLEEP);
	rw_init(&brt->brt_lock, NULL, RW_DEFAULT, NULL);
	brt->brt_spa = spa;
	brt->brt_rangesize = 0;
	brt->brt_nentries = 0;
	brt->brt_vdevs = NULL;
	brt->brt_nvdevs = 0;
	brt_table_alloc(brt);

	spa->spa_brt = brt;
	}

	void
	brt_create(spa_t *spa)
	{

	brt_alloc(spa);
	brt_vdevs_alloc(spa->spa_brt, B_FALSE);
	}

	int
	brt_load(spa_t *spa)
	{

	brt_alloc(spa);
	brt_vdevs_alloc(spa->spa_brt, B_TRUE);

	return (0);
	}

	void
	brt_unload(spa_t *spa)
	{
	brt_t *brt = spa->spa_brt;

	if (brt == NULL)
	return;

	brt_vdevs_free(brt);
	brt_table_free(brt);
	rw_destroy(&brt->brt_lock);
	kmem_free(brt, sizeof (*brt));
	spa->spa_brt = NULL;
	}

	/* BEGIN CSTYLED */
	-ZFS_MODULE_PARAM(zfs_brt, zfs_brt_, prefetch, INT, ZMOD_RW,
	- "Enable prefetching of BRT entries");
	-#ifdef ZFS_BRT_DEBUG
	-ZFS_MODULE_PARAM(zfs_brt, zfs_brt_, debug, INT, ZMOD_RW, "BRT debug");
	-#endif
	+ZFS_MODULE_PARAM(zfs_brt, , brt_zap_prefetch, INT, ZMOD_RW,
	+ "Enable prefetching of BRT ZAP entries");
	+ZFS_MODULE_PARAM(zfs_brt, , brt_zap_default_bs, UINT, ZMOD_RW,
	+ "BRT ZAP leaf blockshift");
	+ZFS_MODULE_PARAM(zfs_brt, , brt_zap_default_ibs, UINT, ZMOD_RW,
	+ "BRT ZAP indirect blockshift");
	/* END CSTYLED */
	diff --git a/sys/contrib/openzfs/module/zfs/dbuf.c b/sys/contrib/openzfs/module/zfs/dbuf.c
	index 6e983093fb97..bb921af6d78a 100644
	--- a/sys/contrib/openzfs/module/zfs/dbuf.c
	+++ b/sys/contrib/openzfs/module/zfs/dbuf.c
	@@ -1,5226 +1,5204 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright 2011 Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2012, 2020 by Delphix. All rights reserved.
	* Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
	* Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
	* Copyright (c) 2019, Klara Inc.
	* Copyright (c) 2019, Allan Jude
	* Copyright (c) 2021, 2022 by Pawel Jakub Dawidek
	*/

	#include <sys/zfs_context.h>
	#include <sys/arc.h>
	#include <sys/dmu.h>
	#include <sys/dmu_send.h>
	#include <sys/dmu_impl.h>
	#include <sys/dbuf.h>
	#include <sys/dmu_objset.h>
	#include <sys/dsl_dataset.h>
	#include <sys/dsl_dir.h>
	#include <sys/dmu_tx.h>
	#include <sys/spa.h>
	#include <sys/zio.h>
	#include <sys/dmu_zfetch.h>
	#include <sys/sa.h>
	#include <sys/sa_impl.h>
	#include <sys/zfeature.h>
	#include <sys/blkptr.h>
	#include <sys/range_tree.h>
	#include <sys/trace_zfs.h>
	#include <sys/callb.h>
	#include <sys/abd.h>
	#include <sys/brt.h>
	#include <sys/vdev.h>
	#include <cityhash.h>
	#include <sys/spa_impl.h>
	#include <sys/wmsum.h>
	#include <sys/vdev_impl.h>

	static kstat_t *dbuf_ksp;

	typedef struct dbuf_stats {
	/*
	* Various statistics about the size of the dbuf cache.
	*/
	kstat_named_t cache_count;
	kstat_named_t cache_size_bytes;
	kstat_named_t cache_size_bytes_max;
	/*
	* Statistics regarding the bounds on the dbuf cache size.
	*/
	kstat_named_t cache_target_bytes;
	kstat_named_t cache_lowater_bytes;
	kstat_named_t cache_hiwater_bytes;
	/*
	* Total number of dbuf cache evictions that have occurred.
	*/
	kstat_named_t cache_total_evicts;
	/*
	* The distribution of dbuf levels in the dbuf cache and
	* the total size of all dbufs at each level.
	*/
	kstat_named_t cache_levels[DN_MAX_LEVELS];
	kstat_named_t cache_levels_bytes[DN_MAX_LEVELS];
	/*
	* Statistics about the dbuf hash table.
	*/
	kstat_named_t hash_hits;
	kstat_named_t hash_misses;
	kstat_named_t hash_collisions;
	kstat_named_t hash_elements;
	kstat_named_t hash_elements_max;
	/*
	* Number of sublists containing more than one dbuf in the dbuf
	* hash table. Keep track of the longest hash chain.
	*/
	kstat_named_t hash_chains;
	kstat_named_t hash_chain_max;
	/*
	* Number of times a dbuf_create() discovers that a dbuf was
	* already created and in the dbuf hash table.
	*/
	kstat_named_t hash_insert_race;
	/*
	* Number of entries in the hash table dbuf and mutex arrays.
	*/
	kstat_named_t hash_table_count;
	kstat_named_t hash_mutex_count;
	/*
	* Statistics about the size of the metadata dbuf cache.
	*/
	kstat_named_t metadata_cache_count;
	kstat_named_t metadata_cache_size_bytes;
	kstat_named_t metadata_cache_size_bytes_max;
	/*
	* For diagnostic purposes, this is incremented whenever we can't add
	* something to the metadata cache because it's full, and instead put
	* the data in the regular dbuf cache.
	*/
	kstat_named_t metadata_cache_overflow;
	} dbuf_stats_t;

	dbuf_stats_t dbuf_stats = {
	{ "cache_count", KSTAT_DATA_UINT64 },
	{ "cache_size_bytes", KSTAT_DATA_UINT64 },
	{ "cache_size_bytes_max", KSTAT_DATA_UINT64 },
	{ "cache_target_bytes", KSTAT_DATA_UINT64 },
	{ "cache_lowater_bytes", KSTAT_DATA_UINT64 },
	{ "cache_hiwater_bytes", KSTAT_DATA_UINT64 },
	{ "cache_total_evicts", KSTAT_DATA_UINT64 },
	{ { "cache_levels_N", KSTAT_DATA_UINT64 } },
	{ { "cache_levels_bytes_N", KSTAT_DATA_UINT64 } },
	{ "hash_hits", KSTAT_DATA_UINT64 },
	{ "hash_misses", KSTAT_DATA_UINT64 },
	{ "hash_collisions", KSTAT_DATA_UINT64 },
	{ "hash_elements", KSTAT_DATA_UINT64 },
	{ "hash_elements_max", KSTAT_DATA_UINT64 },
	{ "hash_chains", KSTAT_DATA_UINT64 },
	{ "hash_chain_max", KSTAT_DATA_UINT64 },
	{ "hash_insert_race", KSTAT_DATA_UINT64 },
	{ "hash_table_count", KSTAT_DATA_UINT64 },
	{ "hash_mutex_count", KSTAT_DATA_UINT64 },
	{ "metadata_cache_count", KSTAT_DATA_UINT64 },
	{ "metadata_cache_size_bytes", KSTAT_DATA_UINT64 },
	{ "metadata_cache_size_bytes_max", KSTAT_DATA_UINT64 },
	{ "metadata_cache_overflow", KSTAT_DATA_UINT64 }
	};

	struct {
	wmsum_t cache_count;
	wmsum_t cache_total_evicts;
	wmsum_t cache_levels[DN_MAX_LEVELS];
	wmsum_t cache_levels_bytes[DN_MAX_LEVELS];
	wmsum_t hash_hits;
	wmsum_t hash_misses;
	wmsum_t hash_collisions;
	wmsum_t hash_chains;
	wmsum_t hash_insert_race;
	wmsum_t metadata_cache_count;
	wmsum_t metadata_cache_overflow;
	} dbuf_sums;

	#define DBUF_STAT_INCR(stat, val) \
	wmsum_add(&dbuf_sums.stat, val);
	#define DBUF_STAT_DECR(stat, val) \
	DBUF_STAT_INCR(stat, -(val));
	#define DBUF_STAT_BUMP(stat) \
	DBUF_STAT_INCR(stat, 1);
	#define DBUF_STAT_BUMPDOWN(stat) \
	DBUF_STAT_INCR(stat, -1);
	#define DBUF_STAT_MAX(stat, v) { \
	uint64_t _m; \
	while ((v) > (_m = dbuf_stats.stat.value.ui64) && \
	(_m != atomic_cas_64(&dbuf_stats.stat.value.ui64, _m, (v))))\
	continue; \
	}

	static void dbuf_write(dbuf_dirty_record_t dr, arc_buf_t data, dmu_tx_t *tx);
	static void dbuf_sync_leaf_verify_bonus_dnode(dbuf_dirty_record_t *dr);
	static int dbuf_read_verify_dnode_crypt(dmu_buf_impl_t *db, uint32_t flags);

	/*
	* Global data structures and functions for the dbuf cache.
	*/
	static kmem_cache_t *dbuf_kmem_cache;
	static taskq_t *dbu_evict_taskq;

	static kthread_t *dbuf_cache_evict_thread;
	static kmutex_t dbuf_evict_lock;
	static kcondvar_t dbuf_evict_cv;
	static boolean_t dbuf_evict_thread_exit;

	/*
	* There are two dbuf caches; each dbuf can only be in one of them at a time.
	*
	* 1. Cache of metadata dbufs, to help make read-heavy administrative commands
	* from /sbin/zfs run faster. The "metadata cache" specifically stores dbufs
	* that represent the metadata that describes filesystems/snapshots/
	* bookmarks/properties/etc. We only evict from this cache when we export a
	* pool, to short-circuit as much I/O as possible for all administrative
	* commands that need the metadata. There is no eviction policy for this
	* cache, because we try to only include types in it which would occupy a
	* very small amount of space per object but create a large impact on the
	* performance of these commands. Instead, after it reaches a maximum size
	* (which should only happen on very small memory systems with a very large
	* number of filesystem objects), we stop taking new dbufs into the
	* metadata cache, instead putting them in the normal dbuf cache.
	*
	* 2. LRU cache of dbufs. The dbuf cache maintains a list of dbufs that
	* are not currently held but have been recently released. These dbufs
	* are not eligible for arc eviction until they are aged out of the cache.
	* Dbufs that are aged out of the cache will be immediately destroyed and
	* become eligible for arc eviction.
	*
	* Dbufs are added to these caches once the last hold is released. If a dbuf is
	* later accessed and still exists in the dbuf cache, then it will be removed
	* from the cache and later re-added to the head of the cache.
	*
	* If a given dbuf meets the requirements for the metadata cache, it will go
	* there, otherwise it will be considered for the generic LRU dbuf cache. The
	* caches and the refcounts tracking their sizes are stored in an array indexed
	* by those caches' matching enum values (from dbuf_cached_state_t).
	*/
	typedef struct dbuf_cache {
	multilist_t cache;
	zfs_refcount_t size ____cacheline_aligned;
	} dbuf_cache_t;
	dbuf_cache_t dbuf_caches[DB_CACHE_MAX];

	/* Size limits for the caches */
	static uint64_t dbuf_cache_max_bytes = UINT64_MAX;
	static uint64_t dbuf_metadata_cache_max_bytes = UINT64_MAX;

	/* Set the default sizes of the caches to log2 fraction of arc size */
	static uint_t dbuf_cache_shift = 5;
	static uint_t dbuf_metadata_cache_shift = 6;

	/* Set the dbuf hash mutex count as log2 shift (dynamic by default) */
	static uint_t dbuf_mutex_cache_shift = 0;

	static unsigned long dbuf_cache_target_bytes(void);
	static unsigned long dbuf_metadata_cache_target_bytes(void);

	/*
	* The LRU dbuf cache uses a three-stage eviction policy:
	* - A low water marker designates when the dbuf eviction thread
	* should stop evicting from the dbuf cache.
	* - When we reach the maximum size (aka mid water mark), we
	* signal the eviction thread to run.
	* - The high water mark indicates when the eviction thread
	* is unable to keep up with the incoming load and eviction must
	* happen in the context of the calling thread.
	*
	* The dbuf cache:
	* (max size)
	* low water mid water hi water
	* +----------------------------------------+----------+----------+
	* \| \| \| \|
	* \| \| \| \|
	* \| \| \| \|
	* \| \| \| \|
	* +----------------------------------------+----------+----------+
	* stop signal evict
	* evicting eviction directly
	* thread
	*
	* The high and low water marks indicate the operating range for the eviction
	* thread. The low water mark is, by default, 90% of the total size of the
	* cache and the high water mark is at 110% (both of these percentages can be
	* changed by setting dbuf_cache_lowater_pct and dbuf_cache_hiwater_pct,
	* respectively). The eviction thread will try to ensure that the cache remains
	* within this range by waking up every second and checking if the cache is
	* above the low water mark. The thread can also be woken up by callers adding
	* elements into the cache if the cache is larger than the mid water (i.e max
	* cache size). Once the eviction thread is woken up and eviction is required,
	* it will continue evicting buffers until it's able to reduce the cache size
	* to the low water mark. If the cache size continues to grow and hits the high
	* water mark, then callers adding elements to the cache will begin to evict
	* directly from the cache until the cache is no longer above the high water
	* mark.
	*/

	/*
	* The percentage above and below the maximum cache size.
	*/
	static uint_t dbuf_cache_hiwater_pct = 10;
	static uint_t dbuf_cache_lowater_pct = 10;

	static int
	dbuf_cons(void vdb, void unused, int kmflag)
	{
	(void) unused, (void) kmflag;
	dmu_buf_impl_t *db = vdb;
	memset(db, 0, sizeof (dmu_buf_impl_t));

	mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL);
	rw_init(&db->db_rwlock, NULL, RW_DEFAULT, NULL);
	cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL);
	multilist_link_init(&db->db_cache_link);
	zfs_refcount_create(&db->db_holds);

	return (0);
	}

	static void
	dbuf_dest(void vdb, void unused)
	{
	(void) unused;
	dmu_buf_impl_t *db = vdb;
	mutex_destroy(&db->db_mtx);
	rw_destroy(&db->db_rwlock);
	cv_destroy(&db->db_changed);
	ASSERT(!multilist_link_active(&db->db_cache_link));
	zfs_refcount_destroy(&db->db_holds);
	}

	/*
	* dbuf hash table routines
	*/
	static dbuf_hash_table_t dbuf_hash_table;

	/*
	* We use Cityhash for this. It's fast, and has good hash properties without
	* requiring any large static buffers.
	*/
	static uint64_t
	dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid)
	{
	return (cityhash4((uintptr_t)os, obj, (uint64_t)lvl, blkid));
	}

	#define DTRACE_SET_STATE(db, why) \
	DTRACE_PROBE2(dbuf__state_change, dmu_buf_impl_t *, db, \
	const char *, why)

	#define DBUF_EQUAL(dbuf, os, obj, level, blkid) \
	((dbuf)->db.db_object == (obj) && \
	(dbuf)->db_objset == (os) && \
	(dbuf)->db_level == (level) && \
	(dbuf)->db_blkid == (blkid))

	dmu_buf_impl_t *
	dbuf_find(objset_t *os, uint64_t obj, uint8_t level, uint64_t blkid,
	uint64_t *hash_out)
	{
	dbuf_hash_table_t *h = &dbuf_hash_table;
	uint64_t hv;
	uint64_t idx;
	dmu_buf_impl_t *db;

	hv = dbuf_hash(os, obj, level, blkid);
	idx = hv & h->hash_table_mask;

	mutex_enter(DBUF_HASH_MUTEX(h, idx));
	for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) {
	if (DBUF_EQUAL(db, os, obj, level, blkid)) {
	mutex_enter(&db->db_mtx);
	if (db->db_state != DB_EVICTING) {
	mutex_exit(DBUF_HASH_MUTEX(h, idx));
	return (db);
	}
	mutex_exit(&db->db_mtx);
	}
	}
	mutex_exit(DBUF_HASH_MUTEX(h, idx));
	if (hash_out != NULL)
	*hash_out = hv;
	return (NULL);
	}

	static dmu_buf_impl_t *
	dbuf_find_bonus(objset_t *os, uint64_t object)
	{
	dnode_t *dn;
	dmu_buf_impl_t *db = NULL;

	if (dnode_hold(os, object, FTAG, &dn) == 0) {
	rw_enter(&dn->dn_struct_rwlock, RW_READER);
	if (dn->dn_bonus != NULL) {
	db = dn->dn_bonus;
	mutex_enter(&db->db_mtx);
	}
	rw_exit(&dn->dn_struct_rwlock);
	dnode_rele(dn, FTAG);
	}
	return (db);
	}

	/*
	* Insert an entry into the hash table. If there is already an element
	* equal to elem in the hash table, then the already existing element
	* will be returned and the new element will not be inserted.
	* Otherwise returns NULL.
	*/
	static dmu_buf_impl_t *
	dbuf_hash_insert(dmu_buf_impl_t *db)
	{
	dbuf_hash_table_t *h = &dbuf_hash_table;
	objset_t *os = db->db_objset;
	uint64_t obj = db->db.db_object;
	int level = db->db_level;
	uint64_t blkid, idx;
	dmu_buf_impl_t *dbf;
	uint32_t i;

	blkid = db->db_blkid;
	ASSERT3U(dbuf_hash(os, obj, level, blkid), ==, db->db_hash);
	idx = db->db_hash & h->hash_table_mask;

	mutex_enter(DBUF_HASH_MUTEX(h, idx));
	for (dbf = h->hash_table[idx], i = 0; dbf != NULL;
	dbf = dbf->db_hash_next, i++) {
	if (DBUF_EQUAL(dbf, os, obj, level, blkid)) {
	mutex_enter(&dbf->db_mtx);
	if (dbf->db_state != DB_EVICTING) {
	mutex_exit(DBUF_HASH_MUTEX(h, idx));
	return (dbf);
	}
	mutex_exit(&dbf->db_mtx);
	}
	}

	if (i > 0) {
	DBUF_STAT_BUMP(hash_collisions);
	if (i == 1)
	DBUF_STAT_BUMP(hash_chains);

	DBUF_STAT_MAX(hash_chain_max, i);
	}

	mutex_enter(&db->db_mtx);
	db->db_hash_next = h->hash_table[idx];
	h->hash_table[idx] = db;
	mutex_exit(DBUF_HASH_MUTEX(h, idx));
	uint64_t he = atomic_inc_64_nv(&dbuf_stats.hash_elements.value.ui64);
	DBUF_STAT_MAX(hash_elements_max, he);

	return (NULL);
	}

	/*
	* This returns whether this dbuf should be stored in the metadata cache, which
	* is based on whether it's from one of the dnode types that store data related
	* to traversing dataset hierarchies.
	*/
	static boolean_t
	dbuf_include_in_metadata_cache(dmu_buf_impl_t *db)
	{
	DB_DNODE_ENTER(db);
	dmu_object_type_t type = DB_DNODE(db)->dn_type;
	DB_DNODE_EXIT(db);

	/* Check if this dbuf is one of the types we care about */
	if (DMU_OT_IS_METADATA_CACHED(type)) {
	/* If we hit this, then we set something up wrong in dmu_ot */
	ASSERT(DMU_OT_IS_METADATA(type));

	/*
	* Sanity check for small-memory systems: don't allocate too
	* much memory for this purpose.
	*/
	if (zfs_refcount_count(
	&dbuf_caches[DB_DBUF_METADATA_CACHE].size) >
	dbuf_metadata_cache_target_bytes()) {
	DBUF_STAT_BUMP(metadata_cache_overflow);
	return (B_FALSE);
	}

	return (B_TRUE);
	}

	return (B_FALSE);
	}

	/*
	* Remove an entry from the hash table. It must be in the EVICTING state.
	*/
	static void
	dbuf_hash_remove(dmu_buf_impl_t *db)
	{
	dbuf_hash_table_t *h = &dbuf_hash_table;
	uint64_t idx;
	dmu_buf_impl_t dbf, *dbp;

	ASSERT3U(dbuf_hash(db->db_objset, db->db.db_object, db->db_level,
	db->db_blkid), ==, db->db_hash);
	idx = db->db_hash & h->hash_table_mask;

	/*
	* We mustn't hold db_mtx to maintain lock ordering:
	* DBUF_HASH_MUTEX > db_mtx.
	*/
	ASSERT(zfs_refcount_is_zero(&db->db_holds));
	ASSERT(db->db_state == DB_EVICTING);
	ASSERT(!MUTEX_HELD(&db->db_mtx));

	mutex_enter(DBUF_HASH_MUTEX(h, idx));
	dbp = &h->hash_table[idx];
	while ((dbf = *dbp) != db) {
	dbp = &dbf->db_hash_next;
	ASSERT(dbf != NULL);
	}
	*dbp = db->db_hash_next;
	db->db_hash_next = NULL;
	if (h->hash_table[idx] &&
	h->hash_table[idx]->db_hash_next == NULL)
	DBUF_STAT_BUMPDOWN(hash_chains);
	mutex_exit(DBUF_HASH_MUTEX(h, idx));
	atomic_dec_64(&dbuf_stats.hash_elements.value.ui64);
	}

	typedef enum {
	DBVU_EVICTING,
	DBVU_NOT_EVICTING
	} dbvu_verify_type_t;

	static void
	dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type)
	{
	#ifdef ZFS_DEBUG
	int64_t holds;

	if (db->db_user == NULL)
	return;

	/* Only data blocks support the attachment of user data. */
	ASSERT(db->db_level == 0);

	/* Clients must resolve a dbuf before attaching user data. */
	ASSERT(db->db.db_data != NULL);
	ASSERT3U(db->db_state, ==, DB_CACHED);

	holds = zfs_refcount_count(&db->db_holds);
	if (verify_type == DBVU_EVICTING) {
	/*
	* Immediate eviction occurs when holds == dirtycnt.
	* For normal eviction buffers, holds is zero on
	* eviction, except when dbuf_fix_old_data() calls
	* dbuf_clear_data(). However, the hold count can grow
	* during eviction even though db_mtx is held (see
	* dmu_bonus_hold() for an example), so we can only
	* test the generic invariant that holds >= dirtycnt.
	*/
	ASSERT3U(holds, >=, db->db_dirtycnt);
	} else {
	if (db->db_user_immediate_evict == TRUE)
	ASSERT3U(holds, >=, db->db_dirtycnt);
	else
	ASSERT3U(holds, >, 0);
	}
	#endif
	}

	static void
	dbuf_evict_user(dmu_buf_impl_t *db)
	{
	dmu_buf_user_t *dbu = db->db_user;

	ASSERT(MUTEX_HELD(&db->db_mtx));

	if (dbu == NULL)
	return;

	dbuf_verify_user(db, DBVU_EVICTING);
	db->db_user = NULL;

	#ifdef ZFS_DEBUG
	if (dbu->dbu_clear_on_evict_dbufp != NULL)
	*dbu->dbu_clear_on_evict_dbufp = NULL;
	#endif

	/*
	* There are two eviction callbacks - one that we call synchronously
	* and one that we invoke via a taskq. The async one is useful for
	* avoiding lock order reversals and limiting stack depth.
	*
	* Note that if we have a sync callback but no async callback,
	* it's likely that the sync callback will free the structure
	* containing the dbu. In that case we need to take care to not
	* dereference dbu after calling the sync evict func.
	*/
	boolean_t has_async = (dbu->dbu_evict_func_async != NULL);

	if (dbu->dbu_evict_func_sync != NULL)
	dbu->dbu_evict_func_sync(dbu);

	if (has_async) {
	taskq_dispatch_ent(dbu_evict_taskq, dbu->dbu_evict_func_async,
	dbu, 0, &dbu->dbu_tqent);
	}
	}

	boolean_t
	dbuf_is_metadata(dmu_buf_impl_t *db)
	{
	/*
	* Consider indirect blocks and spill blocks to be meta data.
	*/
	if (db->db_level > 0 \|\| db->db_blkid == DMU_SPILL_BLKID) {
	return (B_TRUE);
	} else {
	boolean_t is_metadata;

	DB_DNODE_ENTER(db);
	is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type);
	DB_DNODE_EXIT(db);

	return (is_metadata);
	}
	}

	/*
	* We want to exclude buffers that are on a special allocation class from
	* L2ARC.
	*/
	boolean_t
	dbuf_is_l2cacheable(dmu_buf_impl_t *db)
	{
	if (db->db_objset->os_secondary_cache == ZFS_CACHE_ALL \|\|
	(db->db_objset->os_secondary_cache ==
	ZFS_CACHE_METADATA && dbuf_is_metadata(db))) {
	if (l2arc_exclude_special == 0)
	return (B_TRUE);

	blkptr_t *bp = db->db_blkptr;
	if (bp == NULL \|\| BP_IS_HOLE(bp))
	return (B_FALSE);
	uint64_t vdev = DVA_GET_VDEV(bp->blk_dva);
	vdev_t *rvd = db->db_objset->os_spa->spa_root_vdev;
	vdev_t *vd = NULL;

	if (vdev < rvd->vdev_children)
	vd = rvd->vdev_child[vdev];

	if (vd == NULL)
	return (B_TRUE);

	if (vd->vdev_alloc_bias != VDEV_BIAS_SPECIAL &&
	vd->vdev_alloc_bias != VDEV_BIAS_DEDUP)
	return (B_TRUE);
	}
	return (B_FALSE);
	}

	static inline boolean_t
	dnode_level_is_l2cacheable(blkptr_t bp, dnode_t dn, int64_t level)
	{
	if (dn->dn_objset->os_secondary_cache == ZFS_CACHE_ALL \|\|
	(dn->dn_objset->os_secondary_cache == ZFS_CACHE_METADATA &&
	(level > 0 \|\|
	DMU_OT_IS_METADATA(dn->dn_handle->dnh_dnode->dn_type)))) {
	if (l2arc_exclude_special == 0)
	return (B_TRUE);

	if (bp == NULL \|\| BP_IS_HOLE(bp))
	return (B_FALSE);
	uint64_t vdev = DVA_GET_VDEV(bp->blk_dva);
	vdev_t *rvd = dn->dn_objset->os_spa->spa_root_vdev;
	vdev_t *vd = NULL;

	if (vdev < rvd->vdev_children)
	vd = rvd->vdev_child[vdev];

	if (vd == NULL)
	return (B_TRUE);

	if (vd->vdev_alloc_bias != VDEV_BIAS_SPECIAL &&
	vd->vdev_alloc_bias != VDEV_BIAS_DEDUP)
	return (B_TRUE);
	}
	return (B_FALSE);
	}


	/*
	* This function must return indices evenly distributed between all
	* sublists of the multilist. This is needed due to how the dbuf eviction
	* code is laid out; dbuf_evict_thread() assumes dbufs are evenly
	* distributed between all sublists and uses this assumption when
	* deciding which sublist to evict from and how much to evict from it.
	*/
	static unsigned int
	dbuf_cache_multilist_index_func(multilist_t ml, void obj)
	{
	dmu_buf_impl_t *db = obj;

	/*
	* The assumption here, is the hash value for a given
	* dmu_buf_impl_t will remain constant throughout it's lifetime
	* (i.e. it's objset, object, level and blkid fields don't change).
	* Thus, we don't need to store the dbuf's sublist index
	* on insertion, as this index can be recalculated on removal.
	*
	* Also, the low order bits of the hash value are thought to be
	* distributed evenly. Otherwise, in the case that the multilist
	* has a power of two number of sublists, each sublists' usage
	* would not be evenly distributed. In this context full 64bit
	* division would be a waste of time, so limit it to 32 bits.
	*/
	return ((unsigned int)dbuf_hash(db->db_objset, db->db.db_object,
	db->db_level, db->db_blkid) %
	multilist_get_num_sublists(ml));
	}

	/*
	* The target size of the dbuf cache can grow with the ARC target,
	* unless limited by the tunable dbuf_cache_max_bytes.
	*/
	static inline unsigned long
	dbuf_cache_target_bytes(void)
	{
	return (MIN(dbuf_cache_max_bytes,
	arc_target_bytes() >> dbuf_cache_shift));
	}

	/*
	* The target size of the dbuf metadata cache can grow with the ARC target,
	* unless limited by the tunable dbuf_metadata_cache_max_bytes.
	*/
	static inline unsigned long
	dbuf_metadata_cache_target_bytes(void)
	{
	return (MIN(dbuf_metadata_cache_max_bytes,
	arc_target_bytes() >> dbuf_metadata_cache_shift));
	}

	static inline uint64_t
	dbuf_cache_hiwater_bytes(void)
	{
	uint64_t dbuf_cache_target = dbuf_cache_target_bytes();
	return (dbuf_cache_target +
	(dbuf_cache_target * dbuf_cache_hiwater_pct) / 100);
	}

	static inline uint64_t
	dbuf_cache_lowater_bytes(void)
	{
	uint64_t dbuf_cache_target = dbuf_cache_target_bytes();
	return (dbuf_cache_target -
	(dbuf_cache_target * dbuf_cache_lowater_pct) / 100);
	}

	static inline boolean_t
	dbuf_cache_above_lowater(void)
	{
	return (zfs_refcount_count(&dbuf_caches[DB_DBUF_CACHE].size) >
	dbuf_cache_lowater_bytes());
	}

	/*
	* Evict the oldest eligible dbuf from the dbuf cache.
	*/
	static void
	dbuf_evict_one(void)
	{
	int idx = multilist_get_random_index(&dbuf_caches[DB_DBUF_CACHE].cache);
	- multilist_sublist_t *mls = multilist_sublist_lock(
	+ multilist_sublist_t *mls = multilist_sublist_lock_idx(
	&dbuf_caches[DB_DBUF_CACHE].cache, idx);

	ASSERT(!MUTEX_HELD(&dbuf_evict_lock));

	dmu_buf_impl_t *db = multilist_sublist_tail(mls);
	while (db != NULL && mutex_tryenter(&db->db_mtx) == 0) {
	db = multilist_sublist_prev(mls, db);
	}

	DTRACE_PROBE2(dbuf__evict__one, dmu_buf_impl_t *, db,
	multilist_sublist_t *, mls);

	if (db != NULL) {
	multilist_sublist_remove(mls, db);
	multilist_sublist_unlock(mls);
	(void) zfs_refcount_remove_many(
	&dbuf_caches[DB_DBUF_CACHE].size, db->db.db_size, db);
	DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]);
	DBUF_STAT_BUMPDOWN(cache_count);
	DBUF_STAT_DECR(cache_levels_bytes[db->db_level],
	db->db.db_size);
	ASSERT3U(db->db_caching_status, ==, DB_DBUF_CACHE);
	db->db_caching_status = DB_NO_CACHE;
	dbuf_destroy(db);
	DBUF_STAT_BUMP(cache_total_evicts);
	} else {
	multilist_sublist_unlock(mls);
	}
	}

	/*
	* The dbuf evict thread is responsible for aging out dbufs from the
	* cache. Once the cache has reached it's maximum size, dbufs are removed
	* and destroyed. The eviction thread will continue running until the size
	* of the dbuf cache is at or below the maximum size. Once the dbuf is aged
	* out of the cache it is destroyed and becomes eligible for arc eviction.
	*/
	static __attribute__((noreturn)) void
	dbuf_evict_thread(void *unused)
	{
	(void) unused;
	callb_cpr_t cpr;

	CALLB_CPR_INIT(&cpr, &dbuf_evict_lock, callb_generic_cpr, FTAG);

	mutex_enter(&dbuf_evict_lock);
	while (!dbuf_evict_thread_exit) {
	while (!dbuf_cache_above_lowater() && !dbuf_evict_thread_exit) {
	CALLB_CPR_SAFE_BEGIN(&cpr);
	(void) cv_timedwait_idle_hires(&dbuf_evict_cv,
	&dbuf_evict_lock, SEC2NSEC(1), MSEC2NSEC(1), 0);
	CALLB_CPR_SAFE_END(&cpr, &dbuf_evict_lock);
	}
	mutex_exit(&dbuf_evict_lock);

	/*
	* Keep evicting as long as we're above the low water mark
	* for the cache. We do this without holding the locks to
	* minimize lock contention.
	*/
	while (dbuf_cache_above_lowater() && !dbuf_evict_thread_exit) {
	dbuf_evict_one();
	}

	mutex_enter(&dbuf_evict_lock);
	}

	dbuf_evict_thread_exit = B_FALSE;
	cv_broadcast(&dbuf_evict_cv);
	CALLB_CPR_EXIT(&cpr); /* drops dbuf_evict_lock */
	thread_exit();
	}

	/*
	* Wake up the dbuf eviction thread if the dbuf cache is at its max size.
	* If the dbuf cache is at its high water mark, then evict a dbuf from the
	* dbuf cache using the caller's context.
	*/
	static void
	dbuf_evict_notify(uint64_t size)
	{
	/*
	* We check if we should evict without holding the dbuf_evict_lock,
	* because it's OK to occasionally make the wrong decision here,
	* and grabbing the lock results in massive lock contention.
	*/
	if (size > dbuf_cache_target_bytes()) {
	if (size > dbuf_cache_hiwater_bytes())
	dbuf_evict_one();
	cv_signal(&dbuf_evict_cv);
	}
	}

	static int
	dbuf_kstat_update(kstat_t *ksp, int rw)
	{
	dbuf_stats_t *ds = ksp->ks_data;
	dbuf_hash_table_t *h = &dbuf_hash_table;

	if (rw == KSTAT_WRITE)
	return (SET_ERROR(EACCES));

	ds->cache_count.value.ui64 =
	wmsum_value(&dbuf_sums.cache_count);
	ds->cache_size_bytes.value.ui64 =
	zfs_refcount_count(&dbuf_caches[DB_DBUF_CACHE].size);
	ds->cache_target_bytes.value.ui64 = dbuf_cache_target_bytes();
	ds->cache_hiwater_bytes.value.ui64 = dbuf_cache_hiwater_bytes();
	ds->cache_lowater_bytes.value.ui64 = dbuf_cache_lowater_bytes();
	ds->cache_total_evicts.value.ui64 =
	wmsum_value(&dbuf_sums.cache_total_evicts);
	for (int i = 0; i < DN_MAX_LEVELS; i++) {
	ds->cache_levels[i].value.ui64 =
	wmsum_value(&dbuf_sums.cache_levels[i]);
	ds->cache_levels_bytes[i].value.ui64 =
	wmsum_value(&dbuf_sums.cache_levels_bytes[i]);
	}
	ds->hash_hits.value.ui64 =
	wmsum_value(&dbuf_sums.hash_hits);
	ds->hash_misses.value.ui64 =
	wmsum_value(&dbuf_sums.hash_misses);
	ds->hash_collisions.value.ui64 =
	wmsum_value(&dbuf_sums.hash_collisions);
	ds->hash_chains.value.ui64 =
	wmsum_value(&dbuf_sums.hash_chains);
	ds->hash_insert_race.value.ui64 =
	wmsum_value(&dbuf_sums.hash_insert_race);
	ds->hash_table_count.value.ui64 = h->hash_table_mask + 1;
	ds->hash_mutex_count.value.ui64 = h->hash_mutex_mask + 1;
	ds->metadata_cache_count.value.ui64 =
	wmsum_value(&dbuf_sums.metadata_cache_count);
	ds->metadata_cache_size_bytes.value.ui64 = zfs_refcount_count(
	&dbuf_caches[DB_DBUF_METADATA_CACHE].size);
	ds->metadata_cache_overflow.value.ui64 =
	wmsum_value(&dbuf_sums.metadata_cache_overflow);
	return (0);
	}

	void
	dbuf_init(void)
	{
	uint64_t hmsize, hsize = 1ULL << 16;
	dbuf_hash_table_t *h = &dbuf_hash_table;

	/*
	* The hash table is big enough to fill one eighth of physical memory
	* with an average block size of zfs_arc_average_blocksize (default 8K).
	* By default, the table will take up
	* totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
	*/
	while (hsize * zfs_arc_average_blocksize < arc_all_memory() / 8)
	hsize <<= 1;

	h->hash_table = NULL;
	while (h->hash_table == NULL) {
	h->hash_table_mask = hsize - 1;

	h->hash_table = vmem_zalloc(hsize * sizeof (void *), KM_SLEEP);
	if (h->hash_table == NULL)
	hsize >>= 1;

	ASSERT3U(hsize, >=, 1ULL << 10);
	}

	/*
	* The hash table buckets are protected by an array of mutexes where
	* each mutex is reponsible for protecting 128 buckets. A minimum
	* array size of 8192 is targeted to avoid contention.
	*/
	if (dbuf_mutex_cache_shift == 0)
	hmsize = MAX(hsize >> 7, 1ULL << 13);
	else
	hmsize = 1ULL << MIN(dbuf_mutex_cache_shift, 24);

	h->hash_mutexes = NULL;
	while (h->hash_mutexes == NULL) {
	h->hash_mutex_mask = hmsize - 1;

	h->hash_mutexes = vmem_zalloc(hmsize * sizeof (kmutex_t),
	KM_SLEEP);
	if (h->hash_mutexes == NULL)
	hmsize >>= 1;
	}

	dbuf_kmem_cache = kmem_cache_create("dmu_buf_impl_t",
	sizeof (dmu_buf_impl_t),
	0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0);

	for (int i = 0; i < hmsize; i++)
	mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL);

	dbuf_stats_init(h);

	/*
	* All entries are queued via taskq_dispatch_ent(), so min/maxalloc
	* configuration is not required.
	*/
	dbu_evict_taskq = taskq_create("dbu_evict", 1, defclsyspri, 0, 0, 0);

	for (dbuf_cached_state_t dcs = 0; dcs < DB_CACHE_MAX; dcs++) {
	multilist_create(&dbuf_caches[dcs].cache,
	sizeof (dmu_buf_impl_t),
	offsetof(dmu_buf_impl_t, db_cache_link),
	dbuf_cache_multilist_index_func);
	zfs_refcount_create(&dbuf_caches[dcs].size);
	}

	dbuf_evict_thread_exit = B_FALSE;
	mutex_init(&dbuf_evict_lock, NULL, MUTEX_DEFAULT, NULL);
	cv_init(&dbuf_evict_cv, NULL, CV_DEFAULT, NULL);
	dbuf_cache_evict_thread = thread_create(NULL, 0, dbuf_evict_thread,
	NULL, 0, &p0, TS_RUN, minclsyspri);

	wmsum_init(&dbuf_sums.cache_count, 0);
	wmsum_init(&dbuf_sums.cache_total_evicts, 0);
	for (int i = 0; i < DN_MAX_LEVELS; i++) {
	wmsum_init(&dbuf_sums.cache_levels[i], 0);
	wmsum_init(&dbuf_sums.cache_levels_bytes[i], 0);
	}
	wmsum_init(&dbuf_sums.hash_hits, 0);
	wmsum_init(&dbuf_sums.hash_misses, 0);
	wmsum_init(&dbuf_sums.hash_collisions, 0);
	wmsum_init(&dbuf_sums.hash_chains, 0);
	wmsum_init(&dbuf_sums.hash_insert_race, 0);
	wmsum_init(&dbuf_sums.metadata_cache_count, 0);
	wmsum_init(&dbuf_sums.metadata_cache_overflow, 0);

	dbuf_ksp = kstat_create("zfs", 0, "dbufstats", "misc",
	KSTAT_TYPE_NAMED, sizeof (dbuf_stats) / sizeof (kstat_named_t),
	KSTAT_FLAG_VIRTUAL);
	if (dbuf_ksp != NULL) {
	for (int i = 0; i < DN_MAX_LEVELS; i++) {
	snprintf(dbuf_stats.cache_levels[i].name,
	KSTAT_STRLEN, "cache_level_%d", i);
	dbuf_stats.cache_levels[i].data_type =
	KSTAT_DATA_UINT64;
	snprintf(dbuf_stats.cache_levels_bytes[i].name,
	KSTAT_STRLEN, "cache_level_%d_bytes", i);
	dbuf_stats.cache_levels_bytes[i].data_type =
	KSTAT_DATA_UINT64;
	}
	dbuf_ksp->ks_data = &dbuf_stats;
	dbuf_ksp->ks_update = dbuf_kstat_update;
	kstat_install(dbuf_ksp);
	}
	}

	void
	dbuf_fini(void)
	{
	dbuf_hash_table_t *h = &dbuf_hash_table;

	dbuf_stats_destroy();

	for (int i = 0; i < (h->hash_mutex_mask + 1); i++)
	mutex_destroy(&h->hash_mutexes[i]);

	vmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *));
	vmem_free(h->hash_mutexes, (h->hash_mutex_mask + 1) *
	sizeof (kmutex_t));

	kmem_cache_destroy(dbuf_kmem_cache);
	taskq_destroy(dbu_evict_taskq);

	mutex_enter(&dbuf_evict_lock);
	dbuf_evict_thread_exit = B_TRUE;
	while (dbuf_evict_thread_exit) {
	cv_signal(&dbuf_evict_cv);
	cv_wait(&dbuf_evict_cv, &dbuf_evict_lock);
	}
	mutex_exit(&dbuf_evict_lock);

	mutex_destroy(&dbuf_evict_lock);
	cv_destroy(&dbuf_evict_cv);

	for (dbuf_cached_state_t dcs = 0; dcs < DB_CACHE_MAX; dcs++) {
	zfs_refcount_destroy(&dbuf_caches[dcs].size);
	multilist_destroy(&dbuf_caches[dcs].cache);
	}

	if (dbuf_ksp != NULL) {
	kstat_delete(dbuf_ksp);
	dbuf_ksp = NULL;
	}

	wmsum_fini(&dbuf_sums.cache_count);
	wmsum_fini(&dbuf_sums.cache_total_evicts);
	for (int i = 0; i < DN_MAX_LEVELS; i++) {
	wmsum_fini(&dbuf_sums.cache_levels[i]);
	wmsum_fini(&dbuf_sums.cache_levels_bytes[i]);
	}
	wmsum_fini(&dbuf_sums.hash_hits);
	wmsum_fini(&dbuf_sums.hash_misses);
	wmsum_fini(&dbuf_sums.hash_collisions);
	wmsum_fini(&dbuf_sums.hash_chains);
	wmsum_fini(&dbuf_sums.hash_insert_race);
	wmsum_fini(&dbuf_sums.metadata_cache_count);
	wmsum_fini(&dbuf_sums.metadata_cache_overflow);
	}

	/*
	* Other stuff.
	*/

	#ifdef ZFS_DEBUG
	static void
	dbuf_verify(dmu_buf_impl_t *db)
	{
	dnode_t *dn;
	dbuf_dirty_record_t *dr;
	uint32_t txg_prev;

	ASSERT(MUTEX_HELD(&db->db_mtx));

	if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY))
	return;

	ASSERT(db->db_objset != NULL);
	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);
	if (dn == NULL) {
	ASSERT(db->db_parent == NULL);
	ASSERT(db->db_blkptr == NULL);
	} else {
	ASSERT3U(db->db.db_object, ==, dn->dn_object);
	ASSERT3P(db->db_objset, ==, dn->dn_objset);
	ASSERT3U(db->db_level, <, dn->dn_nlevels);
	ASSERT(db->db_blkid == DMU_BONUS_BLKID \|\|
	db->db_blkid == DMU_SPILL_BLKID \|\|
	!avl_is_empty(&dn->dn_dbufs));
	}
	if (db->db_blkid == DMU_BONUS_BLKID) {
	ASSERT(dn != NULL);
	ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
	ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID);
	} else if (db->db_blkid == DMU_SPILL_BLKID) {
	ASSERT(dn != NULL);
	ASSERT0(db->db.db_offset);
	} else {
	ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size);
	}

	if ((dr = list_head(&db->db_dirty_records)) != NULL) {
	ASSERT(dr->dr_dbuf == db);
	txg_prev = dr->dr_txg;
	for (dr = list_next(&db->db_dirty_records, dr); dr != NULL;
	dr = list_next(&db->db_dirty_records, dr)) {
	ASSERT(dr->dr_dbuf == db);
	ASSERT(txg_prev > dr->dr_txg);
	txg_prev = dr->dr_txg;
	}
	}

	/*
	* We can't assert that db_size matches dn_datablksz because it
	* can be momentarily different when another thread is doing
	* dnode_set_blksz().
	*/
	if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) {
	dr = db->db_data_pending;
	/*
	* It should only be modified in syncing context, so
	* make sure we only have one copy of the data.
	*/
	ASSERT(dr == NULL \|\| dr->dt.dl.dr_data == db->db_buf);
	}

	/* verify db->db_blkptr */
	if (db->db_blkptr) {
	if (db->db_parent == dn->dn_dbuf) {
	/* db is pointed to by the dnode */
	/* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */
	if (DMU_OBJECT_IS_SPECIAL(db->db.db_object))
	ASSERT(db->db_parent == NULL);
	else
	ASSERT(db->db_parent != NULL);
	if (db->db_blkid != DMU_SPILL_BLKID)
	ASSERT3P(db->db_blkptr, ==,
	&dn->dn_phys->dn_blkptr[db->db_blkid]);
	} else {
	/* db is pointed to by an indirect block */
	int epb __maybe_unused = db->db_parent->db.db_size >>
	SPA_BLKPTRSHIFT;
	ASSERT3U(db->db_parent->db_level, ==, db->db_level+1);
	ASSERT3U(db->db_parent->db.db_object, ==,
	db->db.db_object);
	/*
	* dnode_grow_indblksz() can make this fail if we don't
	* have the parent's rwlock. XXX indblksz no longer
	* grows. safe to do this now?
	*/
	if (RW_LOCK_HELD(&db->db_parent->db_rwlock)) {
	ASSERT3P(db->db_blkptr, ==,
	((blkptr_t *)db->db_parent->db.db_data +
	db->db_blkid % epb));
	}
	}
	}
	if ((db->db_blkptr == NULL \|\| BP_IS_HOLE(db->db_blkptr)) &&
	(db->db_buf == NULL \|\| db->db_buf->b_data) &&
	db->db.db_data && db->db_blkid != DMU_BONUS_BLKID &&
	db->db_state != DB_FILL && (dn == NULL \|\| !dn->dn_free_txg)) {
	/*
	* If the blkptr isn't set but they have nonzero data,
	* it had better be dirty, otherwise we'll lose that
	* data when we evict this buffer.
	*
	* There is an exception to this rule for indirect blocks; in
	* this case, if the indirect block is a hole, we fill in a few
	* fields on each of the child blocks (importantly, birth time)
	* to prevent hole birth times from being lost when you
	* partially fill in a hole.
	*/
	if (db->db_dirtycnt == 0) {
	if (db->db_level == 0) {
	uint64_t *buf = db->db.db_data;
	int i;

	for (i = 0; i < db->db.db_size >> 3; i++) {
	ASSERT(buf[i] == 0);
	}
	} else {
	blkptr_t *bps = db->db.db_data;
	ASSERT3U(1 << DB_DNODE(db)->dn_indblkshift, ==,
	db->db.db_size);
	/*
	* We want to verify that all the blkptrs in the
	* indirect block are holes, but we may have
	* automatically set up a few fields for them.
	* We iterate through each blkptr and verify
	* they only have those fields set.
	*/
	for (int i = 0;
	i < db->db.db_size / sizeof (blkptr_t);
	i++) {
	blkptr_t *bp = &bps[i];
	ASSERT(ZIO_CHECKSUM_IS_ZERO(
	&bp->blk_cksum));
	ASSERT(
	DVA_IS_EMPTY(&bp->blk_dva[0]) &&
	DVA_IS_EMPTY(&bp->blk_dva[1]) &&
	DVA_IS_EMPTY(&bp->blk_dva[2]));
	ASSERT0(bp->blk_fill);
	ASSERT0(bp->blk_pad[0]);
	ASSERT0(bp->blk_pad[1]);
	ASSERT(!BP_IS_EMBEDDED(bp));
	ASSERT(BP_IS_HOLE(bp));
	ASSERT0(bp->blk_phys_birth);
	}
	}
	}
	}
	DB_DNODE_EXIT(db);
	}
	#endif

	static void
	dbuf_clear_data(dmu_buf_impl_t *db)
	{
	ASSERT(MUTEX_HELD(&db->db_mtx));
	dbuf_evict_user(db);
	ASSERT3P(db->db_buf, ==, NULL);
	db->db.db_data = NULL;
	if (db->db_state != DB_NOFILL) {
	db->db_state = DB_UNCACHED;
	DTRACE_SET_STATE(db, "clear data");
	}
	}

	static void
	dbuf_set_data(dmu_buf_impl_t db, arc_buf_t buf)
	{
	ASSERT(MUTEX_HELD(&db->db_mtx));
	ASSERT(buf != NULL);

	db->db_buf = buf;
	ASSERT(buf->b_data != NULL);
	db->db.db_data = buf->b_data;
	}

	static arc_buf_t *
	dbuf_alloc_arcbuf(dmu_buf_impl_t *db)
	{
	spa_t *spa = db->db_objset->os_spa;

	return (arc_alloc_buf(spa, db, DBUF_GET_BUFC_TYPE(db), db->db.db_size));
	}

	/*
	* Loan out an arc_buf for read. Return the loaned arc_buf.
	*/
	arc_buf_t *
	dbuf_loan_arcbuf(dmu_buf_impl_t *db)
	{
	arc_buf_t *abuf;

	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
	mutex_enter(&db->db_mtx);
	if (arc_released(db->db_buf) \|\| zfs_refcount_count(&db->db_holds) > 1) {
	int blksz = db->db.db_size;
	spa_t *spa = db->db_objset->os_spa;

	mutex_exit(&db->db_mtx);
	abuf = arc_loan_buf(spa, B_FALSE, blksz);
	memcpy(abuf->b_data, db->db.db_data, blksz);
	} else {
	abuf = db->db_buf;
	arc_loan_inuse_buf(abuf, db);
	db->db_buf = NULL;
	dbuf_clear_data(db);
	mutex_exit(&db->db_mtx);
	}
	return (abuf);
	}

	/*
	* Calculate which level n block references the data at the level 0 offset
	* provided.
	*/
	uint64_t
	dbuf_whichblock(const dnode_t *dn, const int64_t level, const uint64_t offset)
	{
	if (dn->dn_datablkshift != 0 && dn->dn_indblkshift != 0) {
	/*
	* The level n blkid is equal to the level 0 blkid divided by
	* the number of level 0s in a level n block.
	*
	* The level 0 blkid is offset >> datablkshift =
	* offset / 2^datablkshift.
	*
	* The number of level 0s in a level n is the number of block
	* pointers in an indirect block, raised to the power of level.
	* This is 2^(indblkshift - SPA_BLKPTRSHIFT)^level =
	* 2^(level*(indblkshift - SPA_BLKPTRSHIFT)).
	*
	* Thus, the level n blkid is: offset /
	* ((2^datablkshift)(2^(level(indblkshift-SPA_BLKPTRSHIFT))))
	* = offset / 2^(datablkshift + level *
	* (indblkshift - SPA_BLKPTRSHIFT))
	* = offset >> (datablkshift + level *
	* (indblkshift - SPA_BLKPTRSHIFT))
	*/

	const unsigned exp = dn->dn_datablkshift +
	level * (dn->dn_indblkshift - SPA_BLKPTRSHIFT);

	if (exp >= 8 * sizeof (offset)) {
	/* This only happens on the highest indirection level */
	ASSERT3U(level, ==, dn->dn_nlevels - 1);
	return (0);
	}

	ASSERT3U(exp, <, 8 * sizeof (offset));

	return (offset >> exp);
	} else {
	ASSERT3U(offset, <, dn->dn_datablksz);
	return (0);
	}
	}

	/*
	* This function is used to lock the parent of the provided dbuf. This should be
	* used when modifying or reading db_blkptr.
	*/
	db_lock_type_t
	dmu_buf_lock_parent(dmu_buf_impl_t db, krw_t rw, const void tag)
	{
	enum db_lock_type ret = DLT_NONE;
	if (db->db_parent != NULL) {
	rw_enter(&db->db_parent->db_rwlock, rw);
	ret = DLT_PARENT;
	} else if (dmu_objset_ds(db->db_objset) != NULL) {
	rrw_enter(&dmu_objset_ds(db->db_objset)->ds_bp_rwlock, rw,
	tag);
	ret = DLT_OBJSET;
	}
	/*
	* We only return a DLT_NONE lock when it's the top-most indirect block
	* of the meta-dnode of the MOS.
	*/
	return (ret);
	}

	/*
	* We need to pass the lock type in because it's possible that the block will
	* move from being the topmost indirect block in a dnode (and thus, have no
	* parent) to not the top-most via an indirection increase. This would cause a
	* panic if we didn't pass the lock type in.
	*/
	void
	dmu_buf_unlock_parent(dmu_buf_impl_t db, db_lock_type_t type, const void tag)
	{
	if (type == DLT_PARENT)
	rw_exit(&db->db_parent->db_rwlock);
	else if (type == DLT_OBJSET)
	rrw_exit(&dmu_objset_ds(db->db_objset)->ds_bp_rwlock, tag);
	}

	static void
	dbuf_read_done(zio_t zio, const zbookmark_phys_t zb, const blkptr_t *bp,
	arc_buf_t buf, void vdb)
	{
	(void) zb, (void) bp;
	dmu_buf_impl_t *db = vdb;

	mutex_enter(&db->db_mtx);
	ASSERT3U(db->db_state, ==, DB_READ);
	/*
	* All reads are synchronous, so we must have a hold on the dbuf
	*/
	ASSERT(zfs_refcount_count(&db->db_holds) > 0);
	ASSERT(db->db_buf == NULL);
	ASSERT(db->db.db_data == NULL);
	if (buf == NULL) {
	/* i/o error */
	ASSERT(zio == NULL \|\| zio->io_error != 0);
	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
	ASSERT3P(db->db_buf, ==, NULL);
	db->db_state = DB_UNCACHED;
	DTRACE_SET_STATE(db, "i/o error");
	} else if (db->db_level == 0 && db->db_freed_in_flight) {
	/* freed in flight */
	ASSERT(zio == NULL \|\| zio->io_error == 0);
	arc_release(buf, db);
	memset(buf->b_data, 0, db->db.db_size);
	arc_buf_freeze(buf);
	db->db_freed_in_flight = FALSE;
	dbuf_set_data(db, buf);
	db->db_state = DB_CACHED;
	DTRACE_SET_STATE(db, "freed in flight");
	} else {
	/* success */
	ASSERT(zio == NULL \|\| zio->io_error == 0);
	dbuf_set_data(db, buf);
	db->db_state = DB_CACHED;
	DTRACE_SET_STATE(db, "successful read");
	}
	cv_broadcast(&db->db_changed);
	dbuf_rele_and_unlock(db, NULL, B_FALSE);
	}

	/*
	* Shortcut for performing reads on bonus dbufs. Returns
	* an error if we fail to verify the dnode associated with
	* a decrypted block. Otherwise success.
	*/
	static int
	dbuf_read_bonus(dmu_buf_impl_t db, dnode_t dn, uint32_t flags)
	{
	int bonuslen, max_bonuslen, err;

	err = dbuf_read_verify_dnode_crypt(db, flags);
	if (err)
	return (err);

	bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen);
	max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots);
	ASSERT(MUTEX_HELD(&db->db_mtx));
	ASSERT(DB_DNODE_HELD(db));
	ASSERT3U(bonuslen, <=, db->db.db_size);
	db->db.db_data = kmem_alloc(max_bonuslen, KM_SLEEP);
	arc_space_consume(max_bonuslen, ARC_SPACE_BONUS);
	if (bonuslen < max_bonuslen)
	memset(db->db.db_data, 0, max_bonuslen);
	if (bonuslen)
	memcpy(db->db.db_data, DN_BONUS(dn->dn_phys), bonuslen);
	db->db_state = DB_CACHED;
	DTRACE_SET_STATE(db, "bonus buffer filled");
	return (0);
	}

	static void
	dbuf_handle_indirect_hole(dmu_buf_impl_t db, dnode_t dn, blkptr_t *dbbp)
	{
	blkptr_t *bps = db->db.db_data;
	uint32_t indbs = 1ULL << dn->dn_indblkshift;
	int n_bps = indbs >> SPA_BLKPTRSHIFT;

	for (int i = 0; i < n_bps; i++) {
	blkptr_t *bp = &bps[i];

	ASSERT3U(BP_GET_LSIZE(dbbp), ==, indbs);
	BP_SET_LSIZE(bp, BP_GET_LEVEL(dbbp) == 1 ?
	dn->dn_datablksz : BP_GET_LSIZE(dbbp));
	BP_SET_TYPE(bp, BP_GET_TYPE(dbbp));
	BP_SET_LEVEL(bp, BP_GET_LEVEL(dbbp) - 1);
	BP_SET_BIRTH(bp, dbbp->blk_birth, 0);
	}
	}

	/*
	* Handle reads on dbufs that are holes, if necessary. This function
	* requires that the dbuf's mutex is held. Returns success (0) if action
	* was taken, ENOENT if no action was taken.
	*/
	static int
	dbuf_read_hole(dmu_buf_impl_t db, dnode_t dn, blkptr_t *bp)
	{
	ASSERT(MUTEX_HELD(&db->db_mtx));

	int is_hole = bp == NULL \|\| BP_IS_HOLE(bp);
	/*
	* For level 0 blocks only, if the above check fails:
	* Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync()
	* processes the delete record and clears the bp while we are waiting
	* for the dn_mtx (resulting in a "no" from block_freed).
	*/
	if (!is_hole && db->db_level == 0)
	is_hole = dnode_block_freed(dn, db->db_blkid) \|\| BP_IS_HOLE(bp);

	if (is_hole) {
	dbuf_set_data(db, dbuf_alloc_arcbuf(db));
	memset(db->db.db_data, 0, db->db.db_size);

	if (bp != NULL && db->db_level > 0 && BP_IS_HOLE(bp) &&
	bp->blk_birth != 0) {
	dbuf_handle_indirect_hole(db, dn, bp);
	}
	db->db_state = DB_CACHED;
	DTRACE_SET_STATE(db, "hole read satisfied");
	return (0);
	}
	return (ENOENT);
	}

	/*
	* This function ensures that, when doing a decrypting read of a block,
	* we make sure we have decrypted the dnode associated with it. We must do
	* this so that we ensure we are fully authenticating the checksum-of-MACs
	* tree from the root of the objset down to this block. Indirect blocks are
	* always verified against their secure checksum-of-MACs assuming that the
	* dnode containing them is correct. Now that we are doing a decrypting read,
	* we can be sure that the key is loaded and verify that assumption. This is
	* especially important considering that we always read encrypted dnode
	* blocks as raw data (without verifying their MACs) to start, and
	* decrypt / authenticate them when we need to read an encrypted bonus buffer.
	*/
	static int
	dbuf_read_verify_dnode_crypt(dmu_buf_impl_t *db, uint32_t flags)
	{
	int err = 0;
	objset_t *os = db->db_objset;
	arc_buf_t *dnode_abuf;
	dnode_t *dn;
	zbookmark_phys_t zb;

	ASSERT(MUTEX_HELD(&db->db_mtx));

	if ((flags & DB_RF_NO_DECRYPT) != 0 \|\|
	!os->os_encrypted \|\| os->os_raw_receive)
	return (0);

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);
	dnode_abuf = (dn->dn_dbuf != NULL) ? dn->dn_dbuf->db_buf : NULL;

	if (dnode_abuf == NULL \|\| !arc_is_encrypted(dnode_abuf)) {
	DB_DNODE_EXIT(db);
	return (0);
	}

	SET_BOOKMARK(&zb, dmu_objset_id(os),
	DMU_META_DNODE_OBJECT, 0, dn->dn_dbuf->db_blkid);
	err = arc_untransform(dnode_abuf, os->os_spa, &zb, B_TRUE);

	/*
	* An error code of EACCES tells us that the key is still not
	* available. This is ok if we are only reading authenticated
	* (and therefore non-encrypted) blocks.
	*/
	if (err == EACCES && ((db->db_blkid != DMU_BONUS_BLKID &&
	!DMU_OT_IS_ENCRYPTED(dn->dn_type)) \|\|
	(db->db_blkid == DMU_BONUS_BLKID &&
	!DMU_OT_IS_ENCRYPTED(dn->dn_bonustype))))
	err = 0;

	DB_DNODE_EXIT(db);

	return (err);
	}

	/*
	* Drops db_mtx and the parent lock specified by dblt and tag before
	* returning.
	*/
	static int
	-dbuf_read_impl(dmu_buf_impl_t db, zio_t zio, uint32_t flags,
	+dbuf_read_impl(dmu_buf_impl_t db, dnode_t dn, zio_t *zio, uint32_t flags,
	db_lock_type_t dblt, const void *tag)
	{
	- dnode_t *dn;
	zbookmark_phys_t zb;
	uint32_t aflags = ARC_FLAG_NOWAIT;
	int err, zio_flags;
	- blkptr_t bp, *bpp;
	+ blkptr_t bp, *bpp = NULL;

	- DB_DNODE_ENTER(db);
	- dn = DB_DNODE(db);
	ASSERT(!zfs_refcount_is_zero(&db->db_holds));
	ASSERT(MUTEX_HELD(&db->db_mtx));
	ASSERT(db->db_state == DB_UNCACHED \|\| db->db_state == DB_NOFILL);
	ASSERT(db->db_buf == NULL);
	ASSERT(db->db_parent == NULL \|\|
	RW_LOCK_HELD(&db->db_parent->db_rwlock));

	if (db->db_blkid == DMU_BONUS_BLKID) {
	err = dbuf_read_bonus(db, dn, flags);
	goto early_unlock;
	}

	- if (db->db_state == DB_UNCACHED) {
	- if (db->db_blkptr == NULL) {
	- bpp = NULL;
	- } else {
	- bp = *db->db_blkptr;
	+ /*
	+ * If we have a pending block clone, we don't want to read the
	+ * underlying block, but the content of the block being cloned,
	+ * pointed by the dirty record, so we have the most recent data.
	+ * If there is no dirty record, then we hit a race in a sync
	+ * process when the dirty record is already removed, while the
	+ * dbuf is not yet destroyed. Such case is equivalent to uncached.
	+ */
	+ if (db->db_state == DB_NOFILL) {
	+ dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records);
	+ if (dr != NULL) {
	+ if (!dr->dt.dl.dr_brtwrite) {
	+ err = EIO;
	+ goto early_unlock;
	+ }
	+ bp = dr->dt.dl.dr_overridden_by;
	bpp = &bp;
	}
	- } else {
	- dbuf_dirty_record_t *dr;
	-
	- ASSERT3S(db->db_state, ==, DB_NOFILL);
	+ }

	- /*
	- * Block cloning: If we have a pending block clone,
	- * we don't want to read the underlying block, but the content
	- * of the block being cloned, so we have the most recent data.
	- */
	- dr = list_head(&db->db_dirty_records);
	- if (dr == NULL \|\| !dr->dt.dl.dr_brtwrite) {
	- err = EIO;
	- goto early_unlock;
	- }
	- bp = dr->dt.dl.dr_overridden_by;
	+ if (bpp == NULL && db->db_blkptr != NULL) {
	+ bp = *db->db_blkptr;
	bpp = &bp;
	}

	err = dbuf_read_hole(db, dn, bpp);
	if (err == 0)
	goto early_unlock;

	ASSERT(bpp != NULL);

	/*
	* Any attempt to read a redacted block should result in an error. This
	* will never happen under normal conditions, but can be useful for
	* debugging purposes.
	*/
	if (BP_IS_REDACTED(bpp)) {
	ASSERT(dsl_dataset_feature_is_active(
	db->db_objset->os_dsl_dataset,
	SPA_FEATURE_REDACTED_DATASETS));
	err = SET_ERROR(EIO);
	goto early_unlock;
	}

	SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset),
	db->db.db_object, db->db_level, db->db_blkid);

	/*
	* All bps of an encrypted os should have the encryption bit set.
	* If this is not true it indicates tampering and we report an error.
	*/
	if (db->db_objset->os_encrypted && !BP_USES_CRYPT(bpp)) {
	spa_log_error(db->db_objset->os_spa, &zb, &bpp->blk_birth);
	err = SET_ERROR(EIO);
	goto early_unlock;
	}

	err = dbuf_read_verify_dnode_crypt(db, flags);
	if (err != 0)
	goto early_unlock;

	- DB_DNODE_EXIT(db);
	-
	db->db_state = DB_READ;
	DTRACE_SET_STATE(db, "read issued");
	mutex_exit(&db->db_mtx);

	if (!DBUF_IS_CACHEABLE(db))
	aflags \|= ARC_FLAG_UNCACHED;
	else if (dbuf_is_l2cacheable(db))
	aflags \|= ARC_FLAG_L2CACHE;

	dbuf_add_ref(db, NULL);

	zio_flags = (flags & DB_RF_CANFAIL) ?
	ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED;

	if ((flags & DB_RF_NO_DECRYPT) && BP_IS_PROTECTED(db->db_blkptr))
	zio_flags \|= ZIO_FLAG_RAW;
	/*
	* The zio layer will copy the provided blkptr later, but we have our
	* own copy so that we can release the parent's rwlock. We have to
	* do that so that if dbuf_read_done is called synchronously (on
	* an l1 cache hit) we don't acquire the db_mtx while holding the
	* parent's rwlock, which would be a lock ordering violation.
	*/
	dmu_buf_unlock_parent(db, dblt, tag);
	- (void) arc_read(zio, db->db_objset->os_spa, bpp,
	+ return (arc_read(zio, db->db_objset->os_spa, bpp,
	dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, zio_flags,
	- &aflags, &zb);
	- return (err);
	+ &aflags, &zb));
	+
	early_unlock:
	- DB_DNODE_EXIT(db);
	mutex_exit(&db->db_mtx);
	dmu_buf_unlock_parent(db, dblt, tag);
	return (err);
	}

	/*
	* This is our just-in-time copy function. It makes a copy of buffers that
	* have been modified in a previous transaction group before we access them in
	* the current active group.
	*
	* This function is used in three places: when we are dirtying a buffer for the
	* first time in a txg, when we are freeing a range in a dnode that includes
	* this buffer, and when we are accessing a buffer which was received compressed
	* and later referenced in a WRITE_BYREF record.
	*
	* Note that when we are called from dbuf_free_range() we do not put a hold on
	* the buffer, we just traverse the active dbuf list for the dnode.
	*/
	static void
	dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
	{
	dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records);

	ASSERT(MUTEX_HELD(&db->db_mtx));
	ASSERT(db->db.db_data != NULL);
	ASSERT(db->db_level == 0);
	ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT);

	if (dr == NULL \|\|
	(dr->dt.dl.dr_data !=
	((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf)))
	return;

	/*
	* If the last dirty record for this dbuf has not yet synced
	* and its referencing the dbuf data, either:
	* reset the reference to point to a new copy,
	* or (if there a no active holders)
	* just null out the current db_data pointer.
	*/
	ASSERT3U(dr->dr_txg, >=, txg - 2);
	if (db->db_blkid == DMU_BONUS_BLKID) {
	dnode_t *dn = DB_DNODE(db);
	int bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots);
	dr->dt.dl.dr_data = kmem_alloc(bonuslen, KM_SLEEP);
	arc_space_consume(bonuslen, ARC_SPACE_BONUS);
	memcpy(dr->dt.dl.dr_data, db->db.db_data, bonuslen);
	} else if (zfs_refcount_count(&db->db_holds) > db->db_dirtycnt) {
	dnode_t *dn = DB_DNODE(db);
	int size = arc_buf_size(db->db_buf);
	arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
	spa_t *spa = db->db_objset->os_spa;
	enum zio_compress compress_type =
	arc_get_compression(db->db_buf);
	uint8_t complevel = arc_get_complevel(db->db_buf);

	if (arc_is_encrypted(db->db_buf)) {
	boolean_t byteorder;
	uint8_t salt[ZIO_DATA_SALT_LEN];
	uint8_t iv[ZIO_DATA_IV_LEN];
	uint8_t mac[ZIO_DATA_MAC_LEN];

	arc_get_raw_params(db->db_buf, &byteorder, salt,
	iv, mac);
	dr->dt.dl.dr_data = arc_alloc_raw_buf(spa, db,
	dmu_objset_id(dn->dn_objset), byteorder, salt, iv,
	mac, dn->dn_type, size, arc_buf_lsize(db->db_buf),
	compress_type, complevel);
	} else if (compress_type != ZIO_COMPRESS_OFF) {
	ASSERT3U(type, ==, ARC_BUFC_DATA);
	dr->dt.dl.dr_data = arc_alloc_compressed_buf(spa, db,
	size, arc_buf_lsize(db->db_buf), compress_type,
	complevel);
	} else {
	dr->dt.dl.dr_data = arc_alloc_buf(spa, db, type, size);
	}
	memcpy(dr->dt.dl.dr_data->b_data, db->db.db_data, size);
	} else {
	db->db_buf = NULL;
	dbuf_clear_data(db);
	}
	}

	int
	-dbuf_read(dmu_buf_impl_t db, zio_t zio, uint32_t flags)
	+dbuf_read(dmu_buf_impl_t db, zio_t pio, uint32_t flags)
	{
	int err = 0;
	boolean_t prefetch;
	dnode_t *dn;

	/*
	* We don't have to hold the mutex to check db_state because it
	* can't be freed while we have a hold on the buffer.
	*/
	ASSERT(!zfs_refcount_is_zero(&db->db_holds));

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);

	prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
	- (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL;
	+ (flags & DB_RF_NOPREFETCH) == 0;

	mutex_enter(&db->db_mtx);
	if (flags & DB_RF_PARTIAL_FIRST)
	db->db_partial_read = B_TRUE;
	else if (!(flags & DB_RF_PARTIAL_MORE))
	db->db_partial_read = B_FALSE;
	if (db->db_state == DB_CACHED) {
	/*
	* Ensure that this block's dnode has been decrypted if
	* the caller has requested decrypted data.
	*/
	err = dbuf_read_verify_dnode_crypt(db, flags);

	/*
	* If the arc buf is compressed or encrypted and the caller
	* requested uncompressed data, we need to untransform it
	* before returning. We also call arc_untransform() on any
	* unauthenticated blocks, which will verify their MAC if
	* the key is now available.
	*/
	if (err == 0 && db->db_buf != NULL &&
	(flags & DB_RF_NO_DECRYPT) == 0 &&
	(arc_is_encrypted(db->db_buf) \|\|
	arc_is_unauthenticated(db->db_buf) \|\|
	arc_get_compression(db->db_buf) != ZIO_COMPRESS_OFF)) {
	spa_t *spa = dn->dn_objset->os_spa;
	zbookmark_phys_t zb;

	SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset),
	db->db.db_object, db->db_level, db->db_blkid);
	dbuf_fix_old_data(db, spa_syncing_txg(spa));
	err = arc_untransform(db->db_buf, spa, &zb, B_FALSE);
	dbuf_set_data(db, db->db_buf);
	}
	mutex_exit(&db->db_mtx);
	if (err == 0 && prefetch) {
	dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE,
	B_FALSE, flags & DB_RF_HAVESTRUCT);
	}
	DB_DNODE_EXIT(db);
	DBUF_STAT_BUMP(hash_hits);
	} else if (db->db_state == DB_UNCACHED \|\| db->db_state == DB_NOFILL) {
	boolean_t need_wait = B_FALSE;

	db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_READER, FTAG);

	- if (zio == NULL && (db->db_state == DB_NOFILL \|\|
	+ if (pio == NULL && (db->db_state == DB_NOFILL \|\|
	(db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)))) {
	spa_t *spa = dn->dn_objset->os_spa;
	- zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL);
	+ pio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL);
	need_wait = B_TRUE;
	}
	- err = dbuf_read_impl(db, zio, flags, dblt, FTAG);
	+ err = dbuf_read_impl(db, dn, pio, flags, dblt, FTAG);
	/*
	* dbuf_read_impl has dropped db_mtx and our parent's rwlock
	* for us
	*/
	if (!err && prefetch) {
	dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE,
	db->db_state != DB_CACHED,
	flags & DB_RF_HAVESTRUCT);
	}

	DB_DNODE_EXIT(db);
	DBUF_STAT_BUMP(hash_misses);

	/*
	* If we created a zio_root we must execute it to avoid
	* leaking it, even if it isn't attached to any work due
	* to an error in dbuf_read_impl().
	*/
	if (need_wait) {
	if (err == 0)
	- err = zio_wait(zio);
	+ err = zio_wait(pio);
	else
	- VERIFY0(zio_wait(zio));
	+ (void) zio_wait(pio);
	+ pio = NULL;
	}
	} else {
	/*
	* Another reader came in while the dbuf was in flight
	* between UNCACHED and CACHED. Either a writer will finish
	* writing the buffer (sending the dbuf to CACHED) or the
	* first reader's request will reach the read_done callback
	* and send the dbuf to CACHED. Otherwise, a failure
	* occurred and the dbuf went to UNCACHED.
	*/
	mutex_exit(&db->db_mtx);
	if (prefetch) {
	dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE,
	B_TRUE, flags & DB_RF_HAVESTRUCT);
	}
	DB_DNODE_EXIT(db);
	DBUF_STAT_BUMP(hash_misses);

	/* Skip the wait per the caller's request. */
	if ((flags & DB_RF_NEVERWAIT) == 0) {
	mutex_enter(&db->db_mtx);
	while (db->db_state == DB_READ \|\|
	db->db_state == DB_FILL) {
	ASSERT(db->db_state == DB_READ \|\|
	(flags & DB_RF_HAVESTRUCT) == 0);
	DTRACE_PROBE2(blocked__read, dmu_buf_impl_t *,
	- db, zio_t *, zio);
	+ db, zio_t *, pio);
	cv_wait(&db->db_changed, &db->db_mtx);
	}
	if (db->db_state == DB_UNCACHED)
	err = SET_ERROR(EIO);
	mutex_exit(&db->db_mtx);
	}
	}

	+ if (pio && err != 0) {
	+ zio_t *zio = zio_null(pio, pio->io_spa, NULL, NULL, NULL,
	+ ZIO_FLAG_CANFAIL);
	+ zio->io_error = err;
	+ zio_nowait(zio);
	+ }
	+
	return (err);
	}

	static void
	dbuf_noread(dmu_buf_impl_t *db)
	{
	ASSERT(!zfs_refcount_is_zero(&db->db_holds));
	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
	mutex_enter(&db->db_mtx);
	while (db->db_state == DB_READ \|\| db->db_state == DB_FILL)
	cv_wait(&db->db_changed, &db->db_mtx);
	if (db->db_state == DB_UNCACHED) {
	ASSERT(db->db_buf == NULL);
	ASSERT(db->db.db_data == NULL);
	dbuf_set_data(db, dbuf_alloc_arcbuf(db));
	db->db_state = DB_FILL;
	DTRACE_SET_STATE(db, "assigning filled buffer");
	} else if (db->db_state == DB_NOFILL) {
	dbuf_clear_data(db);
	} else {
	ASSERT3U(db->db_state, ==, DB_CACHED);
	}
	mutex_exit(&db->db_mtx);
	}

	void
	dbuf_unoverride(dbuf_dirty_record_t *dr)
	{
	dmu_buf_impl_t *db = dr->dr_dbuf;
	blkptr_t *bp = &dr->dt.dl.dr_overridden_by;
	uint64_t txg = dr->dr_txg;

	ASSERT(MUTEX_HELD(&db->db_mtx));
	/*
	* This assert is valid because dmu_sync() expects to be called by
	* a zilog's get_data while holding a range lock. This call only
	* comes from dbuf_dirty() callers who must also hold a range lock.
	*/
	ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC);
	ASSERT(db->db_level == 0);

	if (db->db_blkid == DMU_BONUS_BLKID \|\|
	dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN)
	return;

	ASSERT(db->db_data_pending != dr);

	/* free this block */
	if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite)
	zio_free(db->db_objset->os_spa, txg, bp);

	if (dr->dt.dl.dr_brtwrite) {
	ASSERT(dr->dt.dl.dr_data == NULL);
	dr->dt.dl.dr_data = db->db_buf;
	}
	dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
	dr->dt.dl.dr_nopwrite = B_FALSE;
	dr->dt.dl.dr_brtwrite = B_FALSE;
	dr->dt.dl.dr_has_raw_params = B_FALSE;

	/*
	* Release the already-written buffer, so we leave it in
	* a consistent dirty state. Note that all callers are
	* modifying the buffer, so they will immediately do
	* another (redundant) arc_release(). Therefore, leave
	* the buf thawed to save the effort of freezing &
	* immediately re-thawing it.
	*/
	if (dr->dt.dl.dr_data)
	arc_release(dr->dt.dl.dr_data, db);
	}

	/*
	* Evict (if its unreferenced) or clear (if its referenced) any level-0
	* data blocks in the free range, so that any future readers will find
	* empty blocks.
	*/
	void
	dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid,
	dmu_tx_t *tx)
	{
	dmu_buf_impl_t *db_search;
	dmu_buf_impl_t db, db_next;
	uint64_t txg = tx->tx_txg;
	avl_index_t where;
	dbuf_dirty_record_t *dr;

	if (end_blkid > dn->dn_maxblkid &&
	!(start_blkid == DMU_SPILL_BLKID \|\| end_blkid == DMU_SPILL_BLKID))
	end_blkid = dn->dn_maxblkid;
	dprintf_dnode(dn, "start=%llu end=%llu\n", (u_longlong_t)start_blkid,
	(u_longlong_t)end_blkid);

	db_search = kmem_alloc(sizeof (dmu_buf_impl_t), KM_SLEEP);
	db_search->db_level = 0;
	db_search->db_blkid = start_blkid;
	db_search->db_state = DB_SEARCH;

	mutex_enter(&dn->dn_dbufs_mtx);
	db = avl_find(&dn->dn_dbufs, db_search, &where);
	ASSERT3P(db, ==, NULL);

	db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);

	for (; db != NULL; db = db_next) {
	db_next = AVL_NEXT(&dn->dn_dbufs, db);
	ASSERT(db->db_blkid != DMU_BONUS_BLKID);

	if (db->db_level != 0 \|\| db->db_blkid > end_blkid) {
	break;
	}
	ASSERT3U(db->db_blkid, >=, start_blkid);

	/* found a level 0 buffer in the range */
	mutex_enter(&db->db_mtx);
	if (dbuf_undirty(db, tx)) {
	/* mutex has been dropped and dbuf destroyed */
	continue;
	}

	if (db->db_state == DB_UNCACHED \|\|
	db->db_state == DB_NOFILL \|\|
	db->db_state == DB_EVICTING) {
	ASSERT(db->db.db_data == NULL);
	mutex_exit(&db->db_mtx);
	continue;
	}
	if (db->db_state == DB_READ \|\| db->db_state == DB_FILL) {
	/* will be handled in dbuf_read_done or dbuf_rele */
	db->db_freed_in_flight = TRUE;
	mutex_exit(&db->db_mtx);
	continue;
	}
	if (zfs_refcount_count(&db->db_holds) == 0) {
	ASSERT(db->db_buf);
	dbuf_destroy(db);
	continue;
	}
	/* The dbuf is referenced */

	dr = list_head(&db->db_dirty_records);
	if (dr != NULL) {
	if (dr->dr_txg == txg) {
	/*
	* This buffer is "in-use", re-adjust the file
	* size to reflect that this buffer may
	* contain new data when we sync.
	*/
	if (db->db_blkid != DMU_SPILL_BLKID &&
	db->db_blkid > dn->dn_maxblkid)
	dn->dn_maxblkid = db->db_blkid;
	dbuf_unoverride(dr);
	} else {
	/*
	* This dbuf is not dirty in the open context.
	* Either uncache it (if its not referenced in
	* the open context) or reset its contents to
	* empty.
	*/
	dbuf_fix_old_data(db, txg);
	}
	}
	/* clear the contents if its cached */
	if (db->db_state == DB_CACHED) {
	ASSERT(db->db.db_data != NULL);
	arc_release(db->db_buf, db);
	rw_enter(&db->db_rwlock, RW_WRITER);
	memset(db->db.db_data, 0, db->db.db_size);
	rw_exit(&db->db_rwlock);
	arc_buf_freeze(db->db_buf);
	}

	mutex_exit(&db->db_mtx);
	}

	mutex_exit(&dn->dn_dbufs_mtx);
	kmem_free(db_search, sizeof (dmu_buf_impl_t));
	}

	void
	dbuf_new_size(dmu_buf_impl_t db, int size, dmu_tx_t tx)
	{
	arc_buf_t buf, old_buf;
	dbuf_dirty_record_t *dr;
	int osize = db->db.db_size;
	arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
	dnode_t *dn;

	ASSERT(db->db_blkid != DMU_BONUS_BLKID);

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);

	/*
	* XXX we should be doing a dbuf_read, checking the return
	* value and returning that up to our callers
	*/
	dmu_buf_will_dirty(&db->db, tx);

	/* create the data buffer for the new block */
	buf = arc_alloc_buf(dn->dn_objset->os_spa, db, type, size);

	/* copy old block data to the new block */
	old_buf = db->db_buf;
	memcpy(buf->b_data, old_buf->b_data, MIN(osize, size));
	/* zero the remainder */
	if (size > osize)
	memset((uint8_t *)buf->b_data + osize, 0, size - osize);

	mutex_enter(&db->db_mtx);
	dbuf_set_data(db, buf);
	arc_buf_destroy(old_buf, db);
	db->db.db_size = size;

	dr = list_head(&db->db_dirty_records);
	/* dirty record added by dmu_buf_will_dirty() */
	VERIFY(dr != NULL);
	if (db->db_level == 0)
	dr->dt.dl.dr_data = buf;
	ASSERT3U(dr->dr_txg, ==, tx->tx_txg);
	ASSERT3U(dr->dr_accounted, ==, osize);
	dr->dr_accounted = size;
	mutex_exit(&db->db_mtx);

	dmu_objset_willuse_space(dn->dn_objset, size - osize, tx);
	DB_DNODE_EXIT(db);
	}

	void
	dbuf_release_bp(dmu_buf_impl_t *db)
	{
	objset_t *os __maybe_unused = db->db_objset;

	ASSERT(dsl_pool_sync_context(dmu_objset_pool(os)));
	ASSERT(arc_released(os->os_phys_buf) \|\|
	list_link_active(&os->os_dsl_dataset->ds_synced_link));
	ASSERT(db->db_parent == NULL \|\| arc_released(db->db_parent->db_buf));

	(void) arc_release(db->db_buf, db);
	}

	/*
	* We already have a dirty record for this TXG, and we are being
	* dirtied again.
	*/
	static void
	dbuf_redirty(dbuf_dirty_record_t *dr)
	{
	dmu_buf_impl_t *db = dr->dr_dbuf;

	ASSERT(MUTEX_HELD(&db->db_mtx));

	if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) {
	/*
	* If this buffer has already been written out,
	* we now need to reset its state.
	*/
	dbuf_unoverride(dr);
	if (db->db.db_object != DMU_META_DNODE_OBJECT &&
	db->db_state != DB_NOFILL) {
	/* Already released on initial dirty, so just thaw. */
	ASSERT(arc_released(db->db_buf));
	arc_buf_thaw(db->db_buf);
	}
	}
	}

	dbuf_dirty_record_t *
	dbuf_dirty_lightweight(dnode_t dn, uint64_t blkid, dmu_tx_t tx)
	{
	rw_enter(&dn->dn_struct_rwlock, RW_READER);
	IMPLY(dn->dn_objset->os_raw_receive, dn->dn_maxblkid >= blkid);
	dnode_new_blkid(dn, blkid, tx, B_TRUE, B_FALSE);
	ASSERT(dn->dn_maxblkid >= blkid);

	dbuf_dirty_record_t dr = kmem_zalloc(sizeof (dr), KM_SLEEP);
	list_link_init(&dr->dr_dirty_node);
	list_link_init(&dr->dr_dbuf_node);
	dr->dr_dnode = dn;
	dr->dr_txg = tx->tx_txg;
	dr->dt.dll.dr_blkid = blkid;
	dr->dr_accounted = dn->dn_datablksz;

	/*
	* There should not be any dbuf for the block that we're dirtying.
	* Otherwise the buffer contents could be inconsistent between the
	* dbuf and the lightweight dirty record.
	*/
	ASSERT3P(NULL, ==, dbuf_find(dn->dn_objset, dn->dn_object, 0, blkid,
	NULL));

	mutex_enter(&dn->dn_mtx);
	int txgoff = tx->tx_txg & TXG_MASK;
	if (dn->dn_free_ranges[txgoff] != NULL) {
	range_tree_clear(dn->dn_free_ranges[txgoff], blkid, 1);
	}

	if (dn->dn_nlevels == 1) {
	ASSERT3U(blkid, <, dn->dn_nblkptr);
	list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
	mutex_exit(&dn->dn_mtx);
	rw_exit(&dn->dn_struct_rwlock);
	dnode_setdirty(dn, tx);
	} else {
	mutex_exit(&dn->dn_mtx);

	int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
	dmu_buf_impl_t *parent_db = dbuf_hold_level(dn,
	1, blkid >> epbs, FTAG);
	rw_exit(&dn->dn_struct_rwlock);
	if (parent_db == NULL) {
	kmem_free(dr, sizeof (*dr));
	return (NULL);
	}
	int err = dbuf_read(parent_db, NULL,
	(DB_RF_NOPREFETCH \| DB_RF_CANFAIL));
	if (err != 0) {
	dbuf_rele(parent_db, FTAG);
	kmem_free(dr, sizeof (*dr));
	return (NULL);
	}

	dbuf_dirty_record_t *parent_dr = dbuf_dirty(parent_db, tx);
	dbuf_rele(parent_db, FTAG);
	mutex_enter(&parent_dr->dt.di.dr_mtx);
	ASSERT3U(parent_dr->dr_txg, ==, tx->tx_txg);
	list_insert_tail(&parent_dr->dt.di.dr_children, dr);
	mutex_exit(&parent_dr->dt.di.dr_mtx);
	dr->dr_parent = parent_dr;
	}

	dmu_objset_willuse_space(dn->dn_objset, dr->dr_accounted, tx);

	return (dr);
	}

	dbuf_dirty_record_t *
	dbuf_dirty(dmu_buf_impl_t db, dmu_tx_t tx)
	{
	dnode_t *dn;
	objset_t *os;
	dbuf_dirty_record_t dr, dr_next, *dr_head;
	int txgoff = tx->tx_txg & TXG_MASK;
	boolean_t drop_struct_rwlock = B_FALSE;

	ASSERT(tx->tx_txg != 0);
	ASSERT(!zfs_refcount_is_zero(&db->db_holds));
	DMU_TX_DIRTY_BUF(tx, db);

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);
	/*
	* Shouldn't dirty a regular buffer in syncing context. Private
	* objects may be dirtied in syncing context, but only if they
	* were already pre-dirtied in open context.
	*/
	#ifdef ZFS_DEBUG
	if (dn->dn_objset->os_dsl_dataset != NULL) {
	rrw_enter(&dn->dn_objset->os_dsl_dataset->ds_bp_rwlock,
	RW_READER, FTAG);
	}
	ASSERT(!dmu_tx_is_syncing(tx) \|\|
	BP_IS_HOLE(dn->dn_objset->os_rootbp) \|\|
	DMU_OBJECT_IS_SPECIAL(dn->dn_object) \|\|
	dn->dn_objset->os_dsl_dataset == NULL);
	if (dn->dn_objset->os_dsl_dataset != NULL)
	rrw_exit(&dn->dn_objset->os_dsl_dataset->ds_bp_rwlock, FTAG);
	#endif
	/*
	* We make this assert for private objects as well, but after we
	* check if we're already dirty. They are allowed to re-dirty
	* in syncing context.
	*/
	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT \|\|
	dn->dn_dirtyctx == DN_UNDIRTIED \|\| dn->dn_dirtyctx ==
	(dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));

	mutex_enter(&db->db_mtx);
	/*
	* XXX make this true for indirects too? The problem is that
	* transactions created with dmu_tx_create_assigned() from
	* syncing context don't bother holding ahead.
	*/
	ASSERT(db->db_level != 0 \|\|
	db->db_state == DB_CACHED \|\| db->db_state == DB_FILL \|\|
	db->db_state == DB_NOFILL);

	mutex_enter(&dn->dn_mtx);
	dnode_set_dirtyctx(dn, tx, db);
	if (tx->tx_txg > dn->dn_dirty_txg)
	dn->dn_dirty_txg = tx->tx_txg;
	mutex_exit(&dn->dn_mtx);

	if (db->db_blkid == DMU_SPILL_BLKID)
	dn->dn_have_spill = B_TRUE;

	/*
	* If this buffer is already dirty, we're done.
	*/
	dr_head = list_head(&db->db_dirty_records);
	ASSERT(dr_head == NULL \|\| dr_head->dr_txg <= tx->tx_txg \|\|
	db->db.db_object == DMU_META_DNODE_OBJECT);
	dr_next = dbuf_find_dirty_lte(db, tx->tx_txg);
	if (dr_next && dr_next->dr_txg == tx->tx_txg) {
	DB_DNODE_EXIT(db);

	dbuf_redirty(dr_next);
	mutex_exit(&db->db_mtx);
	return (dr_next);
	}

	/*
	* Only valid if not already dirty.
	*/
	ASSERT(dn->dn_object == 0 \|\|
	dn->dn_dirtyctx == DN_UNDIRTIED \|\| dn->dn_dirtyctx ==
	(dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));

	ASSERT3U(dn->dn_nlevels, >, db->db_level);

	/*
	* We should only be dirtying in syncing context if it's the
	* mos or we're initializing the os or it's a special object.
	* However, we are allowed to dirty in syncing context provided
	* we already dirtied it in open context. Hence we must make
	* this assertion only if we're not already dirty.
	*/
	os = dn->dn_objset;
	VERIFY3U(tx->tx_txg, <=, spa_final_dirty_txg(os->os_spa));
	#ifdef ZFS_DEBUG
	if (dn->dn_objset->os_dsl_dataset != NULL)
	rrw_enter(&os->os_dsl_dataset->ds_bp_rwlock, RW_READER, FTAG);
	ASSERT(!dmu_tx_is_syncing(tx) \|\| DMU_OBJECT_IS_SPECIAL(dn->dn_object) \|\|
	os->os_dsl_dataset == NULL \|\| BP_IS_HOLE(os->os_rootbp));
	if (dn->dn_objset->os_dsl_dataset != NULL)
	rrw_exit(&os->os_dsl_dataset->ds_bp_rwlock, FTAG);
	#endif
	ASSERT(db->db.db_size != 0);

	dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);

	if (db->db_blkid != DMU_BONUS_BLKID && db->db_state != DB_NOFILL) {
	dmu_objset_willuse_space(os, db->db.db_size, tx);
	}

	/*
	* If this buffer is dirty in an old transaction group we need
	* to make a copy of it so that the changes we make in this
	* transaction group won't leak out when we sync the older txg.
	*/
	dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP);
	list_link_init(&dr->dr_dirty_node);
	list_link_init(&dr->dr_dbuf_node);
	dr->dr_dnode = dn;
	if (db->db_level == 0) {
	void *data_old = db->db_buf;

	if (db->db_state != DB_NOFILL) {
	if (db->db_blkid == DMU_BONUS_BLKID) {
	dbuf_fix_old_data(db, tx->tx_txg);
	data_old = db->db.db_data;
	} else if (db->db.db_object != DMU_META_DNODE_OBJECT) {
	/*
	* Release the data buffer from the cache so
	* that we can modify it without impacting
	* possible other users of this cached data
	* block. Note that indirect blocks and
	* private objects are not released until the
	* syncing state (since they are only modified
	* then).
	*/
	arc_release(db->db_buf, db);
	dbuf_fix_old_data(db, tx->tx_txg);
	data_old = db->db_buf;
	}
	ASSERT(data_old != NULL);
	}
	dr->dt.dl.dr_data = data_old;
	} else {
	mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_NOLOCKDEP, NULL);
	list_create(&dr->dt.di.dr_children,
	sizeof (dbuf_dirty_record_t),
	offsetof(dbuf_dirty_record_t, dr_dirty_node));
	}
	if (db->db_blkid != DMU_BONUS_BLKID && db->db_state != DB_NOFILL) {
	dr->dr_accounted = db->db.db_size;
	}
	dr->dr_dbuf = db;
	dr->dr_txg = tx->tx_txg;
	list_insert_before(&db->db_dirty_records, dr_next, dr);

	/*
	* We could have been freed_in_flight between the dbuf_noread
	* and dbuf_dirty. We win, as though the dbuf_noread() had
	* happened after the free.
	*/
	if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
	db->db_blkid != DMU_SPILL_BLKID) {
	mutex_enter(&dn->dn_mtx);
	if (dn->dn_free_ranges[txgoff] != NULL) {
	range_tree_clear(dn->dn_free_ranges[txgoff],
	db->db_blkid, 1);
	}
	mutex_exit(&dn->dn_mtx);
	db->db_freed_in_flight = FALSE;
	}

	/*
	* This buffer is now part of this txg
	*/
	dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg);
	db->db_dirtycnt += 1;
	ASSERT3U(db->db_dirtycnt, <=, 3);

	mutex_exit(&db->db_mtx);

	if (db->db_blkid == DMU_BONUS_BLKID \|\|
	db->db_blkid == DMU_SPILL_BLKID) {
	mutex_enter(&dn->dn_mtx);
	ASSERT(!list_link_active(&dr->dr_dirty_node));
	list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
	mutex_exit(&dn->dn_mtx);
	dnode_setdirty(dn, tx);
	DB_DNODE_EXIT(db);
	return (dr);
	}

	if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
	rw_enter(&dn->dn_struct_rwlock, RW_READER);
	drop_struct_rwlock = B_TRUE;
	}

	/*
	* If we are overwriting a dedup BP, then unless it is snapshotted,
	* when we get to syncing context we will need to decrement its
	* refcount in the DDT. Prefetch the relevant DDT block so that
	* syncing context won't have to wait for the i/o.
	*/
	if (db->db_blkptr != NULL) {
	db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_READER, FTAG);
	ddt_prefetch(os->os_spa, db->db_blkptr);
	dmu_buf_unlock_parent(db, dblt, FTAG);
	}

	/*
	* We need to hold the dn_struct_rwlock to make this assertion,
	* because it protects dn_phys / dn_next_nlevels from changing.
	*/
	ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) \|\|
	dn->dn_phys->dn_nlevels > db->db_level \|\|
	dn->dn_next_nlevels[txgoff] > db->db_level \|\|
	dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level \|\|
	dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level);


	if (db->db_level == 0) {
	ASSERT(!db->db_objset->os_raw_receive \|\|
	dn->dn_maxblkid >= db->db_blkid);
	dnode_new_blkid(dn, db->db_blkid, tx,
	drop_struct_rwlock, B_FALSE);
	ASSERT(dn->dn_maxblkid >= db->db_blkid);
	}

	if (db->db_level+1 < dn->dn_nlevels) {
	dmu_buf_impl_t *parent = db->db_parent;
	dbuf_dirty_record_t *di;
	int parent_held = FALSE;

	if (db->db_parent == NULL \|\| db->db_parent == dn->dn_dbuf) {
	int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
	parent = dbuf_hold_level(dn, db->db_level + 1,
	db->db_blkid >> epbs, FTAG);
	ASSERT(parent != NULL);
	parent_held = TRUE;
	}
	if (drop_struct_rwlock)
	rw_exit(&dn->dn_struct_rwlock);
	ASSERT3U(db->db_level + 1, ==, parent->db_level);
	di = dbuf_dirty(parent, tx);
	if (parent_held)
	dbuf_rele(parent, FTAG);

	mutex_enter(&db->db_mtx);
	/*
	* Since we've dropped the mutex, it's possible that
	* dbuf_undirty() might have changed this out from under us.
	*/
	if (list_head(&db->db_dirty_records) == dr \|\|
	dn->dn_object == DMU_META_DNODE_OBJECT) {
	mutex_enter(&di->dt.di.dr_mtx);
	ASSERT3U(di->dr_txg, ==, tx->tx_txg);
	ASSERT(!list_link_active(&dr->dr_dirty_node));
	list_insert_tail(&di->dt.di.dr_children, dr);
	mutex_exit(&di->dt.di.dr_mtx);
	dr->dr_parent = di;
	}
	mutex_exit(&db->db_mtx);
	} else {
	ASSERT(db->db_level + 1 == dn->dn_nlevels);
	ASSERT(db->db_blkid < dn->dn_nblkptr);
	ASSERT(db->db_parent == NULL \|\| db->db_parent == dn->dn_dbuf);
	mutex_enter(&dn->dn_mtx);
	ASSERT(!list_link_active(&dr->dr_dirty_node));
	list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
	mutex_exit(&dn->dn_mtx);
	if (drop_struct_rwlock)
	rw_exit(&dn->dn_struct_rwlock);
	}

	dnode_setdirty(dn, tx);
	DB_DNODE_EXIT(db);
	return (dr);
	}

	static void
	dbuf_undirty_bonus(dbuf_dirty_record_t *dr)
	{
	dmu_buf_impl_t *db = dr->dr_dbuf;

	if (dr->dt.dl.dr_data != db->db.db_data) {
	struct dnode *dn = dr->dr_dnode;
	int max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots);

	kmem_free(dr->dt.dl.dr_data, max_bonuslen);
	arc_space_return(max_bonuslen, ARC_SPACE_BONUS);
	}
	db->db_data_pending = NULL;
	ASSERT(list_next(&db->db_dirty_records, dr) == NULL);
	list_remove(&db->db_dirty_records, dr);
	if (dr->dr_dbuf->db_level != 0) {
	mutex_destroy(&dr->dt.di.dr_mtx);
	list_destroy(&dr->dt.di.dr_children);
	}
	kmem_free(dr, sizeof (dbuf_dirty_record_t));
	ASSERT3U(db->db_dirtycnt, >, 0);
	db->db_dirtycnt -= 1;
	}

	/*
	* Undirty a buffer in the transaction group referenced by the given
	* transaction. Return whether this evicted the dbuf.
	*/
	boolean_t
	dbuf_undirty(dmu_buf_impl_t db, dmu_tx_t tx)
	{
	uint64_t txg = tx->tx_txg;
	boolean_t brtwrite;

	ASSERT(txg != 0);

	/*
	* Due to our use of dn_nlevels below, this can only be called
	* in open context, unless we are operating on the MOS.
	* From syncing context, dn_nlevels may be different from the
	* dn_nlevels used when dbuf was dirtied.
	*/
	ASSERT(db->db_objset ==
	dmu_objset_pool(db->db_objset)->dp_meta_objset \|\|
	txg != spa_syncing_txg(dmu_objset_spa(db->db_objset)));
	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
	ASSERT0(db->db_level);
	ASSERT(MUTEX_HELD(&db->db_mtx));

	/*
	* If this buffer is not dirty, we're done.
	*/
	dbuf_dirty_record_t *dr = dbuf_find_dirty_eq(db, txg);
	if (dr == NULL)
	return (B_FALSE);
	ASSERT(dr->dr_dbuf == db);

	brtwrite = dr->dt.dl.dr_brtwrite;
	if (brtwrite) {
	/*
	* We are freeing a block that we cloned in the same
	* transaction group.
	*/
	brt_pending_remove(dmu_objset_spa(db->db_objset),
	&dr->dt.dl.dr_overridden_by, tx);
	}

	dnode_t *dn = dr->dr_dnode;

	dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);

	ASSERT(db->db.db_size != 0);

	dsl_pool_undirty_space(dmu_objset_pool(dn->dn_objset),
	dr->dr_accounted, txg);

	list_remove(&db->db_dirty_records, dr);

	/*
	* Note that there are three places in dbuf_dirty()
	* where this dirty record may be put on a list.
	* Make sure to do a list_remove corresponding to
	* every one of those list_insert calls.
	*/
	if (dr->dr_parent) {
	mutex_enter(&dr->dr_parent->dt.di.dr_mtx);
	list_remove(&dr->dr_parent->dt.di.dr_children, dr);
	mutex_exit(&dr->dr_parent->dt.di.dr_mtx);
	} else if (db->db_blkid == DMU_SPILL_BLKID \|\|
	db->db_level + 1 == dn->dn_nlevels) {
	ASSERT(db->db_blkptr == NULL \|\| db->db_parent == dn->dn_dbuf);
	mutex_enter(&dn->dn_mtx);
	list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr);
	mutex_exit(&dn->dn_mtx);
	}

	if (db->db_state != DB_NOFILL && !brtwrite) {
	dbuf_unoverride(dr);

	ASSERT(db->db_buf != NULL);
	ASSERT(dr->dt.dl.dr_data != NULL);
	if (dr->dt.dl.dr_data != db->db_buf)
	arc_buf_destroy(dr->dt.dl.dr_data, db);
	}

	kmem_free(dr, sizeof (dbuf_dirty_record_t));

	ASSERT(db->db_dirtycnt > 0);
	db->db_dirtycnt -= 1;

	if (zfs_refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) {
	ASSERT(db->db_state == DB_NOFILL \|\| brtwrite \|\|
	arc_released(db->db_buf));
	dbuf_destroy(db);
	return (B_TRUE);
	}

	return (B_FALSE);
	}

	static void
	dmu_buf_will_dirty_impl(dmu_buf_t db_fake, int flags, dmu_tx_t tx)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;
	boolean_t undirty = B_FALSE;

	ASSERT(tx->tx_txg != 0);
	ASSERT(!zfs_refcount_is_zero(&db->db_holds));

	/*
	- * Quick check for dirtiness. For already dirty blocks, this
	- * reduces runtime of this function by >90%, and overall performance
	- * by 50% for some workloads (e.g. file deletion with indirect blocks
	- * cached).
	+ * Quick check for dirtiness to improve performance for some workloads
	+ * (e.g. file deletion with indirect blocks cached).
	*/
	mutex_enter(&db->db_mtx);
	-
	if (db->db_state == DB_CACHED \|\| db->db_state == DB_NOFILL) {
	- dbuf_dirty_record_t *dr = dbuf_find_dirty_eq(db, tx->tx_txg);
	/*
	- * It's possible that it is already dirty but not cached,
	+ * It's possible that the dbuf is already dirty but not cached,
	* because there are some calls to dbuf_dirty() that don't
	* go through dmu_buf_will_dirty().
	*/
	+ dbuf_dirty_record_t *dr = dbuf_find_dirty_eq(db, tx->tx_txg);
	if (dr != NULL) {
	- if (dr->dt.dl.dr_brtwrite) {
	+ if (db->db_level == 0 &&
	+ dr->dt.dl.dr_brtwrite) {
	/*
	* Block cloning: If we are dirtying a cloned
	- * block, we cannot simply redirty it, because
	- * this dr has no data associated with it.
	+ * level 0 block, we cannot simply redirty it,
	+ * because this dr has no associated data.
	* We will go through a full undirtying below,
	* before dirtying it again.
	*/
	undirty = B_TRUE;
	} else {
	/* This dbuf is already dirty and cached. */
	dbuf_redirty(dr);
	mutex_exit(&db->db_mtx);
	return;
	}
	}
	}
	mutex_exit(&db->db_mtx);

	DB_DNODE_ENTER(db);
	if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock))
	flags \|= DB_RF_HAVESTRUCT;
	DB_DNODE_EXIT(db);

	/*
	* Block cloning: Do the dbuf_read() before undirtying the dbuf, as we
	* want to make sure dbuf_read() will read the pending cloned block and
	* not the uderlying block that is being replaced. dbuf_undirty() will
	* do dbuf_unoverride(), so we will end up with cloned block content,
	* without overridden BP.
	*/
	(void) dbuf_read(db, NULL, flags);
	if (undirty) {
	mutex_enter(&db->db_mtx);
	VERIFY(!dbuf_undirty(db, tx));
	mutex_exit(&db->db_mtx);
	}
	(void) dbuf_dirty(db, tx);
	}

	void
	dmu_buf_will_dirty(dmu_buf_t db_fake, dmu_tx_t tx)
	{
	dmu_buf_will_dirty_impl(db_fake,
	DB_RF_MUST_SUCCEED \| DB_RF_NOPREFETCH, tx);
	}

	boolean_t
	dmu_buf_is_dirty(dmu_buf_t db_fake, dmu_tx_t tx)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;
	dbuf_dirty_record_t *dr;

	mutex_enter(&db->db_mtx);
	dr = dbuf_find_dirty_eq(db, tx->tx_txg);
	mutex_exit(&db->db_mtx);
	return (dr != NULL);
	}

	void
	dmu_buf_will_clone(dmu_buf_t db_fake, dmu_tx_t tx)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;

	/*
	* Block cloning: We are going to clone into this block, so undirty
	* modifications done to this block so far in this txg. This includes
	* writes and clones into this block.
	*/
	mutex_enter(&db->db_mtx);
	DBUF_VERIFY(db);
	VERIFY(!dbuf_undirty(db, tx));
	- ASSERT3P(dbuf_find_dirty_eq(db, tx->tx_txg), ==, NULL);
	+ ASSERT0P(dbuf_find_dirty_eq(db, tx->tx_txg));
	if (db->db_buf != NULL) {
	arc_buf_destroy(db->db_buf, db);
	db->db_buf = NULL;
	dbuf_clear_data(db);
	}

	db->db_state = DB_NOFILL;
	DTRACE_SET_STATE(db, "allocating NOFILL buffer for clone");

	DBUF_VERIFY(db);
	mutex_exit(&db->db_mtx);

	dbuf_noread(db);
	(void) dbuf_dirty(db, tx);
	}

	void
	dmu_buf_will_not_fill(dmu_buf_t db_fake, dmu_tx_t tx)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;

	mutex_enter(&db->db_mtx);
	db->db_state = DB_NOFILL;
	DTRACE_SET_STATE(db, "allocating NOFILL buffer");
	mutex_exit(&db->db_mtx);

	dbuf_noread(db);
	(void) dbuf_dirty(db, tx);
	}

	void
	dmu_buf_will_fill(dmu_buf_t db_fake, dmu_tx_t tx, boolean_t canfail)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;

	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
	ASSERT(tx->tx_txg != 0);
	ASSERT(db->db_level == 0);
	ASSERT(!zfs_refcount_is_zero(&db->db_holds));

	ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT \|\|
	dmu_tx_private_ok(tx));

	mutex_enter(&db->db_mtx);
	if (db->db_state == DB_NOFILL) {
	/*
	* Block cloning: We will be completely overwriting a block
	* cloned in this transaction group, so let's undirty the
	* pending clone and mark the block as uncached. This will be
	* as if the clone was never done. But if the fill can fail
	* we should have a way to return back to the cloned data.
	*/
	if (canfail && dbuf_find_dirty_eq(db, tx->tx_txg) != NULL) {
	mutex_exit(&db->db_mtx);
	dmu_buf_will_dirty(db_fake, tx);
	return;
	}
	VERIFY(!dbuf_undirty(db, tx));
	db->db_state = DB_UNCACHED;
	}
	mutex_exit(&db->db_mtx);

	dbuf_noread(db);
	(void) dbuf_dirty(db, tx);
	}

	/*
	* This function is effectively the same as dmu_buf_will_dirty(), but
	* indicates the caller expects raw encrypted data in the db, and provides
	* the crypt params (byteorder, salt, iv, mac) which should be stored in the
	* blkptr_t when this dbuf is written. This is only used for blocks of
	* dnodes, during raw receive.
	*/
	void
	dmu_buf_set_crypt_params(dmu_buf_t *db_fake, boolean_t byteorder,
	const uint8_t salt, const uint8_t iv, const uint8_t mac, dmu_tx_t tx)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;
	dbuf_dirty_record_t *dr;

	/*
	* dr_has_raw_params is only processed for blocks of dnodes
	* (see dbuf_sync_dnode_leaf_crypt()).
	*/
	ASSERT3U(db->db.db_object, ==, DMU_META_DNODE_OBJECT);
	ASSERT3U(db->db_level, ==, 0);
	ASSERT(db->db_objset->os_raw_receive);

	dmu_buf_will_dirty_impl(db_fake,
	DB_RF_MUST_SUCCEED \| DB_RF_NOPREFETCH \| DB_RF_NO_DECRYPT, tx);

	dr = dbuf_find_dirty_eq(db, tx->tx_txg);

	ASSERT3P(dr, !=, NULL);

	dr->dt.dl.dr_has_raw_params = B_TRUE;
	dr->dt.dl.dr_byteorder = byteorder;
	memcpy(dr->dt.dl.dr_salt, salt, ZIO_DATA_SALT_LEN);
	memcpy(dr->dt.dl.dr_iv, iv, ZIO_DATA_IV_LEN);
	memcpy(dr->dt.dl.dr_mac, mac, ZIO_DATA_MAC_LEN);
	}

	static void
	dbuf_override_impl(dmu_buf_impl_t db, const blkptr_t bp, dmu_tx_t *tx)
	{
	struct dirty_leaf *dl;
	dbuf_dirty_record_t *dr;

	dr = list_head(&db->db_dirty_records);
	ASSERT3P(dr, !=, NULL);
	ASSERT3U(dr->dr_txg, ==, tx->tx_txg);
	dl = &dr->dt.dl;
	dl->dr_overridden_by = *bp;
	dl->dr_override_state = DR_OVERRIDDEN;
	dl->dr_overridden_by.blk_birth = dr->dr_txg;
	}

	boolean_t
	dmu_buf_fill_done(dmu_buf_t dbuf, dmu_tx_t tx, boolean_t failed)
	{
	(void) tx;
	dmu_buf_impl_t db = (dmu_buf_impl_t )dbuf;
	mutex_enter(&db->db_mtx);
	DBUF_VERIFY(db);

	if (db->db_state == DB_FILL) {
	if (db->db_level == 0 && db->db_freed_in_flight) {
	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
	/* we were freed while filling */
	/* XXX dbuf_undirty? */
	memset(db->db.db_data, 0, db->db.db_size);
	db->db_freed_in_flight = FALSE;
	db->db_state = DB_CACHED;
	DTRACE_SET_STATE(db,
	"fill done handling freed in flight");
	failed = B_FALSE;
	} else if (failed) {
	VERIFY(!dbuf_undirty(db, tx));
	db->db_buf = NULL;
	dbuf_clear_data(db);
	DTRACE_SET_STATE(db, "fill failed");
	} else {
	db->db_state = DB_CACHED;
	DTRACE_SET_STATE(db, "fill done");
	}
	cv_broadcast(&db->db_changed);
	} else {
	db->db_state = DB_CACHED;
	failed = B_FALSE;
	}
	mutex_exit(&db->db_mtx);
	return (failed);
	}

	void
	dmu_buf_write_embedded(dmu_buf_t dbuf, void data,
	bp_embedded_type_t etype, enum zio_compress comp,
	int uncompressed_size, int compressed_size, int byteorder,
	dmu_tx_t *tx)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )dbuf;
	struct dirty_leaf *dl;
	dmu_object_type_t type;
	dbuf_dirty_record_t *dr;

	if (etype == BP_EMBEDDED_TYPE_DATA) {
	ASSERT(spa_feature_is_active(dmu_objset_spa(db->db_objset),
	SPA_FEATURE_EMBEDDED_DATA));
	}

	DB_DNODE_ENTER(db);
	type = DB_DNODE(db)->dn_type;
	DB_DNODE_EXIT(db);

	ASSERT0(db->db_level);
	ASSERT(db->db_blkid != DMU_BONUS_BLKID);

	dmu_buf_will_not_fill(dbuf, tx);

	dr = list_head(&db->db_dirty_records);
	ASSERT3P(dr, !=, NULL);
	ASSERT3U(dr->dr_txg, ==, tx->tx_txg);
	dl = &dr->dt.dl;
	encode_embedded_bp_compressed(&dl->dr_overridden_by,
	data, comp, uncompressed_size, compressed_size);
	BPE_SET_ETYPE(&dl->dr_overridden_by, etype);
	BP_SET_TYPE(&dl->dr_overridden_by, type);
	BP_SET_LEVEL(&dl->dr_overridden_by, 0);
	BP_SET_BYTEORDER(&dl->dr_overridden_by, byteorder);

	dl->dr_override_state = DR_OVERRIDDEN;
	dl->dr_overridden_by.blk_birth = dr->dr_txg;
	}

	void
	dmu_buf_redact(dmu_buf_t dbuf, dmu_tx_t tx)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )dbuf;
	dmu_object_type_t type;
	ASSERT(dsl_dataset_feature_is_active(db->db_objset->os_dsl_dataset,
	SPA_FEATURE_REDACTED_DATASETS));

	DB_DNODE_ENTER(db);
	type = DB_DNODE(db)->dn_type;
	DB_DNODE_EXIT(db);

	ASSERT0(db->db_level);
	dmu_buf_will_not_fill(dbuf, tx);

	blkptr_t bp = { { { {0} } } };
	BP_SET_TYPE(&bp, type);
	BP_SET_LEVEL(&bp, 0);
	BP_SET_BIRTH(&bp, tx->tx_txg, 0);
	BP_SET_REDACTED(&bp);
	BPE_SET_LSIZE(&bp, dbuf->db_size);

	dbuf_override_impl(db, &bp, tx);
	}

	/*
	* Directly assign a provided arc buf to a given dbuf if it's not referenced
	* by anybody except our caller. Otherwise copy arcbuf's contents to dbuf.
	*/
	void
	dbuf_assign_arcbuf(dmu_buf_impl_t db, arc_buf_t buf, dmu_tx_t *tx)
	{
	ASSERT(!zfs_refcount_is_zero(&db->db_holds));
	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
	ASSERT(db->db_level == 0);
	ASSERT3U(dbuf_is_metadata(db), ==, arc_is_metadata(buf));
	ASSERT(buf != NULL);
	ASSERT3U(arc_buf_lsize(buf), ==, db->db.db_size);
	ASSERT(tx->tx_txg != 0);

	arc_return_buf(buf, db);
	ASSERT(arc_released(buf));

	mutex_enter(&db->db_mtx);

	while (db->db_state == DB_READ \|\| db->db_state == DB_FILL)
	cv_wait(&db->db_changed, &db->db_mtx);

	ASSERT(db->db_state == DB_CACHED \|\| db->db_state == DB_UNCACHED \|\|
	db->db_state == DB_NOFILL);

	if (db->db_state == DB_CACHED &&
	zfs_refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) {
	/*
	* In practice, we will never have a case where we have an
	* encrypted arc buffer while additional holds exist on the
	* dbuf. We don't handle this here so we simply assert that
	* fact instead.
	*/
	ASSERT(!arc_is_encrypted(buf));
	mutex_exit(&db->db_mtx);
	(void) dbuf_dirty(db, tx);
	memcpy(db->db.db_data, buf->b_data, db->db.db_size);
	arc_buf_destroy(buf, db);
	return;
	}

	if (db->db_state == DB_CACHED) {
	dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records);

	ASSERT(db->db_buf != NULL);
	if (dr != NULL && dr->dr_txg == tx->tx_txg) {
	ASSERT(dr->dt.dl.dr_data == db->db_buf);

	if (!arc_released(db->db_buf)) {
	ASSERT(dr->dt.dl.dr_override_state ==
	DR_OVERRIDDEN);
	arc_release(db->db_buf, db);
	}
	dr->dt.dl.dr_data = buf;
	arc_buf_destroy(db->db_buf, db);
	} else if (dr == NULL \|\| dr->dt.dl.dr_data != db->db_buf) {
	arc_release(db->db_buf, db);
	arc_buf_destroy(db->db_buf, db);
	}
	db->db_buf = NULL;
	} else if (db->db_state == DB_NOFILL) {
	/*
	* We will be completely replacing the cloned block. In case
	* it was cloned in this transaction group, let's undirty the
	* pending clone and mark the block as uncached. This will be
	* as if the clone was never done.
	*/
	VERIFY(!dbuf_undirty(db, tx));
	db->db_state = DB_UNCACHED;
	}
	ASSERT(db->db_buf == NULL);
	dbuf_set_data(db, buf);
	db->db_state = DB_FILL;
	DTRACE_SET_STATE(db, "filling assigned arcbuf");
	mutex_exit(&db->db_mtx);
	(void) dbuf_dirty(db, tx);
	dmu_buf_fill_done(&db->db, tx, B_FALSE);
	}

	void
	dbuf_destroy(dmu_buf_impl_t *db)
	{
	dnode_t *dn;
	dmu_buf_impl_t *parent = db->db_parent;
	dmu_buf_impl_t *dndb;

	ASSERT(MUTEX_HELD(&db->db_mtx));
	ASSERT(zfs_refcount_is_zero(&db->db_holds));

	if (db->db_buf != NULL) {
	arc_buf_destroy(db->db_buf, db);
	db->db_buf = NULL;
	}

	if (db->db_blkid == DMU_BONUS_BLKID) {
	int slots = DB_DNODE(db)->dn_num_slots;
	int bonuslen = DN_SLOTS_TO_BONUSLEN(slots);
	if (db->db.db_data != NULL) {
	kmem_free(db->db.db_data, bonuslen);
	arc_space_return(bonuslen, ARC_SPACE_BONUS);
	db->db_state = DB_UNCACHED;
	DTRACE_SET_STATE(db, "buffer cleared");
	}
	}

	dbuf_clear_data(db);

	if (multilist_link_active(&db->db_cache_link)) {
	ASSERT(db->db_caching_status == DB_DBUF_CACHE \|\|
	db->db_caching_status == DB_DBUF_METADATA_CACHE);

	multilist_remove(&dbuf_caches[db->db_caching_status].cache, db);
	(void) zfs_refcount_remove_many(
	&dbuf_caches[db->db_caching_status].size,
	db->db.db_size, db);

	if (db->db_caching_status == DB_DBUF_METADATA_CACHE) {
	DBUF_STAT_BUMPDOWN(metadata_cache_count);
	} else {
	DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]);
	DBUF_STAT_BUMPDOWN(cache_count);
	DBUF_STAT_DECR(cache_levels_bytes[db->db_level],
	db->db.db_size);
	}
	db->db_caching_status = DB_NO_CACHE;
	}

	ASSERT(db->db_state == DB_UNCACHED \|\| db->db_state == DB_NOFILL);
	ASSERT(db->db_data_pending == NULL);
	ASSERT(list_is_empty(&db->db_dirty_records));

	db->db_state = DB_EVICTING;
	DTRACE_SET_STATE(db, "buffer eviction started");
	db->db_blkptr = NULL;

	/*
	* Now that db_state is DB_EVICTING, nobody else can find this via
	* the hash table. We can now drop db_mtx, which allows us to
	* acquire the dn_dbufs_mtx.
	*/
	mutex_exit(&db->db_mtx);

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);
	dndb = dn->dn_dbuf;
	if (db->db_blkid != DMU_BONUS_BLKID) {
	boolean_t needlock = !MUTEX_HELD(&dn->dn_dbufs_mtx);
	if (needlock)
	mutex_enter_nested(&dn->dn_dbufs_mtx,
	NESTED_SINGLE);
	avl_remove(&dn->dn_dbufs, db);
	membar_producer();
	DB_DNODE_EXIT(db);
	if (needlock)
	mutex_exit(&dn->dn_dbufs_mtx);
	/*
	* Decrementing the dbuf count means that the hold corresponding
	* to the removed dbuf is no longer discounted in dnode_move(),
	* so the dnode cannot be moved until after we release the hold.
	* The membar_producer() ensures visibility of the decremented
	* value in dnode_move(), since DB_DNODE_EXIT doesn't actually
	* release any lock.
	*/
	mutex_enter(&dn->dn_mtx);
	dnode_rele_and_unlock(dn, db, B_TRUE);
	db->db_dnode_handle = NULL;

	dbuf_hash_remove(db);
	} else {
	DB_DNODE_EXIT(db);
	}

	ASSERT(zfs_refcount_is_zero(&db->db_holds));

	db->db_parent = NULL;

	ASSERT(db->db_buf == NULL);
	ASSERT(db->db.db_data == NULL);
	ASSERT(db->db_hash_next == NULL);
	ASSERT(db->db_blkptr == NULL);
	ASSERT(db->db_data_pending == NULL);
	ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE);
	ASSERT(!multilist_link_active(&db->db_cache_link));

	/*
	* If this dbuf is referenced from an indirect dbuf,
	* decrement the ref count on the indirect dbuf.
	*/
	if (parent && parent != dndb) {
	mutex_enter(&parent->db_mtx);
	dbuf_rele_and_unlock(parent, db, B_TRUE);
	}

	kmem_cache_free(dbuf_kmem_cache, db);
	arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_DBUF);
	}

	/*
	* Note: While bpp will always be updated if the function returns success,
	* parentp will not be updated if the dnode does not have dn_dbuf filled in;
	* this happens when the dnode is the meta-dnode, or {user\|group\|project}used
	* object.
	*/
	__attribute__((always_inline))
	static inline int
	dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse,
	dmu_buf_impl_t parentp, blkptr_t bpp)
	{
	*parentp = NULL;
	*bpp = NULL;

	ASSERT(blkid != DMU_BONUS_BLKID);

	if (blkid == DMU_SPILL_BLKID) {
	mutex_enter(&dn->dn_mtx);
	if (dn->dn_have_spill &&
	(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR))
	*bpp = DN_SPILL_BLKPTR(dn->dn_phys);
	else
	*bpp = NULL;
	dbuf_add_ref(dn->dn_dbuf, NULL);
	*parentp = dn->dn_dbuf;
	mutex_exit(&dn->dn_mtx);
	return (0);
	}

	int nlevels =
	(dn->dn_phys->dn_nlevels == 0) ? 1 : dn->dn_phys->dn_nlevels;
	int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;

	ASSERT3U(level * epbs, <, 64);
	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
	/*
	* This assertion shouldn't trip as long as the max indirect block size
	* is less than 1M. The reason for this is that up to that point,
	* the number of levels required to address an entire object with blocks
	* of size SPA_MINBLOCKSIZE satisfies nlevels * epbs + 1 <= 64. In
	* other words, if N * epbs + 1 > 64, then if (N-1) * epbs + 1 > 55
	* (i.e. we can address the entire object), objects will all use at most
	* N-1 levels and the assertion won't overflow. However, once epbs is
	* 13, 4 * 13 + 1 = 53, but 5 * 13 + 1 = 66. Then, 4 levels will not be
	* enough to address an entire object, so objects will have 5 levels,
	* but then this assertion will overflow.
	*
	* All this is to say that if we ever increase DN_MAX_INDBLKSHIFT, we
	* need to redo this logic to handle overflows.
	*/
	ASSERT(level >= nlevels \|\|
	((nlevels - level - 1) * epbs) +
	highbit64(dn->dn_phys->dn_nblkptr) <= 64);
	if (level >= nlevels \|\|
	blkid >= ((uint64_t)dn->dn_phys->dn_nblkptr <<
	((nlevels - level - 1) * epbs)) \|\|
	(fail_sparse &&
	blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) {
	/* the buffer has no parent yet */
	return (SET_ERROR(ENOENT));
	} else if (level < nlevels-1) {
	/* this block is referenced from an indirect block */
	int err;

	err = dbuf_hold_impl(dn, level + 1,
	blkid >> epbs, fail_sparse, FALSE, NULL, parentp);

	if (err)
	return (err);
	err = dbuf_read(*parentp, NULL,
	(DB_RF_HAVESTRUCT \| DB_RF_NOPREFETCH \| DB_RF_CANFAIL));
	if (err) {
	dbuf_rele(*parentp, NULL);
	*parentp = NULL;
	return (err);
	}
	rw_enter(&(*parentp)->db_rwlock, RW_READER);
	bpp = ((blkptr_t )(*parentp)->db.db_data) +
	(blkid & ((1ULL << epbs) - 1));
	if (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))
	ASSERT(BP_IS_HOLE(*bpp));
	rw_exit(&(*parentp)->db_rwlock);
	return (0);
	} else {
	/* the block is referenced from the dnode */
	ASSERT3U(level, ==, nlevels-1);
	ASSERT(dn->dn_phys->dn_nblkptr == 0 \|\|
	blkid < dn->dn_phys->dn_nblkptr);
	if (dn->dn_dbuf) {
	dbuf_add_ref(dn->dn_dbuf, NULL);
	*parentp = dn->dn_dbuf;
	}
	*bpp = &dn->dn_phys->dn_blkptr[blkid];
	return (0);
	}
	}

	static dmu_buf_impl_t *
	dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
	dmu_buf_impl_t parent, blkptr_t blkptr, uint64_t hash)
	{
	objset_t *os = dn->dn_objset;
	dmu_buf_impl_t db, odb;

	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
	ASSERT(dn->dn_type != DMU_OT_NONE);

	db = kmem_cache_alloc(dbuf_kmem_cache, KM_SLEEP);

	list_create(&db->db_dirty_records, sizeof (dbuf_dirty_record_t),
	offsetof(dbuf_dirty_record_t, dr_dbuf_node));

	db->db_objset = os;
	db->db.db_object = dn->dn_object;
	db->db_level = level;
	db->db_blkid = blkid;
	db->db_dirtycnt = 0;
	db->db_dnode_handle = dn->dn_handle;
	db->db_parent = parent;
	db->db_blkptr = blkptr;
	db->db_hash = hash;

	db->db_user = NULL;
	db->db_user_immediate_evict = FALSE;
	db->db_freed_in_flight = FALSE;
	db->db_pending_evict = FALSE;

	if (blkid == DMU_BONUS_BLKID) {
	ASSERT3P(parent, ==, dn->dn_dbuf);
	db->db.db_size = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) -
	(dn->dn_nblkptr-1) * sizeof (blkptr_t);
	ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
	db->db.db_offset = DMU_BONUS_BLKID;
	db->db_state = DB_UNCACHED;
	DTRACE_SET_STATE(db, "bonus buffer created");
	db->db_caching_status = DB_NO_CACHE;
	/* the bonus dbuf is not placed in the hash table */
	arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_DBUF);
	return (db);
	} else if (blkid == DMU_SPILL_BLKID) {
	db->db.db_size = (blkptr != NULL) ?
	BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE;
	db->db.db_offset = 0;
	} else {
	int blocksize =
	db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz;
	db->db.db_size = blocksize;
	db->db.db_offset = db->db_blkid * blocksize;
	}

	/*
	* Hold the dn_dbufs_mtx while we get the new dbuf
	* in the hash table and added to the dbufs list.
	* This prevents a possible deadlock with someone
	* trying to look up this dbuf before it's added to the
	* dn_dbufs list.
	*/
	mutex_enter(&dn->dn_dbufs_mtx);
	db->db_state = DB_EVICTING; /* not worth logging this state change */
	if ((odb = dbuf_hash_insert(db)) != NULL) {
	/* someone else inserted it first */
	mutex_exit(&dn->dn_dbufs_mtx);
	kmem_cache_free(dbuf_kmem_cache, db);
	DBUF_STAT_BUMP(hash_insert_race);
	return (odb);
	}
	avl_add(&dn->dn_dbufs, db);

	db->db_state = DB_UNCACHED;
	DTRACE_SET_STATE(db, "regular buffer created");
	db->db_caching_status = DB_NO_CACHE;
	mutex_exit(&dn->dn_dbufs_mtx);
	arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_DBUF);

	if (parent && parent != dn->dn_dbuf)
	dbuf_add_ref(parent, db);

	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT \|\|
	zfs_refcount_count(&dn->dn_holds) > 0);
	(void) zfs_refcount_add(&dn->dn_holds, db);

	dprintf_dbuf(db, "db=%p\n", db);

	return (db);
	}

	/*
	* This function returns a block pointer and information about the object,
	* given a dnode and a block. This is a publicly accessible version of
	* dbuf_findbp that only returns some information, rather than the
	* dbuf. Note that the dnode passed in must be held, and the dn_struct_rwlock
	* should be locked as (at least) a reader.
	*/
	int
	dbuf_dnode_findbp(dnode_t *dn, uint64_t level, uint64_t blkid,
	blkptr_t bp, uint16_t datablkszsec, uint8_t *indblkshift)
	{
	dmu_buf_impl_t *dbp = NULL;
	blkptr_t *bp2;
	int err = 0;
	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));

	err = dbuf_findbp(dn, level, blkid, B_FALSE, &dbp, &bp2);
	if (err == 0) {
	ASSERT3P(bp2, !=, NULL);
	bp = bp2;
	if (dbp != NULL)
	dbuf_rele(dbp, NULL);
	if (datablkszsec != NULL)
	*datablkszsec = dn->dn_phys->dn_datablkszsec;
	if (indblkshift != NULL)
	*indblkshift = dn->dn_phys->dn_indblkshift;
	}

	return (err);
	}

	typedef struct dbuf_prefetch_arg {
	spa_t dpa_spa; / The spa to issue the prefetch in. */
	zbookmark_phys_t dpa_zb; /* The target block to prefetch. */
	int dpa_epbs; /* Entries (blkptr_t's) Per Block Shift. */
	int dpa_curlevel; /* The current level that we're reading */
	dnode_t dpa_dnode; / The dnode associated with the prefetch */
	zio_priority_t dpa_prio; /* The priority I/Os should be issued at. */
	zio_t dpa_zio; / The parent zio_t for all prefetches. */
	arc_flags_t dpa_aflags; /* Flags to pass to the final prefetch. */
	dbuf_prefetch_fn dpa_cb; /* prefetch completion callback */
	void dpa_arg; / prefetch completion arg */
	} dbuf_prefetch_arg_t;

	static void
	dbuf_prefetch_fini(dbuf_prefetch_arg_t *dpa, boolean_t io_done)
	{
	if (dpa->dpa_cb != NULL) {
	dpa->dpa_cb(dpa->dpa_arg, dpa->dpa_zb.zb_level,
	dpa->dpa_zb.zb_blkid, io_done);
	}
	kmem_free(dpa, sizeof (*dpa));
	}

	static void
	dbuf_issue_final_prefetch_done(zio_t zio, const zbookmark_phys_t zb,
	const blkptr_t iobp, arc_buf_t abuf, void *private)
	{
	(void) zio, (void) zb, (void) iobp;
	dbuf_prefetch_arg_t *dpa = private;

	if (abuf != NULL)
	arc_buf_destroy(abuf, private);

	dbuf_prefetch_fini(dpa, B_TRUE);
	}

	/*
	* Actually issue the prefetch read for the block given.
	*/
	static void
	dbuf_issue_final_prefetch(dbuf_prefetch_arg_t dpa, blkptr_t bp)
	{
	ASSERT(!BP_IS_REDACTED(bp) \|\|
	dsl_dataset_feature_is_active(
	dpa->dpa_dnode->dn_objset->os_dsl_dataset,
	SPA_FEATURE_REDACTED_DATASETS));

	if (BP_IS_HOLE(bp) \|\| BP_IS_EMBEDDED(bp) \|\| BP_IS_REDACTED(bp))
	return (dbuf_prefetch_fini(dpa, B_FALSE));

	int zio_flags = ZIO_FLAG_CANFAIL \| ZIO_FLAG_SPECULATIVE;
	arc_flags_t aflags =
	dpa->dpa_aflags \| ARC_FLAG_NOWAIT \| ARC_FLAG_PREFETCH \|
	ARC_FLAG_NO_BUF;

	/* dnodes are always read as raw and then converted later */
	if (BP_GET_TYPE(bp) == DMU_OT_DNODE && BP_IS_PROTECTED(bp) &&
	dpa->dpa_curlevel == 0)
	zio_flags \|= ZIO_FLAG_RAW;

	ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp));
	ASSERT3U(dpa->dpa_curlevel, ==, dpa->dpa_zb.zb_level);
	ASSERT(dpa->dpa_zio != NULL);
	(void) arc_read(dpa->dpa_zio, dpa->dpa_spa, bp,
	dbuf_issue_final_prefetch_done, dpa,
	dpa->dpa_prio, zio_flags, &aflags, &dpa->dpa_zb);
	}

	/*
	* Called when an indirect block above our prefetch target is read in. This
	* will either read in the next indirect block down the tree or issue the actual
	* prefetch if the next block down is our target.
	*/
	static void
	dbuf_prefetch_indirect_done(zio_t zio, const zbookmark_phys_t zb,
	const blkptr_t iobp, arc_buf_t abuf, void *private)
	{
	(void) zb, (void) iobp;
	dbuf_prefetch_arg_t *dpa = private;

	ASSERT3S(dpa->dpa_zb.zb_level, <, dpa->dpa_curlevel);
	ASSERT3S(dpa->dpa_curlevel, >, 0);

	if (abuf == NULL) {
	ASSERT(zio == NULL \|\| zio->io_error != 0);
	dbuf_prefetch_fini(dpa, B_TRUE);
	return;
	}
	ASSERT(zio == NULL \|\| zio->io_error == 0);

	/*
	* The dpa_dnode is only valid if we are called with a NULL
	* zio. This indicates that the arc_read() returned without
	* first calling zio_read() to issue a physical read. Once
	* a physical read is made the dpa_dnode must be invalidated
	* as the locks guarding it may have been dropped. If the
	* dpa_dnode is still valid, then we want to add it to the dbuf
	* cache. To do so, we must hold the dbuf associated with the block
	* we just prefetched, read its contents so that we associate it
	* with an arc_buf_t, and then release it.
	*/
	if (zio != NULL) {
	ASSERT3S(BP_GET_LEVEL(zio->io_bp), ==, dpa->dpa_curlevel);
	if (zio->io_flags & ZIO_FLAG_RAW_COMPRESS) {
	ASSERT3U(BP_GET_PSIZE(zio->io_bp), ==, zio->io_size);
	} else {
	ASSERT3U(BP_GET_LSIZE(zio->io_bp), ==, zio->io_size);
	}
	ASSERT3P(zio->io_spa, ==, dpa->dpa_spa);

	dpa->dpa_dnode = NULL;
	} else if (dpa->dpa_dnode != NULL) {
	uint64_t curblkid = dpa->dpa_zb.zb_blkid >>
	(dpa->dpa_epbs * (dpa->dpa_curlevel -
	dpa->dpa_zb.zb_level));
	dmu_buf_impl_t *db = dbuf_hold_level(dpa->dpa_dnode,
	dpa->dpa_curlevel, curblkid, FTAG);
	if (db == NULL) {
	arc_buf_destroy(abuf, private);
	dbuf_prefetch_fini(dpa, B_TRUE);
	return;
	}
	(void) dbuf_read(db, NULL,
	DB_RF_MUST_SUCCEED \| DB_RF_NOPREFETCH \| DB_RF_HAVESTRUCT);
	dbuf_rele(db, FTAG);
	}

	dpa->dpa_curlevel--;
	uint64_t nextblkid = dpa->dpa_zb.zb_blkid >>
	(dpa->dpa_epbs * (dpa->dpa_curlevel - dpa->dpa_zb.zb_level));
	blkptr_t bp = ((blkptr_t )abuf->b_data) +
	P2PHASE(nextblkid, 1ULL << dpa->dpa_epbs);

	ASSERT(!BP_IS_REDACTED(bp) \|\| (dpa->dpa_dnode &&
	dsl_dataset_feature_is_active(
	dpa->dpa_dnode->dn_objset->os_dsl_dataset,
	SPA_FEATURE_REDACTED_DATASETS)));
	if (BP_IS_HOLE(bp) \|\| BP_IS_REDACTED(bp)) {
	arc_buf_destroy(abuf, private);
	dbuf_prefetch_fini(dpa, B_TRUE);
	return;
	} else if (dpa->dpa_curlevel == dpa->dpa_zb.zb_level) {
	ASSERT3U(nextblkid, ==, dpa->dpa_zb.zb_blkid);
	dbuf_issue_final_prefetch(dpa, bp);
	} else {
	arc_flags_t iter_aflags = ARC_FLAG_NOWAIT;
	zbookmark_phys_t zb;

	/* flag if L2ARC eligible, l2arc_noprefetch then decides */
	if (dpa->dpa_aflags & ARC_FLAG_L2CACHE)
	iter_aflags \|= ARC_FLAG_L2CACHE;

	ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp));

	SET_BOOKMARK(&zb, dpa->dpa_zb.zb_objset,
	dpa->dpa_zb.zb_object, dpa->dpa_curlevel, nextblkid);

	(void) arc_read(dpa->dpa_zio, dpa->dpa_spa,
	bp, dbuf_prefetch_indirect_done, dpa,
	ZIO_PRIORITY_SYNC_READ,
	ZIO_FLAG_CANFAIL \| ZIO_FLAG_SPECULATIVE,
	&iter_aflags, &zb);
	}

	arc_buf_destroy(abuf, private);
	}

	/*
	* Issue prefetch reads for the given block on the given level. If the indirect
	* blocks above that block are not in memory, we will read them in
	* asynchronously. As a result, this call never blocks waiting for a read to
	* complete. Note that the prefetch might fail if the dataset is encrypted and
	* the encryption key is unmapped before the IO completes.
	*/
	int
	dbuf_prefetch_impl(dnode_t *dn, int64_t level, uint64_t blkid,
	zio_priority_t prio, arc_flags_t aflags, dbuf_prefetch_fn cb,
	void *arg)
	{
	blkptr_t bp;
	int epbs, nlevels, curlevel;
	uint64_t curblkid;

	ASSERT(blkid != DMU_BONUS_BLKID);
	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));

	if (blkid > dn->dn_maxblkid)
	goto no_issue;

	if (level == 0 && dnode_block_freed(dn, blkid))
	goto no_issue;

	/*
	* This dnode hasn't been written to disk yet, so there's nothing to
	* prefetch.
	*/
	nlevels = dn->dn_phys->dn_nlevels;
	if (level >= nlevels \|\| dn->dn_phys->dn_nblkptr == 0)
	goto no_issue;

	epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
	if (dn->dn_phys->dn_maxblkid < blkid << (epbs * level))
	goto no_issue;

	dmu_buf_impl_t *db = dbuf_find(dn->dn_objset, dn->dn_object,
	level, blkid, NULL);
	if (db != NULL) {
	mutex_exit(&db->db_mtx);
	/*
	* This dbuf already exists. It is either CACHED, or
	* (we assume) about to be read or filled.
	*/
	goto no_issue;
	}

	/*
	* Find the closest ancestor (indirect block) of the target block
	* that is present in the cache. In this indirect block, we will
	* find the bp that is at curlevel, curblkid.
	*/
	curlevel = level;
	curblkid = blkid;
	while (curlevel < nlevels - 1) {
	int parent_level = curlevel + 1;
	uint64_t parent_blkid = curblkid >> epbs;
	dmu_buf_impl_t *db;

	if (dbuf_hold_impl(dn, parent_level, parent_blkid,
	FALSE, TRUE, FTAG, &db) == 0) {
	blkptr_t *bpp = db->db_buf->b_data;
	bp = bpp[P2PHASE(curblkid, 1 << epbs)];
	dbuf_rele(db, FTAG);
	break;
	}

	curlevel = parent_level;
	curblkid = parent_blkid;
	}

	if (curlevel == nlevels - 1) {
	/* No cached indirect blocks found. */
	ASSERT3U(curblkid, <, dn->dn_phys->dn_nblkptr);
	bp = dn->dn_phys->dn_blkptr[curblkid];
	}
	ASSERT(!BP_IS_REDACTED(&bp) \|\|
	dsl_dataset_feature_is_active(dn->dn_objset->os_dsl_dataset,
	SPA_FEATURE_REDACTED_DATASETS));
	if (BP_IS_HOLE(&bp) \|\| BP_IS_REDACTED(&bp))
	goto no_issue;

	ASSERT3U(curlevel, ==, BP_GET_LEVEL(&bp));

	zio_t *pio = zio_root(dmu_objset_spa(dn->dn_objset), NULL, NULL,
	ZIO_FLAG_CANFAIL);

	dbuf_prefetch_arg_t dpa = kmem_zalloc(sizeof (dpa), KM_SLEEP);
	dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;
	SET_BOOKMARK(&dpa->dpa_zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET,
	dn->dn_object, level, blkid);
	dpa->dpa_curlevel = curlevel;
	dpa->dpa_prio = prio;
	dpa->dpa_aflags = aflags;
	dpa->dpa_spa = dn->dn_objset->os_spa;
	dpa->dpa_dnode = dn;
	dpa->dpa_epbs = epbs;
	dpa->dpa_zio = pio;
	dpa->dpa_cb = cb;
	dpa->dpa_arg = arg;

	if (!DNODE_LEVEL_IS_CACHEABLE(dn, level))
	dpa->dpa_aflags \|= ARC_FLAG_UNCACHED;
	else if (dnode_level_is_l2cacheable(&bp, dn, level))
	dpa->dpa_aflags \|= ARC_FLAG_L2CACHE;

	/*
	* If we have the indirect just above us, no need to do the asynchronous
	* prefetch chain; we'll just run the last step ourselves. If we're at
	* a higher level, though, we want to issue the prefetches for all the
	* indirect blocks asynchronously, so we can go on with whatever we were
	* doing.
	*/
	if (curlevel == level) {
	ASSERT3U(curblkid, ==, blkid);
	dbuf_issue_final_prefetch(dpa, &bp);
	} else {
	arc_flags_t iter_aflags = ARC_FLAG_NOWAIT;
	zbookmark_phys_t zb;

	/* flag if L2ARC eligible, l2arc_noprefetch then decides */
	if (dnode_level_is_l2cacheable(&bp, dn, level))
	iter_aflags \|= ARC_FLAG_L2CACHE;

	SET_BOOKMARK(&zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET,
	dn->dn_object, curlevel, curblkid);
	(void) arc_read(dpa->dpa_zio, dpa->dpa_spa,
	&bp, dbuf_prefetch_indirect_done, dpa,
	ZIO_PRIORITY_SYNC_READ,
	ZIO_FLAG_CANFAIL \| ZIO_FLAG_SPECULATIVE,
	&iter_aflags, &zb);
	}
	/*
	* We use pio here instead of dpa_zio since it's possible that
	* dpa may have already been freed.
	*/
	zio_nowait(pio);
	return (1);
	no_issue:
	if (cb != NULL)
	cb(arg, level, blkid, B_FALSE);
	return (0);
	}

	int
	dbuf_prefetch(dnode_t *dn, int64_t level, uint64_t blkid, zio_priority_t prio,
	arc_flags_t aflags)
	{

	return (dbuf_prefetch_impl(dn, level, blkid, prio, aflags, NULL, NULL));
	}

	/*
	* Helper function for dbuf_hold_impl() to copy a buffer. Handles
	* the case of encrypted, compressed and uncompressed buffers by
	* allocating the new buffer, respectively, with arc_alloc_raw_buf(),
	* arc_alloc_compressed_buf() or arc_alloc_buf().*
	*
	* NOTE: Declared noinline to avoid stack bloat in dbuf_hold_impl().
	*/
	noinline static void
	dbuf_hold_copy(dnode_t dn, dmu_buf_impl_t db)
	{
	dbuf_dirty_record_t *dr = db->db_data_pending;
	arc_buf_t *data = dr->dt.dl.dr_data;
	enum zio_compress compress_type = arc_get_compression(data);
	uint8_t complevel = arc_get_complevel(data);

	if (arc_is_encrypted(data)) {
	boolean_t byteorder;
	uint8_t salt[ZIO_DATA_SALT_LEN];
	uint8_t iv[ZIO_DATA_IV_LEN];
	uint8_t mac[ZIO_DATA_MAC_LEN];

	arc_get_raw_params(data, &byteorder, salt, iv, mac);
	dbuf_set_data(db, arc_alloc_raw_buf(dn->dn_objset->os_spa, db,
	dmu_objset_id(dn->dn_objset), byteorder, salt, iv, mac,
	dn->dn_type, arc_buf_size(data), arc_buf_lsize(data),
	compress_type, complevel));
	} else if (compress_type != ZIO_COMPRESS_OFF) {
	dbuf_set_data(db, arc_alloc_compressed_buf(
	dn->dn_objset->os_spa, db, arc_buf_size(data),
	arc_buf_lsize(data), compress_type, complevel));
	} else {
	dbuf_set_data(db, arc_alloc_buf(dn->dn_objset->os_spa, db,
	DBUF_GET_BUFC_TYPE(db), db->db.db_size));
	}

	rw_enter(&db->db_rwlock, RW_WRITER);
	memcpy(db->db.db_data, data->b_data, arc_buf_size(data));
	rw_exit(&db->db_rwlock);
	}

	/*
	* Returns with db_holds incremented, and db_mtx not held.
	* Note: dn_struct_rwlock must be held.
	*/
	int
	dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid,
	boolean_t fail_sparse, boolean_t fail_uncached,
	const void tag, dmu_buf_impl_t *dbp)
	{
	dmu_buf_impl_t db, parent = NULL;
	uint64_t hv;

	/* If the pool has been created, verify the tx_sync_lock is not held */
	spa_t *spa = dn->dn_objset->os_spa;
	dsl_pool_t *dp = spa->spa_dsl_pool;
	if (dp != NULL) {
	ASSERT(!MUTEX_HELD(&dp->dp_tx.tx_sync_lock));
	}

	ASSERT(blkid != DMU_BONUS_BLKID);
	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
	ASSERT3U(dn->dn_nlevels, >, level);

	*dbp = NULL;

	/* dbuf_find() returns with db_mtx held */
	db = dbuf_find(dn->dn_objset, dn->dn_object, level, blkid, &hv);

	if (db == NULL) {
	blkptr_t *bp = NULL;
	int err;

	if (fail_uncached)
	return (SET_ERROR(ENOENT));

	ASSERT3P(parent, ==, NULL);
	err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp);
	if (fail_sparse) {
	if (err == 0 && bp && BP_IS_HOLE(bp))
	err = SET_ERROR(ENOENT);
	if (err) {
	if (parent)
	dbuf_rele(parent, NULL);
	return (err);
	}
	}
	if (err && err != ENOENT)
	return (err);
	db = dbuf_create(dn, level, blkid, parent, bp, hv);
	}

	if (fail_uncached && db->db_state != DB_CACHED) {
	mutex_exit(&db->db_mtx);
	return (SET_ERROR(ENOENT));
	}

	if (db->db_buf != NULL) {
	arc_buf_access(db->db_buf);
	ASSERT3P(db->db.db_data, ==, db->db_buf->b_data);
	}

	ASSERT(db->db_buf == NULL \|\| arc_referenced(db->db_buf));

	/*
	* If this buffer is currently syncing out, and we are
	* still referencing it from db_data, we need to make a copy
	* of it in case we decide we want to dirty it again in this txg.
	*/
	if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
	dn->dn_object != DMU_META_DNODE_OBJECT &&
	db->db_state == DB_CACHED && db->db_data_pending) {
	dbuf_dirty_record_t *dr = db->db_data_pending;
	if (dr->dt.dl.dr_data == db->db_buf) {
	ASSERT3P(db->db_buf, !=, NULL);
	dbuf_hold_copy(dn, db);
	}
	}

	if (multilist_link_active(&db->db_cache_link)) {
	ASSERT(zfs_refcount_is_zero(&db->db_holds));
	ASSERT(db->db_caching_status == DB_DBUF_CACHE \|\|
	db->db_caching_status == DB_DBUF_METADATA_CACHE);

	multilist_remove(&dbuf_caches[db->db_caching_status].cache, db);
	(void) zfs_refcount_remove_many(
	&dbuf_caches[db->db_caching_status].size,
	db->db.db_size, db);

	if (db->db_caching_status == DB_DBUF_METADATA_CACHE) {
	DBUF_STAT_BUMPDOWN(metadata_cache_count);
	} else {
	DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]);
	DBUF_STAT_BUMPDOWN(cache_count);
	DBUF_STAT_DECR(cache_levels_bytes[db->db_level],
	db->db.db_size);
	}
	db->db_caching_status = DB_NO_CACHE;
	}
	(void) zfs_refcount_add(&db->db_holds, tag);
	DBUF_VERIFY(db);
	mutex_exit(&db->db_mtx);

	/* NOTE: we can't rele the parent until after we drop the db_mtx */
	if (parent)
	dbuf_rele(parent, NULL);

	ASSERT3P(DB_DNODE(db), ==, dn);
	ASSERT3U(db->db_blkid, ==, blkid);
	ASSERT3U(db->db_level, ==, level);
	*dbp = db;

	return (0);
	}

	dmu_buf_impl_t *
	dbuf_hold(dnode_t dn, uint64_t blkid, const void tag)
	{
	return (dbuf_hold_level(dn, 0, blkid, tag));
	}

	dmu_buf_impl_t *
	dbuf_hold_level(dnode_t dn, int level, uint64_t blkid, const void tag)
	{
	dmu_buf_impl_t *db;
	int err = dbuf_hold_impl(dn, level, blkid, FALSE, FALSE, tag, &db);
	return (err ? NULL : db);
	}

	void
	dbuf_create_bonus(dnode_t *dn)
	{
	ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));

	ASSERT(dn->dn_bonus == NULL);
	dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL,
	dbuf_hash(dn->dn_objset, dn->dn_object, 0, DMU_BONUS_BLKID));
	}

	int
	dbuf_spill_set_blksz(dmu_buf_t db_fake, uint64_t blksz, dmu_tx_t tx)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;

	if (db->db_blkid != DMU_SPILL_BLKID)
	return (SET_ERROR(ENOTSUP));
	if (blksz == 0)
	blksz = SPA_MINBLOCKSIZE;
	ASSERT3U(blksz, <=, spa_maxblocksize(dmu_objset_spa(db->db_objset)));
	blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE);

	dbuf_new_size(db, blksz, tx);

	return (0);
	}

	void
	dbuf_rm_spill(dnode_t dn, dmu_tx_t tx)
	{
	dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx);
	}

	#pragma weak dmu_buf_add_ref = dbuf_add_ref
	void
	dbuf_add_ref(dmu_buf_impl_t db, const void tag)
	{
	int64_t holds = zfs_refcount_add(&db->db_holds, tag);
	VERIFY3S(holds, >, 1);
	}

	#pragma weak dmu_buf_try_add_ref = dbuf_try_add_ref
	boolean_t
	dbuf_try_add_ref(dmu_buf_t db_fake, objset_t os, uint64_t obj, uint64_t blkid,
	const void *tag)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;
	dmu_buf_impl_t *found_db;
	boolean_t result = B_FALSE;

	if (blkid == DMU_BONUS_BLKID)
	found_db = dbuf_find_bonus(os, obj);
	else
	found_db = dbuf_find(os, obj, 0, blkid, NULL);

	if (found_db != NULL) {
	if (db == found_db && dbuf_refcount(db) > db->db_dirtycnt) {
	(void) zfs_refcount_add(&db->db_holds, tag);
	result = B_TRUE;
	}
	mutex_exit(&found_db->db_mtx);
	}
	return (result);
	}

	/*
	* If you call dbuf_rele() you had better not be referencing the dnode handle
	* unless you have some other direct or indirect hold on the dnode. (An indirect
	* hold is a hold on one of the dnode's dbufs, including the bonus buffer.)
	* Without that, the dbuf_rele() could lead to a dnode_rele() followed by the
	* dnode's parent dbuf evicting its dnode handles.
	*/
	void
	dbuf_rele(dmu_buf_impl_t db, const void tag)
	{
	mutex_enter(&db->db_mtx);
	dbuf_rele_and_unlock(db, tag, B_FALSE);
	}

	void
	dmu_buf_rele(dmu_buf_t db, const void tag)
	{
	dbuf_rele((dmu_buf_impl_t *)db, tag);
	}

	/*
	* dbuf_rele() for an already-locked dbuf. This is necessary to allow
	* db_dirtycnt and db_holds to be updated atomically. The 'evicting'
	* argument should be set if we are already in the dbuf-evicting code
	* path, in which case we don't want to recursively evict. This allows us to
	* avoid deeply nested stacks that would have a call flow similar to this:
	*
	* dbuf_rele()-->dbuf_rele_and_unlock()-->dbuf_evict_notify()
	* ^ \|
	* \| \|
	* +-----dbuf_destroy()<--dbuf_evict_one()<--------+
	*
	*/
	void
	dbuf_rele_and_unlock(dmu_buf_impl_t db, const void tag, boolean_t evicting)
	{
	int64_t holds;
	uint64_t size;

	ASSERT(MUTEX_HELD(&db->db_mtx));
	DBUF_VERIFY(db);

	/*
	* Remove the reference to the dbuf before removing its hold on the
	* dnode so we can guarantee in dnode_move() that a referenced bonus
	* buffer has a corresponding dnode hold.
	*/
	holds = zfs_refcount_remove(&db->db_holds, tag);
	ASSERT(holds >= 0);

	/*
	* We can't freeze indirects if there is a possibility that they
	* may be modified in the current syncing context.
	*/
	if (db->db_buf != NULL &&
	holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) {
	arc_buf_freeze(db->db_buf);
	}

	if (holds == db->db_dirtycnt &&
	db->db_level == 0 && db->db_user_immediate_evict)
	dbuf_evict_user(db);

	if (holds == 0) {
	if (db->db_blkid == DMU_BONUS_BLKID) {
	dnode_t *dn;
	boolean_t evict_dbuf = db->db_pending_evict;

	/*
	* If the dnode moves here, we cannot cross this
	* barrier until the move completes.
	*/
	DB_DNODE_ENTER(db);

	dn = DB_DNODE(db);
	atomic_dec_32(&dn->dn_dbufs_count);

	/*
	* Decrementing the dbuf count means that the bonus
	* buffer's dnode hold is no longer discounted in
	* dnode_move(). The dnode cannot move until after
	* the dnode_rele() below.
	*/
	DB_DNODE_EXIT(db);

	/*
	* Do not reference db after its lock is dropped.
	* Another thread may evict it.
	*/
	mutex_exit(&db->db_mtx);

	if (evict_dbuf)
	dnode_evict_bonus(dn);

	dnode_rele(dn, db);
	} else if (db->db_buf == NULL) {
	/*
	* This is a special case: we never associated this
	* dbuf with any data allocated from the ARC.
	*/
	ASSERT(db->db_state == DB_UNCACHED \|\|
	db->db_state == DB_NOFILL);
	dbuf_destroy(db);
	} else if (arc_released(db->db_buf)) {
	/*
	* This dbuf has anonymous data associated with it.
	*/
	dbuf_destroy(db);
	} else if (!(DBUF_IS_CACHEABLE(db) \|\| db->db_partial_read) \|\|
	db->db_pending_evict) {
	dbuf_destroy(db);
	} else if (!multilist_link_active(&db->db_cache_link)) {
	ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE);

	dbuf_cached_state_t dcs =
	dbuf_include_in_metadata_cache(db) ?
	DB_DBUF_METADATA_CACHE : DB_DBUF_CACHE;
	db->db_caching_status = dcs;

	multilist_insert(&dbuf_caches[dcs].cache, db);
	uint64_t db_size = db->db.db_size;
	size = zfs_refcount_add_many(
	&dbuf_caches[dcs].size, db_size, db);
	uint8_t db_level = db->db_level;
	mutex_exit(&db->db_mtx);

	if (dcs == DB_DBUF_METADATA_CACHE) {
	DBUF_STAT_BUMP(metadata_cache_count);
	DBUF_STAT_MAX(metadata_cache_size_bytes_max,
	size);
	} else {
	DBUF_STAT_BUMP(cache_count);
	DBUF_STAT_MAX(cache_size_bytes_max, size);
	DBUF_STAT_BUMP(cache_levels[db_level]);
	DBUF_STAT_INCR(cache_levels_bytes[db_level],
	db_size);
	}

	if (dcs == DB_DBUF_CACHE && !evicting)
	dbuf_evict_notify(size);
	}
	} else {
	mutex_exit(&db->db_mtx);
	}

	}

	#pragma weak dmu_buf_refcount = dbuf_refcount
	uint64_t
	dbuf_refcount(dmu_buf_impl_t *db)
	{
	return (zfs_refcount_count(&db->db_holds));
	}

	uint64_t
	dmu_buf_user_refcount(dmu_buf_t *db_fake)
	{
	uint64_t holds;
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;

	mutex_enter(&db->db_mtx);
	ASSERT3U(zfs_refcount_count(&db->db_holds), >=, db->db_dirtycnt);
	holds = zfs_refcount_count(&db->db_holds) - db->db_dirtycnt;
	mutex_exit(&db->db_mtx);

	return (holds);
	}

	void *
	dmu_buf_replace_user(dmu_buf_t db_fake, dmu_buf_user_t old_user,
	dmu_buf_user_t *new_user)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;

	mutex_enter(&db->db_mtx);
	dbuf_verify_user(db, DBVU_NOT_EVICTING);
	if (db->db_user == old_user)
	db->db_user = new_user;
	else
	old_user = db->db_user;
	dbuf_verify_user(db, DBVU_NOT_EVICTING);
	mutex_exit(&db->db_mtx);

	return (old_user);
	}

	void *
	dmu_buf_set_user(dmu_buf_t db_fake, dmu_buf_user_t user)
	{
	return (dmu_buf_replace_user(db_fake, NULL, user));
	}

	void *
	dmu_buf_set_user_ie(dmu_buf_t db_fake, dmu_buf_user_t user)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;

	db->db_user_immediate_evict = TRUE;
	return (dmu_buf_set_user(db_fake, user));
	}

	void *
	dmu_buf_remove_user(dmu_buf_t db_fake, dmu_buf_user_t user)
	{
	return (dmu_buf_replace_user(db_fake, user, NULL));
	}

	void *
	dmu_buf_get_user(dmu_buf_t *db_fake)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;

	dbuf_verify_user(db, DBVU_NOT_EVICTING);
	return (db->db_user);
	}

	void
	dmu_buf_user_evict_wait(void)
	{
	taskq_wait(dbu_evict_taskq);
	}

	blkptr_t *
	dmu_buf_get_blkptr(dmu_buf_t *db)
	{
	dmu_buf_impl_t dbi = (dmu_buf_impl_t )db;
	return (dbi->db_blkptr);
	}

	objset_t *
	dmu_buf_get_objset(dmu_buf_t *db)
	{
	dmu_buf_impl_t dbi = (dmu_buf_impl_t )db;
	return (dbi->db_objset);
	}

	-dnode_t *
	-dmu_buf_dnode_enter(dmu_buf_t *db)
	-{
	- dmu_buf_impl_t dbi = (dmu_buf_impl_t )db;
	- DB_DNODE_ENTER(dbi);
	- return (DB_DNODE(dbi));
	-}
	-
	-void
	-dmu_buf_dnode_exit(dmu_buf_t *db)
	-{
	- dmu_buf_impl_t dbi = (dmu_buf_impl_t )db;
	- DB_DNODE_EXIT(dbi);
	-}
	-
	static void
	dbuf_check_blkptr(dnode_t dn, dmu_buf_impl_t db)
	{
	/* ASSERT(dmu_tx_is_syncing(tx) */
	ASSERT(MUTEX_HELD(&db->db_mtx));

	if (db->db_blkptr != NULL)
	return;

	if (db->db_blkid == DMU_SPILL_BLKID) {
	db->db_blkptr = DN_SPILL_BLKPTR(dn->dn_phys);
	BP_ZERO(db->db_blkptr);
	return;
	}
	if (db->db_level == dn->dn_phys->dn_nlevels-1) {
	/*
	* This buffer was allocated at a time when there was
	* no available blkptrs from the dnode, or it was
	* inappropriate to hook it in (i.e., nlevels mismatch).
	*/
	ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr);
	ASSERT(db->db_parent == NULL);
	db->db_parent = dn->dn_dbuf;
	db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid];
	DBUF_VERIFY(db);
	} else {
	dmu_buf_impl_t *parent = db->db_parent;
	int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;

	ASSERT(dn->dn_phys->dn_nlevels > 1);
	if (parent == NULL) {
	mutex_exit(&db->db_mtx);
	rw_enter(&dn->dn_struct_rwlock, RW_READER);
	parent = dbuf_hold_level(dn, db->db_level + 1,
	db->db_blkid >> epbs, db);
	rw_exit(&dn->dn_struct_rwlock);
	mutex_enter(&db->db_mtx);
	db->db_parent = parent;
	}
	db->db_blkptr = (blkptr_t *)parent->db.db_data +
	(db->db_blkid & ((1ULL << epbs) - 1));
	DBUF_VERIFY(db);
	}
	}

	static void
	dbuf_sync_bonus(dbuf_dirty_record_t dr, dmu_tx_t tx)
	{
	dmu_buf_impl_t *db = dr->dr_dbuf;
	void *data = dr->dt.dl.dr_data;

	ASSERT0(db->db_level);
	ASSERT(MUTEX_HELD(&db->db_mtx));
	ASSERT(db->db_blkid == DMU_BONUS_BLKID);
	ASSERT(data != NULL);

	dnode_t *dn = dr->dr_dnode;
	ASSERT3U(DN_MAX_BONUS_LEN(dn->dn_phys), <=,
	DN_SLOTS_TO_BONUSLEN(dn->dn_phys->dn_extra_slots + 1));
	memcpy(DN_BONUS(dn->dn_phys), data, DN_MAX_BONUS_LEN(dn->dn_phys));

	dbuf_sync_leaf_verify_bonus_dnode(dr);

	dbuf_undirty_bonus(dr);
	dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg, B_FALSE);
	}

	/*
	* When syncing out a blocks of dnodes, adjust the block to deal with
	* encryption. Normally, we make sure the block is decrypted before writing
	* it. If we have crypt params, then we are writing a raw (encrypted) block,
	* from a raw receive. In this case, set the ARC buf's crypt params so
	* that the BP will be filled with the correct byteorder, salt, iv, and mac.
	*/
	static void
	dbuf_prepare_encrypted_dnode_leaf(dbuf_dirty_record_t *dr)
	{
	int err;
	dmu_buf_impl_t *db = dr->dr_dbuf;

	ASSERT(MUTEX_HELD(&db->db_mtx));
	ASSERT3U(db->db.db_object, ==, DMU_META_DNODE_OBJECT);
	ASSERT3U(db->db_level, ==, 0);

	if (!db->db_objset->os_raw_receive && arc_is_encrypted(db->db_buf)) {
	zbookmark_phys_t zb;

	/*
	* Unfortunately, there is currently no mechanism for
	* syncing context to handle decryption errors. An error
	* here is only possible if an attacker maliciously
	* changed a dnode block and updated the associated
	* checksums going up the block tree.
	*/
	SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset),
	db->db.db_object, db->db_level, db->db_blkid);
	err = arc_untransform(db->db_buf, db->db_objset->os_spa,
	&zb, B_TRUE);
	if (err)
	panic("Invalid dnode block MAC");
	} else if (dr->dt.dl.dr_has_raw_params) {
	(void) arc_release(dr->dt.dl.dr_data, db);
	arc_convert_to_raw(dr->dt.dl.dr_data,
	dmu_objset_id(db->db_objset),
	dr->dt.dl.dr_byteorder, DMU_OT_DNODE,
	dr->dt.dl.dr_salt, dr->dt.dl.dr_iv, dr->dt.dl.dr_mac);
	}
	}

	/*
	* dbuf_sync_indirect() is called recursively from dbuf_sync_list() so it
	* is critical the we not allow the compiler to inline this function in to
	* dbuf_sync_list() thereby drastically bloating the stack usage.
	*/
	noinline static void
	dbuf_sync_indirect(dbuf_dirty_record_t dr, dmu_tx_t tx)
	{
	dmu_buf_impl_t *db = dr->dr_dbuf;
	dnode_t *dn = dr->dr_dnode;

	ASSERT(dmu_tx_is_syncing(tx));

	dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);

	mutex_enter(&db->db_mtx);

	ASSERT(db->db_level > 0);
	DBUF_VERIFY(db);

	/* Read the block if it hasn't been read yet. */
	if (db->db_buf == NULL) {
	mutex_exit(&db->db_mtx);
	(void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED);
	mutex_enter(&db->db_mtx);
	}
	ASSERT3U(db->db_state, ==, DB_CACHED);
	ASSERT(db->db_buf != NULL);

	/* Indirect block size must match what the dnode thinks it is. */
	ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
	dbuf_check_blkptr(dn, db);

	/* Provide the pending dirty record to child dbufs */
	db->db_data_pending = dr;

	mutex_exit(&db->db_mtx);

	dbuf_write(dr, db->db_buf, tx);

	zio_t *zio = dr->dr_zio;
	mutex_enter(&dr->dt.di.dr_mtx);
	dbuf_sync_list(&dr->dt.di.dr_children, db->db_level - 1, tx);
	ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
	mutex_exit(&dr->dt.di.dr_mtx);
	zio_nowait(zio);
	}

	/*
	* Verify that the size of the data in our bonus buffer does not exceed
	* its recorded size.
	*
	* The purpose of this verification is to catch any cases in development
	* where the size of a phys structure (i.e space_map_phys_t) grows and,
	* due to incorrect feature management, older pools expect to read more
	* data even though they didn't actually write it to begin with.
	*
	* For a example, this would catch an error in the feature logic where we
	* open an older pool and we expect to write the space map histogram of
	* a space map with size SPACE_MAP_SIZE_V0.
	*/
	static void
	dbuf_sync_leaf_verify_bonus_dnode(dbuf_dirty_record_t *dr)
	{
	#ifdef ZFS_DEBUG
	dnode_t *dn = dr->dr_dnode;

	/*
	* Encrypted bonus buffers can have data past their bonuslen.
	* Skip the verification of these blocks.
	*/
	if (DMU_OT_IS_ENCRYPTED(dn->dn_bonustype))
	return;

	uint16_t bonuslen = dn->dn_phys->dn_bonuslen;
	uint16_t maxbonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots);
	ASSERT3U(bonuslen, <=, maxbonuslen);

	arc_buf_t *datap = dr->dt.dl.dr_data;
	char datap_end = ((char )datap) + bonuslen;
	char datap_max = ((char )datap) + maxbonuslen;

	/* ensure that everything is zero after our data */
	for (; datap_end < datap_max; datap_end++)
	ASSERT(*datap_end == 0);
	#endif
	}

	static blkptr_t *
	dbuf_lightweight_bp(dbuf_dirty_record_t *dr)
	{
	/* This must be a lightweight dirty record. */
	ASSERT3P(dr->dr_dbuf, ==, NULL);
	dnode_t *dn = dr->dr_dnode;

	if (dn->dn_phys->dn_nlevels == 1) {
	VERIFY3U(dr->dt.dll.dr_blkid, <, dn->dn_phys->dn_nblkptr);
	return (&dn->dn_phys->dn_blkptr[dr->dt.dll.dr_blkid]);
	} else {
	dmu_buf_impl_t *parent_db = dr->dr_parent->dr_dbuf;
	int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
	VERIFY3U(parent_db->db_level, ==, 1);
	VERIFY3P(parent_db->db_dnode_handle->dnh_dnode, ==, dn);
	VERIFY3U(dr->dt.dll.dr_blkid >> epbs, ==, parent_db->db_blkid);
	blkptr_t *bp = parent_db->db.db_data;
	return (&bp[dr->dt.dll.dr_blkid & ((1 << epbs) - 1)]);
	}
	}

	static void
	dbuf_lightweight_ready(zio_t *zio)
	{
	dbuf_dirty_record_t *dr = zio->io_private;
	blkptr_t *bp = zio->io_bp;

	if (zio->io_error != 0)
	return;

	dnode_t *dn = dr->dr_dnode;

	blkptr_t *bp_orig = dbuf_lightweight_bp(dr);
	spa_t *spa = dmu_objset_spa(dn->dn_objset);
	int64_t delta = bp_get_dsize_sync(spa, bp) -
	bp_get_dsize_sync(spa, bp_orig);
	dnode_diduse_space(dn, delta);

	uint64_t blkid = dr->dt.dll.dr_blkid;
	mutex_enter(&dn->dn_mtx);
	if (blkid > dn->dn_phys->dn_maxblkid) {
	ASSERT0(dn->dn_objset->os_raw_receive);
	dn->dn_phys->dn_maxblkid = blkid;
	}
	mutex_exit(&dn->dn_mtx);

	if (!BP_IS_EMBEDDED(bp)) {
	uint64_t fill = BP_IS_HOLE(bp) ? 0 : 1;
	BP_SET_FILL(bp, fill);
	}

	dmu_buf_impl_t *parent_db;
	EQUIV(dr->dr_parent == NULL, dn->dn_phys->dn_nlevels == 1);
	if (dr->dr_parent == NULL) {
	parent_db = dn->dn_dbuf;
	} else {
	parent_db = dr->dr_parent->dr_dbuf;
	}
	rw_enter(&parent_db->db_rwlock, RW_WRITER);
	bp_orig = bp;
	rw_exit(&parent_db->db_rwlock);
	}

	static void
	dbuf_lightweight_done(zio_t *zio)
	{
	dbuf_dirty_record_t *dr = zio->io_private;

	VERIFY0(zio->io_error);

	objset_t *os = dr->dr_dnode->dn_objset;
	dmu_tx_t *tx = os->os_synctx;

	if (zio->io_flags & (ZIO_FLAG_IO_REWRITE \| ZIO_FLAG_NOPWRITE)) {
	ASSERT(BP_EQUAL(zio->io_bp, &zio->io_bp_orig));
	} else {
	dsl_dataset_t *ds = os->os_dsl_dataset;
	(void) dsl_dataset_block_kill(ds, &zio->io_bp_orig, tx, B_TRUE);
	dsl_dataset_block_born(ds, zio->io_bp, tx);
	}

	dsl_pool_undirty_space(dmu_objset_pool(os), dr->dr_accounted,
	zio->io_txg);

	abd_free(dr->dt.dll.dr_abd);
	kmem_free(dr, sizeof (*dr));
	}

	noinline static void
	dbuf_sync_lightweight(dbuf_dirty_record_t dr, dmu_tx_t tx)
	{
	dnode_t *dn = dr->dr_dnode;
	zio_t *pio;
	if (dn->dn_phys->dn_nlevels == 1) {
	pio = dn->dn_zio;
	} else {
	pio = dr->dr_parent->dr_zio;
	}

	zbookmark_phys_t zb = {
	.zb_objset = dmu_objset_id(dn->dn_objset),
	.zb_object = dn->dn_object,
	.zb_level = 0,
	.zb_blkid = dr->dt.dll.dr_blkid,
	};

	/*
	* See comment in dbuf_write(). This is so that zio->io_bp_orig
	* will have the old BP in dbuf_lightweight_done().
	*/
	dr->dr_bp_copy = *dbuf_lightweight_bp(dr);

	dr->dr_zio = zio_write(pio, dmu_objset_spa(dn->dn_objset),
	dmu_tx_get_txg(tx), &dr->dr_bp_copy, dr->dt.dll.dr_abd,
	dn->dn_datablksz, abd_get_size(dr->dt.dll.dr_abd),
	&dr->dt.dll.dr_props, dbuf_lightweight_ready, NULL,
	dbuf_lightweight_done, dr, ZIO_PRIORITY_ASYNC_WRITE,
	ZIO_FLAG_MUSTSUCCEED \| dr->dt.dll.dr_flags, &zb);

	zio_nowait(dr->dr_zio);
	}

	/*
	* dbuf_sync_leaf() is called recursively from dbuf_sync_list() so it is
	* critical the we not allow the compiler to inline this function in to
	* dbuf_sync_list() thereby drastically bloating the stack usage.
	*/
	noinline static void
	dbuf_sync_leaf(dbuf_dirty_record_t dr, dmu_tx_t tx)
	{
	arc_buf_t **datap = &dr->dt.dl.dr_data;
	dmu_buf_impl_t *db = dr->dr_dbuf;
	dnode_t *dn = dr->dr_dnode;
	objset_t *os;
	uint64_t txg = tx->tx_txg;

	ASSERT(dmu_tx_is_syncing(tx));

	dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);

	mutex_enter(&db->db_mtx);
	/*
	* To be synced, we must be dirtied. But we
	* might have been freed after the dirty.
	*/
	if (db->db_state == DB_UNCACHED) {
	/* This buffer has been freed since it was dirtied */
	ASSERT(db->db.db_data == NULL);
	} else if (db->db_state == DB_FILL) {
	/* This buffer was freed and is now being re-filled */
	ASSERT(db->db.db_data != dr->dt.dl.dr_data);
	} else if (db->db_state == DB_READ) {
	/*
	* This buffer has a clone we need to write, and an in-flight
	* read on the BP we're about to clone. Its safe to issue the
	* write here because the read has already been issued and the
	* contents won't change.
	*/
	ASSERT(dr->dt.dl.dr_brtwrite &&
	dr->dt.dl.dr_override_state == DR_OVERRIDDEN);
	} else {
	ASSERT(db->db_state == DB_CACHED \|\| db->db_state == DB_NOFILL);
	}
	DBUF_VERIFY(db);

	if (db->db_blkid == DMU_SPILL_BLKID) {
	mutex_enter(&dn->dn_mtx);
	if (!(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) {
	/*
	* In the previous transaction group, the bonus buffer
	* was entirely used to store the attributes for the
	* dnode which overrode the dn_spill field. However,
	* when adding more attributes to the file a spill
	* block was required to hold the extra attributes.
	*
	* Make sure to clear the garbage left in the dn_spill
	* field from the previous attributes in the bonus
	* buffer. Otherwise, after writing out the spill
	* block to the new allocated dva, it will free
	* the old block pointed to by the invalid dn_spill.
	*/
	db->db_blkptr = NULL;
	}
	dn->dn_phys->dn_flags \|= DNODE_FLAG_SPILL_BLKPTR;
	mutex_exit(&dn->dn_mtx);
	}

	/*
	* If this is a bonus buffer, simply copy the bonus data into the
	* dnode. It will be written out when the dnode is synced (and it
	* will be synced, since it must have been dirty for dbuf_sync to
	* be called).
	*/
	if (db->db_blkid == DMU_BONUS_BLKID) {
	ASSERT(dr->dr_dbuf == db);
	dbuf_sync_bonus(dr, tx);
	return;
	}

	os = dn->dn_objset;

	/*
	* This function may have dropped the db_mtx lock allowing a dmu_sync
	* operation to sneak in. As a result, we need to ensure that we
	* don't check the dr_override_state until we have returned from
	* dbuf_check_blkptr.
	*/
	dbuf_check_blkptr(dn, db);

	/*
	* If this buffer is in the middle of an immediate write,
	* wait for the synchronous IO to complete.
	*/
	while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) {
	ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT);
	cv_wait(&db->db_changed, &db->db_mtx);
	}

	/*
	* If this is a dnode block, ensure it is appropriately encrypted
	* or decrypted, depending on what we are writing to it this txg.
	*/
	if (os->os_encrypted && dn->dn_object == DMU_META_DNODE_OBJECT)
	dbuf_prepare_encrypted_dnode_leaf(dr);

	- if (db->db_state != DB_NOFILL &&
	+ if (datap != NULL && datap == db->db_buf &&
	dn->dn_object != DMU_META_DNODE_OBJECT &&
	zfs_refcount_count(&db->db_holds) > 1 &&
	- dr->dt.dl.dr_override_state != DR_OVERRIDDEN &&
	- *datap == db->db_buf) {
	+ dr->dt.dl.dr_override_state != DR_OVERRIDDEN) {
	/*
	* If this buffer is currently "in use" (i.e., there
	* are active holds and db_data still references it),
	* then make a copy before we start the write so that
	* any modifications from the open txg will not leak
	* into this write.
	*
	* NOTE: this copy does not need to be made for
	* objects only modified in the syncing context (e.g.
	* DNONE_DNODE blocks).
	*/
	int psize = arc_buf_size(*datap);
	int lsize = arc_buf_lsize(*datap);
	arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
	enum zio_compress compress_type = arc_get_compression(*datap);
	uint8_t complevel = arc_get_complevel(*datap);

	if (arc_is_encrypted(*datap)) {
	boolean_t byteorder;
	uint8_t salt[ZIO_DATA_SALT_LEN];
	uint8_t iv[ZIO_DATA_IV_LEN];
	uint8_t mac[ZIO_DATA_MAC_LEN];

	arc_get_raw_params(*datap, &byteorder, salt, iv, mac);
	*datap = arc_alloc_raw_buf(os->os_spa, db,
	dmu_objset_id(os), byteorder, salt, iv, mac,
	dn->dn_type, psize, lsize, compress_type,
	complevel);
	} else if (compress_type != ZIO_COMPRESS_OFF) {
	ASSERT3U(type, ==, ARC_BUFC_DATA);
	*datap = arc_alloc_compressed_buf(os->os_spa, db,
	psize, lsize, compress_type, complevel);
	} else {
	*datap = arc_alloc_buf(os->os_spa, db, type, psize);
	}
	memcpy((*datap)->b_data, db->db.db_data, psize);
	}
	db->db_data_pending = dr;

	mutex_exit(&db->db_mtx);

	dbuf_write(dr, *datap, tx);

	ASSERT(!list_link_active(&dr->dr_dirty_node));
	if (dn->dn_object == DMU_META_DNODE_OBJECT) {
	list_insert_tail(&dn->dn_dirty_records[txg & TXG_MASK], dr);
	} else {
	zio_nowait(dr->dr_zio);
	}
	}

	void
	dbuf_sync_list(list_t list, int level, dmu_tx_t tx)
	{
	dbuf_dirty_record_t *dr;

	while ((dr = list_head(list))) {
	if (dr->dr_zio != NULL) {
	/*
	* If we find an already initialized zio then we
	* are processing the meta-dnode, and we have finished.
	* The dbufs for all dnodes are put back on the list
	* during processing, so that we can zio_wait()
	* these IOs after initiating all child IOs.
	*/
	ASSERT3U(dr->dr_dbuf->db.db_object, ==,
	DMU_META_DNODE_OBJECT);
	break;
	}
	list_remove(list, dr);
	if (dr->dr_dbuf == NULL) {
	dbuf_sync_lightweight(dr, tx);
	} else {
	if (dr->dr_dbuf->db_blkid != DMU_BONUS_BLKID &&
	dr->dr_dbuf->db_blkid != DMU_SPILL_BLKID) {
	VERIFY3U(dr->dr_dbuf->db_level, ==, level);
	}
	if (dr->dr_dbuf->db_level > 0)
	dbuf_sync_indirect(dr, tx);
	else
	dbuf_sync_leaf(dr, tx);
	}
	}
	}

	static void
	dbuf_write_ready(zio_t zio, arc_buf_t buf, void *vdb)
	{
	(void) buf;
	dmu_buf_impl_t *db = vdb;
	dnode_t *dn;
	blkptr_t *bp = zio->io_bp;
	blkptr_t *bp_orig = &zio->io_bp_orig;
	spa_t *spa = zio->io_spa;
	int64_t delta;
	uint64_t fill = 0;
	int i;

	ASSERT3P(db->db_blkptr, !=, NULL);
	ASSERT3P(&db->db_data_pending->dr_bp_copy, ==, bp);

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);
	delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig);
	dnode_diduse_space(dn, delta - zio->io_prev_space_delta);
	zio->io_prev_space_delta = delta;

	if (bp->blk_birth != 0) {
	ASSERT((db->db_blkid != DMU_SPILL_BLKID &&
	BP_GET_TYPE(bp) == dn->dn_type) \|\|
	(db->db_blkid == DMU_SPILL_BLKID &&
	BP_GET_TYPE(bp) == dn->dn_bonustype) \|\|
	BP_IS_EMBEDDED(bp));
	ASSERT(BP_GET_LEVEL(bp) == db->db_level);
	}

	mutex_enter(&db->db_mtx);

	#ifdef ZFS_DEBUG
	if (db->db_blkid == DMU_SPILL_BLKID) {
	ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR);
	ASSERT(!(BP_IS_HOLE(bp)) &&
	db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys));
	}
	#endif

	if (db->db_level == 0) {
	mutex_enter(&dn->dn_mtx);
	if (db->db_blkid > dn->dn_phys->dn_maxblkid &&
	db->db_blkid != DMU_SPILL_BLKID) {
	ASSERT0(db->db_objset->os_raw_receive);
	dn->dn_phys->dn_maxblkid = db->db_blkid;
	}
	mutex_exit(&dn->dn_mtx);

	if (dn->dn_type == DMU_OT_DNODE) {
	i = 0;
	while (i < db->db.db_size) {
	dnode_phys_t *dnp =
	(void )(((char )db->db.db_data) + i);

	i += DNODE_MIN_SIZE;
	if (dnp->dn_type != DMU_OT_NONE) {
	fill++;
	for (int j = 0; j < dnp->dn_nblkptr;
	j++) {
	(void) zfs_blkptr_verify(spa,
	&dnp->dn_blkptr[j],
	BLK_CONFIG_SKIP,
	BLK_VERIFY_HALT);
	}
	if (dnp->dn_flags &
	DNODE_FLAG_SPILL_BLKPTR) {
	(void) zfs_blkptr_verify(spa,
	DN_SPILL_BLKPTR(dnp),
	BLK_CONFIG_SKIP,
	BLK_VERIFY_HALT);
	}
	i += dnp->dn_extra_slots *
	DNODE_MIN_SIZE;
	}
	}
	} else {
	if (BP_IS_HOLE(bp)) {
	fill = 0;
	} else {
	fill = 1;
	}
	}
	} else {
	blkptr_t *ibp = db->db.db_data;
	ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
	for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) {
	if (BP_IS_HOLE(ibp))
	continue;
	(void) zfs_blkptr_verify(spa, ibp,
	BLK_CONFIG_SKIP, BLK_VERIFY_HALT);
	fill += BP_GET_FILL(ibp);
	}
	}
	DB_DNODE_EXIT(db);

	if (!BP_IS_EMBEDDED(bp))
	BP_SET_FILL(bp, fill);

	mutex_exit(&db->db_mtx);

	db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_WRITER, FTAG);
	db->db_blkptr = bp;
	dmu_buf_unlock_parent(db, dblt, FTAG);
	}

	/*
	* This function gets called just prior to running through the compression
	* stage of the zio pipeline. If we're an indirect block comprised of only
	* holes, then we want this indirect to be compressed away to a hole. In
	* order to do that we must zero out any information about the holes that
	* this indirect points to prior to before we try to compress it.
	*/
	static void
	dbuf_write_children_ready(zio_t zio, arc_buf_t buf, void *vdb)
	{
	(void) zio, (void) buf;
	dmu_buf_impl_t *db = vdb;
	dnode_t *dn;
	blkptr_t *bp;
	unsigned int epbs, i;

	ASSERT3U(db->db_level, >, 0);
	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);
	epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
	ASSERT3U(epbs, <, 31);

	/* Determine if all our children are holes */
	for (i = 0, bp = db->db.db_data; i < 1ULL << epbs; i++, bp++) {
	if (!BP_IS_HOLE(bp))
	break;
	}

	/*
	* If all the children are holes, then zero them all out so that
	* we may get compressed away.
	*/
	if (i == 1ULL << epbs) {
	/*
	* We only found holes. Grab the rwlock to prevent
	* anybody from reading the blocks we're about to
	* zero out.
	*/
	rw_enter(&db->db_rwlock, RW_WRITER);
	memset(db->db.db_data, 0, db->db.db_size);
	rw_exit(&db->db_rwlock);
	}
	DB_DNODE_EXIT(db);
	}

	static void
	dbuf_write_done(zio_t zio, arc_buf_t buf, void *vdb)
	{
	(void) buf;
	dmu_buf_impl_t *db = vdb;
	blkptr_t *bp_orig = &zio->io_bp_orig;
	blkptr_t *bp = db->db_blkptr;
	objset_t *os = db->db_objset;
	dmu_tx_t *tx = os->os_synctx;

	ASSERT0(zio->io_error);
	ASSERT(db->db_blkptr == bp);

	/*
	* For nopwrites and rewrites we ensure that the bp matches our
	* original and bypass all the accounting.
	*/
	if (zio->io_flags & (ZIO_FLAG_IO_REWRITE \| ZIO_FLAG_NOPWRITE)) {
	ASSERT(BP_EQUAL(bp, bp_orig));
	} else {
	dsl_dataset_t *ds = os->os_dsl_dataset;
	(void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE);
	dsl_dataset_block_born(ds, bp, tx);
	}

	mutex_enter(&db->db_mtx);

	DBUF_VERIFY(db);

	dbuf_dirty_record_t *dr = db->db_data_pending;
	dnode_t *dn = dr->dr_dnode;
	ASSERT(!list_link_active(&dr->dr_dirty_node));
	ASSERT(dr->dr_dbuf == db);
	ASSERT(list_next(&db->db_dirty_records, dr) == NULL);
	list_remove(&db->db_dirty_records, dr);

	#ifdef ZFS_DEBUG
	if (db->db_blkid == DMU_SPILL_BLKID) {
	ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR);
	ASSERT(!(BP_IS_HOLE(db->db_blkptr)) &&
	db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys));
	}
	#endif

	if (db->db_level == 0) {
	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
	ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN);
	- if (db->db_state != DB_NOFILL) {
	- if (dr->dt.dl.dr_data != NULL &&
	- dr->dt.dl.dr_data != db->db_buf) {
	- arc_buf_destroy(dr->dt.dl.dr_data, db);
	- }
	+ if (dr->dt.dl.dr_data != NULL &&
	+ dr->dt.dl.dr_data != db->db_buf) {
	+ arc_buf_destroy(dr->dt.dl.dr_data, db);
	}
	} else {
	ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
	ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift);
	if (!BP_IS_HOLE(db->db_blkptr)) {
	int epbs __maybe_unused = dn->dn_phys->dn_indblkshift -
	SPA_BLKPTRSHIFT;
	ASSERT3U(db->db_blkid, <=,
	dn->dn_phys->dn_maxblkid >> (db->db_level * epbs));
	ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==,
	db->db.db_size);
	}
	mutex_destroy(&dr->dt.di.dr_mtx);
	list_destroy(&dr->dt.di.dr_children);
	}

	cv_broadcast(&db->db_changed);
	ASSERT(db->db_dirtycnt > 0);
	db->db_dirtycnt -= 1;
	db->db_data_pending = NULL;
	dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg, B_FALSE);

	dsl_pool_undirty_space(dmu_objset_pool(os), dr->dr_accounted,
	zio->io_txg);

	kmem_free(dr, sizeof (dbuf_dirty_record_t));
	}

	static void
	dbuf_write_nofill_ready(zio_t *zio)
	{
	dbuf_write_ready(zio, NULL, zio->io_private);
	}

	static void
	dbuf_write_nofill_done(zio_t *zio)
	{
	dbuf_write_done(zio, NULL, zio->io_private);
	}

	static void
	dbuf_write_override_ready(zio_t *zio)
	{
	dbuf_dirty_record_t *dr = zio->io_private;
	dmu_buf_impl_t *db = dr->dr_dbuf;

	dbuf_write_ready(zio, NULL, db);
	}

	static void
	dbuf_write_override_done(zio_t *zio)
	{
	dbuf_dirty_record_t *dr = zio->io_private;
	dmu_buf_impl_t *db = dr->dr_dbuf;
	blkptr_t *obp = &dr->dt.dl.dr_overridden_by;

	mutex_enter(&db->db_mtx);
	if (!BP_EQUAL(zio->io_bp, obp)) {
	if (!BP_IS_HOLE(obp))
	dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp);
	arc_release(dr->dt.dl.dr_data, db);
	}
	mutex_exit(&db->db_mtx);

	dbuf_write_done(zio, NULL, db);

	if (zio->io_abd != NULL)
	abd_free(zio->io_abd);
	}

	typedef struct dbuf_remap_impl_callback_arg {
	objset_t *drica_os;
	uint64_t drica_blk_birth;
	dmu_tx_t *drica_tx;
	} dbuf_remap_impl_callback_arg_t;

	static void
	dbuf_remap_impl_callback(uint64_t vdev, uint64_t offset, uint64_t size,
	void *arg)
	{
	dbuf_remap_impl_callback_arg_t *drica = arg;
	objset_t *os = drica->drica_os;
	spa_t *spa = dmu_objset_spa(os);
	dmu_tx_t *tx = drica->drica_tx;

	ASSERT(dsl_pool_sync_context(spa_get_dsl(spa)));

	if (os == spa_meta_objset(spa)) {
	spa_vdev_indirect_mark_obsolete(spa, vdev, offset, size, tx);
	} else {
	dsl_dataset_block_remapped(dmu_objset_ds(os), vdev, offset,
	size, drica->drica_blk_birth, tx);
	}
	}

	static void
	dbuf_remap_impl(dnode_t dn, blkptr_t bp, krwlock_t rw, dmu_tx_t tx)
	{
	blkptr_t bp_copy = *bp;
	spa_t *spa = dmu_objset_spa(dn->dn_objset);
	dbuf_remap_impl_callback_arg_t drica;

	ASSERT(dsl_pool_sync_context(spa_get_dsl(spa)));

	drica.drica_os = dn->dn_objset;
	drica.drica_blk_birth = bp->blk_birth;
	drica.drica_tx = tx;
	if (spa_remap_blkptr(spa, &bp_copy, dbuf_remap_impl_callback,
	&drica)) {
	/*
	* If the blkptr being remapped is tracked by a livelist,
	* then we need to make sure the livelist reflects the update.
	* First, cancel out the old blkptr by appending a 'FREE'
	* entry. Next, add an 'ALLOC' to track the new version. This
	* way we avoid trying to free an inaccurate blkptr at delete.
	* Note that embedded blkptrs are not tracked in livelists.
	*/
	if (dn->dn_objset != spa_meta_objset(spa)) {
	dsl_dataset_t *ds = dmu_objset_ds(dn->dn_objset);
	if (dsl_deadlist_is_open(&ds->ds_dir->dd_livelist) &&
	bp->blk_birth > ds->ds_dir->dd_origin_txg) {
	ASSERT(!BP_IS_EMBEDDED(bp));
	ASSERT(dsl_dir_is_clone(ds->ds_dir));
	ASSERT(spa_feature_is_enabled(spa,
	SPA_FEATURE_LIVELIST));
	bplist_append(&ds->ds_dir->dd_pending_frees,
	bp);
	bplist_append(&ds->ds_dir->dd_pending_allocs,
	&bp_copy);
	}
	}

	/*
	* The db_rwlock prevents dbuf_read_impl() from
	* dereferencing the BP while we are changing it. To
	* avoid lock contention, only grab it when we are actually
	* changing the BP.
	*/
	if (rw != NULL)
	rw_enter(rw, RW_WRITER);
	*bp = bp_copy;
	if (rw != NULL)
	rw_exit(rw);
	}
	}

	/*
	* Remap any existing BP's to concrete vdevs, if possible.
	*/
	static void
	dbuf_remap(dnode_t dn, dmu_buf_impl_t db, dmu_tx_t *tx)
	{
	spa_t *spa = dmu_objset_spa(db->db_objset);
	ASSERT(dsl_pool_sync_context(spa_get_dsl(spa)));

	if (!spa_feature_is_active(spa, SPA_FEATURE_DEVICE_REMOVAL))
	return;

	if (db->db_level > 0) {
	blkptr_t *bp = db->db.db_data;
	for (int i = 0; i < db->db.db_size >> SPA_BLKPTRSHIFT; i++) {
	dbuf_remap_impl(dn, &bp[i], &db->db_rwlock, tx);
	}
	} else if (db->db.db_object == DMU_META_DNODE_OBJECT) {
	dnode_phys_t *dnp = db->db.db_data;
	ASSERT3U(db->db_dnode_handle->dnh_dnode->dn_type, ==,
	DMU_OT_DNODE);
	for (int i = 0; i < db->db.db_size >> DNODE_SHIFT;
	i += dnp[i].dn_extra_slots + 1) {
	for (int j = 0; j < dnp[i].dn_nblkptr; j++) {
	krwlock_t *lock = (dn->dn_dbuf == NULL ? NULL :
	&dn->dn_dbuf->db_rwlock);
	dbuf_remap_impl(dn, &dnp[i].dn_blkptr[j], lock,
	tx);
	}
	}
	}
	}


	/* Issue I/O to commit a dirty buffer to disk. */
	static void
	dbuf_write(dbuf_dirty_record_t dr, arc_buf_t data, dmu_tx_t *tx)
	{
	dmu_buf_impl_t *db = dr->dr_dbuf;
	dnode_t *dn = dr->dr_dnode;
	objset_t *os;
	dmu_buf_impl_t *parent = db->db_parent;
	uint64_t txg = tx->tx_txg;
	zbookmark_phys_t zb;
	zio_prop_t zp;
	zio_t pio; / parent I/O */
	int wp_flag = 0;

	ASSERT(dmu_tx_is_syncing(tx));

	os = dn->dn_objset;

	- if (db->db_state != DB_NOFILL) {
	- if (db->db_level > 0 \|\| dn->dn_type == DMU_OT_DNODE) {
	- /*
	- * Private object buffers are released here rather
	- * than in dbuf_dirty() since they are only modified
	- * in the syncing context and we don't want the
	- * overhead of making multiple copies of the data.
	- */
	- if (BP_IS_HOLE(db->db_blkptr)) {
	- arc_buf_thaw(data);
	- } else {
	- dbuf_release_bp(db);
	- }
	- dbuf_remap(dn, db, tx);
	- }
	+ if (db->db_level > 0 \|\| dn->dn_type == DMU_OT_DNODE) {
	+ /*
	+ * Private object buffers are released here rather than in
	+ * dbuf_dirty() since they are only modified in the syncing
	+ * context and we don't want the overhead of making multiple
	+ * copies of the data.
	+ */
	+ if (BP_IS_HOLE(db->db_blkptr))
	+ arc_buf_thaw(data);
	+ else
	+ dbuf_release_bp(db);
	+ dbuf_remap(dn, db, tx);
	}

	if (parent != dn->dn_dbuf) {
	/* Our parent is an indirect block. */
	/* We have a dirty parent that has been scheduled for write. */
	ASSERT(parent && parent->db_data_pending);
	/* Our parent's buffer is one level closer to the dnode. */
	ASSERT(db->db_level == parent->db_level-1);
	/*
	* We're about to modify our parent's db_data by modifying
	* our block pointer, so the parent must be released.
	*/
	ASSERT(arc_released(parent->db_buf));
	pio = parent->db_data_pending->dr_zio;
	} else {
	/* Our parent is the dnode itself. */
	ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 &&
	db->db_blkid != DMU_SPILL_BLKID) \|\|
	(db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0));
	if (db->db_blkid != DMU_SPILL_BLKID)
	ASSERT3P(db->db_blkptr, ==,
	&dn->dn_phys->dn_blkptr[db->db_blkid]);
	pio = dn->dn_zio;
	}

	ASSERT(db->db_level == 0 \|\| data == db->db_buf);
	ASSERT3U(db->db_blkptr->blk_birth, <=, txg);
	ASSERT(pio);

	SET_BOOKMARK(&zb, os->os_dsl_dataset ?
	os->os_dsl_dataset->ds_object : DMU_META_OBJSET,
	db->db.db_object, db->db_level, db->db_blkid);

	if (db->db_blkid == DMU_SPILL_BLKID)
	wp_flag = WP_SPILL;
	- wp_flag \|= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0;
	+ wp_flag \|= (data == NULL) ? WP_NOFILL : 0;

	dmu_write_policy(os, dn, db->db_level, wp_flag, &zp);

	/*
	* We copy the blkptr now (rather than when we instantiate the dirty
	* record), because its value can change between open context and
	* syncing context. We do not need to hold dn_struct_rwlock to read
	* db_blkptr because we are in syncing context.
	*/
	dr->dr_bp_copy = *db->db_blkptr;

	if (db->db_level == 0 &&
	dr->dt.dl.dr_override_state == DR_OVERRIDDEN) {
	/*
	* The BP for this block has been provided by open context
	* (by dmu_sync() or dmu_buf_write_embedded()).
	*/
	abd_t *contents = (data != NULL) ?
	abd_get_from_buf(data->b_data, arc_buf_size(data)) : NULL;

	dr->dr_zio = zio_write(pio, os->os_spa, txg, &dr->dr_bp_copy,
	contents, db->db.db_size, db->db.db_size, &zp,
	dbuf_write_override_ready, NULL,
	dbuf_write_override_done,
	dr, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb);
	mutex_enter(&db->db_mtx);
	dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
	zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by,
	dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite,
	dr->dt.dl.dr_brtwrite);
	mutex_exit(&db->db_mtx);
	- } else if (db->db_state == DB_NOFILL) {
	+ } else if (data == NULL) {
	ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF \|\|
	zp.zp_checksum == ZIO_CHECKSUM_NOPARITY);
	dr->dr_zio = zio_write(pio, os->os_spa, txg,
	&dr->dr_bp_copy, NULL, db->db.db_size, db->db.db_size, &zp,
	dbuf_write_nofill_ready, NULL,
	dbuf_write_nofill_done, db,
	ZIO_PRIORITY_ASYNC_WRITE,
	ZIO_FLAG_MUSTSUCCEED \| ZIO_FLAG_NODATA, &zb);
	} else {
	ASSERT(arc_released(data));

	/*
	* For indirect blocks, we want to setup the children
	* ready callback so that we can properly handle an indirect
	* block that only contains holes.
	*/
	arc_write_done_func_t *children_ready_cb = NULL;
	if (db->db_level != 0)
	children_ready_cb = dbuf_write_children_ready;

	dr->dr_zio = arc_write(pio, os->os_spa, txg,
	&dr->dr_bp_copy, data, !DBUF_IS_CACHEABLE(db),
	dbuf_is_l2cacheable(db), &zp, dbuf_write_ready,
	children_ready_cb, dbuf_write_done, db,
	ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb);
	}
	}

	EXPORT_SYMBOL(dbuf_find);
	EXPORT_SYMBOL(dbuf_is_metadata);
	EXPORT_SYMBOL(dbuf_destroy);
	EXPORT_SYMBOL(dbuf_loan_arcbuf);
	EXPORT_SYMBOL(dbuf_whichblock);
	EXPORT_SYMBOL(dbuf_read);
	EXPORT_SYMBOL(dbuf_unoverride);
	EXPORT_SYMBOL(dbuf_free_range);
	EXPORT_SYMBOL(dbuf_new_size);
	EXPORT_SYMBOL(dbuf_release_bp);
	EXPORT_SYMBOL(dbuf_dirty);
	EXPORT_SYMBOL(dmu_buf_set_crypt_params);
	EXPORT_SYMBOL(dmu_buf_will_dirty);
	EXPORT_SYMBOL(dmu_buf_is_dirty);
	EXPORT_SYMBOL(dmu_buf_will_clone);
	EXPORT_SYMBOL(dmu_buf_will_not_fill);
	EXPORT_SYMBOL(dmu_buf_will_fill);
	EXPORT_SYMBOL(dmu_buf_fill_done);
	EXPORT_SYMBOL(dmu_buf_rele);
	EXPORT_SYMBOL(dbuf_assign_arcbuf);
	EXPORT_SYMBOL(dbuf_prefetch);
	EXPORT_SYMBOL(dbuf_hold_impl);
	EXPORT_SYMBOL(dbuf_hold);
	EXPORT_SYMBOL(dbuf_hold_level);
	EXPORT_SYMBOL(dbuf_create_bonus);
	EXPORT_SYMBOL(dbuf_spill_set_blksz);
	EXPORT_SYMBOL(dbuf_rm_spill);
	EXPORT_SYMBOL(dbuf_add_ref);
	EXPORT_SYMBOL(dbuf_rele);
	EXPORT_SYMBOL(dbuf_rele_and_unlock);
	EXPORT_SYMBOL(dbuf_refcount);
	EXPORT_SYMBOL(dbuf_sync_list);
	EXPORT_SYMBOL(dmu_buf_set_user);
	EXPORT_SYMBOL(dmu_buf_set_user_ie);
	EXPORT_SYMBOL(dmu_buf_get_user);
	EXPORT_SYMBOL(dmu_buf_get_blkptr);

	ZFS_MODULE_PARAM(zfs_dbuf_cache, dbuf_cache_, max_bytes, U64, ZMOD_RW,
	"Maximum size in bytes of the dbuf cache.");

	ZFS_MODULE_PARAM(zfs_dbuf_cache, dbuf_cache_, hiwater_pct, UINT, ZMOD_RW,
	"Percentage over dbuf_cache_max_bytes for direct dbuf eviction.");

	ZFS_MODULE_PARAM(zfs_dbuf_cache, dbuf_cache_, lowater_pct, UINT, ZMOD_RW,
	"Percentage below dbuf_cache_max_bytes when dbuf eviction stops.");

	ZFS_MODULE_PARAM(zfs_dbuf, dbuf_, metadata_cache_max_bytes, U64, ZMOD_RW,
	"Maximum size in bytes of dbuf metadata cache.");

	ZFS_MODULE_PARAM(zfs_dbuf, dbuf_, cache_shift, UINT, ZMOD_RW,
	"Set size of dbuf cache to log2 fraction of arc size.");

	ZFS_MODULE_PARAM(zfs_dbuf, dbuf_, metadata_cache_shift, UINT, ZMOD_RW,
	"Set size of dbuf metadata cache to log2 fraction of arc size.");

	ZFS_MODULE_PARAM(zfs_dbuf, dbuf_, mutex_cache_shift, UINT, ZMOD_RD,
	"Set size of dbuf cache mutex array as log2 shift.");
	diff --git a/sys/contrib/openzfs/module/zfs/dmu.c b/sys/contrib/openzfs/module/zfs/dmu.c
	index 3215ab1c2a14..d8d5cfdbd230 100644
	--- a/sys/contrib/openzfs/module/zfs/dmu.c
	+++ b/sys/contrib/openzfs/module/zfs/dmu.c
	@@ -1,2586 +1,2617 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
	* Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
	* Copyright (c) 2013, Joyent, Inc. All rights reserved.
	* Copyright (c) 2016, Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2015 by Chunwei Chen. All rights reserved.
	* Copyright (c) 2019 Datto Inc.
	* Copyright (c) 2019, Klara Inc.
	* Copyright (c) 2019, Allan Jude
	* Copyright (c) 2022 Hewlett Packard Enterprise Development LP.
	* Copyright (c) 2021, 2022 by Pawel Jakub Dawidek
	*/

	#include <sys/dmu.h>
	#include <sys/dmu_impl.h>
	#include <sys/dmu_tx.h>
	#include <sys/dbuf.h>
	#include <sys/dnode.h>
	#include <sys/zfs_context.h>
	#include <sys/dmu_objset.h>
	#include <sys/dmu_traverse.h>
	#include <sys/dsl_dataset.h>
	#include <sys/dsl_dir.h>
	#include <sys/dsl_pool.h>
	#include <sys/dsl_synctask.h>
	#include <sys/dsl_prop.h>
	#include <sys/dmu_zfetch.h>
	#include <sys/zfs_ioctl.h>
	#include <sys/zap.h>
	#include <sys/zio_checksum.h>
	#include <sys/zio_compress.h>
	#include <sys/sa.h>
	#include <sys/zfeature.h>
	#include <sys/abd.h>
	#include <sys/brt.h>
	#include <sys/trace_zfs.h>
	#include <sys/zfs_racct.h>
	#include <sys/zfs_rlock.h>
	#ifdef _KERNEL
	#include <sys/vmsystm.h>
	#include <sys/zfs_znode.h>
	#endif

	/*
	* Enable/disable nopwrite feature.
	*/
	static int zfs_nopwrite_enabled = 1;

	/*
	* Tunable to control percentage of dirtied L1 blocks from frees allowed into
	* one TXG. After this threshold is crossed, additional dirty blocks from frees
	* will wait until the next TXG.
	* A value of zero will disable this throttle.
	*/
	static uint_t zfs_per_txg_dirty_frees_percent = 30;

	/*
	* Enable/disable forcing txg sync when dirty checking for holes with lseek().
	* By default this is enabled to ensure accurate hole reporting, it can result
	* in a significant performance penalty for lseek(SEEK_HOLE) heavy workloads.
	* Disabling this option will result in holes never being reported in dirty
	* files which is always safe.
	*/
	static int zfs_dmu_offset_next_sync = 1;

	/*
	* Limit the amount we can prefetch with one call to this amount. This
	* helps to limit the amount of memory that can be used by prefetching.
	* Larger objects should be prefetched a bit at a time.
	*/
	#ifdef _ILP32
	uint_t dmu_prefetch_max = 8 * 1024 * 1024;
	#else
	uint_t dmu_prefetch_max = 8 * SPA_MAXBLOCKSIZE;
	#endif

	const dmu_object_type_info_t dmu_ot[DMU_OT_NUMTYPES] = {
	{DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "unallocated" },
	{DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "object directory" },
	{DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "object array" },
	{DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "packed nvlist" },
	{DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "packed nvlist size" },
	{DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj" },
	{DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj header" },
	{DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA space map header" },
	{DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA space map" },
	{DMU_BSWAP_UINT64, TRUE, FALSE, TRUE, "ZIL intent log" },
	{DMU_BSWAP_DNODE, TRUE, FALSE, TRUE, "DMU dnode" },
	{DMU_BSWAP_OBJSET, TRUE, TRUE, FALSE, "DMU objset" },
	{DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL directory" },
	{DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL directory child map"},
	{DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dataset snap map" },
	{DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL props" },
	{DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL dataset" },
	{DMU_BSWAP_ZNODE, TRUE, FALSE, FALSE, "ZFS znode" },
	{DMU_BSWAP_OLDACL, TRUE, FALSE, TRUE, "ZFS V0 ACL" },
	{DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "ZFS plain file" },
	{DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS directory" },
	{DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "ZFS master node" },
	{DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS delete queue" },
	{DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "zvol object" },
	{DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "zvol prop" },
	{DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "other uint8[]" },
	{DMU_BSWAP_UINT64, FALSE, FALSE, TRUE, "other uint64[]" },
	{DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "other ZAP" },
	{DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "persistent error log" },
	{DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "SPA history" },
	{DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA history offsets" },
	{DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "Pool properties" },
	{DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL permissions" },
	{DMU_BSWAP_ACL, TRUE, FALSE, TRUE, "ZFS ACL" },
	{DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "ZFS SYSACL" },
	{DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "FUID table" },
	{DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "FUID table size" },
	{DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dataset next clones"},
	{DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "scan work queue" },
	{DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS user/group/project used" },
	{DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS user/group/project quota"},
	{DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "snapshot refcount tags"},
	{DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "DDT ZAP algorithm" },
	{DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "DDT statistics" },
	{DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "System attributes" },
	{DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA master node" },
	{DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA attr registration" },
	{DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA attr layouts" },
	{DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "scan translations" },
	{DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "deduplicated block" },
	{DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL deadlist map" },
	{DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL deadlist map hdr" },
	{DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dir clones" },
	{DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj subobj" }
	};

	dmu_object_byteswap_info_t dmu_ot_byteswap[DMU_BSWAP_NUMFUNCS] = {
	{ byteswap_uint8_array, "uint8" },
	{ byteswap_uint16_array, "uint16" },
	{ byteswap_uint32_array, "uint32" },
	{ byteswap_uint64_array, "uint64" },
	{ zap_byteswap, "zap" },
	{ dnode_buf_byteswap, "dnode" },
	{ dmu_objset_byteswap, "objset" },
	{ zfs_znode_byteswap, "znode" },
	{ zfs_oldacl_byteswap, "oldacl" },
	{ zfs_acl_byteswap, "acl" }
	};

	int
	dmu_buf_hold_noread_by_dnode(dnode_t *dn, uint64_t offset,
	const void tag, dmu_buf_t *dbp)
	{
	uint64_t blkid;
	dmu_buf_impl_t *db;

	rw_enter(&dn->dn_struct_rwlock, RW_READER);
	blkid = dbuf_whichblock(dn, 0, offset);
	db = dbuf_hold(dn, blkid, tag);
	rw_exit(&dn->dn_struct_rwlock);

	if (db == NULL) {
	*dbp = NULL;
	return (SET_ERROR(EIO));
	}

	*dbp = &db->db;
	return (0);
	}

	int
	dmu_buf_hold_noread(objset_t *os, uint64_t object, uint64_t offset,
	const void tag, dmu_buf_t *dbp)
	{
	dnode_t *dn;
	uint64_t blkid;
	dmu_buf_impl_t *db;
	int err;

	err = dnode_hold(os, object, FTAG, &dn);
	if (err)
	return (err);
	rw_enter(&dn->dn_struct_rwlock, RW_READER);
	blkid = dbuf_whichblock(dn, 0, offset);
	db = dbuf_hold(dn, blkid, tag);
	rw_exit(&dn->dn_struct_rwlock);
	dnode_rele(dn, FTAG);

	if (db == NULL) {
	*dbp = NULL;
	return (SET_ERROR(EIO));
	}

	*dbp = &db->db;
	return (err);
	}

	int
	dmu_buf_hold_by_dnode(dnode_t *dn, uint64_t offset,
	const void tag, dmu_buf_t *dbp, int flags)
	{
	int err;
	int db_flags = DB_RF_CANFAIL;

	if (flags & DMU_READ_NO_PREFETCH)
	db_flags \|= DB_RF_NOPREFETCH;
	if (flags & DMU_READ_NO_DECRYPT)
	db_flags \|= DB_RF_NO_DECRYPT;

	err = dmu_buf_hold_noread_by_dnode(dn, offset, tag, dbp);
	if (err == 0) {
	dmu_buf_impl_t db = (dmu_buf_impl_t )(*dbp);
	err = dbuf_read(db, NULL, db_flags);
	if (err != 0) {
	dbuf_rele(db, tag);
	*dbp = NULL;
	}
	}

	return (err);
	}

	int
	dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset,
	const void tag, dmu_buf_t *dbp, int flags)
	{
	int err;
	int db_flags = DB_RF_CANFAIL;

	if (flags & DMU_READ_NO_PREFETCH)
	db_flags \|= DB_RF_NOPREFETCH;
	if (flags & DMU_READ_NO_DECRYPT)
	db_flags \|= DB_RF_NO_DECRYPT;

	err = dmu_buf_hold_noread(os, object, offset, tag, dbp);
	if (err == 0) {
	dmu_buf_impl_t db = (dmu_buf_impl_t )(*dbp);
	err = dbuf_read(db, NULL, db_flags);
	if (err != 0) {
	dbuf_rele(db, tag);
	*dbp = NULL;
	}
	}

	return (err);
	}

	int
	dmu_bonus_max(void)
	{
	return (DN_OLD_MAX_BONUSLEN);
	}

	int
	dmu_set_bonus(dmu_buf_t db_fake, int newsize, dmu_tx_t tx)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;
	dnode_t *dn;
	int error;

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);

	if (dn->dn_bonus != db) {
	error = SET_ERROR(EINVAL);
	} else if (newsize < 0 \|\| newsize > db_fake->db_size) {
	error = SET_ERROR(EINVAL);
	} else {
	dnode_setbonuslen(dn, newsize, tx);
	error = 0;
	}

	DB_DNODE_EXIT(db);
	return (error);
	}

	int
	dmu_set_bonustype(dmu_buf_t db_fake, dmu_object_type_t type, dmu_tx_t tx)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;
	dnode_t *dn;
	int error;

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);

	if (!DMU_OT_IS_VALID(type)) {
	error = SET_ERROR(EINVAL);
	} else if (dn->dn_bonus != db) {
	error = SET_ERROR(EINVAL);
	} else {
	dnode_setbonus_type(dn, type, tx);
	error = 0;
	}

	DB_DNODE_EXIT(db);
	return (error);
	}

	dmu_object_type_t
	dmu_get_bonustype(dmu_buf_t *db_fake)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;
	dnode_t *dn;
	dmu_object_type_t type;

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);
	type = dn->dn_bonustype;
	DB_DNODE_EXIT(db);

	return (type);
	}

	int
	dmu_rm_spill(objset_t os, uint64_t object, dmu_tx_t tx)
	{
	dnode_t *dn;
	int error;

	error = dnode_hold(os, object, FTAG, &dn);
	dbuf_rm_spill(dn, tx);
	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
	dnode_rm_spill(dn, tx);
	rw_exit(&dn->dn_struct_rwlock);
	dnode_rele(dn, FTAG);
	return (error);
	}

	/*
	* Lookup and hold the bonus buffer for the provided dnode. If the dnode
	* has not yet been allocated a new bonus dbuf a will be allocated.
	* Returns ENOENT, EIO, or 0.
	*/
	int dmu_bonus_hold_by_dnode(dnode_t dn, const void tag, dmu_buf_t **dbp,
	uint32_t flags)
	{
	dmu_buf_impl_t *db;
	int error;
	uint32_t db_flags = DB_RF_MUST_SUCCEED;

	if (flags & DMU_READ_NO_PREFETCH)
	db_flags \|= DB_RF_NOPREFETCH;
	if (flags & DMU_READ_NO_DECRYPT)
	db_flags \|= DB_RF_NO_DECRYPT;

	rw_enter(&dn->dn_struct_rwlock, RW_READER);
	if (dn->dn_bonus == NULL) {
	if (!rw_tryupgrade(&dn->dn_struct_rwlock)) {
	rw_exit(&dn->dn_struct_rwlock);
	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
	}
	if (dn->dn_bonus == NULL)
	dbuf_create_bonus(dn);
	}
	db = dn->dn_bonus;

	/* as long as the bonus buf is held, the dnode will be held */
	if (zfs_refcount_add(&db->db_holds, tag) == 1) {
	VERIFY(dnode_add_ref(dn, db));
	atomic_inc_32(&dn->dn_dbufs_count);
	}

	/*
	* Wait to drop dn_struct_rwlock until after adding the bonus dbuf's
	* hold and incrementing the dbuf count to ensure that dnode_move() sees
	* a dnode hold for every dbuf.
	*/
	rw_exit(&dn->dn_struct_rwlock);

	error = dbuf_read(db, NULL, db_flags);
	if (error) {
	dnode_evict_bonus(dn);
	dbuf_rele(db, tag);
	*dbp = NULL;
	return (error);
	}

	*dbp = &db->db;
	return (0);
	}

	int
	dmu_bonus_hold(objset_t os, uint64_t object, const void tag, dmu_buf_t **dbp)
	{
	dnode_t *dn;
	int error;

	error = dnode_hold(os, object, FTAG, &dn);
	if (error)
	return (error);

	error = dmu_bonus_hold_by_dnode(dn, tag, dbp, DMU_READ_NO_PREFETCH);
	dnode_rele(dn, FTAG);

	return (error);
	}

	/*
	* returns ENOENT, EIO, or 0.
	*
	* This interface will allocate a blank spill dbuf when a spill blk
	* doesn't already exist on the dnode.
	*
	* if you only want to find an already existing spill db, then
	* dmu_spill_hold_existing() should be used.
	*/
	int
	dmu_spill_hold_by_dnode(dnode_t dn, uint32_t flags, const void tag,
	dmu_buf_t **dbp)
	{
	dmu_buf_impl_t *db = NULL;
	int err;

	if ((flags & DB_RF_HAVESTRUCT) == 0)
	rw_enter(&dn->dn_struct_rwlock, RW_READER);

	db = dbuf_hold(dn, DMU_SPILL_BLKID, tag);

	if ((flags & DB_RF_HAVESTRUCT) == 0)
	rw_exit(&dn->dn_struct_rwlock);

	if (db == NULL) {
	*dbp = NULL;
	return (SET_ERROR(EIO));
	}
	err = dbuf_read(db, NULL, flags);
	if (err == 0)
	*dbp = &db->db;
	else {
	dbuf_rele(db, tag);
	*dbp = NULL;
	}
	return (err);
	}

	int
	dmu_spill_hold_existing(dmu_buf_t bonus, const void tag, dmu_buf_t **dbp)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )bonus;
	dnode_t *dn;
	int err;

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);

	if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_SA) {
	err = SET_ERROR(EINVAL);
	} else {
	rw_enter(&dn->dn_struct_rwlock, RW_READER);

	if (!dn->dn_have_spill) {
	err = SET_ERROR(ENOENT);
	} else {
	err = dmu_spill_hold_by_dnode(dn,
	DB_RF_HAVESTRUCT \| DB_RF_CANFAIL, tag, dbp);
	}

	rw_exit(&dn->dn_struct_rwlock);
	}

	DB_DNODE_EXIT(db);
	return (err);
	}

	int
	dmu_spill_hold_by_bonus(dmu_buf_t bonus, uint32_t flags, const void tag,
	dmu_buf_t **dbp)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )bonus;
	dnode_t *dn;
	int err;
	uint32_t db_flags = DB_RF_CANFAIL;

	if (flags & DMU_READ_NO_DECRYPT)
	db_flags \|= DB_RF_NO_DECRYPT;

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);
	err = dmu_spill_hold_by_dnode(dn, db_flags, tag, dbp);
	DB_DNODE_EXIT(db);

	return (err);
	}

	/*
	* Note: longer-term, we should modify all of the dmu_buf_*() interfaces
	* to take a held dnode rather than <os, object> -- the lookup is wasteful,
	* and can induce severe lock contention when writing to several files
	* whose dnodes are in the same block.
	*/
	int
	dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length,
	boolean_t read, const void tag, int numbufsp, dmu_buf_t ***dbpp,
	uint32_t flags)
	{
	dmu_buf_t **dbp;
	zstream_t *zs = NULL;
	uint64_t blkid, nblks, i;
	uint32_t dbuf_flags;
	int err;
	zio_t *zio = NULL;
	boolean_t missed = B_FALSE;

	ASSERT(!read \|\| length <= DMU_MAX_ACCESS);

	/*
	* Note: We directly notify the prefetch code of this read, so that
	* we can tell it about the multi-block read. dbuf_read() only knows
	* about the one block it is accessing.
	*/
	dbuf_flags = DB_RF_CANFAIL \| DB_RF_NEVERWAIT \| DB_RF_HAVESTRUCT \|
	DB_RF_NOPREFETCH;

	if ((flags & DMU_READ_NO_DECRYPT) != 0)
	dbuf_flags \|= DB_RF_NO_DECRYPT;

	rw_enter(&dn->dn_struct_rwlock, RW_READER);
	if (dn->dn_datablkshift) {
	int blkshift = dn->dn_datablkshift;
	nblks = (P2ROUNDUP(offset + length, 1ULL << blkshift) -
	P2ALIGN(offset, 1ULL << blkshift)) >> blkshift;
	} else {
	if (offset + length > dn->dn_datablksz) {
	zfs_panic_recover("zfs: accessing past end of object "
	"%llx/%llx (size=%u access=%llu+%llu)",
	(longlong_t)dn->dn_objset->
	os_dsl_dataset->ds_object,
	(longlong_t)dn->dn_object, dn->dn_datablksz,
	(longlong_t)offset, (longlong_t)length);
	rw_exit(&dn->dn_struct_rwlock);
	return (SET_ERROR(EIO));
	}
	nblks = 1;
	}
	dbp = kmem_zalloc(sizeof (dmu_buf_t ) nblks, KM_SLEEP);

	if (read)
	zio = zio_root(dn->dn_objset->os_spa, NULL, NULL,
	ZIO_FLAG_CANFAIL);
	blkid = dbuf_whichblock(dn, 0, offset);
	if ((flags & DMU_READ_NO_PREFETCH) == 0) {
	/*
	* Prepare the zfetch before initiating the demand reads, so
	* that if multiple threads block on same indirect block, we
	* base predictions on the original less racy request order.
	*/
	zs = dmu_zfetch_prepare(&dn->dn_zfetch, blkid, nblks, read,
	B_TRUE);
	}
	for (i = 0; i < nblks; i++) {
	dmu_buf_impl_t *db = dbuf_hold(dn, blkid + i, tag);
	if (db == NULL) {
	- if (zs)
	- dmu_zfetch_run(zs, missed, B_TRUE);
	+ if (zs) {
	+ dmu_zfetch_run(&dn->dn_zfetch, zs, missed,
	+ B_TRUE);
	+ }
	rw_exit(&dn->dn_struct_rwlock);
	dmu_buf_rele_array(dbp, nblks, tag);
	if (read)
	zio_nowait(zio);
	return (SET_ERROR(EIO));
	}

	/*
	* Initiate async demand data read.
	* We check the db_state after calling dbuf_read() because
	* (1) dbuf_read() may change the state to CACHED due to a
	* hit in the ARC, and (2) on a cache miss, a child will
	* have been added to "zio" but not yet completed, so the
	* state will not yet be CACHED.
	*/
	if (read) {
	if (i == nblks - 1 && blkid + i < dn->dn_maxblkid &&
	offset + length < db->db.db_offset +
	db->db.db_size) {
	if (offset <= db->db.db_offset)
	dbuf_flags \|= DB_RF_PARTIAL_FIRST;
	else
	dbuf_flags \|= DB_RF_PARTIAL_MORE;
	}
	(void) dbuf_read(db, zio, dbuf_flags);
	if (db->db_state != DB_CACHED)
	missed = B_TRUE;
	}
	dbp[i] = &db->db;
	}

	if (!read)
	zfs_racct_write(length, nblks);

	if (zs)
	- dmu_zfetch_run(zs, missed, B_TRUE);
	+ dmu_zfetch_run(&dn->dn_zfetch, zs, missed, B_TRUE);
	rw_exit(&dn->dn_struct_rwlock);

	if (read) {
	/* wait for async read i/o */
	err = zio_wait(zio);
	if (err) {
	dmu_buf_rele_array(dbp, nblks, tag);
	return (err);
	}

	/* wait for other io to complete */
	for (i = 0; i < nblks; i++) {
	dmu_buf_impl_t db = (dmu_buf_impl_t )dbp[i];
	mutex_enter(&db->db_mtx);
	while (db->db_state == DB_READ \|\|
	db->db_state == DB_FILL)
	cv_wait(&db->db_changed, &db->db_mtx);
	if (db->db_state == DB_UNCACHED)
	err = SET_ERROR(EIO);
	mutex_exit(&db->db_mtx);
	if (err) {
	dmu_buf_rele_array(dbp, nblks, tag);
	return (err);
	}
	}
	}

	*numbufsp = nblks;
	*dbpp = dbp;
	return (0);
	}

	int
	dmu_buf_hold_array(objset_t *os, uint64_t object, uint64_t offset,
	uint64_t length, int read, const void tag, int numbufsp,
	dmu_buf_t ***dbpp)
	{
	dnode_t *dn;
	int err;

	err = dnode_hold(os, object, FTAG, &dn);
	if (err)
	return (err);

	err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag,
	numbufsp, dbpp, DMU_READ_PREFETCH);

	dnode_rele(dn, FTAG);

	return (err);
	}

	int
	dmu_buf_hold_array_by_bonus(dmu_buf_t *db_fake, uint64_t offset,
	uint64_t length, boolean_t read, const void tag, int numbufsp,
	dmu_buf_t ***dbpp)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;
	dnode_t *dn;
	int err;

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);
	err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag,
	numbufsp, dbpp, DMU_READ_PREFETCH);
	DB_DNODE_EXIT(db);

	return (err);
	}

	void
	dmu_buf_rele_array(dmu_buf_t *dbp_fake, int numbufs, const void tag)
	{
	int i;
	dmu_buf_impl_t dbp = (dmu_buf_impl_t )dbp_fake;

	if (numbufs == 0)
	return;

	for (i = 0; i < numbufs; i++) {
	if (dbp[i])
	dbuf_rele(dbp[i], tag);
	}

	kmem_free(dbp, sizeof (dmu_buf_t ) numbufs);
	}

	/*
	- * Issue prefetch i/os for the given blocks. If level is greater than 0, the
	+ * Issue prefetch I/Os for the given blocks. If level is greater than 0, the
	* indirect blocks prefetched will be those that point to the blocks containing
	- * the data starting at offset, and continuing to offset + len.
	+ * the data starting at offset, and continuing to offset + len. If the range
	+ * it too long, prefetch the first dmu_prefetch_max bytes as requested, while
	+ * for the rest only a higher level, also fitting within dmu_prefetch_max. It
	+ * should primarily help random reads, since for long sequential reads there is
	+ * a speculative prefetcher.
	*
	* Note that if the indirect blocks above the blocks being prefetched are not
	- * in cache, they will be asynchronously read in.
	+ * in cache, they will be asynchronously read in. Dnode read by dnode_hold()
	+ * is currently synchronous.
	*/
	void
	dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset,
	uint64_t len, zio_priority_t pri)
	{
	dnode_t *dn;
	- uint64_t blkid;
	- int nblks, err;
	-
	- if (len == 0) { /* they're interested in the bonus buffer */
	- dn = DMU_META_DNODE(os);

	- if (object == 0 \|\| object >= DN_MAX_OBJECT)
	- return;
	-
	- rw_enter(&dn->dn_struct_rwlock, RW_READER);
	- blkid = dbuf_whichblock(dn, level,
	- object * sizeof (dnode_phys_t));
	- dbuf_prefetch(dn, level, blkid, pri, 0);
	- rw_exit(&dn->dn_struct_rwlock);
	+ if (dmu_prefetch_max == 0 \|\| len == 0) {
	+ dmu_prefetch_dnode(os, object, pri);
	return;
	}

	- /*
	- * See comment before the definition of dmu_prefetch_max.
	- */
	- len = MIN(len, dmu_prefetch_max);
	-
	- /*
	- * XXX - Note, if the dnode for the requested object is not
	- * already cached, we will do a synchronous read in the
	- * dnode_hold() call. The same is true for any indirects.
	- */
	- err = dnode_hold(os, object, FTAG, &dn);
	- if (err != 0)
	+ if (dnode_hold(os, object, FTAG, &dn) != 0)
	return;

	+ dmu_prefetch_by_dnode(dn, level, offset, len, pri);
	+
	+ dnode_rele(dn, FTAG);
	+}
	+
	+void
	+dmu_prefetch_by_dnode(dnode_t *dn, int64_t level, uint64_t offset,
	+ uint64_t len, zio_priority_t pri)
	+{
	+ int64_t level2 = level;
	+ uint64_t start, end, start2, end2;
	+
	/*
	- * offset + len - 1 is the last byte we want to prefetch for, and offset
	- * is the first. Then dbuf_whichblk(dn, level, off + len - 1) is the
	- * last block we want to prefetch, and dbuf_whichblock(dn, level,
	- * offset) is the first. Then the number we need to prefetch is the
	- * last - first + 1.
	+ * Depending on len we may do two prefetches: blocks [start, end) at
	+ * level, and following blocks [start2, end2) at higher level2.
	*/
	rw_enter(&dn->dn_struct_rwlock, RW_READER);
	- if (level > 0 \|\| dn->dn_datablkshift != 0) {
	- nblks = dbuf_whichblock(dn, level, offset + len - 1) -
	- dbuf_whichblock(dn, level, offset) + 1;
	+ if (dn->dn_datablkshift != 0) {
	+ /*
	+ * The object has multiple blocks. Calculate the full range
	+ * of blocks [start, end2) and then split it into two parts,
	+ * so that the first [start, end) fits into dmu_prefetch_max.
	+ */
	+ start = dbuf_whichblock(dn, level, offset);
	+ end2 = dbuf_whichblock(dn, level, offset + len - 1) + 1;
	+ uint8_t ibs = dn->dn_indblkshift;
	+ uint8_t bs = (level == 0) ? dn->dn_datablkshift : ibs;
	+ uint_t limit = P2ROUNDUP(dmu_prefetch_max, 1 << bs) >> bs;
	+ start2 = end = MIN(end2, start + limit);
	+
	+ /*
	+ * Find level2 where [start2, end2) fits into dmu_prefetch_max.
	+ */
	+ uint8_t ibps = ibs - SPA_BLKPTRSHIFT;
	+ limit = P2ROUNDUP(dmu_prefetch_max, 1 << ibs) >> ibs;
	+ do {
	+ level2++;
	+ start2 = P2ROUNDUP(start2, 1 << ibps) >> ibps;
	+ end2 = P2ROUNDUP(end2, 1 << ibps) >> ibps;
	+ } while (end2 - start2 > limit);
	} else {
	- nblks = (offset < dn->dn_datablksz);
	+ /* There is only one block. Prefetch it or nothing. */
	+ start = start2 = end2 = 0;
	+ end = start + (level == 0 && offset < dn->dn_datablksz);
	}

	- if (nblks != 0) {
	- blkid = dbuf_whichblock(dn, level, offset);
	- for (int i = 0; i < nblks; i++)
	- dbuf_prefetch(dn, level, blkid + i, pri, 0);
	- }
	+ for (uint64_t i = start; i < end; i++)
	+ dbuf_prefetch(dn, level, i, pri, 0);
	+ for (uint64_t i = start2; i < end2; i++)
	+ dbuf_prefetch(dn, level2, i, pri, 0);
	rw_exit(&dn->dn_struct_rwlock);
	+}

	- dnode_rele(dn, FTAG);
	+/*
	+ * Issue prefetch I/Os for the given object's dnode.
	+ */
	+void
	+dmu_prefetch_dnode(objset_t *os, uint64_t object, zio_priority_t pri)
	+{
	+ if (object == 0 \|\| object >= DN_MAX_OBJECT)
	+ return;
	+
	+ dnode_t *dn = DMU_META_DNODE(os);
	+ rw_enter(&dn->dn_struct_rwlock, RW_READER);
	+ uint64_t blkid = dbuf_whichblock(dn, 0, object * sizeof (dnode_phys_t));
	+ dbuf_prefetch(dn, 0, blkid, pri, 0);
	+ rw_exit(&dn->dn_struct_rwlock);
	}

	/*
	* Get the next "chunk" of file data to free. We traverse the file from
	* the end so that the file gets shorter over time (if we crashes in the
	* middle, this will leave us in a better state). We find allocated file
	* data by simply searching the allocated level 1 indirects.
	*
	* On input, *start should be the first offset that does not need to be
	* freed (e.g. "offset + length"). On return, *start will be the first
	* offset that should be freed and l1blks is set to the number of level 1
	* indirect blocks found within the chunk.
	*/
	static int
	get_next_chunk(dnode_t dn, uint64_t start, uint64_t minimum, uint64_t *l1blks)
	{
	uint64_t blks;
	uint64_t maxblks = DMU_MAX_ACCESS >> (dn->dn_indblkshift + 1);
	/* bytes of data covered by a level-1 indirect block */
	uint64_t iblkrange = (uint64_t)dn->dn_datablksz *
	EPB(dn->dn_indblkshift, SPA_BLKPTRSHIFT);

	ASSERT3U(minimum, <=, *start);

	/*
	* Check if we can free the entire range assuming that all of the
	* L1 blocks in this range have data. If we can, we use this
	* worst case value as an estimate so we can avoid having to look
	* at the object's actual data.
	*/
	uint64_t total_l1blks =
	(roundup(start, iblkrange) - (minimum / iblkrange iblkrange)) /
	iblkrange;
	if (total_l1blks <= maxblks) {
	*l1blks = total_l1blks;
	*start = minimum;
	return (0);
	}
	ASSERT(ISP2(iblkrange));

	for (blks = 0; *start > minimum && blks < maxblks; blks++) {
	int err;

	/*
	* dnode_next_offset(BACKWARDS) will find an allocated L1
	* indirect block at or before the input offset. We must
	* decrement *start so that it is at the end of the region
	* to search.
	*/
	(*start)--;

	err = dnode_next_offset(dn,
	DNODE_FIND_BACKWARDS, start, 2, 1, 0);

	/* if there are no indirect blocks before start, we are done */
	if (err == ESRCH) {
	*start = minimum;
	break;
	} else if (err != 0) {
	*l1blks = blks;
	return (err);
	}

	/* set start to the beginning of this L1 indirect */
	start = P2ALIGN(start, iblkrange);
	}
	if (*start < minimum)
	*start = minimum;
	*l1blks = blks;

	return (0);
	}

	/*
	* If this objset is of type OST_ZFS return true if vfs's unmounted flag is set,
	* otherwise return false.
	* Used below in dmu_free_long_range_impl() to enable abort when unmounting
	*/
	static boolean_t
	dmu_objset_zfs_unmounting(objset_t *os)
	{
	#ifdef _KERNEL
	if (dmu_objset_type(os) == DMU_OST_ZFS)
	return (zfs_get_vfs_flag_unmounted(os));
	#else
	(void) os;
	#endif
	return (B_FALSE);
	}

	static int
	dmu_free_long_range_impl(objset_t os, dnode_t dn, uint64_t offset,
	uint64_t length)
	{
	uint64_t object_size;
	int err;
	uint64_t dirty_frees_threshold;
	dsl_pool_t *dp = dmu_objset_pool(os);

	if (dn == NULL)
	return (SET_ERROR(EINVAL));

	object_size = (dn->dn_maxblkid + 1) * dn->dn_datablksz;
	if (offset >= object_size)
	return (0);

	if (zfs_per_txg_dirty_frees_percent <= 100)
	dirty_frees_threshold =
	zfs_per_txg_dirty_frees_percent * zfs_dirty_data_max / 100;
	else
	dirty_frees_threshold = zfs_dirty_data_max / 20;

	if (length == DMU_OBJECT_END \|\| offset + length > object_size)
	length = object_size - offset;

	while (length != 0) {
	uint64_t chunk_end, chunk_begin, chunk_len;
	uint64_t l1blks;
	dmu_tx_t *tx;

	if (dmu_objset_zfs_unmounting(dn->dn_objset))
	return (SET_ERROR(EINTR));

	chunk_end = chunk_begin = offset + length;

	/* move chunk_begin backwards to the beginning of this chunk */
	err = get_next_chunk(dn, &chunk_begin, offset, &l1blks);
	if (err)
	return (err);
	ASSERT3U(chunk_begin, >=, offset);
	ASSERT3U(chunk_begin, <=, chunk_end);

	chunk_len = chunk_end - chunk_begin;

	tx = dmu_tx_create(os);
	dmu_tx_hold_free(tx, dn->dn_object, chunk_begin, chunk_len);

	/*
	* Mark this transaction as typically resulting in a net
	* reduction in space used.
	*/
	dmu_tx_mark_netfree(tx);
	err = dmu_tx_assign(tx, TXG_WAIT);
	if (err) {
	dmu_tx_abort(tx);
	return (err);
	}

	uint64_t txg = dmu_tx_get_txg(tx);

	mutex_enter(&dp->dp_lock);
	uint64_t long_free_dirty =
	dp->dp_long_free_dirty_pertxg[txg & TXG_MASK];
	mutex_exit(&dp->dp_lock);

	/*
	* To avoid filling up a TXG with just frees, wait for
	* the next TXG to open before freeing more chunks if
	* we have reached the threshold of frees.
	*/
	if (dirty_frees_threshold != 0 &&
	long_free_dirty >= dirty_frees_threshold) {
	DMU_TX_STAT_BUMP(dmu_tx_dirty_frees_delay);
	dmu_tx_commit(tx);
	txg_wait_open(dp, 0, B_TRUE);
	continue;
	}

	/*
	* In order to prevent unnecessary write throttling, for each
	* TXG, we track the cumulative size of L1 blocks being dirtied
	* in dnode_free_range() below. We compare this number to a
	* tunable threshold, past which we prevent new L1 dirty freeing
	* blocks from being added into the open TXG. See
	* dmu_free_long_range_impl() for details. The threshold
	* prevents write throttle activation due to dirty freeing L1
	* blocks taking up a large percentage of zfs_dirty_data_max.
	*/
	mutex_enter(&dp->dp_lock);
	dp->dp_long_free_dirty_pertxg[txg & TXG_MASK] +=
	l1blks << dn->dn_indblkshift;
	mutex_exit(&dp->dp_lock);
	DTRACE_PROBE3(free__long__range,
	uint64_t, long_free_dirty, uint64_t, chunk_len,
	uint64_t, txg);
	dnode_free_range(dn, chunk_begin, chunk_len, tx);

	dmu_tx_commit(tx);

	length -= chunk_len;
	}
	return (0);
	}

	int
	dmu_free_long_range(objset_t *os, uint64_t object,
	uint64_t offset, uint64_t length)
	{
	dnode_t *dn;
	int err;

	err = dnode_hold(os, object, FTAG, &dn);
	if (err != 0)
	return (err);
	err = dmu_free_long_range_impl(os, dn, offset, length);

	/*
	* It is important to zero out the maxblkid when freeing the entire
	* file, so that (a) subsequent calls to dmu_free_long_range_impl()
	* will take the fast path, and (b) dnode_reallocate() can verify
	* that the entire file has been freed.
	*/
	if (err == 0 && offset == 0 && length == DMU_OBJECT_END)
	dn->dn_maxblkid = 0;

	dnode_rele(dn, FTAG);
	return (err);
	}

	int
	dmu_free_long_object(objset_t *os, uint64_t object)
	{
	dmu_tx_t *tx;
	int err;

	err = dmu_free_long_range(os, object, 0, DMU_OBJECT_END);
	if (err != 0)
	return (err);

	tx = dmu_tx_create(os);
	dmu_tx_hold_bonus(tx, object);
	dmu_tx_hold_free(tx, object, 0, DMU_OBJECT_END);
	dmu_tx_mark_netfree(tx);
	err = dmu_tx_assign(tx, TXG_WAIT);
	if (err == 0) {
	err = dmu_object_free(os, object, tx);
	dmu_tx_commit(tx);
	} else {
	dmu_tx_abort(tx);
	}

	return (err);
	}

	int
	dmu_free_range(objset_t *os, uint64_t object, uint64_t offset,
	uint64_t size, dmu_tx_t *tx)
	{
	dnode_t *dn;
	int err = dnode_hold(os, object, FTAG, &dn);
	if (err)
	return (err);
	ASSERT(offset < UINT64_MAX);
	ASSERT(size == DMU_OBJECT_END \|\| size <= UINT64_MAX - offset);
	dnode_free_range(dn, offset, size, tx);
	dnode_rele(dn, FTAG);
	return (0);
	}

	static int
	dmu_read_impl(dnode_t *dn, uint64_t offset, uint64_t size,
	void *buf, uint32_t flags)
	{
	dmu_buf_t **dbp;
	int numbufs, err = 0;

	/*
	* Deal with odd block sizes, where there can't be data past the first
	* block. If we ever do the tail block optimization, we will need to
	* handle that here as well.
	*/
	if (dn->dn_maxblkid == 0) {
	uint64_t newsz = offset > dn->dn_datablksz ? 0 :
	MIN(size, dn->dn_datablksz - offset);
	memset((char *)buf + newsz, 0, size - newsz);
	size = newsz;
	}

	while (size > 0) {
	uint64_t mylen = MIN(size, DMU_MAX_ACCESS / 2);
	int i;

	/*
	* NB: we could do this block-at-a-time, but it's nice
	* to be reading in parallel.
	*/
	err = dmu_buf_hold_array_by_dnode(dn, offset, mylen,
	TRUE, FTAG, &numbufs, &dbp, flags);
	if (err)
	break;

	for (i = 0; i < numbufs; i++) {
	uint64_t tocpy;
	int64_t bufoff;
	dmu_buf_t *db = dbp[i];

	ASSERT(size > 0);

	bufoff = offset - db->db_offset;
	tocpy = MIN(db->db_size - bufoff, size);

	(void) memcpy(buf, (char *)db->db_data + bufoff, tocpy);

	offset += tocpy;
	size -= tocpy;
	buf = (char *)buf + tocpy;
	}
	dmu_buf_rele_array(dbp, numbufs, FTAG);
	}
	return (err);
	}

	int
	dmu_read(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
	void *buf, uint32_t flags)
	{
	dnode_t *dn;
	int err;

	err = dnode_hold(os, object, FTAG, &dn);
	if (err != 0)
	return (err);

	err = dmu_read_impl(dn, offset, size, buf, flags);
	dnode_rele(dn, FTAG);
	return (err);
	}

	int
	dmu_read_by_dnode(dnode_t dn, uint64_t offset, uint64_t size, void buf,
	uint32_t flags)
	{
	return (dmu_read_impl(dn, offset, size, buf, flags));
	}

	static void
	dmu_write_impl(dmu_buf_t **dbp, int numbufs, uint64_t offset, uint64_t size,
	const void buf, dmu_tx_t tx)
	{
	int i;

	for (i = 0; i < numbufs; i++) {
	uint64_t tocpy;
	int64_t bufoff;
	dmu_buf_t *db = dbp[i];

	ASSERT(size > 0);

	bufoff = offset - db->db_offset;
	tocpy = MIN(db->db_size - bufoff, size);

	ASSERT(i == 0 \|\| i == numbufs-1 \|\| tocpy == db->db_size);

	if (tocpy == db->db_size)
	dmu_buf_will_fill(db, tx, B_FALSE);
	else
	dmu_buf_will_dirty(db, tx);

	(void) memcpy((char *)db->db_data + bufoff, buf, tocpy);

	if (tocpy == db->db_size)
	dmu_buf_fill_done(db, tx, B_FALSE);

	offset += tocpy;
	size -= tocpy;
	buf = (char *)buf + tocpy;
	}
	}

	void
	dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
	const void buf, dmu_tx_t tx)
	{
	dmu_buf_t **dbp;
	int numbufs;

	if (size == 0)
	return;

	VERIFY0(dmu_buf_hold_array(os, object, offset, size,
	FALSE, FTAG, &numbufs, &dbp));
	dmu_write_impl(dbp, numbufs, offset, size, buf, tx);
	dmu_buf_rele_array(dbp, numbufs, FTAG);
	}

	/*
	* Note: Lustre is an external consumer of this interface.
	*/
	void
	dmu_write_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size,
	const void buf, dmu_tx_t tx)
	{
	dmu_buf_t **dbp;
	int numbufs;

	if (size == 0)
	return;

	VERIFY0(dmu_buf_hold_array_by_dnode(dn, offset, size,
	FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH));
	dmu_write_impl(dbp, numbufs, offset, size, buf, tx);
	dmu_buf_rele_array(dbp, numbufs, FTAG);
	}

	void
	dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
	dmu_tx_t *tx)
	{
	dmu_buf_t **dbp;
	int numbufs, i;

	if (size == 0)
	return;

	VERIFY(0 == dmu_buf_hold_array(os, object, offset, size,
	FALSE, FTAG, &numbufs, &dbp));

	for (i = 0; i < numbufs; i++) {
	dmu_buf_t *db = dbp[i];

	dmu_buf_will_not_fill(db, tx);
	}
	dmu_buf_rele_array(dbp, numbufs, FTAG);
	}

	void
	dmu_write_embedded(objset_t *os, uint64_t object, uint64_t offset,
	void *data, uint8_t etype, uint8_t comp, int uncompressed_size,
	int compressed_size, int byteorder, dmu_tx_t *tx)
	{
	dmu_buf_t *db;

	ASSERT3U(etype, <, NUM_BP_EMBEDDED_TYPES);
	ASSERT3U(comp, <, ZIO_COMPRESS_FUNCTIONS);
	VERIFY0(dmu_buf_hold_noread(os, object, offset,
	FTAG, &db));

	dmu_buf_write_embedded(db,
	data, (bp_embedded_type_t)etype, (enum zio_compress)comp,
	uncompressed_size, compressed_size, byteorder, tx);

	dmu_buf_rele(db, FTAG);
	}

	void
	dmu_redact(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
	dmu_tx_t *tx)
	{
	int numbufs, i;
	dmu_buf_t **dbp;

	VERIFY0(dmu_buf_hold_array(os, object, offset, size, FALSE, FTAG,
	&numbufs, &dbp));
	for (i = 0; i < numbufs; i++)
	dmu_buf_redact(dbp[i], tx);
	dmu_buf_rele_array(dbp, numbufs, FTAG);
	}

	#ifdef _KERNEL
	int
	dmu_read_uio_dnode(dnode_t dn, zfs_uio_t uio, uint64_t size)
	{
	dmu_buf_t **dbp;
	int numbufs, i, err;

	/*
	* NB: we could do this block-at-a-time, but it's nice
	* to be reading in parallel.
	*/
	err = dmu_buf_hold_array_by_dnode(dn, zfs_uio_offset(uio), size,
	TRUE, FTAG, &numbufs, &dbp, 0);
	if (err)
	return (err);

	for (i = 0; i < numbufs; i++) {
	uint64_t tocpy;
	int64_t bufoff;
	dmu_buf_t *db = dbp[i];

	ASSERT(size > 0);

	bufoff = zfs_uio_offset(uio) - db->db_offset;
	tocpy = MIN(db->db_size - bufoff, size);

	err = zfs_uio_fault_move((char *)db->db_data + bufoff, tocpy,
	UIO_READ, uio);

	if (err)
	break;

	size -= tocpy;
	}
	dmu_buf_rele_array(dbp, numbufs, FTAG);

	return (err);
	}

	/*
	* Read 'size' bytes into the uio buffer.
	* From object zdb->db_object.
	* Starting at zfs_uio_offset(uio).
	*
	* If the caller already has a dbuf in the target object
	* (e.g. its bonus buffer), this routine is faster than dmu_read_uio(),
	* because we don't have to find the dnode_t for the object.
	*/
	int
	dmu_read_uio_dbuf(dmu_buf_t zdb, zfs_uio_t uio, uint64_t size)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )zdb;
	dnode_t *dn;
	int err;

	if (size == 0)
	return (0);

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);
	err = dmu_read_uio_dnode(dn, uio, size);
	DB_DNODE_EXIT(db);

	return (err);
	}

	/*
	* Read 'size' bytes into the uio buffer.
	* From the specified object
	* Starting at offset zfs_uio_offset(uio).
	*/
	int
	dmu_read_uio(objset_t os, uint64_t object, zfs_uio_t uio, uint64_t size)
	{
	dnode_t *dn;
	int err;

	if (size == 0)
	return (0);

	err = dnode_hold(os, object, FTAG, &dn);
	if (err)
	return (err);

	err = dmu_read_uio_dnode(dn, uio, size);

	dnode_rele(dn, FTAG);

	return (err);
	}

	int
	dmu_write_uio_dnode(dnode_t dn, zfs_uio_t uio, uint64_t size, dmu_tx_t *tx)
	{
	dmu_buf_t **dbp;
	int numbufs;
	int err = 0;
	int i;

	err = dmu_buf_hold_array_by_dnode(dn, zfs_uio_offset(uio), size,
	FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH);
	if (err)
	return (err);

	for (i = 0; i < numbufs; i++) {
	uint64_t tocpy;
	int64_t bufoff;
	dmu_buf_t *db = dbp[i];

	ASSERT(size > 0);

	offset_t off = zfs_uio_offset(uio);
	bufoff = off - db->db_offset;
	tocpy = MIN(db->db_size - bufoff, size);

	ASSERT(i == 0 \|\| i == numbufs-1 \|\| tocpy == db->db_size);

	if (tocpy == db->db_size)
	dmu_buf_will_fill(db, tx, B_TRUE);
	else
	dmu_buf_will_dirty(db, tx);

	err = zfs_uio_fault_move((char *)db->db_data + bufoff,
	tocpy, UIO_WRITE, uio);

	if (tocpy == db->db_size && dmu_buf_fill_done(db, tx, err)) {
	/* The fill was reverted. Undo any uio progress. */
	zfs_uio_advance(uio, off - zfs_uio_offset(uio));
	}

	if (err)
	break;

	size -= tocpy;
	}

	dmu_buf_rele_array(dbp, numbufs, FTAG);
	return (err);
	}

	/*
	* Write 'size' bytes from the uio buffer.
	* To object zdb->db_object.
	* Starting at offset zfs_uio_offset(uio).
	*
	* If the caller already has a dbuf in the target object
	* (e.g. its bonus buffer), this routine is faster than dmu_write_uio(),
	* because we don't have to find the dnode_t for the object.
	*/
	int
	dmu_write_uio_dbuf(dmu_buf_t zdb, zfs_uio_t uio, uint64_t size,
	dmu_tx_t *tx)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )zdb;
	dnode_t *dn;
	int err;

	if (size == 0)
	return (0);

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);
	err = dmu_write_uio_dnode(dn, uio, size, tx);
	DB_DNODE_EXIT(db);

	return (err);
	}

	/*
	* Write 'size' bytes from the uio buffer.
	* To the specified object.
	* Starting at offset zfs_uio_offset(uio).
	*/
	int
	dmu_write_uio(objset_t os, uint64_t object, zfs_uio_t uio, uint64_t size,
	dmu_tx_t *tx)
	{
	dnode_t *dn;
	int err;

	if (size == 0)
	return (0);

	err = dnode_hold(os, object, FTAG, &dn);
	if (err)
	return (err);

	err = dmu_write_uio_dnode(dn, uio, size, tx);

	dnode_rele(dn, FTAG);

	return (err);
	}
	#endif /* _KERNEL */

	/*
	* Allocate a loaned anonymous arc buffer.
	*/
	arc_buf_t *
	dmu_request_arcbuf(dmu_buf_t *handle, int size)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )handle;

	return (arc_loan_buf(db->db_objset->os_spa, B_FALSE, size));
	}

	/*
	* Free a loaned arc buffer.
	*/
	void
	dmu_return_arcbuf(arc_buf_t *buf)
	{
	arc_return_buf(buf, FTAG);
	arc_buf_destroy(buf, FTAG);
	}

	/*
	* A "lightweight" write is faster than a regular write (e.g.
	* dmu_write_by_dnode() or dmu_assign_arcbuf_by_dnode()), because it avoids the
	* CPU cost of creating a dmu_buf_impl_t and arc_buf_[hdr_]_t. However, the
	* data can not be read or overwritten until the transaction's txg has been
	* synced. This makes it appropriate for workloads that are known to be
	* (temporarily) write-only, like "zfs receive".
	*
	* A single block is written, starting at the specified offset in bytes. If
	* the call is successful, it returns 0 and the provided abd has been
	* consumed (the caller should not free it).
	*/
	int
	dmu_lightweight_write_by_dnode(dnode_t dn, uint64_t offset, abd_t abd,
	const zio_prop_t zp, zio_flag_t flags, dmu_tx_t tx)
	{
	dbuf_dirty_record_t *dr =
	dbuf_dirty_lightweight(dn, dbuf_whichblock(dn, 0, offset), tx);
	if (dr == NULL)
	return (SET_ERROR(EIO));
	dr->dt.dll.dr_abd = abd;
	dr->dt.dll.dr_props = *zp;
	dr->dt.dll.dr_flags = flags;
	return (0);
	}

	/*
	* When possible directly assign passed loaned arc buffer to a dbuf.
	* If this is not possible copy the contents of passed arc buf via
	* dmu_write().
	*/
	int
	dmu_assign_arcbuf_by_dnode(dnode_t dn, uint64_t offset, arc_buf_t buf,
	dmu_tx_t *tx)
	{
	dmu_buf_impl_t *db;
	objset_t *os = dn->dn_objset;
	uint64_t object = dn->dn_object;
	uint32_t blksz = (uint32_t)arc_buf_lsize(buf);
	uint64_t blkid;

	rw_enter(&dn->dn_struct_rwlock, RW_READER);
	blkid = dbuf_whichblock(dn, 0, offset);
	db = dbuf_hold(dn, blkid, FTAG);
	rw_exit(&dn->dn_struct_rwlock);
	if (db == NULL)
	return (SET_ERROR(EIO));

	/*
	* We can only assign if the offset is aligned and the arc buf is the
	* same size as the dbuf.
	*/
	if (offset == db->db.db_offset && blksz == db->db.db_size) {
	zfs_racct_write(blksz, 1);
	dbuf_assign_arcbuf(db, buf, tx);
	dbuf_rele(db, FTAG);
	} else {
	/* compressed bufs must always be assignable to their dbuf */
	ASSERT3U(arc_get_compression(buf), ==, ZIO_COMPRESS_OFF);
	ASSERT(!(buf->b_flags & ARC_BUF_FLAG_COMPRESSED));

	dbuf_rele(db, FTAG);
	dmu_write(os, object, offset, blksz, buf->b_data, tx);
	dmu_return_arcbuf(buf);
	}

	return (0);
	}

	int
	dmu_assign_arcbuf_by_dbuf(dmu_buf_t handle, uint64_t offset, arc_buf_t buf,
	dmu_tx_t *tx)
	{
	int err;
	dmu_buf_impl_t dbuf = (dmu_buf_impl_t )handle;

	DB_DNODE_ENTER(dbuf);
	err = dmu_assign_arcbuf_by_dnode(DB_DNODE(dbuf), offset, buf, tx);
	DB_DNODE_EXIT(dbuf);

	return (err);
	}

	typedef struct {
	dbuf_dirty_record_t *dsa_dr;
	dmu_sync_cb_t *dsa_done;
	zgd_t *dsa_zgd;
	dmu_tx_t *dsa_tx;
	} dmu_sync_arg_t;

	static void
	dmu_sync_ready(zio_t zio, arc_buf_t buf, void *varg)
	{
	(void) buf;
	dmu_sync_arg_t *dsa = varg;
	dmu_buf_t *db = dsa->dsa_zgd->zgd_db;
	blkptr_t *bp = zio->io_bp;

	if (zio->io_error == 0) {
	if (BP_IS_HOLE(bp)) {
	/*
	* A block of zeros may compress to a hole, but the
	* block size still needs to be known for replay.
	*/
	BP_SET_LSIZE(bp, db->db_size);
	} else if (!BP_IS_EMBEDDED(bp)) {
	ASSERT(BP_GET_LEVEL(bp) == 0);
	BP_SET_FILL(bp, 1);
	}
	}
	}

	static void
	dmu_sync_late_arrival_ready(zio_t *zio)
	{
	dmu_sync_ready(zio, NULL, zio->io_private);
	}

	static void
	dmu_sync_done(zio_t zio, arc_buf_t buf, void *varg)
	{
	(void) buf;
	dmu_sync_arg_t *dsa = varg;
	dbuf_dirty_record_t *dr = dsa->dsa_dr;
	dmu_buf_impl_t *db = dr->dr_dbuf;
	zgd_t *zgd = dsa->dsa_zgd;

	/*
	* Record the vdev(s) backing this blkptr so they can be flushed after
	* the writes for the lwb have completed.
	*/
	if (zio->io_error == 0) {
	zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp);
	}

	mutex_enter(&db->db_mtx);
	ASSERT(dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC);
	if (zio->io_error == 0) {
	dr->dt.dl.dr_nopwrite = !!(zio->io_flags & ZIO_FLAG_NOPWRITE);
	if (dr->dt.dl.dr_nopwrite) {
	blkptr_t *bp = zio->io_bp;
	blkptr_t *bp_orig = &zio->io_bp_orig;
	uint8_t chksum = BP_GET_CHECKSUM(bp_orig);

	ASSERT(BP_EQUAL(bp, bp_orig));
	VERIFY(BP_EQUAL(bp, db->db_blkptr));
	ASSERT(zio->io_prop.zp_compress != ZIO_COMPRESS_OFF);
	VERIFY(zio_checksum_table[chksum].ci_flags &
	ZCHECKSUM_FLAG_NOPWRITE);
	}
	dr->dt.dl.dr_overridden_by = *zio->io_bp;
	dr->dt.dl.dr_override_state = DR_OVERRIDDEN;
	dr->dt.dl.dr_copies = zio->io_prop.zp_copies;

	/*
	* Old style holes are filled with all zeros, whereas
	* new-style holes maintain their lsize, type, level,
	* and birth time (see zio_write_compress). While we
	* need to reset the BP_SET_LSIZE() call that happened
	* in dmu_sync_ready for old style holes, we do not
	* want to wipe out the information contained in new
	* style holes. Thus, only zero out the block pointer if
	* it's an old style hole.
	*/
	if (BP_IS_HOLE(&dr->dt.dl.dr_overridden_by) &&
	dr->dt.dl.dr_overridden_by.blk_birth == 0)
	BP_ZERO(&dr->dt.dl.dr_overridden_by);
	} else {
	dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
	}
	cv_broadcast(&db->db_changed);
	mutex_exit(&db->db_mtx);

	dsa->dsa_done(dsa->dsa_zgd, zio->io_error);

	kmem_free(dsa, sizeof (*dsa));
	}

	static void
	dmu_sync_late_arrival_done(zio_t *zio)
	{
	blkptr_t *bp = zio->io_bp;
	dmu_sync_arg_t *dsa = zio->io_private;
	zgd_t *zgd = dsa->dsa_zgd;

	if (zio->io_error == 0) {
	/*
	* Record the vdev(s) backing this blkptr so they can be
	* flushed after the writes for the lwb have completed.
	*/
	zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp);

	if (!BP_IS_HOLE(bp)) {
	blkptr_t *bp_orig __maybe_unused = &zio->io_bp_orig;
	ASSERT(!(zio->io_flags & ZIO_FLAG_NOPWRITE));
	ASSERT(BP_IS_HOLE(bp_orig) \|\| !BP_EQUAL(bp, bp_orig));
	ASSERT(zio->io_bp->blk_birth == zio->io_txg);
	ASSERT(zio->io_txg > spa_syncing_txg(zio->io_spa));
	zio_free(zio->io_spa, zio->io_txg, zio->io_bp);
	}
	}

	dmu_tx_commit(dsa->dsa_tx);

	dsa->dsa_done(dsa->dsa_zgd, zio->io_error);

	abd_free(zio->io_abd);
	kmem_free(dsa, sizeof (*dsa));
	}

	static int
	dmu_sync_late_arrival(zio_t pio, objset_t os, dmu_sync_cb_t done, zgd_t zgd,
	zio_prop_t zp, zbookmark_phys_t zb)
	{
	dmu_sync_arg_t *dsa;
	dmu_tx_t *tx;
	int error;

	error = dbuf_read((dmu_buf_impl_t *)zgd->zgd_db, NULL,
	DB_RF_CANFAIL \| DB_RF_NOPREFETCH);
	if (error != 0)
	return (error);

	tx = dmu_tx_create(os);
	dmu_tx_hold_space(tx, zgd->zgd_db->db_size);
	/*
	* This transaction does not produce any dirty data or log blocks, so
	* it should not be throttled. All other cases wait for TXG sync, by
	* which time the log block we are writing will be obsolete, so we can
	* skip waiting and just return error here instead.
	*/
	if (dmu_tx_assign(tx, TXG_NOWAIT \| TXG_NOTHROTTLE) != 0) {
	dmu_tx_abort(tx);
	/* Make zl_get_data do txg_waited_synced() */
	return (SET_ERROR(EIO));
	}

	/*
	* In order to prevent the zgd's lwb from being free'd prior to
	* dmu_sync_late_arrival_done() being called, we have to ensure
	* the lwb's "max txg" takes this tx's txg into account.
	*/
	zil_lwb_add_txg(zgd->zgd_lwb, dmu_tx_get_txg(tx));

	dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP);
	dsa->dsa_dr = NULL;
	dsa->dsa_done = done;
	dsa->dsa_zgd = zgd;
	dsa->dsa_tx = tx;

	/*
	* Since we are currently syncing this txg, it's nontrivial to
	* determine what BP to nopwrite against, so we disable nopwrite.
	*
	* When syncing, the db_blkptr is initially the BP of the previous
	* txg. We can not nopwrite against it because it will be changed
	* (this is similar to the non-late-arrival case where the dbuf is
	* dirty in a future txg).
	*
	* Then dbuf_write_ready() sets bp_blkptr to the location we will write.
	* We can not nopwrite against it because although the BP will not
	* (typically) be changed, the data has not yet been persisted to this
	* location.
	*
	* Finally, when dbuf_write_done() is called, it is theoretically
	* possible to always nopwrite, because the data that was written in
	* this txg is the same data that we are trying to write. However we
	* would need to check that this dbuf is not dirty in any future
	* txg's (as we do in the normal dmu_sync() path). For simplicity, we
	* don't nopwrite in this case.
	*/
	zp->zp_nopwrite = B_FALSE;

	zio_nowait(zio_write(pio, os->os_spa, dmu_tx_get_txg(tx), zgd->zgd_bp,
	abd_get_from_buf(zgd->zgd_db->db_data, zgd->zgd_db->db_size),
	zgd->zgd_db->db_size, zgd->zgd_db->db_size, zp,
	dmu_sync_late_arrival_ready, NULL, dmu_sync_late_arrival_done,
	dsa, ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, zb));

	return (0);
	}

	/*
	* Intent log support: sync the block associated with db to disk.
	* N.B. and XXX: the caller is responsible for making sure that the
	* data isn't changing while dmu_sync() is writing it.
	*
	* Return values:
	*
	* EEXIST: this txg has already been synced, so there's nothing to do.
	* The caller should not log the write.
	*
	* ENOENT: the block was dbuf_free_range()'d, so there's nothing to do.
	* The caller should not log the write.
	*
	* EALREADY: this block is already in the process of being synced.
	* The caller should track its progress (somehow).
	*
	* EIO: could not do the I/O.
	* The caller should do a txg_wait_synced().
	*
	* 0: the I/O has been initiated.
	* The caller should log this blkptr in the done callback.
	* It is possible that the I/O will fail, in which case
	* the error will be reported to the done callback and
	* propagated to pio from zio_done().
	*/
	int
	dmu_sync(zio_t pio, uint64_t txg, dmu_sync_cb_t done, zgd_t *zgd)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )zgd->zgd_db;
	objset_t *os = db->db_objset;
	dsl_dataset_t *ds = os->os_dsl_dataset;
	dbuf_dirty_record_t dr, dr_next;
	dmu_sync_arg_t *dsa;
	zbookmark_phys_t zb;
	zio_prop_t zp;
	dnode_t *dn;

	ASSERT(pio != NULL);
	ASSERT(txg != 0);

	SET_BOOKMARK(&zb, ds->ds_object,
	db->db.db_object, db->db_level, db->db_blkid);

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);
	dmu_write_policy(os, dn, db->db_level, WP_DMU_SYNC, &zp);
	DB_DNODE_EXIT(db);

	/*
	* If we're frozen (running ziltest), we always need to generate a bp.
	*/
	if (txg > spa_freeze_txg(os->os_spa))
	return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb));

	/*
	* Grabbing db_mtx now provides a barrier between dbuf_sync_leaf()
	* and us. If we determine that this txg is not yet syncing,
	* but it begins to sync a moment later, that's OK because the
	* sync thread will block in dbuf_sync_leaf() until we drop db_mtx.
	*/
	mutex_enter(&db->db_mtx);

	if (txg <= spa_last_synced_txg(os->os_spa)) {
	/*
	* This txg has already synced. There's nothing to do.
	*/
	mutex_exit(&db->db_mtx);
	return (SET_ERROR(EEXIST));
	}

	if (txg <= spa_syncing_txg(os->os_spa)) {
	/*
	* This txg is currently syncing, so we can't mess with
	* the dirty record anymore; just write a new log block.
	*/
	mutex_exit(&db->db_mtx);
	return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb));
	}

	dr = dbuf_find_dirty_eq(db, txg);

	if (dr == NULL) {
	/*
	* There's no dr for this dbuf, so it must have been freed.
	* There's no need to log writes to freed blocks, so we're done.
	*/
	mutex_exit(&db->db_mtx);
	return (SET_ERROR(ENOENT));
	}

	dr_next = list_next(&db->db_dirty_records, dr);
	ASSERT(dr_next == NULL \|\| dr_next->dr_txg < txg);

	if (db->db_blkptr != NULL) {
	/*
	* We need to fill in zgd_bp with the current blkptr so that
	* the nopwrite code can check if we're writing the same
	* data that's already on disk. We can only nopwrite if we
	* are sure that after making the copy, db_blkptr will not
	* change until our i/o completes. We ensure this by
	* holding the db_mtx, and only allowing nopwrite if the
	* block is not already dirty (see below). This is verified
	* by dmu_sync_done(), which VERIFYs that the db_blkptr has
	* not changed.
	*/
	zgd->zgd_bp = db->db_blkptr;
	}

	/*
	* Assume the on-disk data is X, the current syncing data (in
	* txg - 1) is Y, and the current in-memory data is Z (currently
	* in dmu_sync).
	*
	* We usually want to perform a nopwrite if X and Z are the
	* same. However, if Y is different (i.e. the BP is going to
	* change before this write takes effect), then a nopwrite will
	* be incorrect - we would override with X, which could have
	* been freed when Y was written.
	*
	* (Note that this is not a concern when we are nop-writing from
	* syncing context, because X and Y must be identical, because
	* all previous txgs have been synced.)
	*
	* Therefore, we disable nopwrite if the current BP could change
	* before this TXG. There are two ways it could change: by
	* being dirty (dr_next is non-NULL), or by being freed
	* (dnode_block_freed()). This behavior is verified by
	* zio_done(), which VERIFYs that the override BP is identical
	* to the on-disk BP.
	*/
	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);
	if (dr_next != NULL \|\| dnode_block_freed(dn, db->db_blkid))
	zp.zp_nopwrite = B_FALSE;
	DB_DNODE_EXIT(db);

	ASSERT(dr->dr_txg == txg);
	if (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC \|\|
	dr->dt.dl.dr_override_state == DR_OVERRIDDEN) {
	/*
	* We have already issued a sync write for this buffer,
	* or this buffer has already been synced. It could not
	* have been dirtied since, or we would have cleared the state.
	*/
	mutex_exit(&db->db_mtx);
	return (SET_ERROR(EALREADY));
	}

	ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN);
	dr->dt.dl.dr_override_state = DR_IN_DMU_SYNC;
	mutex_exit(&db->db_mtx);

	dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP);
	dsa->dsa_dr = dr;
	dsa->dsa_done = done;
	dsa->dsa_zgd = zgd;
	dsa->dsa_tx = NULL;

	zio_nowait(arc_write(pio, os->os_spa, txg, zgd->zgd_bp,
	dr->dt.dl.dr_data, !DBUF_IS_CACHEABLE(db), dbuf_is_l2cacheable(db),
	&zp, dmu_sync_ready, NULL, dmu_sync_done, dsa,
	ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, &zb));

	return (0);
	}

	int
	dmu_object_set_nlevels(objset_t os, uint64_t object, int nlevels, dmu_tx_t tx)
	{
	dnode_t *dn;
	int err;

	err = dnode_hold(os, object, FTAG, &dn);
	if (err)
	return (err);
	err = dnode_set_nlevels(dn, nlevels, tx);
	dnode_rele(dn, FTAG);
	return (err);
	}

	int
	dmu_object_set_blocksize(objset_t *os, uint64_t object, uint64_t size, int ibs,
	dmu_tx_t *tx)
	{
	dnode_t *dn;
	int err;

	err = dnode_hold(os, object, FTAG, &dn);
	if (err)
	return (err);
	err = dnode_set_blksz(dn, size, ibs, tx);
	dnode_rele(dn, FTAG);
	return (err);
	}

	int
	dmu_object_set_maxblkid(objset_t *os, uint64_t object, uint64_t maxblkid,
	dmu_tx_t *tx)
	{
	dnode_t *dn;
	int err;

	err = dnode_hold(os, object, FTAG, &dn);
	if (err)
	return (err);
	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
	dnode_new_blkid(dn, maxblkid, tx, B_FALSE, B_TRUE);
	rw_exit(&dn->dn_struct_rwlock);
	dnode_rele(dn, FTAG);
	return (0);
	}

	void
	dmu_object_set_checksum(objset_t *os, uint64_t object, uint8_t checksum,
	dmu_tx_t *tx)
	{
	dnode_t *dn;

	/*
	* Send streams include each object's checksum function. This
	* check ensures that the receiving system can understand the
	* checksum function transmitted.
	*/
	ASSERT3U(checksum, <, ZIO_CHECKSUM_LEGACY_FUNCTIONS);

	VERIFY0(dnode_hold(os, object, FTAG, &dn));
	ASSERT3U(checksum, <, ZIO_CHECKSUM_FUNCTIONS);
	dn->dn_checksum = checksum;
	dnode_setdirty(dn, tx);
	dnode_rele(dn, FTAG);
	}

	void
	dmu_object_set_compress(objset_t *os, uint64_t object, uint8_t compress,
	dmu_tx_t *tx)
	{
	dnode_t *dn;

	/*
	* Send streams include each object's compression function. This
	* check ensures that the receiving system can understand the
	* compression function transmitted.
	*/
	ASSERT3U(compress, <, ZIO_COMPRESS_LEGACY_FUNCTIONS);

	VERIFY0(dnode_hold(os, object, FTAG, &dn));
	dn->dn_compress = compress;
	dnode_setdirty(dn, tx);
	dnode_rele(dn, FTAG);
	}

	/*
	* When the "redundant_metadata" property is set to "most", only indirect
	* blocks of this level and higher will have an additional ditto block.
	*/
	static const int zfs_redundant_metadata_most_ditto_level = 2;

	void
	dmu_write_policy(objset_t os, dnode_t dn, int level, int wp, zio_prop_t *zp)
	{
	dmu_object_type_t type = dn ? dn->dn_type : DMU_OT_OBJSET;
	boolean_t ismd = (level > 0 \|\| DMU_OT_IS_METADATA(type) \|\|
	(wp & WP_SPILL));
	enum zio_checksum checksum = os->os_checksum;
	enum zio_compress compress = os->os_compress;
	uint8_t complevel = os->os_complevel;
	enum zio_checksum dedup_checksum = os->os_dedup_checksum;
	boolean_t dedup = B_FALSE;
	boolean_t nopwrite = B_FALSE;
	boolean_t dedup_verify = os->os_dedup_verify;
	boolean_t encrypt = B_FALSE;
	int copies = os->os_copies;

	/*
	* We maintain different write policies for each of the following
	* types of data:
	* 1. metadata
	* 2. preallocated blocks (i.e. level-0 blocks of a dump device)
	* 3. all other level 0 blocks
	*/
	if (ismd) {
	/*
	* XXX -- we should design a compression algorithm
	* that specializes in arrays of bps.
	*/
	compress = zio_compress_select(os->os_spa,
	ZIO_COMPRESS_ON, ZIO_COMPRESS_ON);

	/*
	* Metadata always gets checksummed. If the data
	* checksum is multi-bit correctable, and it's not a
	* ZBT-style checksum, then it's suitable for metadata
	* as well. Otherwise, the metadata checksum defaults
	* to fletcher4.
	*/
	if (!(zio_checksum_table[checksum].ci_flags &
	ZCHECKSUM_FLAG_METADATA) \|\|
	(zio_checksum_table[checksum].ci_flags &
	ZCHECKSUM_FLAG_EMBEDDED))
	checksum = ZIO_CHECKSUM_FLETCHER_4;

	switch (os->os_redundant_metadata) {
	case ZFS_REDUNDANT_METADATA_ALL:
	copies++;
	break;
	case ZFS_REDUNDANT_METADATA_MOST:
	if (level >= zfs_redundant_metadata_most_ditto_level \|\|
	DMU_OT_IS_METADATA(type) \|\| (wp & WP_SPILL))
	copies++;
	break;
	case ZFS_REDUNDANT_METADATA_SOME:
	if (DMU_OT_IS_CRITICAL(type))
	copies++;
	break;
	case ZFS_REDUNDANT_METADATA_NONE:
	break;
	}
	} else if (wp & WP_NOFILL) {
	ASSERT(level == 0);

	/*
	* If we're writing preallocated blocks, we aren't actually
	* writing them so don't set any policy properties. These
	* blocks are currently only used by an external subsystem
	* outside of zfs (i.e. dump) and not written by the zio
	* pipeline.
	*/
	compress = ZIO_COMPRESS_OFF;
	checksum = ZIO_CHECKSUM_OFF;
	} else {
	compress = zio_compress_select(os->os_spa, dn->dn_compress,
	compress);
	complevel = zio_complevel_select(os->os_spa, compress,
	complevel, complevel);

	checksum = (dedup_checksum == ZIO_CHECKSUM_OFF) ?
	zio_checksum_select(dn->dn_checksum, checksum) :
	dedup_checksum;

	/*
	* Determine dedup setting. If we are in dmu_sync(),
	* we won't actually dedup now because that's all
	* done in syncing context; but we do want to use the
	* dedup checksum. If the checksum is not strong
	* enough to ensure unique signatures, force
	* dedup_verify.
	*/
	if (dedup_checksum != ZIO_CHECKSUM_OFF) {
	dedup = (wp & WP_DMU_SYNC) ? B_FALSE : B_TRUE;
	if (!(zio_checksum_table[checksum].ci_flags &
	ZCHECKSUM_FLAG_DEDUP))
	dedup_verify = B_TRUE;
	}

	/*
	* Enable nopwrite if we have secure enough checksum
	* algorithm (see comment in zio_nop_write) and
	* compression is enabled. We don't enable nopwrite if
	* dedup is enabled as the two features are mutually
	* exclusive.
	*/
	nopwrite = (!dedup && (zio_checksum_table[checksum].ci_flags &
	ZCHECKSUM_FLAG_NOPWRITE) &&
	compress != ZIO_COMPRESS_OFF && zfs_nopwrite_enabled);
	}

	/*
	* All objects in an encrypted objset are protected from modification
	* via a MAC. Encrypted objects store their IV and salt in the last DVA
	* in the bp, so we cannot use all copies. Encrypted objects are also
	* not subject to nopwrite since writing the same data will still
	* result in a new ciphertext. Only encrypted blocks can be dedup'd
	* to avoid ambiguity in the dedup code since the DDT does not store
	* object types.
	*/
	if (os->os_encrypted && (wp & WP_NOFILL) == 0) {
	encrypt = B_TRUE;

	if (DMU_OT_IS_ENCRYPTED(type)) {
	copies = MIN(copies, SPA_DVAS_PER_BP - 1);
	nopwrite = B_FALSE;
	} else {
	dedup = B_FALSE;
	}

	if (level <= 0 &&
	(type == DMU_OT_DNODE \|\| type == DMU_OT_OBJSET)) {
	compress = ZIO_COMPRESS_EMPTY;
	}
	}

	zp->zp_compress = compress;
	zp->zp_complevel = complevel;
	zp->zp_checksum = checksum;
	zp->zp_type = (wp & WP_SPILL) ? dn->dn_bonustype : type;
	zp->zp_level = level;
	zp->zp_copies = MIN(copies, spa_max_replication(os->os_spa));
	zp->zp_dedup = dedup;
	zp->zp_dedup_verify = dedup && dedup_verify;
	zp->zp_nopwrite = nopwrite;
	zp->zp_encrypt = encrypt;
	zp->zp_byteorder = ZFS_HOST_BYTEORDER;
	memset(zp->zp_salt, 0, ZIO_DATA_SALT_LEN);
	memset(zp->zp_iv, 0, ZIO_DATA_IV_LEN);
	memset(zp->zp_mac, 0, ZIO_DATA_MAC_LEN);
	zp->zp_zpl_smallblk = DMU_OT_IS_FILE(zp->zp_type) ?
	os->os_zpl_special_smallblock : 0;

	ASSERT3U(zp->zp_compress, !=, ZIO_COMPRESS_INHERIT);
	}

	/*
	* Reports the location of data and holes in an object. In order to
	* accurately report holes all dirty data must be synced to disk. This
	* causes extremely poor performance when seeking for holes in a dirty file.
	* As a compromise, only provide hole data when the dnode is clean. When
	* a dnode is dirty report the dnode as having no holes by returning EBUSY
	* which is always safe to do.
	*/
	int
	dmu_offset_next(objset_t os, uint64_t object, boolean_t hole, uint64_t off)
	{
	dnode_t *dn;
	int restarted = 0, err;

	restart:
	err = dnode_hold(os, object, FTAG, &dn);
	if (err)
	return (err);

	rw_enter(&dn->dn_struct_rwlock, RW_READER);

	if (dnode_is_dirty(dn)) {
	/*
	* If the zfs_dmu_offset_next_sync module option is enabled
	* then hole reporting has been requested. Dirty dnodes
	* must be synced to disk to accurately report holes.
	*
	* Provided a RL_READER rangelock spanning 0-UINT64_MAX is
	* held by the caller only a single restart will be required.
	* We tolerate callers which do not hold the rangelock by
	* returning EBUSY and not reporting holes after one restart.
	*/
	if (zfs_dmu_offset_next_sync) {
	rw_exit(&dn->dn_struct_rwlock);
	dnode_rele(dn, FTAG);

	if (restarted)
	return (SET_ERROR(EBUSY));

	txg_wait_synced(dmu_objset_pool(os), 0);
	restarted = 1;
	goto restart;
	}

	err = SET_ERROR(EBUSY);
	} else {
	err = dnode_next_offset(dn, DNODE_FIND_HAVELOCK \|
	(hole ? DNODE_FIND_HOLE : 0), off, 1, 1, 0);
	}

	rw_exit(&dn->dn_struct_rwlock);
	dnode_rele(dn, FTAG);

	return (err);
	}

	int
	dmu_read_l0_bps(objset_t *os, uint64_t object, uint64_t offset, uint64_t length,
	blkptr_t bps, size_t nbpsp)
	{
	dmu_buf_t *dbp, dbuf;
	dmu_buf_impl_t *db;
	blkptr_t *bp;
	int error, numbufs;

	error = dmu_buf_hold_array(os, object, offset, length, FALSE, FTAG,
	&numbufs, &dbp);
	if (error != 0) {
	if (error == ESRCH) {
	error = SET_ERROR(ENXIO);
	}
	return (error);
	}

	ASSERT3U(numbufs, <=, *nbpsp);

	for (int i = 0; i < numbufs; i++) {
	dbuf = dbp[i];
	db = (dmu_buf_impl_t *)dbuf;

	mutex_enter(&db->db_mtx);

	if (!list_is_empty(&db->db_dirty_records)) {
	dbuf_dirty_record_t *dr;

	dr = list_head(&db->db_dirty_records);
	if (dr->dt.dl.dr_brtwrite) {
	/*
	* This is very special case where we clone a
	* block and in the same transaction group we
	* read its BP (most likely to clone the clone).
	*/
	bp = &dr->dt.dl.dr_overridden_by;
	} else {
	/*
	* The block was modified in the same
	* transaction group.
	*/
	mutex_exit(&db->db_mtx);
	error = SET_ERROR(EAGAIN);
	goto out;
	}
	} else {
	bp = db->db_blkptr;
	}

	mutex_exit(&db->db_mtx);

	if (bp == NULL) {
	/*
	- * The block was created in this transaction group,
	- * so it has no BP yet.
	+ * The file size was increased, but the block was never
	+ * written, otherwise we would either have the block
	+ * pointer or the dirty record and would not get here.
	+ * It is effectively a hole, so report it as such.
	*/
	- error = SET_ERROR(EAGAIN);
	- goto out;
	+ BP_ZERO(&bps[i]);
	+ continue;
	}
	/*
	* Make sure we clone only data blocks.
	*/
	if (BP_IS_METADATA(bp) && !BP_IS_HOLE(bp)) {
	error = SET_ERROR(EINVAL);
	goto out;
	}

	/*
	* If the block was allocated in transaction group that is not
	* yet synced, we could clone it, but we couldn't write this
	* operation into ZIL, or it may be impossible to replay, since
	* the block may appear not yet allocated at that point.
	*/
	if (BP_PHYSICAL_BIRTH(bp) > spa_freeze_txg(os->os_spa)) {
	error = SET_ERROR(EINVAL);
	goto out;
	}
	if (BP_PHYSICAL_BIRTH(bp) > spa_last_synced_txg(os->os_spa)) {
	error = SET_ERROR(EAGAIN);
	goto out;
	}

	bps[i] = *bp;
	}

	*nbpsp = numbufs;
	out:
	dmu_buf_rele_array(dbp, numbufs, FTAG);

	return (error);
	}

	int
	dmu_brt_clone(objset_t *os, uint64_t object, uint64_t offset, uint64_t length,
	dmu_tx_t tx, const blkptr_t bps, size_t nbps)
	{
	spa_t *spa;
	dmu_buf_t *dbp, dbuf;
	dmu_buf_impl_t *db;
	struct dirty_leaf *dl;
	dbuf_dirty_record_t *dr;
	const blkptr_t *bp;
	int error = 0, i, numbufs;

	spa = os->os_spa;

	VERIFY0(dmu_buf_hold_array(os, object, offset, length, FALSE, FTAG,
	&numbufs, &dbp));
	ASSERT3U(nbps, ==, numbufs);

	/*
	* Before we start cloning make sure that the dbufs sizes match new BPs
	* sizes. If they don't, that's a no-go, as we are not able to shrink
	* dbufs.
	*/
	for (i = 0; i < numbufs; i++) {
	dbuf = dbp[i];
	db = (dmu_buf_impl_t *)dbuf;
	bp = &bps[i];

	ASSERT0(db->db_level);
	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
	ASSERT(db->db_blkid != DMU_SPILL_BLKID);

	if (!BP_IS_HOLE(bp) && BP_GET_LSIZE(bp) != dbuf->db_size) {
	error = SET_ERROR(EXDEV);
	goto out;
	}
	}

	for (i = 0; i < numbufs; i++) {
	dbuf = dbp[i];
	db = (dmu_buf_impl_t *)dbuf;
	bp = &bps[i];

	ASSERT0(db->db_level);
	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
	ASSERT(db->db_blkid != DMU_SPILL_BLKID);
	ASSERT(BP_IS_HOLE(bp) \|\| dbuf->db_size == BP_GET_LSIZE(bp));

	dmu_buf_will_clone(dbuf, tx);

	mutex_enter(&db->db_mtx);

	dr = list_head(&db->db_dirty_records);
	VERIFY(dr != NULL);
	ASSERT3U(dr->dr_txg, ==, tx->tx_txg);
	dl = &dr->dt.dl;
	dl->dr_overridden_by = *bp;
	- dl->dr_brtwrite = B_TRUE;
	- dl->dr_override_state = DR_OVERRIDDEN;
	- if (BP_IS_HOLE(bp)) {
	- dl->dr_overridden_by.blk_birth = 0;
	- dl->dr_overridden_by.blk_phys_birth = 0;
	- } else {
	- dl->dr_overridden_by.blk_birth = dr->dr_txg;
	+ if (!BP_IS_HOLE(bp) \|\| bp->blk_birth != 0) {
	if (!BP_IS_EMBEDDED(bp)) {
	- dl->dr_overridden_by.blk_phys_birth =
	- BP_PHYSICAL_BIRTH(bp);
	+ BP_SET_BIRTH(&dl->dr_overridden_by, dr->dr_txg,
	+ BP_PHYSICAL_BIRTH(bp));
	+ } else {
	+ dl->dr_overridden_by.blk_birth = dr->dr_txg;
	}
	}
	+ dl->dr_brtwrite = B_TRUE;
	+ dl->dr_override_state = DR_OVERRIDDEN;

	mutex_exit(&db->db_mtx);

	/*
	* When data in embedded into BP there is no need to create
	* BRT entry as there is no data block. Just copy the BP as
	* it contains the data.
	*/
	if (!BP_IS_HOLE(bp) && !BP_IS_EMBEDDED(bp)) {
	brt_pending_add(spa, bp, tx);
	}
	}
	out:
	dmu_buf_rele_array(dbp, numbufs, FTAG);

	return (error);
	}

	void
	__dmu_object_info_from_dnode(dnode_t dn, dmu_object_info_t doi)
	{
	dnode_phys_t *dnp = dn->dn_phys;

	doi->doi_data_block_size = dn->dn_datablksz;
	doi->doi_metadata_block_size = dn->dn_indblkshift ?
	1ULL << dn->dn_indblkshift : 0;
	doi->doi_type = dn->dn_type;
	doi->doi_bonus_type = dn->dn_bonustype;
	doi->doi_bonus_size = dn->dn_bonuslen;
	doi->doi_dnodesize = dn->dn_num_slots << DNODE_SHIFT;
	doi->doi_indirection = dn->dn_nlevels;
	doi->doi_checksum = dn->dn_checksum;
	doi->doi_compress = dn->dn_compress;
	doi->doi_nblkptr = dn->dn_nblkptr;
	doi->doi_physical_blocks_512 = (DN_USED_BYTES(dnp) + 256) >> 9;
	doi->doi_max_offset = (dn->dn_maxblkid + 1) * dn->dn_datablksz;
	doi->doi_fill_count = 0;
	for (int i = 0; i < dnp->dn_nblkptr; i++)
	doi->doi_fill_count += BP_GET_FILL(&dnp->dn_blkptr[i]);
	}

	void
	dmu_object_info_from_dnode(dnode_t dn, dmu_object_info_t doi)
	{
	rw_enter(&dn->dn_struct_rwlock, RW_READER);
	mutex_enter(&dn->dn_mtx);

	__dmu_object_info_from_dnode(dn, doi);

	mutex_exit(&dn->dn_mtx);
	rw_exit(&dn->dn_struct_rwlock);
	}

	/*
	* Get information on a DMU object.
	* If doi is NULL, just indicates whether the object exists.
	*/
	int
	dmu_object_info(objset_t os, uint64_t object, dmu_object_info_t doi)
	{
	dnode_t *dn;
	int err = dnode_hold(os, object, FTAG, &dn);

	if (err)
	return (err);

	if (doi != NULL)
	dmu_object_info_from_dnode(dn, doi);

	dnode_rele(dn, FTAG);
	return (0);
	}

	/*
	* As above, but faster; can be used when you have a held dbuf in hand.
	*/
	void
	dmu_object_info_from_db(dmu_buf_t db_fake, dmu_object_info_t doi)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;

	DB_DNODE_ENTER(db);
	dmu_object_info_from_dnode(DB_DNODE(db), doi);
	DB_DNODE_EXIT(db);
	}

	/*
	* Faster still when you only care about the size.
	*/
	void
	dmu_object_size_from_db(dmu_buf_t db_fake, uint32_t blksize,
	u_longlong_t *nblk512)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;
	dnode_t *dn;

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);

	*blksize = dn->dn_datablksz;
	/* add in number of slots used for the dnode itself */
	*nblk512 = ((DN_USED_BYTES(dn->dn_phys) + SPA_MINBLOCKSIZE/2) >>
	SPA_MINBLOCKSHIFT) + dn->dn_num_slots;
	DB_DNODE_EXIT(db);
	}

	void
	dmu_object_dnsize_from_db(dmu_buf_t db_fake, int dnsize)
	{
	dmu_buf_impl_t db = (dmu_buf_impl_t )db_fake;
	dnode_t *dn;

	DB_DNODE_ENTER(db);
	dn = DB_DNODE(db);
	*dnsize = dn->dn_num_slots << DNODE_SHIFT;
	DB_DNODE_EXIT(db);
	}

	void
	byteswap_uint64_array(void *vbuf, size_t size)
	{
	uint64_t *buf = vbuf;
	size_t count = size >> 3;
	int i;

	ASSERT((size & 7) == 0);

	for (i = 0; i < count; i++)
	buf[i] = BSWAP_64(buf[i]);
	}

	void
	byteswap_uint32_array(void *vbuf, size_t size)
	{
	uint32_t *buf = vbuf;
	size_t count = size >> 2;
	int i;

	ASSERT((size & 3) == 0);

	for (i = 0; i < count; i++)
	buf[i] = BSWAP_32(buf[i]);
	}

	void
	byteswap_uint16_array(void *vbuf, size_t size)
	{
	uint16_t *buf = vbuf;
	size_t count = size >> 1;
	int i;

	ASSERT((size & 1) == 0);

	for (i = 0; i < count; i++)
	buf[i] = BSWAP_16(buf[i]);
	}

	void
	byteswap_uint8_array(void *vbuf, size_t size)
	{
	(void) vbuf, (void) size;
	}

	void
	dmu_init(void)
	{
	abd_init();
	zfs_dbgmsg_init();
	sa_cache_init();
	dmu_objset_init();
	dnode_init();
	zfetch_init();
	dmu_tx_init();
	l2arc_init();
	arc_init();
	dbuf_init();
	}

	void
	dmu_fini(void)
	{
	arc_fini(); /* arc depends on l2arc, so arc must go first */
	l2arc_fini();
	dmu_tx_fini();
	zfetch_fini();
	dbuf_fini();
	dnode_fini();
	dmu_objset_fini();
	sa_cache_fini();
	zfs_dbgmsg_fini();
	abd_fini();
	}

	EXPORT_SYMBOL(dmu_bonus_hold);
	EXPORT_SYMBOL(dmu_bonus_hold_by_dnode);
	EXPORT_SYMBOL(dmu_buf_hold_array_by_bonus);
	EXPORT_SYMBOL(dmu_buf_rele_array);
	EXPORT_SYMBOL(dmu_prefetch);
	+EXPORT_SYMBOL(dmu_prefetch_by_dnode);
	+EXPORT_SYMBOL(dmu_prefetch_dnode);
	EXPORT_SYMBOL(dmu_free_range);
	EXPORT_SYMBOL(dmu_free_long_range);
	EXPORT_SYMBOL(dmu_free_long_object);
	EXPORT_SYMBOL(dmu_read);
	EXPORT_SYMBOL(dmu_read_by_dnode);
	EXPORT_SYMBOL(dmu_write);
	EXPORT_SYMBOL(dmu_write_by_dnode);
	EXPORT_SYMBOL(dmu_prealloc);
	EXPORT_SYMBOL(dmu_object_info);
	EXPORT_SYMBOL(dmu_object_info_from_dnode);
	EXPORT_SYMBOL(dmu_object_info_from_db);
	EXPORT_SYMBOL(dmu_object_size_from_db);
	EXPORT_SYMBOL(dmu_object_dnsize_from_db);
	EXPORT_SYMBOL(dmu_object_set_nlevels);
	EXPORT_SYMBOL(dmu_object_set_blocksize);
	EXPORT_SYMBOL(dmu_object_set_maxblkid);
	EXPORT_SYMBOL(dmu_object_set_checksum);
	EXPORT_SYMBOL(dmu_object_set_compress);
	EXPORT_SYMBOL(dmu_offset_next);
	EXPORT_SYMBOL(dmu_write_policy);
	EXPORT_SYMBOL(dmu_sync);
	EXPORT_SYMBOL(dmu_request_arcbuf);
	EXPORT_SYMBOL(dmu_return_arcbuf);
	EXPORT_SYMBOL(dmu_assign_arcbuf_by_dnode);
	EXPORT_SYMBOL(dmu_assign_arcbuf_by_dbuf);
	EXPORT_SYMBOL(dmu_buf_hold);
	EXPORT_SYMBOL(dmu_ot);

	ZFS_MODULE_PARAM(zfs, zfs_, nopwrite_enabled, INT, ZMOD_RW,
	"Enable NOP writes");

	ZFS_MODULE_PARAM(zfs, zfs_, per_txg_dirty_frees_percent, UINT, ZMOD_RW,
	"Percentage of dirtied blocks from frees in one TXG");

	ZFS_MODULE_PARAM(zfs, zfs_, dmu_offset_next_sync, INT, ZMOD_RW,
	"Enable forcing txg sync to find holes");

	/* CSTYLED */
	ZFS_MODULE_PARAM(zfs, , dmu_prefetch_max, UINT, ZMOD_RW,
	"Limit one prefetch call to this size");
	diff --git a/sys/contrib/openzfs/module/zfs/dmu_objset.c b/sys/contrib/openzfs/module/zfs/dmu_objset.c
	index d134d4958f7c..ef966d6703c9 100644
	--- a/sys/contrib/openzfs/module/zfs/dmu_objset.c
	+++ b/sys/contrib/openzfs/module/zfs/dmu_objset.c
	@@ -1,3083 +1,3102 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2012, 2020 by Delphix. All rights reserved.
	* Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
	* Copyright (c) 2013, Joyent, Inc. All rights reserved.
	* Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
	* Copyright (c) 2015, STRATO AG, Inc. All rights reserved.
	* Copyright (c) 2016 Actifio, Inc. All rights reserved.
	* Copyright 2017 Nexenta Systems, Inc.
	* Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
	* Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
	* Copyright (c) 2019, Klara Inc.
	* Copyright (c) 2019, Allan Jude
	* Copyright (c) 2022 Hewlett Packard Enterprise Development LP.
	*/

	/* Portions Copyright 2010 Robert Milkowski */

	#include <sys/cred.h>
	#include <sys/zfs_context.h>
	#include <sys/dmu_objset.h>
	#include <sys/dsl_dir.h>
	#include <sys/dsl_dataset.h>
	#include <sys/dsl_prop.h>
	#include <sys/dsl_pool.h>
	#include <sys/dsl_synctask.h>
	#include <sys/dsl_deleg.h>
	#include <sys/dnode.h>
	#include <sys/dbuf.h>
	#include <sys/zvol.h>
	#include <sys/dmu_tx.h>
	#include <sys/zap.h>
	#include <sys/zil.h>
	#include <sys/dmu_impl.h>
	#include <sys/zfs_ioctl.h>
	#include <sys/sa.h>
	#include <sys/zfs_onexit.h>
	#include <sys/dsl_destroy.h>
	#include <sys/vdev.h>
	#include <sys/zfeature.h>
	#include <sys/policy.h>
	#include <sys/spa_impl.h>
	#include <sys/dmu_recv.h>
	#include <sys/zfs_project.h>
	#include "zfs_namecheck.h"
	#include <sys/vdev_impl.h>
	#include <sys/arc.h>

	/*
	* Needed to close a window in dnode_move() that allows the objset to be freed
	* before it can be safely accessed.
	*/
	krwlock_t os_lock;

	/*
	* Tunable to overwrite the maximum number of threads for the parallelization
	* of dmu_objset_find_dp, needed to speed up the import of pools with many
	* datasets.
	* Default is 4 times the number of leaf vdevs.
	*/
	static const int dmu_find_threads = 0;

	/*
	* Backfill lower metadnode objects after this many have been freed.
	* Backfilling negatively impacts object creation rates, so only do it
	* if there are enough holes to fill.
	*/
	static const int dmu_rescan_dnode_threshold = 1 << DN_MAX_INDBLKSHIFT;

	static const char *upgrade_tag = "upgrade_tag";

	static void dmu_objset_find_dp_cb(void *arg);

	static void dmu_objset_upgrade(objset_t *os, dmu_objset_upgrade_cb_t cb);
	static void dmu_objset_upgrade_stop(objset_t *os);

	void
	dmu_objset_init(void)
	{
	rw_init(&os_lock, NULL, RW_DEFAULT, NULL);
	}

	void
	dmu_objset_fini(void)
	{
	rw_destroy(&os_lock);
	}

	spa_t *
	dmu_objset_spa(objset_t *os)
	{
	return (os->os_spa);
	}

	zilog_t *
	dmu_objset_zil(objset_t *os)
	{
	return (os->os_zil);
	}

	dsl_pool_t *
	dmu_objset_pool(objset_t *os)
	{
	dsl_dataset_t *ds;

	if ((ds = os->os_dsl_dataset) != NULL && ds->ds_dir)
	return (ds->ds_dir->dd_pool);
	else
	return (spa_get_dsl(os->os_spa));
	}

	dsl_dataset_t *
	dmu_objset_ds(objset_t *os)
	{
	return (os->os_dsl_dataset);
	}

	dmu_objset_type_t
	dmu_objset_type(objset_t *os)
	{
	return (os->os_phys->os_type);
	}

	void
	dmu_objset_name(objset_t os, char buf)
	{
	dsl_dataset_name(os->os_dsl_dataset, buf);
	}

	uint64_t
	dmu_objset_id(objset_t *os)
	{
	dsl_dataset_t *ds = os->os_dsl_dataset;

	return (ds ? ds->ds_object : 0);
	}

	uint64_t
	dmu_objset_dnodesize(objset_t *os)
	{
	return (os->os_dnodesize);
	}

	zfs_sync_type_t
	dmu_objset_syncprop(objset_t *os)
	{
	return (os->os_sync);
	}

	zfs_logbias_op_t
	dmu_objset_logbias(objset_t *os)
	{
	return (os->os_logbias);
	}

	static void
	checksum_changed_cb(void *arg, uint64_t newval)
	{
	objset_t *os = arg;

	/*
	* Inheritance should have been done by now.
	*/
	ASSERT(newval != ZIO_CHECKSUM_INHERIT);

	os->os_checksum = zio_checksum_select(newval, ZIO_CHECKSUM_ON_VALUE);
	}

	static void
	compression_changed_cb(void *arg, uint64_t newval)
	{
	objset_t *os = arg;

	/*
	* Inheritance and range checking should have been done by now.
	*/
	ASSERT(newval != ZIO_COMPRESS_INHERIT);

	os->os_compress = zio_compress_select(os->os_spa,
	ZIO_COMPRESS_ALGO(newval), ZIO_COMPRESS_ON);
	os->os_complevel = zio_complevel_select(os->os_spa, os->os_compress,
	ZIO_COMPRESS_LEVEL(newval), ZIO_COMPLEVEL_DEFAULT);
	}

	static void
	copies_changed_cb(void *arg, uint64_t newval)
	{
	objset_t *os = arg;

	/*
	* Inheritance and range checking should have been done by now.
	*/
	ASSERT(newval > 0);
	ASSERT(newval <= spa_max_replication(os->os_spa));

	os->os_copies = newval;
	}

	static void
	dedup_changed_cb(void *arg, uint64_t newval)
	{
	objset_t *os = arg;
	spa_t *spa = os->os_spa;
	enum zio_checksum checksum;

	/*
	* Inheritance should have been done by now.
	*/
	ASSERT(newval != ZIO_CHECKSUM_INHERIT);

	checksum = zio_checksum_dedup_select(spa, newval, ZIO_CHECKSUM_OFF);

	os->os_dedup_checksum = checksum & ZIO_CHECKSUM_MASK;
	os->os_dedup_verify = !!(checksum & ZIO_CHECKSUM_VERIFY);
	}

	static void
	primary_cache_changed_cb(void *arg, uint64_t newval)
	{
	objset_t *os = arg;

	/*
	* Inheritance and range checking should have been done by now.
	*/
	ASSERT(newval == ZFS_CACHE_ALL \|\| newval == ZFS_CACHE_NONE \|\|
	newval == ZFS_CACHE_METADATA);

	os->os_primary_cache = newval;
	}

	static void
	secondary_cache_changed_cb(void *arg, uint64_t newval)
	{
	objset_t *os = arg;

	/*
	* Inheritance and range checking should have been done by now.
	*/
	ASSERT(newval == ZFS_CACHE_ALL \|\| newval == ZFS_CACHE_NONE \|\|
	newval == ZFS_CACHE_METADATA);

	os->os_secondary_cache = newval;
	}

	+static void
	+prefetch_changed_cb(void *arg, uint64_t newval)
	+{
	+ objset_t *os = arg;
	+
	+ /*
	+ * Inheritance should have been done by now.
	+ */
	+ ASSERT(newval == ZFS_PREFETCH_ALL \|\| newval == ZFS_PREFETCH_NONE \|\|
	+ newval == ZFS_PREFETCH_METADATA);
	+ os->os_prefetch = newval;
	+}
	+
	static void
	sync_changed_cb(void *arg, uint64_t newval)
	{
	objset_t *os = arg;

	/*
	* Inheritance and range checking should have been done by now.
	*/
	ASSERT(newval == ZFS_SYNC_STANDARD \|\| newval == ZFS_SYNC_ALWAYS \|\|
	newval == ZFS_SYNC_DISABLED);

	os->os_sync = newval;
	if (os->os_zil)
	zil_set_sync(os->os_zil, newval);
	}

	static void
	redundant_metadata_changed_cb(void *arg, uint64_t newval)
	{
	objset_t *os = arg;

	/*
	* Inheritance and range checking should have been done by now.
	*/
	ASSERT(newval == ZFS_REDUNDANT_METADATA_ALL \|\|
	newval == ZFS_REDUNDANT_METADATA_MOST \|\|
	newval == ZFS_REDUNDANT_METADATA_SOME \|\|
	newval == ZFS_REDUNDANT_METADATA_NONE);

	os->os_redundant_metadata = newval;
	}

	static void
	dnodesize_changed_cb(void *arg, uint64_t newval)
	{
	objset_t *os = arg;

	switch (newval) {
	case ZFS_DNSIZE_LEGACY:
	os->os_dnodesize = DNODE_MIN_SIZE;
	break;
	case ZFS_DNSIZE_AUTO:
	/*
	* Choose a dnode size that will work well for most
	* workloads if the user specified "auto". Future code
	* improvements could dynamically select a dnode size
	* based on observed workload patterns.
	*/
	os->os_dnodesize = DNODE_MIN_SIZE * 2;
	break;
	case ZFS_DNSIZE_1K:
	case ZFS_DNSIZE_2K:
	case ZFS_DNSIZE_4K:
	case ZFS_DNSIZE_8K:
	case ZFS_DNSIZE_16K:
	os->os_dnodesize = newval;
	break;
	}
	}

	static void
	smallblk_changed_cb(void *arg, uint64_t newval)
	{
	objset_t *os = arg;

	/*
	* Inheritance and range checking should have been done by now.
	*/
	ASSERT(newval <= SPA_MAXBLOCKSIZE);
	ASSERT(ISP2(newval));

	os->os_zpl_special_smallblock = newval;
	}

	static void
	logbias_changed_cb(void *arg, uint64_t newval)
	{
	objset_t *os = arg;

	ASSERT(newval == ZFS_LOGBIAS_LATENCY \|\|
	newval == ZFS_LOGBIAS_THROUGHPUT);
	os->os_logbias = newval;
	if (os->os_zil)
	zil_set_logbias(os->os_zil, newval);
	}

	static void
	recordsize_changed_cb(void *arg, uint64_t newval)
	{
	objset_t *os = arg;

	os->os_recordsize = newval;
	}

	void
	dmu_objset_byteswap(void *buf, size_t size)
	{
	objset_phys_t *osp = buf;

	ASSERT(size == OBJSET_PHYS_SIZE_V1 \|\| size == OBJSET_PHYS_SIZE_V2 \|\|
	size == sizeof (objset_phys_t));
	dnode_byteswap(&osp->os_meta_dnode);
	byteswap_uint64_array(&osp->os_zil_header, sizeof (zil_header_t));
	osp->os_type = BSWAP_64(osp->os_type);
	osp->os_flags = BSWAP_64(osp->os_flags);
	if (size >= OBJSET_PHYS_SIZE_V2) {
	dnode_byteswap(&osp->os_userused_dnode);
	dnode_byteswap(&osp->os_groupused_dnode);
	if (size >= sizeof (objset_phys_t))
	dnode_byteswap(&osp->os_projectused_dnode);
	}
	}

	/*
	* The hash is a CRC-based hash of the objset_t pointer and the object number.
	*/
	static uint64_t
	dnode_hash(const objset_t *os, uint64_t obj)
	{
	uintptr_t osv = (uintptr_t)os;
	uint64_t crc = -1ULL;

	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
	/*
	* The low 6 bits of the pointer don't have much entropy, because
	* the objset_t is larger than 2^6 bytes long.
	*/
	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF];
	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF];
	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF];
	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 16)) & 0xFF];

	crc ^= (osv>>14) ^ (obj>>24);

	return (crc);
	}

	static unsigned int
	dnode_multilist_index_func(multilist_t ml, void obj)
	{
	dnode_t *dn = obj;

	/*
	* The low order bits of the hash value are thought to be
	* distributed evenly. Otherwise, in the case that the multilist
	* has a power of two number of sublists, each sublists' usage
	* would not be evenly distributed. In this context full 64bit
	* division would be a waste of time, so limit it to 32 bits.
	*/
	return ((unsigned int)dnode_hash(dn->dn_objset, dn->dn_object) %
	multilist_get_num_sublists(ml));
	}

	static inline boolean_t
	dmu_os_is_l2cacheable(objset_t *os)
	{
	if (os->os_secondary_cache == ZFS_CACHE_ALL \|\|
	os->os_secondary_cache == ZFS_CACHE_METADATA) {
	if (l2arc_exclude_special == 0)
	return (B_TRUE);

	blkptr_t *bp = os->os_rootbp;
	if (bp == NULL \|\| BP_IS_HOLE(bp))
	return (B_FALSE);
	uint64_t vdev = DVA_GET_VDEV(bp->blk_dva);
	vdev_t *rvd = os->os_spa->spa_root_vdev;
	vdev_t *vd = NULL;

	if (vdev < rvd->vdev_children)
	vd = rvd->vdev_child[vdev];

	if (vd == NULL)
	return (B_TRUE);

	if (vd->vdev_alloc_bias != VDEV_BIAS_SPECIAL &&
	vd->vdev_alloc_bias != VDEV_BIAS_DEDUP)
	return (B_TRUE);
	}
	return (B_FALSE);
	}

	/*
	* Instantiates the objset_t in-memory structure corresponding to the
	* objset_phys_t that's pointed to by the specified blkptr_t.
	*/
	int
	dmu_objset_open_impl(spa_t spa, dsl_dataset_t ds, blkptr_t *bp,
	objset_t **osp)
	{
	objset_t *os;
	int i, err;

	ASSERT(ds == NULL \|\| MUTEX_HELD(&ds->ds_opening_lock));
	ASSERT(!BP_IS_REDACTED(bp));

	/*
	* We need the pool config lock to get properties.
	*/
	ASSERT(ds == NULL \|\| dsl_pool_config_held(ds->ds_dir->dd_pool));

	/*
	* The $ORIGIN dataset (if it exists) doesn't have an associated
	* objset, so there's no reason to open it. The $ORIGIN dataset
	* will not exist on pools older than SPA_VERSION_ORIGIN.
	*/
	if (ds != NULL && spa_get_dsl(spa) != NULL &&
	spa_get_dsl(spa)->dp_origin_snap != NULL) {
	ASSERT3P(ds->ds_dir, !=,
	spa_get_dsl(spa)->dp_origin_snap->ds_dir);
	}

	os = kmem_zalloc(sizeof (objset_t), KM_SLEEP);
	os->os_dsl_dataset = ds;
	os->os_spa = spa;
	os->os_rootbp = bp;
	if (!BP_IS_HOLE(os->os_rootbp)) {
	arc_flags_t aflags = ARC_FLAG_WAIT;
	zbookmark_phys_t zb;
	int size;
	zio_flag_t zio_flags = ZIO_FLAG_CANFAIL;
	SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET,
	ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);

	if (dmu_os_is_l2cacheable(os))
	aflags \|= ARC_FLAG_L2CACHE;

	if (ds != NULL && ds->ds_dir->dd_crypto_obj != 0) {
	ASSERT3U(BP_GET_COMPRESS(bp), ==, ZIO_COMPRESS_OFF);
	ASSERT(BP_IS_AUTHENTICATED(bp));
	zio_flags \|= ZIO_FLAG_RAW;
	}

	dprintf_bp(os->os_rootbp, "reading %s", "");
	err = arc_read(NULL, spa, os->os_rootbp,
	arc_getbuf_func, &os->os_phys_buf,
	ZIO_PRIORITY_SYNC_READ, zio_flags, &aflags, &zb);
	if (err != 0) {
	kmem_free(os, sizeof (objset_t));
	/* convert checksum errors into IO errors */
	if (err == ECKSUM)
	err = SET_ERROR(EIO);
	return (err);
	}

	if (spa_version(spa) < SPA_VERSION_USERSPACE)
	size = OBJSET_PHYS_SIZE_V1;
	else if (!spa_feature_is_enabled(spa,
	SPA_FEATURE_PROJECT_QUOTA))
	size = OBJSET_PHYS_SIZE_V2;
	else
	size = sizeof (objset_phys_t);

	/* Increase the blocksize if we are permitted. */
	if (arc_buf_size(os->os_phys_buf) < size) {
	arc_buf_t *buf = arc_alloc_buf(spa, &os->os_phys_buf,
	ARC_BUFC_METADATA, size);
	memset(buf->b_data, 0, size);
	memcpy(buf->b_data, os->os_phys_buf->b_data,
	arc_buf_size(os->os_phys_buf));
	arc_buf_destroy(os->os_phys_buf, &os->os_phys_buf);
	os->os_phys_buf = buf;
	}

	os->os_phys = os->os_phys_buf->b_data;
	os->os_flags = os->os_phys->os_flags;
	} else {
	int size = spa_version(spa) >= SPA_VERSION_USERSPACE ?
	sizeof (objset_phys_t) : OBJSET_PHYS_SIZE_V1;
	os->os_phys_buf = arc_alloc_buf(spa, &os->os_phys_buf,
	ARC_BUFC_METADATA, size);
	os->os_phys = os->os_phys_buf->b_data;
	memset(os->os_phys, 0, size);
	}
	/*
	* These properties will be filled in by the logic in zfs_get_zplprop()
	* when they are queried for the first time.
	*/
	os->os_version = OBJSET_PROP_UNINITIALIZED;
	os->os_normalization = OBJSET_PROP_UNINITIALIZED;
	os->os_utf8only = OBJSET_PROP_UNINITIALIZED;
	os->os_casesensitivity = OBJSET_PROP_UNINITIALIZED;

	/*
	* Note: the changed_cb will be called once before the register
	* func returns, thus changing the checksum/compression from the
	* default (fletcher2/off). Snapshots don't need to know about
	* checksum/compression/copies.
	*/
	if (ds != NULL) {
	os->os_encrypted = (ds->ds_dir->dd_crypto_obj != 0);

	err = dsl_prop_register(ds,
	zfs_prop_to_name(ZFS_PROP_PRIMARYCACHE),
	primary_cache_changed_cb, os);
	if (err == 0) {
	err = dsl_prop_register(ds,
	zfs_prop_to_name(ZFS_PROP_SECONDARYCACHE),
	secondary_cache_changed_cb, os);
	}
	+ if (err == 0) {
	+ err = dsl_prop_register(ds,
	+ zfs_prop_to_name(ZFS_PROP_PREFETCH),
	+ prefetch_changed_cb, os);
	+ }
	if (!ds->ds_is_snapshot) {
	if (err == 0) {
	err = dsl_prop_register(ds,
	zfs_prop_to_name(ZFS_PROP_CHECKSUM),
	checksum_changed_cb, os);
	}
	if (err == 0) {
	err = dsl_prop_register(ds,
	zfs_prop_to_name(ZFS_PROP_COMPRESSION),
	compression_changed_cb, os);
	}
	if (err == 0) {
	err = dsl_prop_register(ds,
	zfs_prop_to_name(ZFS_PROP_COPIES),
	copies_changed_cb, os);
	}
	if (err == 0) {
	err = dsl_prop_register(ds,
	zfs_prop_to_name(ZFS_PROP_DEDUP),
	dedup_changed_cb, os);
	}
	if (err == 0) {
	err = dsl_prop_register(ds,
	zfs_prop_to_name(ZFS_PROP_LOGBIAS),
	logbias_changed_cb, os);
	}
	if (err == 0) {
	err = dsl_prop_register(ds,
	zfs_prop_to_name(ZFS_PROP_SYNC),
	sync_changed_cb, os);
	}
	if (err == 0) {
	err = dsl_prop_register(ds,
	zfs_prop_to_name(
	ZFS_PROP_REDUNDANT_METADATA),
	redundant_metadata_changed_cb, os);
	}
	if (err == 0) {
	err = dsl_prop_register(ds,
	zfs_prop_to_name(ZFS_PROP_RECORDSIZE),
	recordsize_changed_cb, os);
	}
	if (err == 0) {
	err = dsl_prop_register(ds,
	zfs_prop_to_name(ZFS_PROP_DNODESIZE),
	dnodesize_changed_cb, os);
	}
	if (err == 0) {
	err = dsl_prop_register(ds,
	zfs_prop_to_name(
	ZFS_PROP_SPECIAL_SMALL_BLOCKS),
	smallblk_changed_cb, os);
	}
	}
	if (err != 0) {
	arc_buf_destroy(os->os_phys_buf, &os->os_phys_buf);
	kmem_free(os, sizeof (objset_t));
	return (err);
	}
	} else {
	/* It's the meta-objset. */
	os->os_checksum = ZIO_CHECKSUM_FLETCHER_4;
	os->os_compress = ZIO_COMPRESS_ON;
	os->os_complevel = ZIO_COMPLEVEL_DEFAULT;
	os->os_encrypted = B_FALSE;
	os->os_copies = spa_max_replication(spa);
	os->os_dedup_checksum = ZIO_CHECKSUM_OFF;
	os->os_dedup_verify = B_FALSE;
	os->os_logbias = ZFS_LOGBIAS_LATENCY;
	os->os_sync = ZFS_SYNC_STANDARD;
	os->os_primary_cache = ZFS_CACHE_ALL;
	os->os_secondary_cache = ZFS_CACHE_ALL;
	os->os_dnodesize = DNODE_MIN_SIZE;
	+ os->os_prefetch = ZFS_PREFETCH_ALL;
	}

	if (ds == NULL \|\| !ds->ds_is_snapshot)
	os->os_zil_header = os->os_phys->os_zil_header;
	os->os_zil = zil_alloc(os, &os->os_zil_header);

	for (i = 0; i < TXG_SIZE; i++) {
	multilist_create(&os->os_dirty_dnodes[i], sizeof (dnode_t),
	offsetof(dnode_t, dn_dirty_link[i]),
	dnode_multilist_index_func);
	}
	list_create(&os->os_dnodes, sizeof (dnode_t),
	offsetof(dnode_t, dn_link));
	list_create(&os->os_downgraded_dbufs, sizeof (dmu_buf_impl_t),
	offsetof(dmu_buf_impl_t, db_link));

	list_link_init(&os->os_evicting_node);

	mutex_init(&os->os_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&os->os_userused_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&os->os_obj_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&os->os_user_ptr_lock, NULL, MUTEX_DEFAULT, NULL);
	os->os_obj_next_percpu_len = boot_ncpus;
	os->os_obj_next_percpu = kmem_zalloc(os->os_obj_next_percpu_len *
	sizeof (os->os_obj_next_percpu[0]), KM_SLEEP);

	dnode_special_open(os, &os->os_phys->os_meta_dnode,
	DMU_META_DNODE_OBJECT, &os->os_meta_dnode);
	if (OBJSET_BUF_HAS_USERUSED(os->os_phys_buf)) {
	dnode_special_open(os, &os->os_phys->os_userused_dnode,
	DMU_USERUSED_OBJECT, &os->os_userused_dnode);
	dnode_special_open(os, &os->os_phys->os_groupused_dnode,
	DMU_GROUPUSED_OBJECT, &os->os_groupused_dnode);
	if (OBJSET_BUF_HAS_PROJECTUSED(os->os_phys_buf))
	dnode_special_open(os,
	&os->os_phys->os_projectused_dnode,
	DMU_PROJECTUSED_OBJECT, &os->os_projectused_dnode);
	}

	mutex_init(&os->os_upgrade_lock, NULL, MUTEX_DEFAULT, NULL);

	*osp = os;
	return (0);
	}

	int
	dmu_objset_from_ds(dsl_dataset_t ds, objset_t *osp)
	{
	int err = 0;

	/*
	* We need the pool_config lock to manipulate the dsl_dataset_t.
	* Even if the dataset is long-held, we need the pool_config lock
	* to open the objset, as it needs to get properties.
	*/
	ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool));

	mutex_enter(&ds->ds_opening_lock);
	if (ds->ds_objset == NULL) {
	objset_t *os;
	rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
	err = dmu_objset_open_impl(dsl_dataset_get_spa(ds),
	ds, dsl_dataset_get_blkptr(ds), &os);
	rrw_exit(&ds->ds_bp_rwlock, FTAG);

	if (err == 0) {
	mutex_enter(&ds->ds_lock);
	ASSERT(ds->ds_objset == NULL);
	ds->ds_objset = os;
	mutex_exit(&ds->ds_lock);
	}
	}
	*osp = ds->ds_objset;
	mutex_exit(&ds->ds_opening_lock);
	return (err);
	}

	/*
	* Holds the pool while the objset is held. Therefore only one objset
	* can be held at a time.
	*/
	int
	dmu_objset_hold_flags(const char name, boolean_t decrypt, const void tag,
	objset_t **osp)
	{
	dsl_pool_t *dp;
	dsl_dataset_t *ds;
	int err;
	ds_hold_flags_t flags;

	flags = (decrypt) ? DS_HOLD_FLAG_DECRYPT : DS_HOLD_FLAG_NONE;
	err = dsl_pool_hold(name, tag, &dp);
	if (err != 0)
	return (err);
	err = dsl_dataset_hold_flags(dp, name, flags, tag, &ds);
	if (err != 0) {
	dsl_pool_rele(dp, tag);
	return (err);
	}

	err = dmu_objset_from_ds(ds, osp);
	if (err != 0) {
	dsl_dataset_rele(ds, tag);
	dsl_pool_rele(dp, tag);
	}

	return (err);
	}

	int
	dmu_objset_hold(const char name, const void tag, objset_t **osp)
	{
	return (dmu_objset_hold_flags(name, B_FALSE, tag, osp));
	}

	static int
	dmu_objset_own_impl(dsl_dataset_t *ds, dmu_objset_type_t type,
	boolean_t readonly, boolean_t decrypt, const void tag, objset_t *osp)
	{
	(void) tag;

	int err = dmu_objset_from_ds(ds, osp);
	if (err != 0) {
	return (err);
	} else if (type != DMU_OST_ANY && type != (*osp)->os_phys->os_type) {
	return (SET_ERROR(EINVAL));
	} else if (!readonly && dsl_dataset_is_snapshot(ds)) {
	return (SET_ERROR(EROFS));
	} else if (!readonly && decrypt &&
	dsl_dir_incompatible_encryption_version(ds->ds_dir)) {
	return (SET_ERROR(EROFS));
	}

	/* if we are decrypting, we can now check MACs in os->os_phys_buf */
	if (decrypt && arc_is_unauthenticated((*osp)->os_phys_buf)) {
	zbookmark_phys_t zb;

	SET_BOOKMARK(&zb, ds->ds_object, ZB_ROOT_OBJECT,
	ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
	err = arc_untransform((osp)->os_phys_buf, (osp)->os_spa,
	&zb, B_FALSE);
	if (err != 0)
	return (err);

	ASSERT0(arc_is_unauthenticated((*osp)->os_phys_buf));
	}

	return (0);
	}

	/*
	* dsl_pool must not be held when this is called.
	* Upon successful return, there will be a longhold on the dataset,
	* and the dsl_pool will not be held.
	*/
	int
	dmu_objset_own(const char *name, dmu_objset_type_t type,
	boolean_t readonly, boolean_t decrypt, const void tag, objset_t *osp)
	{
	dsl_pool_t *dp;
	dsl_dataset_t *ds;
	int err;
	ds_hold_flags_t flags;

	flags = (decrypt) ? DS_HOLD_FLAG_DECRYPT : DS_HOLD_FLAG_NONE;
	err = dsl_pool_hold(name, FTAG, &dp);
	if (err != 0)
	return (err);
	err = dsl_dataset_own(dp, name, flags, tag, &ds);
	if (err != 0) {
	dsl_pool_rele(dp, FTAG);
	return (err);
	}
	err = dmu_objset_own_impl(ds, type, readonly, decrypt, tag, osp);
	if (err != 0) {
	dsl_dataset_disown(ds, flags, tag);
	dsl_pool_rele(dp, FTAG);
	return (err);
	}

	/*
	* User accounting requires the dataset to be decrypted and rw.
	* We also don't begin user accounting during claiming to help
	* speed up pool import times and to keep this txg reserved
	* completely for recovery work.
	*/
	if (!readonly && !dp->dp_spa->spa_claiming &&
	(ds->ds_dir->dd_crypto_obj == 0 \|\| decrypt)) {
	if (dmu_objset_userobjspace_upgradable(*osp) \|\|
	dmu_objset_projectquota_upgradable(*osp)) {
	dmu_objset_id_quota_upgrade(*osp);
	} else if (dmu_objset_userused_enabled(*osp)) {
	dmu_objset_userspace_upgrade(*osp);
	}
	}

	dsl_pool_rele(dp, FTAG);
	return (0);
	}

	int
	dmu_objset_own_obj(dsl_pool_t *dp, uint64_t obj, dmu_objset_type_t type,
	boolean_t readonly, boolean_t decrypt, const void tag, objset_t *osp)
	{
	dsl_dataset_t *ds;
	int err;
	ds_hold_flags_t flags;

	flags = (decrypt) ? DS_HOLD_FLAG_DECRYPT : DS_HOLD_FLAG_NONE;
	err = dsl_dataset_own_obj(dp, obj, flags, tag, &ds);
	if (err != 0)
	return (err);

	err = dmu_objset_own_impl(ds, type, readonly, decrypt, tag, osp);
	if (err != 0) {
	dsl_dataset_disown(ds, flags, tag);
	return (err);
	}

	return (0);
	}

	void
	dmu_objset_rele_flags(objset_t os, boolean_t decrypt, const void tag)
	{
	ds_hold_flags_t flags;
	dsl_pool_t *dp = dmu_objset_pool(os);

	flags = (decrypt) ? DS_HOLD_FLAG_DECRYPT : DS_HOLD_FLAG_NONE;
	dsl_dataset_rele_flags(os->os_dsl_dataset, flags, tag);
	dsl_pool_rele(dp, tag);
	}

	void
	dmu_objset_rele(objset_t os, const void tag)
	{
	dmu_objset_rele_flags(os, B_FALSE, tag);
	}

	/*
	* When we are called, os MUST refer to an objset associated with a dataset
	* that is owned by 'tag'; that is, is held and long held by 'tag' and ds_owner
	* == tag. We will then release and reacquire ownership of the dataset while
	* holding the pool config_rwlock to avoid intervening namespace or ownership
	* changes may occur.
	*
	* This exists solely to accommodate zfs_ioc_userspace_upgrade()'s desire to
	* release the hold on its dataset and acquire a new one on the dataset of the
	* same name so that it can be partially torn down and reconstructed.
	*/
	void
	dmu_objset_refresh_ownership(dsl_dataset_t ds, dsl_dataset_t *newds,
	boolean_t decrypt, const void *tag)
	{
	dsl_pool_t *dp;
	char name[ZFS_MAX_DATASET_NAME_LEN];
	ds_hold_flags_t flags;

	flags = (decrypt) ? DS_HOLD_FLAG_DECRYPT : DS_HOLD_FLAG_NONE;
	VERIFY3P(ds, !=, NULL);
	VERIFY3P(ds->ds_owner, ==, tag);
	VERIFY(dsl_dataset_long_held(ds));

	dsl_dataset_name(ds, name);
	dp = ds->ds_dir->dd_pool;
	dsl_pool_config_enter(dp, FTAG);
	dsl_dataset_disown(ds, flags, tag);
	VERIFY0(dsl_dataset_own(dp, name, flags, tag, newds));
	dsl_pool_config_exit(dp, FTAG);
	}

	void
	dmu_objset_disown(objset_t os, boolean_t decrypt, const void tag)
	{
	ds_hold_flags_t flags;

	flags = (decrypt) ? DS_HOLD_FLAG_DECRYPT : DS_HOLD_FLAG_NONE;
	/*
	* Stop upgrading thread
	*/
	dmu_objset_upgrade_stop(os);
	dsl_dataset_disown(os->os_dsl_dataset, flags, tag);
	}

	void
	dmu_objset_evict_dbufs(objset_t *os)
	{
	dnode_t *dn_marker;
	dnode_t *dn;

	dn_marker = kmem_alloc(sizeof (dnode_t), KM_SLEEP);

	mutex_enter(&os->os_lock);
	dn = list_head(&os->os_dnodes);
	while (dn != NULL) {
	/*
	* Skip dnodes without holds. We have to do this dance
	* because dnode_add_ref() only works if there is already a
	* hold. If the dnode has no holds, then it has no dbufs.
	*/
	if (dnode_add_ref(dn, FTAG)) {
	list_insert_after(&os->os_dnodes, dn, dn_marker);
	mutex_exit(&os->os_lock);

	dnode_evict_dbufs(dn);
	dnode_rele(dn, FTAG);

	mutex_enter(&os->os_lock);
	dn = list_next(&os->os_dnodes, dn_marker);
	list_remove(&os->os_dnodes, dn_marker);
	} else {
	dn = list_next(&os->os_dnodes, dn);
	}
	}
	mutex_exit(&os->os_lock);

	kmem_free(dn_marker, sizeof (dnode_t));

	if (DMU_USERUSED_DNODE(os) != NULL) {
	if (DMU_PROJECTUSED_DNODE(os) != NULL)
	dnode_evict_dbufs(DMU_PROJECTUSED_DNODE(os));
	dnode_evict_dbufs(DMU_GROUPUSED_DNODE(os));
	dnode_evict_dbufs(DMU_USERUSED_DNODE(os));
	}
	dnode_evict_dbufs(DMU_META_DNODE(os));
	}

	/*
	* Objset eviction processing is split into into two pieces.
	* The first marks the objset as evicting, evicts any dbufs that
	* have a refcount of zero, and then queues up the objset for the
	* second phase of eviction. Once os->os_dnodes has been cleared by
	* dnode_buf_pageout()->dnode_destroy(), the second phase is executed.
	* The second phase closes the special dnodes, dequeues the objset from
	* the list of those undergoing eviction, and finally frees the objset.
	*
	* NOTE: Due to asynchronous eviction processing (invocation of
	* dnode_buf_pageout()), it is possible for the meta dnode for the
	* objset to have no holds even though os->os_dnodes is not empty.
	*/
	void
	dmu_objset_evict(objset_t *os)
	{
	dsl_dataset_t *ds = os->os_dsl_dataset;

	for (int t = 0; t < TXG_SIZE; t++)
	ASSERT(!dmu_objset_is_dirty(os, t));

	if (ds)
	dsl_prop_unregister_all(ds, os);

	if (os->os_sa)
	sa_tear_down(os);

	dmu_objset_evict_dbufs(os);

	mutex_enter(&os->os_lock);
	spa_evicting_os_register(os->os_spa, os);
	if (list_is_empty(&os->os_dnodes)) {
	mutex_exit(&os->os_lock);
	dmu_objset_evict_done(os);
	} else {
	mutex_exit(&os->os_lock);
	}


	}

	void
	dmu_objset_evict_done(objset_t *os)
	{
	ASSERT3P(list_head(&os->os_dnodes), ==, NULL);

	dnode_special_close(&os->os_meta_dnode);
	if (DMU_USERUSED_DNODE(os)) {
	if (DMU_PROJECTUSED_DNODE(os))
	dnode_special_close(&os->os_projectused_dnode);
	dnode_special_close(&os->os_userused_dnode);
	dnode_special_close(&os->os_groupused_dnode);
	}
	zil_free(os->os_zil);

	arc_buf_destroy(os->os_phys_buf, &os->os_phys_buf);

	/*
	* This is a barrier to prevent the objset from going away in
	* dnode_move() until we can safely ensure that the objset is still in
	* use. We consider the objset valid before the barrier and invalid
	* after the barrier.
	*/
	rw_enter(&os_lock, RW_READER);
	rw_exit(&os_lock);

	kmem_free(os->os_obj_next_percpu,
	os->os_obj_next_percpu_len * sizeof (os->os_obj_next_percpu[0]));

	mutex_destroy(&os->os_lock);
	mutex_destroy(&os->os_userused_lock);
	mutex_destroy(&os->os_obj_lock);
	mutex_destroy(&os->os_user_ptr_lock);
	mutex_destroy(&os->os_upgrade_lock);
	for (int i = 0; i < TXG_SIZE; i++)
	multilist_destroy(&os->os_dirty_dnodes[i]);
	spa_evicting_os_deregister(os->os_spa, os);
	kmem_free(os, sizeof (objset_t));
	}

	inode_timespec_t
	dmu_objset_snap_cmtime(objset_t *os)
	{
	return (dsl_dir_snap_cmtime(os->os_dsl_dataset->ds_dir));
	}

	objset_t *
	dmu_objset_create_impl_dnstats(spa_t spa, dsl_dataset_t ds, blkptr_t *bp,
	dmu_objset_type_t type, int levels, int blksz, int ibs, dmu_tx_t *tx)
	{
	objset_t *os;
	dnode_t *mdn;

	ASSERT(dmu_tx_is_syncing(tx));

	if (blksz == 0)
	blksz = DNODE_BLOCK_SIZE;
	if (ibs == 0)
	ibs = DN_MAX_INDBLKSHIFT;

	if (ds != NULL)
	VERIFY0(dmu_objset_from_ds(ds, &os));
	else
	VERIFY0(dmu_objset_open_impl(spa, NULL, bp, &os));

	mdn = DMU_META_DNODE(os);

	dnode_allocate(mdn, DMU_OT_DNODE, blksz, ibs, DMU_OT_NONE, 0,
	DNODE_MIN_SLOTS, tx);

	/*
	* We don't want to have to increase the meta-dnode's nlevels
	* later, because then we could do it in quiescing context while
	* we are also accessing it in open context.
	*
	* This precaution is not necessary for the MOS (ds == NULL),
	* because the MOS is only updated in syncing context.
	* This is most fortunate: the MOS is the only objset that
	* needs to be synced multiple times as spa_sync() iterates
	* to convergence, so minimizing its dn_nlevels matters.
	*/
	if (ds != NULL) {
	if (levels == 0) {
	levels = 1;

	/*
	* Determine the number of levels necessary for the
	* meta-dnode to contain DN_MAX_OBJECT dnodes. Note
	* that in order to ensure that we do not overflow
	* 64 bits, there has to be a nlevels that gives us a
	* number of blocks > DN_MAX_OBJECT but < 2^64.
	* Therefore, (mdn->dn_indblkshift - SPA_BLKPTRSHIFT)
	* (10) must be less than (64 - log2(DN_MAX_OBJECT))
	* (16).
	*/
	while ((uint64_t)mdn->dn_nblkptr <<
	(mdn->dn_datablkshift - DNODE_SHIFT + (levels - 1) *
	(mdn->dn_indblkshift - SPA_BLKPTRSHIFT)) <
	DN_MAX_OBJECT)
	levels++;
	}

	mdn->dn_next_nlevels[tx->tx_txg & TXG_MASK] =
	mdn->dn_nlevels = levels;
	}

	ASSERT(type != DMU_OST_NONE);
	ASSERT(type != DMU_OST_ANY);
	ASSERT(type < DMU_OST_NUMTYPES);
	os->os_phys->os_type = type;

	/*
	* Enable user accounting if it is enabled and this is not an
	* encrypted receive.
	*/
	if (dmu_objset_userused_enabled(os) &&
	(!os->os_encrypted \|\| !dmu_objset_is_receiving(os))) {
	os->os_phys->os_flags \|= OBJSET_FLAG_USERACCOUNTING_COMPLETE;
	if (dmu_objset_userobjused_enabled(os)) {
	ASSERT3P(ds, !=, NULL);
	ds->ds_feature_activation[
	SPA_FEATURE_USEROBJ_ACCOUNTING] = (void *)B_TRUE;
	os->os_phys->os_flags \|=
	OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE;
	}
	if (dmu_objset_projectquota_enabled(os)) {
	ASSERT3P(ds, !=, NULL);
	ds->ds_feature_activation[
	SPA_FEATURE_PROJECT_QUOTA] = (void *)B_TRUE;
	os->os_phys->os_flags \|=
	OBJSET_FLAG_PROJECTQUOTA_COMPLETE;
	}
	os->os_flags = os->os_phys->os_flags;
	}

	dsl_dataset_dirty(ds, tx);

	return (os);
	}

	/* called from dsl for meta-objset */
	objset_t *
	dmu_objset_create_impl(spa_t spa, dsl_dataset_t ds, blkptr_t *bp,
	dmu_objset_type_t type, dmu_tx_t *tx)
	{
	return (dmu_objset_create_impl_dnstats(spa, ds, bp, type, 0, 0, 0, tx));
	}

	typedef struct dmu_objset_create_arg {
	const char *doca_name;
	cred_t *doca_cred;
	proc_t *doca_proc;
	void (doca_userfunc)(objset_t os, void *arg,
	cred_t cr, dmu_tx_t tx);
	void *doca_userarg;
	dmu_objset_type_t doca_type;
	uint64_t doca_flags;
	dsl_crypto_params_t *doca_dcp;
	} dmu_objset_create_arg_t;

	static int
	dmu_objset_create_check(void arg, dmu_tx_t tx)
	{
	dmu_objset_create_arg_t *doca = arg;
	dsl_pool_t *dp = dmu_tx_pool(tx);
	dsl_dir_t *pdd;
	dsl_dataset_t *parentds;
	objset_t *parentos;
	const char *tail;
	int error;

	if (strchr(doca->doca_name, '@') != NULL)
	return (SET_ERROR(EINVAL));

	if (strlen(doca->doca_name) >= ZFS_MAX_DATASET_NAME_LEN)
	return (SET_ERROR(ENAMETOOLONG));

	if (dataset_nestcheck(doca->doca_name) != 0)
	return (SET_ERROR(ENAMETOOLONG));

	error = dsl_dir_hold(dp, doca->doca_name, FTAG, &pdd, &tail);
	if (error != 0)
	return (error);
	if (tail == NULL) {
	dsl_dir_rele(pdd, FTAG);
	return (SET_ERROR(EEXIST));
	}

	error = dmu_objset_create_crypt_check(pdd, doca->doca_dcp, NULL);
	if (error != 0) {
	dsl_dir_rele(pdd, FTAG);
	return (error);
	}

	error = dsl_fs_ss_limit_check(pdd, 1, ZFS_PROP_FILESYSTEM_LIMIT, NULL,
	doca->doca_cred, doca->doca_proc);
	if (error != 0) {
	dsl_dir_rele(pdd, FTAG);
	return (error);
	}

	/* can't create below anything but filesystems (eg. no ZVOLs) */
	error = dsl_dataset_hold_obj(pdd->dd_pool,
	dsl_dir_phys(pdd)->dd_head_dataset_obj, FTAG, &parentds);
	if (error != 0) {
	dsl_dir_rele(pdd, FTAG);
	return (error);
	}
	error = dmu_objset_from_ds(parentds, &parentos);
	if (error != 0) {
	dsl_dataset_rele(parentds, FTAG);
	dsl_dir_rele(pdd, FTAG);
	return (error);
	}
	if (dmu_objset_type(parentos) != DMU_OST_ZFS) {
	dsl_dataset_rele(parentds, FTAG);
	dsl_dir_rele(pdd, FTAG);
	return (SET_ERROR(ZFS_ERR_WRONG_PARENT));
	}
	dsl_dataset_rele(parentds, FTAG);
	dsl_dir_rele(pdd, FTAG);

	return (error);
	}

	static void
	dmu_objset_create_sync(void arg, dmu_tx_t tx)
	{
	dmu_objset_create_arg_t *doca = arg;
	dsl_pool_t *dp = dmu_tx_pool(tx);
	spa_t *spa = dp->dp_spa;
	dsl_dir_t *pdd;
	const char *tail;
	dsl_dataset_t *ds;
	uint64_t obj;
	blkptr_t *bp;
	objset_t *os;
	zio_t *rzio;

	VERIFY0(dsl_dir_hold(dp, doca->doca_name, FTAG, &pdd, &tail));

	obj = dsl_dataset_create_sync(pdd, tail, NULL, doca->doca_flags,
	doca->doca_cred, doca->doca_dcp, tx);

	VERIFY0(dsl_dataset_hold_obj_flags(pdd->dd_pool, obj,
	DS_HOLD_FLAG_DECRYPT, FTAG, &ds));
	rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
	bp = dsl_dataset_get_blkptr(ds);
	os = dmu_objset_create_impl(spa, ds, bp, doca->doca_type, tx);
	rrw_exit(&ds->ds_bp_rwlock, FTAG);

	if (doca->doca_userfunc != NULL) {
	doca->doca_userfunc(os, doca->doca_userarg,
	doca->doca_cred, tx);
	}

	/*
	* The doca_userfunc() may write out some data that needs to be
	* encrypted if the dataset is encrypted (specifically the root
	* directory). This data must be written out before the encryption
	* key mapping is removed by dsl_dataset_rele_flags(). Force the
	* I/O to occur immediately by invoking the relevant sections of
	* dsl_pool_sync().
	*/
	if (os->os_encrypted) {
	dsl_dataset_t *tmpds = NULL;
	boolean_t need_sync_done = B_FALSE;

	mutex_enter(&ds->ds_lock);
	ds->ds_owner = FTAG;
	mutex_exit(&ds->ds_lock);

	rzio = zio_root(spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
	tmpds = txg_list_remove_this(&dp->dp_dirty_datasets, ds,
	tx->tx_txg);
	if (tmpds != NULL) {
	dsl_dataset_sync(ds, rzio, tx);
	need_sync_done = B_TRUE;
	}
	VERIFY0(zio_wait(rzio));

	dmu_objset_sync_done(os, tx);
	taskq_wait(dp->dp_sync_taskq);
	if (txg_list_member(&dp->dp_dirty_datasets, ds, tx->tx_txg)) {
	ASSERT3P(ds->ds_key_mapping, !=, NULL);
	key_mapping_rele(spa, ds->ds_key_mapping, ds);
	}

	rzio = zio_root(spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
	tmpds = txg_list_remove_this(&dp->dp_dirty_datasets, ds,
	tx->tx_txg);
	if (tmpds != NULL) {
	dmu_buf_rele(ds->ds_dbuf, ds);
	dsl_dataset_sync(ds, rzio, tx);
	}
	VERIFY0(zio_wait(rzio));

	if (need_sync_done) {
	ASSERT3P(ds->ds_key_mapping, !=, NULL);
	key_mapping_rele(spa, ds->ds_key_mapping, ds);
	dsl_dataset_sync_done(ds, tx);
	dmu_buf_rele(ds->ds_dbuf, ds);
	}

	mutex_enter(&ds->ds_lock);
	ds->ds_owner = NULL;
	mutex_exit(&ds->ds_lock);
	}

	spa_history_log_internal_ds(ds, "create", tx, " ");

	dsl_dataset_rele_flags(ds, DS_HOLD_FLAG_DECRYPT, FTAG);
	dsl_dir_rele(pdd, FTAG);
	}

	int
	dmu_objset_create(const char *name, dmu_objset_type_t type, uint64_t flags,
	dsl_crypto_params_t dcp, dmu_objset_create_sync_func_t func, void arg)
	{
	dmu_objset_create_arg_t doca;
	dsl_crypto_params_t tmp_dcp = { 0 };

	doca.doca_name = name;
	doca.doca_cred = CRED();
	doca.doca_proc = curproc;
	doca.doca_flags = flags;
	doca.doca_userfunc = func;
	doca.doca_userarg = arg;
	doca.doca_type = type;

	/*
	* Some callers (mostly for testing) do not provide a dcp on their
	* own but various code inside the sync task will require it to be
	* allocated. Rather than adding NULL checks throughout this code
	* or adding dummy dcp's to all of the callers we simply create a
	* dummy one here and use that. This zero dcp will have the same
	* effect as asking for inheritance of all encryption params.
	*/
	doca.doca_dcp = (dcp != NULL) ? dcp : &tmp_dcp;

	int rv = dsl_sync_task(name,
	dmu_objset_create_check, dmu_objset_create_sync, &doca,
	6, ZFS_SPACE_CHECK_NORMAL);

	if (rv == 0)
	zvol_create_minor(name);
	return (rv);
	}

	typedef struct dmu_objset_clone_arg {
	const char *doca_clone;
	const char *doca_origin;
	cred_t *doca_cred;
	proc_t *doca_proc;
	} dmu_objset_clone_arg_t;

	static int
	dmu_objset_clone_check(void arg, dmu_tx_t tx)
	{
	dmu_objset_clone_arg_t *doca = arg;
	dsl_dir_t *pdd;
	const char *tail;
	int error;
	dsl_dataset_t *origin;
	dsl_pool_t *dp = dmu_tx_pool(tx);

	if (strchr(doca->doca_clone, '@') != NULL)
	return (SET_ERROR(EINVAL));

	if (strlen(doca->doca_clone) >= ZFS_MAX_DATASET_NAME_LEN)
	return (SET_ERROR(ENAMETOOLONG));

	error = dsl_dir_hold(dp, doca->doca_clone, FTAG, &pdd, &tail);
	if (error != 0)
	return (error);
	if (tail == NULL) {
	dsl_dir_rele(pdd, FTAG);
	return (SET_ERROR(EEXIST));
	}

	error = dsl_fs_ss_limit_check(pdd, 1, ZFS_PROP_FILESYSTEM_LIMIT, NULL,
	doca->doca_cred, doca->doca_proc);
	if (error != 0) {
	dsl_dir_rele(pdd, FTAG);
	return (SET_ERROR(EDQUOT));
	}

	error = dsl_dataset_hold(dp, doca->doca_origin, FTAG, &origin);
	if (error != 0) {
	dsl_dir_rele(pdd, FTAG);
	return (error);
	}

	/* You can only clone snapshots, not the head datasets. */
	if (!origin->ds_is_snapshot) {
	dsl_dataset_rele(origin, FTAG);
	dsl_dir_rele(pdd, FTAG);
	return (SET_ERROR(EINVAL));
	}

	dsl_dataset_rele(origin, FTAG);
	dsl_dir_rele(pdd, FTAG);

	return (0);
	}

	static void
	dmu_objset_clone_sync(void arg, dmu_tx_t tx)
	{
	dmu_objset_clone_arg_t *doca = arg;
	dsl_pool_t *dp = dmu_tx_pool(tx);
	dsl_dir_t *pdd;
	const char *tail;
	dsl_dataset_t origin, ds;
	uint64_t obj;
	char namebuf[ZFS_MAX_DATASET_NAME_LEN];

	VERIFY0(dsl_dir_hold(dp, doca->doca_clone, FTAG, &pdd, &tail));
	VERIFY0(dsl_dataset_hold(dp, doca->doca_origin, FTAG, &origin));

	obj = dsl_dataset_create_sync(pdd, tail, origin, 0,
	doca->doca_cred, NULL, tx);

	VERIFY0(dsl_dataset_hold_obj(pdd->dd_pool, obj, FTAG, &ds));
	dsl_dataset_name(origin, namebuf);
	spa_history_log_internal_ds(ds, "clone", tx,
	"origin=%s (%llu)", namebuf, (u_longlong_t)origin->ds_object);
	dsl_dataset_rele(ds, FTAG);
	dsl_dataset_rele(origin, FTAG);
	dsl_dir_rele(pdd, FTAG);
	}

	int
	dmu_objset_clone(const char clone, const char origin)
	{
	dmu_objset_clone_arg_t doca;

	doca.doca_clone = clone;
	doca.doca_origin = origin;
	doca.doca_cred = CRED();
	doca.doca_proc = curproc;

	int rv = dsl_sync_task(clone,
	dmu_objset_clone_check, dmu_objset_clone_sync, &doca,
	6, ZFS_SPACE_CHECK_NORMAL);

	if (rv == 0)
	zvol_create_minor(clone);

	return (rv);
	}

	int
	dmu_objset_snapshot_one(const char fsname, const char snapname)
	{
	int err;
	char *longsnap = kmem_asprintf("%s@%s", fsname, snapname);
	nvlist_t *snaps = fnvlist_alloc();

	fnvlist_add_boolean(snaps, longsnap);
	kmem_strfree(longsnap);
	err = dsl_dataset_snapshot(snaps, NULL, NULL);
	fnvlist_free(snaps);
	return (err);
	}

	static void
	dmu_objset_upgrade_task_cb(void *data)
	{
	objset_t *os = data;

	mutex_enter(&os->os_upgrade_lock);
	os->os_upgrade_status = EINTR;
	if (!os->os_upgrade_exit) {
	int status;

	mutex_exit(&os->os_upgrade_lock);

	status = os->os_upgrade_cb(os);

	mutex_enter(&os->os_upgrade_lock);

	os->os_upgrade_status = status;
	}
	os->os_upgrade_exit = B_TRUE;
	os->os_upgrade_id = 0;
	mutex_exit(&os->os_upgrade_lock);
	dsl_dataset_long_rele(dmu_objset_ds(os), upgrade_tag);
	}

	static void
	dmu_objset_upgrade(objset_t *os, dmu_objset_upgrade_cb_t cb)
	{
	if (os->os_upgrade_id != 0)
	return;

	ASSERT(dsl_pool_config_held(dmu_objset_pool(os)));
	dsl_dataset_long_hold(dmu_objset_ds(os), upgrade_tag);

	mutex_enter(&os->os_upgrade_lock);
	if (os->os_upgrade_id == 0 && os->os_upgrade_status == 0) {
	os->os_upgrade_exit = B_FALSE;
	os->os_upgrade_cb = cb;
	os->os_upgrade_id = taskq_dispatch(
	os->os_spa->spa_upgrade_taskq,
	dmu_objset_upgrade_task_cb, os, TQ_SLEEP);
	if (os->os_upgrade_id == TASKQID_INVALID) {
	dsl_dataset_long_rele(dmu_objset_ds(os), upgrade_tag);
	os->os_upgrade_status = ENOMEM;
	}
	} else {
	dsl_dataset_long_rele(dmu_objset_ds(os), upgrade_tag);
	}
	mutex_exit(&os->os_upgrade_lock);
	}

	static void
	dmu_objset_upgrade_stop(objset_t *os)
	{
	mutex_enter(&os->os_upgrade_lock);
	os->os_upgrade_exit = B_TRUE;
	if (os->os_upgrade_id != 0) {
	taskqid_t id = os->os_upgrade_id;

	os->os_upgrade_id = 0;
	mutex_exit(&os->os_upgrade_lock);

	if ((taskq_cancel_id(os->os_spa->spa_upgrade_taskq, id)) == 0) {
	dsl_dataset_long_rele(dmu_objset_ds(os), upgrade_tag);
	}
	txg_wait_synced(os->os_spa->spa_dsl_pool, 0);
	} else {
	mutex_exit(&os->os_upgrade_lock);
	}
	}

	static void
	dmu_objset_sync_dnodes(multilist_sublist_t list, dmu_tx_t tx)
	{
	dnode_t *dn;

	while ((dn = multilist_sublist_head(list)) != NULL) {
	ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT);
	ASSERT(dn->dn_dbuf->db_data_pending);
	/*
	* Initialize dn_zio outside dnode_sync() because the
	* meta-dnode needs to set it outside dnode_sync().
	*/
	dn->dn_zio = dn->dn_dbuf->db_data_pending->dr_zio;
	ASSERT(dn->dn_zio);

	ASSERT3U(dn->dn_nlevels, <=, DN_MAX_LEVELS);
	multilist_sublist_remove(list, dn);

	/*
	* See the comment above dnode_rele_task() for an explanation
	* of why this dnode hold is always needed (even when not
	* doing user accounting).
	*/
	multilist_t *newlist = &dn->dn_objset->os_synced_dnodes;
	(void) dnode_add_ref(dn, newlist);
	multilist_insert(newlist, dn);

	dnode_sync(dn, tx);
	}
	}

	static void
	dmu_objset_write_ready(zio_t zio, arc_buf_t abuf, void *arg)
	{
	(void) abuf;
	blkptr_t *bp = zio->io_bp;
	objset_t *os = arg;
	dnode_phys_t *dnp = &os->os_phys->os_meta_dnode;
	uint64_t fill = 0;

	ASSERT(!BP_IS_EMBEDDED(bp));
	ASSERT3U(BP_GET_TYPE(bp), ==, DMU_OT_OBJSET);
	ASSERT0(BP_GET_LEVEL(bp));

	/*
	* Update rootbp fill count: it should be the number of objects
	* allocated in the object set (not counting the "special"
	* objects that are stored in the objset_phys_t -- the meta
	* dnode and user/group/project accounting objects).
	*/
	for (int i = 0; i < dnp->dn_nblkptr; i++)
	fill += BP_GET_FILL(&dnp->dn_blkptr[i]);

	BP_SET_FILL(bp, fill);

	if (os->os_dsl_dataset != NULL)
	rrw_enter(&os->os_dsl_dataset->ds_bp_rwlock, RW_WRITER, FTAG);
	os->os_rootbp = bp;
	if (os->os_dsl_dataset != NULL)
	rrw_exit(&os->os_dsl_dataset->ds_bp_rwlock, FTAG);
	}

	static void
	dmu_objset_write_done(zio_t zio, arc_buf_t abuf, void *arg)
	{
	(void) abuf;
	blkptr_t *bp = zio->io_bp;
	blkptr_t *bp_orig = &zio->io_bp_orig;
	objset_t *os = arg;

	if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
	ASSERT(BP_EQUAL(bp, bp_orig));
	} else {
	dsl_dataset_t *ds = os->os_dsl_dataset;
	dmu_tx_t *tx = os->os_synctx;

	(void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE);
	dsl_dataset_block_born(ds, bp, tx);
	}
	kmem_free(bp, sizeof (*bp));
	}

	typedef struct sync_dnodes_arg {
	multilist_t *sda_list;
	int sda_sublist_idx;
	multilist_t *sda_newlist;
	dmu_tx_t *sda_tx;
	} sync_dnodes_arg_t;

	static void
	sync_dnodes_task(void *arg)
	{
	sync_dnodes_arg_t *sda = arg;

	multilist_sublist_t *ms =
	- multilist_sublist_lock(sda->sda_list, sda->sda_sublist_idx);
	+ multilist_sublist_lock_idx(sda->sda_list, sda->sda_sublist_idx);

	dmu_objset_sync_dnodes(ms, sda->sda_tx);

	multilist_sublist_unlock(ms);

	kmem_free(sda, sizeof (*sda));
	}


	/* called from dsl */
	void
	dmu_objset_sync(objset_t os, zio_t pio, dmu_tx_t *tx)
	{
	int txgoff;
	zbookmark_phys_t zb;
	zio_prop_t zp;
	zio_t *zio;
	list_t *list;
	dbuf_dirty_record_t *dr;
	int num_sublists;
	multilist_t *ml;
	blkptr_t blkptr_copy = kmem_alloc(sizeof (os->os_rootbp), KM_SLEEP);
	blkptr_copy = os->os_rootbp;

	dprintf_ds(os->os_dsl_dataset, "txg=%llu\n", (u_longlong_t)tx->tx_txg);

	ASSERT(dmu_tx_is_syncing(tx));
	/* XXX the write_done callback should really give us the tx... */
	os->os_synctx = tx;

	if (os->os_dsl_dataset == NULL) {
	/*
	* This is the MOS. If we have upgraded,
	* spa_max_replication() could change, so reset
	* os_copies here.
	*/
	os->os_copies = spa_max_replication(os->os_spa);
	}

	/*
	* Create the root block IO
	*/
	SET_BOOKMARK(&zb, os->os_dsl_dataset ?
	os->os_dsl_dataset->ds_object : DMU_META_OBJSET,
	ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
	arc_release(os->os_phys_buf, &os->os_phys_buf);

	dmu_write_policy(os, NULL, 0, 0, &zp);

	/*
	* If we are either claiming the ZIL or doing a raw receive, write
	* out the os_phys_buf raw. Neither of these actions will effect the
	* MAC at this point.
	*/
	if (os->os_raw_receive \|\|
	os->os_next_write_raw[tx->tx_txg & TXG_MASK]) {
	ASSERT(os->os_encrypted);
	arc_convert_to_raw(os->os_phys_buf,
	os->os_dsl_dataset->ds_object, ZFS_HOST_BYTEORDER,
	DMU_OT_OBJSET, NULL, NULL, NULL);
	}

	zio = arc_write(pio, os->os_spa, tx->tx_txg,
	blkptr_copy, os->os_phys_buf, B_FALSE, dmu_os_is_l2cacheable(os),
	&zp, dmu_objset_write_ready, NULL, dmu_objset_write_done,
	os, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb);

	/*
	* Sync special dnodes - the parent IO for the sync is the root block
	*/
	DMU_META_DNODE(os)->dn_zio = zio;
	dnode_sync(DMU_META_DNODE(os), tx);

	os->os_phys->os_flags = os->os_flags;

	if (DMU_USERUSED_DNODE(os) &&
	DMU_USERUSED_DNODE(os)->dn_type != DMU_OT_NONE) {
	DMU_USERUSED_DNODE(os)->dn_zio = zio;
	dnode_sync(DMU_USERUSED_DNODE(os), tx);
	DMU_GROUPUSED_DNODE(os)->dn_zio = zio;
	dnode_sync(DMU_GROUPUSED_DNODE(os), tx);
	}

	if (DMU_PROJECTUSED_DNODE(os) &&
	DMU_PROJECTUSED_DNODE(os)->dn_type != DMU_OT_NONE) {
	DMU_PROJECTUSED_DNODE(os)->dn_zio = zio;
	dnode_sync(DMU_PROJECTUSED_DNODE(os), tx);
	}

	txgoff = tx->tx_txg & TXG_MASK;

	/*
	* We must create the list here because it uses the
	* dn_dirty_link[] of this txg. But it may already
	* exist because we call dsl_dataset_sync() twice per txg.
	*/
	if (os->os_synced_dnodes.ml_sublists == NULL) {
	multilist_create(&os->os_synced_dnodes, sizeof (dnode_t),
	offsetof(dnode_t, dn_dirty_link[txgoff]),
	dnode_multilist_index_func);
	} else {
	ASSERT3U(os->os_synced_dnodes.ml_offset, ==,
	offsetof(dnode_t, dn_dirty_link[txgoff]));
	}

	ml = &os->os_dirty_dnodes[txgoff];
	num_sublists = multilist_get_num_sublists(ml);
	for (int i = 0; i < num_sublists; i++) {
	if (multilist_sublist_is_empty_idx(ml, i))
	continue;
	sync_dnodes_arg_t sda = kmem_alloc(sizeof (sda), KM_SLEEP);
	sda->sda_list = ml;
	sda->sda_sublist_idx = i;
	sda->sda_tx = tx;
	(void) taskq_dispatch(dmu_objset_pool(os)->dp_sync_taskq,
	sync_dnodes_task, sda, 0);
	/* callback frees sda */
	}
	taskq_wait(dmu_objset_pool(os)->dp_sync_taskq);

	list = &DMU_META_DNODE(os)->dn_dirty_records[txgoff];
	while ((dr = list_remove_head(list)) != NULL) {
	ASSERT0(dr->dr_dbuf->db_level);
	zio_nowait(dr->dr_zio);
	}

	/* Enable dnode backfill if enough objects have been freed. */
	if (os->os_freed_dnodes >= dmu_rescan_dnode_threshold) {
	os->os_rescan_dnodes = B_TRUE;
	os->os_freed_dnodes = 0;
	}

	/*
	* Free intent log blocks up to this tx.
	*/
	zil_sync(os->os_zil, tx);
	os->os_phys->os_zil_header = os->os_zil_header;
	zio_nowait(zio);
	}

	boolean_t
	dmu_objset_is_dirty(objset_t *os, uint64_t txg)
	{
	return (!multilist_is_empty(&os->os_dirty_dnodes[txg & TXG_MASK]));
	}

	static file_info_cb_t *file_cbs[DMU_OST_NUMTYPES];

	void
	dmu_objset_register_type(dmu_objset_type_t ost, file_info_cb_t *cb)
	{
	file_cbs[ost] = cb;
	}

	int
	dmu_get_file_info(objset_t os, dmu_object_type_t bonustype, const void data,
	zfs_file_info_t *zfi)
	{
	file_info_cb_t *cb = file_cbs[os->os_phys->os_type];
	if (cb == NULL)
	return (EINVAL);
	return (cb(bonustype, data, zfi));
	}

	boolean_t
	dmu_objset_userused_enabled(objset_t *os)
	{
	return (spa_version(os->os_spa) >= SPA_VERSION_USERSPACE &&
	file_cbs[os->os_phys->os_type] != NULL &&
	DMU_USERUSED_DNODE(os) != NULL);
	}

	boolean_t
	dmu_objset_userobjused_enabled(objset_t *os)
	{
	return (dmu_objset_userused_enabled(os) &&
	spa_feature_is_enabled(os->os_spa, SPA_FEATURE_USEROBJ_ACCOUNTING));
	}

	boolean_t
	dmu_objset_projectquota_enabled(objset_t *os)
	{
	return (file_cbs[os->os_phys->os_type] != NULL &&
	DMU_PROJECTUSED_DNODE(os) != NULL &&
	spa_feature_is_enabled(os->os_spa, SPA_FEATURE_PROJECT_QUOTA));
	}

	typedef struct userquota_node {
	/* must be in the first filed, see userquota_update_cache() */
	char uqn_id[20 + DMU_OBJACCT_PREFIX_LEN];
	int64_t uqn_delta;
	avl_node_t uqn_node;
	} userquota_node_t;

	typedef struct userquota_cache {
	avl_tree_t uqc_user_deltas;
	avl_tree_t uqc_group_deltas;
	avl_tree_t uqc_project_deltas;
	} userquota_cache_t;

	static int
	userquota_compare(const void l, const void r)
	{
	const userquota_node_t *luqn = l;
	const userquota_node_t *ruqn = r;
	int rv;

	/*
	* NB: can only access uqn_id because userquota_update_cache() doesn't
	* pass in an entire userquota_node_t.
	*/
	rv = strcmp(luqn->uqn_id, ruqn->uqn_id);

	return (TREE_ISIGN(rv));
	}

	static void
	do_userquota_cacheflush(objset_t os, userquota_cache_t cache, dmu_tx_t *tx)
	{
	void *cookie;
	userquota_node_t *uqn;

	ASSERT(dmu_tx_is_syncing(tx));

	cookie = NULL;
	while ((uqn = avl_destroy_nodes(&cache->uqc_user_deltas,
	&cookie)) != NULL) {
	/*
	* os_userused_lock protects against concurrent calls to
	* zap_increment_int(). It's needed because zap_increment_int()
	* is not thread-safe (i.e. not atomic).
	*/
	mutex_enter(&os->os_userused_lock);
	VERIFY0(zap_increment(os, DMU_USERUSED_OBJECT,
	uqn->uqn_id, uqn->uqn_delta, tx));
	mutex_exit(&os->os_userused_lock);
	kmem_free(uqn, sizeof (*uqn));
	}
	avl_destroy(&cache->uqc_user_deltas);

	cookie = NULL;
	while ((uqn = avl_destroy_nodes(&cache->uqc_group_deltas,
	&cookie)) != NULL) {
	mutex_enter(&os->os_userused_lock);
	VERIFY0(zap_increment(os, DMU_GROUPUSED_OBJECT,
	uqn->uqn_id, uqn->uqn_delta, tx));
	mutex_exit(&os->os_userused_lock);
	kmem_free(uqn, sizeof (*uqn));
	}
	avl_destroy(&cache->uqc_group_deltas);

	if (dmu_objset_projectquota_enabled(os)) {
	cookie = NULL;
	while ((uqn = avl_destroy_nodes(&cache->uqc_project_deltas,
	&cookie)) != NULL) {
	mutex_enter(&os->os_userused_lock);
	VERIFY0(zap_increment(os, DMU_PROJECTUSED_OBJECT,
	uqn->uqn_id, uqn->uqn_delta, tx));
	mutex_exit(&os->os_userused_lock);
	kmem_free(uqn, sizeof (*uqn));
	}
	avl_destroy(&cache->uqc_project_deltas);
	}
	}

	static void
	userquota_update_cache(avl_tree_t avl, const char id, int64_t delta)
	{
	userquota_node_t *uqn;
	avl_index_t idx;

	ASSERT(strlen(id) < sizeof (uqn->uqn_id));
	/*
	* Use id directly for searching because uqn_id is the first field of
	* userquota_node_t and fields after uqn_id won't be accessed in
	* avl_find().
	*/
	uqn = avl_find(avl, (const void *)id, &idx);
	if (uqn == NULL) {
	uqn = kmem_zalloc(sizeof (*uqn), KM_SLEEP);
	strlcpy(uqn->uqn_id, id, sizeof (uqn->uqn_id));
	avl_insert(avl, uqn, idx);
	}
	uqn->uqn_delta += delta;
	}

	static void
	do_userquota_update(objset_t os, userquota_cache_t cache, uint64_t used,
	uint64_t flags, uint64_t user, uint64_t group, uint64_t project,
	boolean_t subtract)
	{
	if (flags & DNODE_FLAG_USERUSED_ACCOUNTED) {
	int64_t delta = DNODE_MIN_SIZE + used;
	char name[20];

	if (subtract)
	delta = -delta;

	(void) snprintf(name, sizeof (name), "%llx", (longlong_t)user);
	userquota_update_cache(&cache->uqc_user_deltas, name, delta);

	(void) snprintf(name, sizeof (name), "%llx", (longlong_t)group);
	userquota_update_cache(&cache->uqc_group_deltas, name, delta);

	if (dmu_objset_projectquota_enabled(os)) {
	(void) snprintf(name, sizeof (name), "%llx",
	(longlong_t)project);
	userquota_update_cache(&cache->uqc_project_deltas,
	name, delta);
	}
	}
	}

	static void
	do_userobjquota_update(objset_t os, userquota_cache_t cache, uint64_t flags,
	uint64_t user, uint64_t group, uint64_t project, boolean_t subtract)
	{
	if (flags & DNODE_FLAG_USEROBJUSED_ACCOUNTED) {
	char name[20 + DMU_OBJACCT_PREFIX_LEN];
	int delta = subtract ? -1 : 1;

	(void) snprintf(name, sizeof (name), DMU_OBJACCT_PREFIX "%llx",
	(longlong_t)user);
	userquota_update_cache(&cache->uqc_user_deltas, name, delta);

	(void) snprintf(name, sizeof (name), DMU_OBJACCT_PREFIX "%llx",
	(longlong_t)group);
	userquota_update_cache(&cache->uqc_group_deltas, name, delta);

	if (dmu_objset_projectquota_enabled(os)) {
	(void) snprintf(name, sizeof (name),
	DMU_OBJACCT_PREFIX "%llx", (longlong_t)project);
	userquota_update_cache(&cache->uqc_project_deltas,
	name, delta);
	}
	}
	}

	typedef struct userquota_updates_arg {
	objset_t *uua_os;
	int uua_sublist_idx;
	dmu_tx_t *uua_tx;
	} userquota_updates_arg_t;

	static void
	userquota_updates_task(void *arg)
	{
	userquota_updates_arg_t *uua = arg;
	objset_t *os = uua->uua_os;
	dmu_tx_t *tx = uua->uua_tx;
	dnode_t *dn;
	userquota_cache_t cache = { { 0 } };

	- multilist_sublist_t *list =
	- multilist_sublist_lock(&os->os_synced_dnodes, uua->uua_sublist_idx);
	+ multilist_sublist_t *list = multilist_sublist_lock_idx(
	+ &os->os_synced_dnodes, uua->uua_sublist_idx);

	ASSERT(multilist_sublist_head(list) == NULL \|\|
	dmu_objset_userused_enabled(os));
	avl_create(&cache.uqc_user_deltas, userquota_compare,
	sizeof (userquota_node_t), offsetof(userquota_node_t, uqn_node));
	avl_create(&cache.uqc_group_deltas, userquota_compare,
	sizeof (userquota_node_t), offsetof(userquota_node_t, uqn_node));
	if (dmu_objset_projectquota_enabled(os))
	avl_create(&cache.uqc_project_deltas, userquota_compare,
	sizeof (userquota_node_t), offsetof(userquota_node_t,
	uqn_node));

	while ((dn = multilist_sublist_head(list)) != NULL) {
	int flags;
	ASSERT(!DMU_OBJECT_IS_SPECIAL(dn->dn_object));
	ASSERT(dn->dn_phys->dn_type == DMU_OT_NONE \|\|
	dn->dn_phys->dn_flags &
	DNODE_FLAG_USERUSED_ACCOUNTED);

	flags = dn->dn_id_flags;
	ASSERT(flags);
	if (flags & DN_ID_OLD_EXIST) {
	do_userquota_update(os, &cache, dn->dn_oldused,
	dn->dn_oldflags, dn->dn_olduid, dn->dn_oldgid,
	dn->dn_oldprojid, B_TRUE);
	do_userobjquota_update(os, &cache, dn->dn_oldflags,
	dn->dn_olduid, dn->dn_oldgid,
	dn->dn_oldprojid, B_TRUE);
	}
	if (flags & DN_ID_NEW_EXIST) {
	do_userquota_update(os, &cache,
	DN_USED_BYTES(dn->dn_phys), dn->dn_phys->dn_flags,
	dn->dn_newuid, dn->dn_newgid,
	dn->dn_newprojid, B_FALSE);
	do_userobjquota_update(os, &cache,
	dn->dn_phys->dn_flags, dn->dn_newuid, dn->dn_newgid,
	dn->dn_newprojid, B_FALSE);
	}

	mutex_enter(&dn->dn_mtx);
	dn->dn_oldused = 0;
	dn->dn_oldflags = 0;
	if (dn->dn_id_flags & DN_ID_NEW_EXIST) {
	dn->dn_olduid = dn->dn_newuid;
	dn->dn_oldgid = dn->dn_newgid;
	dn->dn_oldprojid = dn->dn_newprojid;
	dn->dn_id_flags \|= DN_ID_OLD_EXIST;
	if (dn->dn_bonuslen == 0)
	dn->dn_id_flags \|= DN_ID_CHKED_SPILL;
	else
	dn->dn_id_flags \|= DN_ID_CHKED_BONUS;
	}
	dn->dn_id_flags &= ~(DN_ID_NEW_EXIST);
	mutex_exit(&dn->dn_mtx);

	multilist_sublist_remove(list, dn);
	dnode_rele(dn, &os->os_synced_dnodes);
	}
	do_userquota_cacheflush(os, &cache, tx);
	multilist_sublist_unlock(list);
	kmem_free(uua, sizeof (*uua));
	}

	/*
	* Release dnode holds from dmu_objset_sync_dnodes(). When the dnode is being
	* synced (i.e. we have issued the zio's for blocks in the dnode), it can't be
	* evicted because the block containing the dnode can't be evicted until it is
	* written out. However, this hold is necessary to prevent the dnode_t from
	* being moved (via dnode_move()) while it's still referenced by
	* dbuf_dirty_record_t:dr_dnode. And dr_dnode is needed for
	* dirty_lightweight_leaf-type dirty records.
	*
	* If we are doing user-object accounting, the dnode_rele() happens from
	* userquota_updates_task() instead.
	*/
	static void
	dnode_rele_task(void *arg)
	{
	userquota_updates_arg_t *uua = arg;
	objset_t *os = uua->uua_os;

	- multilist_sublist_t *list =
	- multilist_sublist_lock(&os->os_synced_dnodes, uua->uua_sublist_idx);
	+ multilist_sublist_t *list = multilist_sublist_lock_idx(
	+ &os->os_synced_dnodes, uua->uua_sublist_idx);

	dnode_t *dn;
	while ((dn = multilist_sublist_head(list)) != NULL) {
	multilist_sublist_remove(list, dn);
	dnode_rele(dn, &os->os_synced_dnodes);
	}
	multilist_sublist_unlock(list);
	kmem_free(uua, sizeof (*uua));
	}

	/*
	* Return TRUE if userquota updates are needed.
	*/
	static boolean_t
	dmu_objset_do_userquota_updates_prep(objset_t os, dmu_tx_t tx)
	{
	if (!dmu_objset_userused_enabled(os))
	return (B_FALSE);

	/*
	* If this is a raw receive just return and handle accounting
	* later when we have the keys loaded. We also don't do user
	* accounting during claiming since the datasets are not owned
	* for the duration of claiming and this txg should only be
	* used for recovery.
	*/
	if (os->os_encrypted && dmu_objset_is_receiving(os))
	return (B_FALSE);

	if (tx->tx_txg <= os->os_spa->spa_claim_max_txg)
	return (B_FALSE);

	/* Allocate the user/group/project used objects if necessary. */
	if (DMU_USERUSED_DNODE(os)->dn_type == DMU_OT_NONE) {
	VERIFY0(zap_create_claim(os,
	DMU_USERUSED_OBJECT,
	DMU_OT_USERGROUP_USED, DMU_OT_NONE, 0, tx));
	VERIFY0(zap_create_claim(os,
	DMU_GROUPUSED_OBJECT,
	DMU_OT_USERGROUP_USED, DMU_OT_NONE, 0, tx));
	}

	if (dmu_objset_projectquota_enabled(os) &&
	DMU_PROJECTUSED_DNODE(os)->dn_type == DMU_OT_NONE) {
	VERIFY0(zap_create_claim(os, DMU_PROJECTUSED_OBJECT,
	DMU_OT_USERGROUP_USED, DMU_OT_NONE, 0, tx));
	}
	return (B_TRUE);
	}

	/*
	* Dispatch taskq tasks to dp_sync_taskq to update the user accounting, and
	* also release the holds on the dnodes from dmu_objset_sync_dnodes().
	* The caller must taskq_wait(dp_sync_taskq).
	*/
	void
	dmu_objset_sync_done(objset_t os, dmu_tx_t tx)
	{
	boolean_t need_userquota = dmu_objset_do_userquota_updates_prep(os, tx);

	int num_sublists = multilist_get_num_sublists(&os->os_synced_dnodes);
	for (int i = 0; i < num_sublists; i++) {
	userquota_updates_arg_t *uua =
	kmem_alloc(sizeof (*uua), KM_SLEEP);
	uua->uua_os = os;
	uua->uua_sublist_idx = i;
	uua->uua_tx = tx;

	/*
	* If we don't need to update userquotas, use
	* dnode_rele_task() to call dnode_rele()
	*/
	(void) taskq_dispatch(dmu_objset_pool(os)->dp_sync_taskq,
	need_userquota ? userquota_updates_task : dnode_rele_task,
	uua, 0);
	/* callback frees uua */
	}
	}


	/*
	* Returns a pointer to data to find uid/gid from
	*
	* If a dirty record for transaction group that is syncing can't
	* be found then NULL is returned. In the NULL case it is assumed
	* the uid/gid aren't changing.
	*/
	static void *
	dmu_objset_userquota_find_data(dmu_buf_impl_t db, dmu_tx_t tx)
	{
	dbuf_dirty_record_t *dr;
	void *data;

	if (db->db_dirtycnt == 0)
	return (db->db.db_data); /* Nothing is changing */

	dr = dbuf_find_dirty_eq(db, tx->tx_txg);

	if (dr == NULL) {
	data = NULL;
	} else {
	if (dr->dr_dnode->dn_bonuslen == 0 &&
	dr->dr_dbuf->db_blkid == DMU_SPILL_BLKID)
	data = dr->dt.dl.dr_data->b_data;
	else
	data = dr->dt.dl.dr_data;
	}

	return (data);
	}

	void
	dmu_objset_userquota_get_ids(dnode_t dn, boolean_t before, dmu_tx_t tx)
	{
	objset_t *os = dn->dn_objset;
	void *data = NULL;
	dmu_buf_impl_t *db = NULL;
	int flags = dn->dn_id_flags;
	int error;
	boolean_t have_spill = B_FALSE;

	if (!dmu_objset_userused_enabled(dn->dn_objset))
	return;

	/*
	* Raw receives introduce a problem with user accounting. Raw
	* receives cannot update the user accounting info because the
	* user ids and the sizes are encrypted. To guarantee that we
	* never end up with bad user accounting, we simply disable it
	* during raw receives. We also disable this for normal receives
	* so that an incremental raw receive may be done on top of an
	* existing non-raw receive.
	*/
	if (os->os_encrypted && dmu_objset_is_receiving(os))
	return;

	if (before && (flags & (DN_ID_CHKED_BONUS\|DN_ID_OLD_EXIST\|
	DN_ID_CHKED_SPILL)))
	return;

	if (before && dn->dn_bonuslen != 0)
	data = DN_BONUS(dn->dn_phys);
	else if (!before && dn->dn_bonuslen != 0) {
	if (dn->dn_bonus) {
	db = dn->dn_bonus;
	mutex_enter(&db->db_mtx);
	data = dmu_objset_userquota_find_data(db, tx);
	} else {
	data = DN_BONUS(dn->dn_phys);
	}
	} else if (dn->dn_bonuslen == 0 && dn->dn_bonustype == DMU_OT_SA) {
	int rf = 0;

	if (RW_WRITE_HELD(&dn->dn_struct_rwlock))
	rf \|= DB_RF_HAVESTRUCT;
	error = dmu_spill_hold_by_dnode(dn,
	rf \| DB_RF_MUST_SUCCEED,
	FTAG, (dmu_buf_t **)&db);
	ASSERT(error == 0);
	mutex_enter(&db->db_mtx);
	data = (before) ? db->db.db_data :
	dmu_objset_userquota_find_data(db, tx);
	have_spill = B_TRUE;
	} else {
	mutex_enter(&dn->dn_mtx);
	dn->dn_id_flags \|= DN_ID_CHKED_BONUS;
	mutex_exit(&dn->dn_mtx);
	return;
	}

	/*
	* Must always call the callback in case the object
	* type has changed and that type isn't an object type to track
	*/
	zfs_file_info_t zfi;
	error = file_cbs[os->os_phys->os_type](dn->dn_bonustype, data, &zfi);

	if (before) {
	ASSERT(data);
	dn->dn_olduid = zfi.zfi_user;
	dn->dn_oldgid = zfi.zfi_group;
	dn->dn_oldprojid = zfi.zfi_project;
	} else if (data) {
	dn->dn_newuid = zfi.zfi_user;
	dn->dn_newgid = zfi.zfi_group;
	dn->dn_newprojid = zfi.zfi_project;
	}

	/*
	* Preserve existing uid/gid when the callback can't determine
	* what the new uid/gid are and the callback returned EEXIST.
	* The EEXIST error tells us to just use the existing uid/gid.
	* If we don't know what the old values are then just assign
	* them to 0, since that is a new file being created.
	*/
	if (!before && data == NULL && error == EEXIST) {
	if (flags & DN_ID_OLD_EXIST) {
	dn->dn_newuid = dn->dn_olduid;
	dn->dn_newgid = dn->dn_oldgid;
	dn->dn_newprojid = dn->dn_oldprojid;
	} else {
	dn->dn_newuid = 0;
	dn->dn_newgid = 0;
	dn->dn_newprojid = ZFS_DEFAULT_PROJID;
	}
	error = 0;
	}

	if (db)
	mutex_exit(&db->db_mtx);

	mutex_enter(&dn->dn_mtx);
	if (error == 0 && before)
	dn->dn_id_flags \|= DN_ID_OLD_EXIST;
	if (error == 0 && !before)
	dn->dn_id_flags \|= DN_ID_NEW_EXIST;

	if (have_spill) {
	dn->dn_id_flags \|= DN_ID_CHKED_SPILL;
	} else {
	dn->dn_id_flags \|= DN_ID_CHKED_BONUS;
	}
	mutex_exit(&dn->dn_mtx);
	if (have_spill)
	dmu_buf_rele((dmu_buf_t *)db, FTAG);
	}

	boolean_t
	dmu_objset_userspace_present(objset_t *os)
	{
	return (os->os_phys->os_flags &
	OBJSET_FLAG_USERACCOUNTING_COMPLETE);
	}

	boolean_t
	dmu_objset_userobjspace_present(objset_t *os)
	{
	return (os->os_phys->os_flags &
	OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE);
	}

	boolean_t
	dmu_objset_projectquota_present(objset_t *os)
	{
	return (os->os_phys->os_flags &
	OBJSET_FLAG_PROJECTQUOTA_COMPLETE);
	}

	static int
	dmu_objset_space_upgrade(objset_t *os)
	{
	uint64_t obj;
	int err = 0;

	/*
	* We simply need to mark every object dirty, so that it will be
	* synced out and now accounted. If this is called
	* concurrently, or if we already did some work before crashing,
	* that's fine, since we track each object's accounted state
	* independently.
	*/

	for (obj = 0; err == 0; err = dmu_object_next(os, &obj, FALSE, 0)) {
	dmu_tx_t *tx;
	dmu_buf_t *db;
	int objerr;

	mutex_enter(&os->os_upgrade_lock);
	if (os->os_upgrade_exit)
	err = SET_ERROR(EINTR);
	mutex_exit(&os->os_upgrade_lock);
	if (err != 0)
	return (err);

	if (issig(JUSTLOOKING) && issig(FORREAL))
	return (SET_ERROR(EINTR));

	objerr = dmu_bonus_hold(os, obj, FTAG, &db);
	if (objerr != 0)
	continue;
	tx = dmu_tx_create(os);
	dmu_tx_hold_bonus(tx, obj);
	objerr = dmu_tx_assign(tx, TXG_WAIT);
	if (objerr != 0) {
	dmu_buf_rele(db, FTAG);
	dmu_tx_abort(tx);
	continue;
	}
	dmu_buf_will_dirty(db, tx);
	dmu_buf_rele(db, FTAG);
	dmu_tx_commit(tx);
	}
	return (0);
	}

	static int
	dmu_objset_userspace_upgrade_cb(objset_t *os)
	{
	int err = 0;

	if (dmu_objset_userspace_present(os))
	return (0);
	if (dmu_objset_is_snapshot(os))
	return (SET_ERROR(EINVAL));
	if (!dmu_objset_userused_enabled(os))
	return (SET_ERROR(ENOTSUP));

	err = dmu_objset_space_upgrade(os);
	if (err)
	return (err);

	os->os_flags \|= OBJSET_FLAG_USERACCOUNTING_COMPLETE;
	txg_wait_synced(dmu_objset_pool(os), 0);
	return (0);
	}

	void
	dmu_objset_userspace_upgrade(objset_t *os)
	{
	dmu_objset_upgrade(os, dmu_objset_userspace_upgrade_cb);
	}

	static int
	dmu_objset_id_quota_upgrade_cb(objset_t *os)
	{
	int err = 0;

	if (dmu_objset_userobjspace_present(os) &&
	dmu_objset_projectquota_present(os))
	return (0);
	if (dmu_objset_is_snapshot(os))
	return (SET_ERROR(EINVAL));
	if (!dmu_objset_userused_enabled(os))
	return (SET_ERROR(ENOTSUP));
	if (!dmu_objset_projectquota_enabled(os) &&
	dmu_objset_userobjspace_present(os))
	return (SET_ERROR(ENOTSUP));

	if (dmu_objset_userobjused_enabled(os))
	dmu_objset_ds(os)->ds_feature_activation[
	SPA_FEATURE_USEROBJ_ACCOUNTING] = (void *)B_TRUE;
	if (dmu_objset_projectquota_enabled(os))
	dmu_objset_ds(os)->ds_feature_activation[
	SPA_FEATURE_PROJECT_QUOTA] = (void *)B_TRUE;

	err = dmu_objset_space_upgrade(os);
	if (err)
	return (err);

	os->os_flags \|= OBJSET_FLAG_USERACCOUNTING_COMPLETE;
	if (dmu_objset_userobjused_enabled(os))
	os->os_flags \|= OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE;
	if (dmu_objset_projectquota_enabled(os))
	os->os_flags \|= OBJSET_FLAG_PROJECTQUOTA_COMPLETE;

	txg_wait_synced(dmu_objset_pool(os), 0);
	return (0);
	}

	void
	dmu_objset_id_quota_upgrade(objset_t *os)
	{
	dmu_objset_upgrade(os, dmu_objset_id_quota_upgrade_cb);
	}

	boolean_t
	dmu_objset_userobjspace_upgradable(objset_t *os)
	{
	return (dmu_objset_type(os) == DMU_OST_ZFS &&
	!dmu_objset_is_snapshot(os) &&
	dmu_objset_userobjused_enabled(os) &&
	!dmu_objset_userobjspace_present(os) &&
	spa_writeable(dmu_objset_spa(os)));
	}

	boolean_t
	dmu_objset_projectquota_upgradable(objset_t *os)
	{
	return (dmu_objset_type(os) == DMU_OST_ZFS &&
	!dmu_objset_is_snapshot(os) &&
	dmu_objset_projectquota_enabled(os) &&
	!dmu_objset_projectquota_present(os) &&
	spa_writeable(dmu_objset_spa(os)));
	}

	void
	dmu_objset_space(objset_t os, uint64_t refdbytesp, uint64_t *availbytesp,
	uint64_t usedobjsp, uint64_t availobjsp)
	{
	dsl_dataset_space(os->os_dsl_dataset, refdbytesp, availbytesp,
	usedobjsp, availobjsp);
	}

	uint64_t
	dmu_objset_fsid_guid(objset_t *os)
	{
	return (dsl_dataset_fsid_guid(os->os_dsl_dataset));
	}

	void
	dmu_objset_fast_stat(objset_t os, dmu_objset_stats_t stat)
	{
	stat->dds_type = os->os_phys->os_type;
	if (os->os_dsl_dataset)
	dsl_dataset_fast_stat(os->os_dsl_dataset, stat);
	}

	void
	dmu_objset_stats(objset_t os, nvlist_t nv)
	{
	ASSERT(os->os_dsl_dataset \|\|
	os->os_phys->os_type == DMU_OST_META);

	if (os->os_dsl_dataset != NULL)
	dsl_dataset_stats(os->os_dsl_dataset, nv);

	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_TYPE,
	os->os_phys->os_type);
	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USERACCOUNTING,
	dmu_objset_userspace_present(os));
	}

	int
	dmu_objset_is_snapshot(objset_t *os)
	{
	if (os->os_dsl_dataset != NULL)
	return (os->os_dsl_dataset->ds_is_snapshot);
	else
	return (B_FALSE);
	}

	int
	dmu_snapshot_realname(objset_t os, const char name, char *real, int maxlen,
	boolean_t *conflict)
	{
	dsl_dataset_t *ds = os->os_dsl_dataset;
	uint64_t ignored;

	if (dsl_dataset_phys(ds)->ds_snapnames_zapobj == 0)
	return (SET_ERROR(ENOENT));

	return (zap_lookup_norm(ds->ds_dir->dd_pool->dp_meta_objset,
	dsl_dataset_phys(ds)->ds_snapnames_zapobj, name, 8, 1, &ignored,
	MT_NORMALIZE, real, maxlen, conflict));
	}

	int
	dmu_snapshot_list_next(objset_t os, int namelen, char name,
	uint64_t idp, uint64_t offp, boolean_t *case_conflict)
	{
	dsl_dataset_t *ds = os->os_dsl_dataset;
	zap_cursor_t cursor;
	zap_attribute_t attr;

	ASSERT(dsl_pool_config_held(dmu_objset_pool(os)));

	if (dsl_dataset_phys(ds)->ds_snapnames_zapobj == 0)
	return (SET_ERROR(ENOENT));

	zap_cursor_init_serialized(&cursor,
	ds->ds_dir->dd_pool->dp_meta_objset,
	dsl_dataset_phys(ds)->ds_snapnames_zapobj, *offp);

	if (zap_cursor_retrieve(&cursor, &attr) != 0) {
	zap_cursor_fini(&cursor);
	return (SET_ERROR(ENOENT));
	}

	if (strlen(attr.za_name) + 1 > namelen) {
	zap_cursor_fini(&cursor);
	return (SET_ERROR(ENAMETOOLONG));
	}

	(void) strlcpy(name, attr.za_name, namelen);
	if (idp)
	*idp = attr.za_first_integer;
	if (case_conflict)
	*case_conflict = attr.za_normalization_conflict;
	zap_cursor_advance(&cursor);
	*offp = zap_cursor_serialize(&cursor);
	zap_cursor_fini(&cursor);

	return (0);
	}

	int
	dmu_snapshot_lookup(objset_t os, const char name, uint64_t *value)
	{
	return (dsl_dataset_snap_lookup(os->os_dsl_dataset, name, value));
	}

	int
	dmu_dir_list_next(objset_t os, int namelen, char name,
	uint64_t idp, uint64_t offp)
	{
	dsl_dir_t *dd = os->os_dsl_dataset->ds_dir;
	zap_cursor_t cursor;
	zap_attribute_t attr;

	/* there is no next dir on a snapshot! */
	if (os->os_dsl_dataset->ds_object !=
	dsl_dir_phys(dd)->dd_head_dataset_obj)
	return (SET_ERROR(ENOENT));

	zap_cursor_init_serialized(&cursor,
	dd->dd_pool->dp_meta_objset,
	dsl_dir_phys(dd)->dd_child_dir_zapobj, *offp);

	if (zap_cursor_retrieve(&cursor, &attr) != 0) {
	zap_cursor_fini(&cursor);
	return (SET_ERROR(ENOENT));
	}

	if (strlen(attr.za_name) + 1 > namelen) {
	zap_cursor_fini(&cursor);
	return (SET_ERROR(ENAMETOOLONG));
	}

	(void) strlcpy(name, attr.za_name, namelen);
	if (idp)
	*idp = attr.za_first_integer;
	zap_cursor_advance(&cursor);
	*offp = zap_cursor_serialize(&cursor);
	zap_cursor_fini(&cursor);

	return (0);
	}

	typedef struct dmu_objset_find_ctx {
	taskq_t *dc_tq;
	dsl_pool_t *dc_dp;
	uint64_t dc_ddobj;
	char dc_ddname; / last component of ddobj's name */
	int (dc_func)(dsl_pool_t , dsl_dataset_t , void );
	void *dc_arg;
	int dc_flags;
	kmutex_t *dc_error_lock;
	int *dc_error;
	} dmu_objset_find_ctx_t;

	static void
	dmu_objset_find_dp_impl(dmu_objset_find_ctx_t *dcp)
	{
	dsl_pool_t *dp = dcp->dc_dp;
	dsl_dir_t *dd;
	dsl_dataset_t *ds;
	zap_cursor_t zc;
	zap_attribute_t *attr;
	uint64_t thisobj;
	int err = 0;

	/* don't process if there already was an error */
	if (*dcp->dc_error != 0)
	goto out;

	/*
	* Note: passing the name (dc_ddname) here is optional, but it
	* improves performance because we don't need to call
	* zap_value_search() to determine the name.
	*/
	err = dsl_dir_hold_obj(dp, dcp->dc_ddobj, dcp->dc_ddname, FTAG, &dd);
	if (err != 0)
	goto out;

	/* Don't visit hidden ($MOS & $ORIGIN) objsets. */
	if (dd->dd_myname[0] == '$') {
	dsl_dir_rele(dd, FTAG);
	goto out;
	}

	thisobj = dsl_dir_phys(dd)->dd_head_dataset_obj;
	attr = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP);

	/*
	* Iterate over all children.
	*/
	if (dcp->dc_flags & DS_FIND_CHILDREN) {
	for (zap_cursor_init(&zc, dp->dp_meta_objset,
	dsl_dir_phys(dd)->dd_child_dir_zapobj);
	zap_cursor_retrieve(&zc, attr) == 0;
	(void) zap_cursor_advance(&zc)) {
	ASSERT3U(attr->za_integer_length, ==,
	sizeof (uint64_t));
	ASSERT3U(attr->za_num_integers, ==, 1);

	dmu_objset_find_ctx_t *child_dcp =
	kmem_alloc(sizeof (*child_dcp), KM_SLEEP);
	child_dcp = dcp;
	child_dcp->dc_ddobj = attr->za_first_integer;
	child_dcp->dc_ddname = spa_strdup(attr->za_name);
	if (dcp->dc_tq != NULL)
	(void) taskq_dispatch(dcp->dc_tq,
	dmu_objset_find_dp_cb, child_dcp, TQ_SLEEP);
	else
	dmu_objset_find_dp_impl(child_dcp);
	}
	zap_cursor_fini(&zc);
	}

	/*
	* Iterate over all snapshots.
	*/
	if (dcp->dc_flags & DS_FIND_SNAPSHOTS) {
	dsl_dataset_t *ds;
	err = dsl_dataset_hold_obj(dp, thisobj, FTAG, &ds);

	if (err == 0) {
	uint64_t snapobj;

	snapobj = dsl_dataset_phys(ds)->ds_snapnames_zapobj;
	dsl_dataset_rele(ds, FTAG);

	for (zap_cursor_init(&zc, dp->dp_meta_objset, snapobj);
	zap_cursor_retrieve(&zc, attr) == 0;
	(void) zap_cursor_advance(&zc)) {
	ASSERT3U(attr->za_integer_length, ==,
	sizeof (uint64_t));
	ASSERT3U(attr->za_num_integers, ==, 1);

	err = dsl_dataset_hold_obj(dp,
	attr->za_first_integer, FTAG, &ds);
	if (err != 0)
	break;
	err = dcp->dc_func(dp, ds, dcp->dc_arg);
	dsl_dataset_rele(ds, FTAG);
	if (err != 0)
	break;
	}
	zap_cursor_fini(&zc);
	}
	}

	kmem_free(attr, sizeof (zap_attribute_t));

	if (err != 0) {
	dsl_dir_rele(dd, FTAG);
	goto out;
	}

	/*
	* Apply to self.
	*/
	err = dsl_dataset_hold_obj(dp, thisobj, FTAG, &ds);

	/*
	* Note: we hold the dir while calling dsl_dataset_hold_obj() so
	* that the dir will remain cached, and we won't have to re-instantiate
	* it (which could be expensive due to finding its name via
	* zap_value_search()).
	*/
	dsl_dir_rele(dd, FTAG);
	if (err != 0)
	goto out;
	err = dcp->dc_func(dp, ds, dcp->dc_arg);
	dsl_dataset_rele(ds, FTAG);

	out:
	if (err != 0) {
	mutex_enter(dcp->dc_error_lock);
	/* only keep first error */
	if (*dcp->dc_error == 0)
	*dcp->dc_error = err;
	mutex_exit(dcp->dc_error_lock);
	}

	if (dcp->dc_ddname != NULL)
	spa_strfree(dcp->dc_ddname);
	kmem_free(dcp, sizeof (*dcp));
	}

	static void
	dmu_objset_find_dp_cb(void *arg)
	{
	dmu_objset_find_ctx_t *dcp = arg;
	dsl_pool_t *dp = dcp->dc_dp;

	/*
	* We need to get a pool_config_lock here, as there are several
	* assert(pool_config_held) down the stack. Getting a lock via
	* dsl_pool_config_enter is risky, as it might be stalled by a
	* pending writer. This would deadlock, as the write lock can
	* only be granted when our parent thread gives up the lock.
	* The _prio interface gives us priority over a pending writer.
	*/
	dsl_pool_config_enter_prio(dp, FTAG);

	dmu_objset_find_dp_impl(dcp);

	dsl_pool_config_exit(dp, FTAG);
	}

	/*
	* Find objsets under and including ddobj, call func(ds) on each.
	* The order for the enumeration is completely undefined.
	* func is called with dsl_pool_config held.
	*/
	int
	dmu_objset_find_dp(dsl_pool_t *dp, uint64_t ddobj,
	int func(dsl_pool_t , dsl_dataset_t , void ), void arg, int flags)
	{
	int error = 0;
	taskq_t *tq = NULL;
	int ntasks;
	dmu_objset_find_ctx_t *dcp;
	kmutex_t err_lock;

	mutex_init(&err_lock, NULL, MUTEX_DEFAULT, NULL);
	dcp = kmem_alloc(sizeof (*dcp), KM_SLEEP);
	dcp->dc_tq = NULL;
	dcp->dc_dp = dp;
	dcp->dc_ddobj = ddobj;
	dcp->dc_ddname = NULL;
	dcp->dc_func = func;
	dcp->dc_arg = arg;
	dcp->dc_flags = flags;
	dcp->dc_error_lock = &err_lock;
	dcp->dc_error = &error;

	if ((flags & DS_FIND_SERIALIZE) \|\| dsl_pool_config_held_writer(dp)) {
	/*
	* In case a write lock is held we can't make use of
	* parallelism, as down the stack of the worker threads
	* the lock is asserted via dsl_pool_config_held.
	* In case of a read lock this is solved by getting a read
	* lock in each worker thread, which isn't possible in case
	* of a writer lock. So we fall back to the synchronous path
	* here.
	* In the future it might be possible to get some magic into
	* dsl_pool_config_held in a way that it returns true for
	* the worker threads so that a single lock held from this
	* thread suffices. For now, stay single threaded.
	*/
	dmu_objset_find_dp_impl(dcp);
	mutex_destroy(&err_lock);

	return (error);
	}

	ntasks = dmu_find_threads;
	if (ntasks == 0)
	ntasks = vdev_count_leaves(dp->dp_spa) * 4;
	tq = taskq_create("dmu_objset_find", ntasks, maxclsyspri, ntasks,
	INT_MAX, 0);
	if (tq == NULL) {
	kmem_free(dcp, sizeof (*dcp));
	mutex_destroy(&err_lock);

	return (SET_ERROR(ENOMEM));
	}
	dcp->dc_tq = tq;

	/* dcp will be freed by task */
	(void) taskq_dispatch(tq, dmu_objset_find_dp_cb, dcp, TQ_SLEEP);

	/*
	* PORTING: this code relies on the property of taskq_wait to wait
	* until no more tasks are queued and no more tasks are active. As
	* we always queue new tasks from within other tasks, task_wait
	* reliably waits for the full recursion to finish, even though we
	* enqueue new tasks after taskq_wait has been called.
	* On platforms other than illumos, taskq_wait may not have this
	* property.
	*/
	taskq_wait(tq);
	taskq_destroy(tq);
	mutex_destroy(&err_lock);

	return (error);
	}

	/*
	* Find all objsets under name, and for each, call 'func(child_name, arg)'.
	* The dp_config_rwlock must not be held when this is called, and it
	* will not be held when the callback is called.
	* Therefore this function should only be used when the pool is not changing
	* (e.g. in syncing context), or the callback can deal with the possible races.
	*/
	static int
	dmu_objset_find_impl(spa_t spa, const char name,
	int func(const char , void ), void *arg, int flags)
	{
	dsl_dir_t *dd;
	dsl_pool_t *dp = spa_get_dsl(spa);
	dsl_dataset_t *ds;
	zap_cursor_t zc;
	zap_attribute_t *attr;
	char *child;
	uint64_t thisobj;
	int err;

	dsl_pool_config_enter(dp, FTAG);

	err = dsl_dir_hold(dp, name, FTAG, &dd, NULL);
	if (err != 0) {
	dsl_pool_config_exit(dp, FTAG);
	return (err);
	}

	/* Don't visit hidden ($MOS & $ORIGIN) objsets. */
	if (dd->dd_myname[0] == '$') {
	dsl_dir_rele(dd, FTAG);
	dsl_pool_config_exit(dp, FTAG);
	return (0);
	}

	thisobj = dsl_dir_phys(dd)->dd_head_dataset_obj;
	attr = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP);

	/*
	* Iterate over all children.
	*/
	if (flags & DS_FIND_CHILDREN) {
	for (zap_cursor_init(&zc, dp->dp_meta_objset,
	dsl_dir_phys(dd)->dd_child_dir_zapobj);
	zap_cursor_retrieve(&zc, attr) == 0;
	(void) zap_cursor_advance(&zc)) {
	ASSERT3U(attr->za_integer_length, ==,
	sizeof (uint64_t));
	ASSERT3U(attr->za_num_integers, ==, 1);

	child = kmem_asprintf("%s/%s", name, attr->za_name);
	dsl_pool_config_exit(dp, FTAG);
	err = dmu_objset_find_impl(spa, child,
	func, arg, flags);
	dsl_pool_config_enter(dp, FTAG);
	kmem_strfree(child);
	if (err != 0)
	break;
	}
	zap_cursor_fini(&zc);

	if (err != 0) {
	dsl_dir_rele(dd, FTAG);
	dsl_pool_config_exit(dp, FTAG);
	kmem_free(attr, sizeof (zap_attribute_t));
	return (err);
	}
	}

	/*
	* Iterate over all snapshots.
	*/
	if (flags & DS_FIND_SNAPSHOTS) {
	err = dsl_dataset_hold_obj(dp, thisobj, FTAG, &ds);

	if (err == 0) {
	uint64_t snapobj;

	snapobj = dsl_dataset_phys(ds)->ds_snapnames_zapobj;
	dsl_dataset_rele(ds, FTAG);

	for (zap_cursor_init(&zc, dp->dp_meta_objset, snapobj);
	zap_cursor_retrieve(&zc, attr) == 0;
	(void) zap_cursor_advance(&zc)) {
	ASSERT3U(attr->za_integer_length, ==,
	sizeof (uint64_t));
	ASSERT3U(attr->za_num_integers, ==, 1);

	child = kmem_asprintf("%s@%s",
	name, attr->za_name);
	dsl_pool_config_exit(dp, FTAG);
	err = func(child, arg);
	dsl_pool_config_enter(dp, FTAG);
	kmem_strfree(child);
	if (err != 0)
	break;
	}
	zap_cursor_fini(&zc);
	}
	}

	dsl_dir_rele(dd, FTAG);
	kmem_free(attr, sizeof (zap_attribute_t));
	dsl_pool_config_exit(dp, FTAG);

	if (err != 0)
	return (err);

	/* Apply to self. */
	return (func(name, arg));
	}

	/*
	* See comment above dmu_objset_find_impl().
	*/
	int
	dmu_objset_find(const char name, int func(const char , void ), void arg,
	int flags)
	{
	spa_t *spa;
	int error;

	error = spa_open(name, &spa, FTAG);
	if (error != 0)
	return (error);
	error = dmu_objset_find_impl(spa, name, func, arg, flags);
	spa_close(spa, FTAG);
	return (error);
	}

	boolean_t
	dmu_objset_incompatible_encryption_version(objset_t *os)
	{
	return (dsl_dir_incompatible_encryption_version(
	os->os_dsl_dataset->ds_dir));
	}

	void
	dmu_objset_set_user(objset_t os, void user_ptr)
	{
	ASSERT(MUTEX_HELD(&os->os_user_ptr_lock));
	os->os_user_ptr = user_ptr;
	}

	void *
	dmu_objset_get_user(objset_t *os)
	{
	ASSERT(MUTEX_HELD(&os->os_user_ptr_lock));
	return (os->os_user_ptr);
	}

	/*
	* Determine name of filesystem, given name of snapshot.
	* buf must be at least ZFS_MAX_DATASET_NAME_LEN bytes
	*/
	int
	dmu_fsname(const char snapname, char buf)
	{
	char *atp = strchr(snapname, '@');
	if (atp == NULL)
	return (SET_ERROR(EINVAL));
	if (atp - snapname >= ZFS_MAX_DATASET_NAME_LEN)
	return (SET_ERROR(ENAMETOOLONG));
	(void) strlcpy(buf, snapname, atp - snapname + 1);
	return (0);
	}

	/*
	* Call when we think we're going to write/free space in open context
	* to track the amount of dirty data in the open txg, which is also the
	* amount of memory that can not be evicted until this txg syncs.
	*
	* Note that there are two conditions where this can be called from
	* syncing context:
	*
	* [1] When we just created the dataset, in which case we go on with
	* updating any accounting of dirty data as usual.
	* [2] When we are dirtying MOS data, in which case we only update the
	* pool's accounting of dirty data.
	*/
	void
	dmu_objset_willuse_space(objset_t os, int64_t space, dmu_tx_t tx)
	{
	dsl_dataset_t *ds = os->os_dsl_dataset;
	int64_t aspace = spa_get_worst_case_asize(os->os_spa, space);

	if (ds != NULL) {
	dsl_dir_willuse_space(ds->ds_dir, aspace, tx);
	}

	dsl_pool_dirty_space(dmu_tx_pool(tx), space, tx);
	}

	#if defined(_KERNEL)
	EXPORT_SYMBOL(dmu_objset_zil);
	EXPORT_SYMBOL(dmu_objset_pool);
	EXPORT_SYMBOL(dmu_objset_ds);
	EXPORT_SYMBOL(dmu_objset_type);
	EXPORT_SYMBOL(dmu_objset_name);
	EXPORT_SYMBOL(dmu_objset_hold);
	EXPORT_SYMBOL(dmu_objset_hold_flags);
	EXPORT_SYMBOL(dmu_objset_own);
	EXPORT_SYMBOL(dmu_objset_rele);
	EXPORT_SYMBOL(dmu_objset_rele_flags);
	EXPORT_SYMBOL(dmu_objset_disown);
	EXPORT_SYMBOL(dmu_objset_from_ds);
	EXPORT_SYMBOL(dmu_objset_create);
	EXPORT_SYMBOL(dmu_objset_clone);
	EXPORT_SYMBOL(dmu_objset_stats);
	EXPORT_SYMBOL(dmu_objset_fast_stat);
	EXPORT_SYMBOL(dmu_objset_spa);
	EXPORT_SYMBOL(dmu_objset_space);
	EXPORT_SYMBOL(dmu_objset_fsid_guid);
	EXPORT_SYMBOL(dmu_objset_find);
	EXPORT_SYMBOL(dmu_objset_byteswap);
	EXPORT_SYMBOL(dmu_objset_evict_dbufs);
	EXPORT_SYMBOL(dmu_objset_snap_cmtime);
	EXPORT_SYMBOL(dmu_objset_dnodesize);

	EXPORT_SYMBOL(dmu_objset_sync);
	EXPORT_SYMBOL(dmu_objset_is_dirty);
	EXPORT_SYMBOL(dmu_objset_create_impl_dnstats);
	EXPORT_SYMBOL(dmu_objset_create_impl);
	EXPORT_SYMBOL(dmu_objset_open_impl);
	EXPORT_SYMBOL(dmu_objset_evict);
	EXPORT_SYMBOL(dmu_objset_register_type);
	EXPORT_SYMBOL(dmu_objset_sync_done);
	EXPORT_SYMBOL(dmu_objset_userquota_get_ids);
	EXPORT_SYMBOL(dmu_objset_userused_enabled);
	EXPORT_SYMBOL(dmu_objset_userspace_upgrade);
	EXPORT_SYMBOL(dmu_objset_userspace_present);
	EXPORT_SYMBOL(dmu_objset_userobjused_enabled);
	EXPORT_SYMBOL(dmu_objset_userobjspace_upgradable);
	EXPORT_SYMBOL(dmu_objset_userobjspace_present);
	EXPORT_SYMBOL(dmu_objset_projectquota_enabled);
	EXPORT_SYMBOL(dmu_objset_projectquota_present);
	EXPORT_SYMBOL(dmu_objset_projectquota_upgradable);
	EXPORT_SYMBOL(dmu_objset_id_quota_upgrade);
	#endif
	diff --git a/sys/contrib/openzfs/module/zfs/dmu_recv.c b/sys/contrib/openzfs/module/zfs/dmu_recv.c
	index 54aa60259ea1..9f1c25f866f7 100644
	--- a/sys/contrib/openzfs/module/zfs/dmu_recv.c
	+++ b/sys/contrib/openzfs/module/zfs/dmu_recv.c
	@@ -1,3801 +1,3806 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright 2011 Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
	* Copyright (c) 2014, Joyent, Inc. All rights reserved.
	* Copyright 2014 HybridCluster. All rights reserved.
	* Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
	* Copyright (c) 2019, Klara Inc.
	* Copyright (c) 2019, Allan Jude
	* Copyright (c) 2019 Datto Inc.
	* Copyright (c) 2022 Axcient.
	*/

	#include <sys/arc.h>
	#include <sys/spa_impl.h>
	#include <sys/dmu.h>
	#include <sys/dmu_impl.h>
	#include <sys/dmu_send.h>
	#include <sys/dmu_recv.h>
	#include <sys/dmu_tx.h>
	#include <sys/dbuf.h>
	#include <sys/dnode.h>
	#include <sys/zfs_context.h>
	#include <sys/dmu_objset.h>
	#include <sys/dmu_traverse.h>
	#include <sys/dsl_dataset.h>
	#include <sys/dsl_dir.h>
	#include <sys/dsl_prop.h>
	#include <sys/dsl_pool.h>
	#include <sys/dsl_synctask.h>
	#include <sys/zfs_ioctl.h>
	#include <sys/zap.h>
	#include <sys/zvol.h>
	#include <sys/zio_checksum.h>
	#include <sys/zfs_znode.h>
	#include <zfs_fletcher.h>
	#include <sys/avl.h>
	#include <sys/ddt.h>
	#include <sys/zfs_onexit.h>
	#include <sys/dsl_destroy.h>
	#include <sys/blkptr.h>
	#include <sys/dsl_bookmark.h>
	#include <sys/zfeature.h>
	#include <sys/bqueue.h>
	#include <sys/objlist.h>
	#ifdef _KERNEL
	#include <sys/zfs_vfsops.h>
	#endif
	#include <sys/zfs_file.h>

	static uint_t zfs_recv_queue_length = SPA_MAXBLOCKSIZE;
	static uint_t zfs_recv_queue_ff = 20;
	static uint_t zfs_recv_write_batch_size = 1024 * 1024;
	static int zfs_recv_best_effort_corrective = 0;

	static const void *const dmu_recv_tag = "dmu_recv_tag";
	const char *const recv_clone_name = "%recv";

	typedef enum {
	ORNS_NO,
	ORNS_YES,
	ORNS_MAYBE
	} or_need_sync_t;

	static int receive_read_payload_and_next_header(dmu_recv_cookie_t *ra, int len,
	void *buf);

	struct receive_record_arg {
	dmu_replay_record_t header;
	void payload; / Pointer to a buffer containing the payload */
	/*
	* If the record is a WRITE or SPILL, pointer to the abd containing the
	* payload.
	*/
	abd_t *abd;
	int payload_size;
	uint64_t bytes_read; /* bytes read from stream when record created */
	boolean_t eos_marker; /* Marks the end of the stream */
	bqueue_node_t node;
	};

	struct receive_writer_arg {
	objset_t *os;
	boolean_t byteswap;
	bqueue_t q;

	/*
	* These three members are used to signal to the main thread when
	* we're done.
	*/
	kmutex_t mutex;
	kcondvar_t cv;
	boolean_t done;

	int err;
	const char *tofs;
	boolean_t heal;
	boolean_t resumable;
	boolean_t raw; /* DMU_BACKUP_FEATURE_RAW set */
	boolean_t spill; /* DRR_FLAG_SPILL_BLOCK set */
	boolean_t full; /* this is a full send stream */
	uint64_t last_object;
	uint64_t last_offset;
	uint64_t max_object; /* highest object ID referenced in stream */
	uint64_t bytes_read; /* bytes read when current record created */

	list_t write_batch;

	/* Encryption parameters for the last received DRR_OBJECT_RANGE */
	boolean_t or_crypt_params_present;
	uint64_t or_firstobj;
	uint64_t or_numslots;
	uint8_t or_salt[ZIO_DATA_SALT_LEN];
	uint8_t or_iv[ZIO_DATA_IV_LEN];
	uint8_t or_mac[ZIO_DATA_MAC_LEN];
	boolean_t or_byteorder;
	zio_t *heal_pio;

	/* Keep track of DRR_FREEOBJECTS right after DRR_OBJECT_RANGE */
	or_need_sync_t or_need_sync;
	};

	typedef struct dmu_recv_begin_arg {
	const char *drba_origin;
	dmu_recv_cookie_t *drba_cookie;
	cred_t *drba_cred;
	proc_t *drba_proc;
	dsl_crypto_params_t *drba_dcp;
	} dmu_recv_begin_arg_t;

	static void
	byteswap_record(dmu_replay_record_t *drr)
	{
	#define DO64(X) (drr->drr_u.X = BSWAP_64(drr->drr_u.X))
	#define DO32(X) (drr->drr_u.X = BSWAP_32(drr->drr_u.X))
	drr->drr_type = BSWAP_32(drr->drr_type);
	drr->drr_payloadlen = BSWAP_32(drr->drr_payloadlen);

	switch (drr->drr_type) {
	case DRR_BEGIN:
	DO64(drr_begin.drr_magic);
	DO64(drr_begin.drr_versioninfo);
	DO64(drr_begin.drr_creation_time);
	DO32(drr_begin.drr_type);
	DO32(drr_begin.drr_flags);
	DO64(drr_begin.drr_toguid);
	DO64(drr_begin.drr_fromguid);
	break;
	case DRR_OBJECT:
	DO64(drr_object.drr_object);
	DO32(drr_object.drr_type);
	DO32(drr_object.drr_bonustype);
	DO32(drr_object.drr_blksz);
	DO32(drr_object.drr_bonuslen);
	DO32(drr_object.drr_raw_bonuslen);
	DO64(drr_object.drr_toguid);
	DO64(drr_object.drr_maxblkid);
	break;
	case DRR_FREEOBJECTS:
	DO64(drr_freeobjects.drr_firstobj);
	DO64(drr_freeobjects.drr_numobjs);
	DO64(drr_freeobjects.drr_toguid);
	break;
	case DRR_WRITE:
	DO64(drr_write.drr_object);
	DO32(drr_write.drr_type);
	DO64(drr_write.drr_offset);
	DO64(drr_write.drr_logical_size);
	DO64(drr_write.drr_toguid);
	ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_write.drr_key.ddk_cksum);
	DO64(drr_write.drr_key.ddk_prop);
	DO64(drr_write.drr_compressed_size);
	break;
	case DRR_WRITE_EMBEDDED:
	DO64(drr_write_embedded.drr_object);
	DO64(drr_write_embedded.drr_offset);
	DO64(drr_write_embedded.drr_length);
	DO64(drr_write_embedded.drr_toguid);
	DO32(drr_write_embedded.drr_lsize);
	DO32(drr_write_embedded.drr_psize);
	break;
	case DRR_FREE:
	DO64(drr_free.drr_object);
	DO64(drr_free.drr_offset);
	DO64(drr_free.drr_length);
	DO64(drr_free.drr_toguid);
	break;
	case DRR_SPILL:
	DO64(drr_spill.drr_object);
	DO64(drr_spill.drr_length);
	DO64(drr_spill.drr_toguid);
	DO64(drr_spill.drr_compressed_size);
	DO32(drr_spill.drr_type);
	break;
	case DRR_OBJECT_RANGE:
	DO64(drr_object_range.drr_firstobj);
	DO64(drr_object_range.drr_numslots);
	DO64(drr_object_range.drr_toguid);
	break;
	case DRR_REDACT:
	DO64(drr_redact.drr_object);
	DO64(drr_redact.drr_offset);
	DO64(drr_redact.drr_length);
	DO64(drr_redact.drr_toguid);
	break;
	case DRR_END:
	DO64(drr_end.drr_toguid);
	ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_end.drr_checksum);
	break;
	default:
	break;
	}

	if (drr->drr_type != DRR_BEGIN) {
	ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_checksum.drr_checksum);
	}

	#undef DO64
	#undef DO32
	}

	static boolean_t
	redact_snaps_contains(uint64_t *snaps, uint64_t num_snaps, uint64_t guid)
	{
	for (int i = 0; i < num_snaps; i++) {
	if (snaps[i] == guid)
	return (B_TRUE);
	}
	return (B_FALSE);
	}

	/*
	* Check that the new stream we're trying to receive is redacted with respect to
	* a subset of the snapshots that the origin was redacted with respect to. For
	* the reasons behind this, see the man page on redacted zfs sends and receives.
	*/
	static boolean_t
	compatible_redact_snaps(uint64_t *origin_snaps, uint64_t origin_num_snaps,
	uint64_t *redact_snaps, uint64_t num_redact_snaps)
	{
	/*
	* Short circuit the comparison; if we are redacted with respect to
	* more snapshots than the origin, we can't be redacted with respect
	* to a subset.
	*/
	if (num_redact_snaps > origin_num_snaps) {
	return (B_FALSE);
	}

	for (int i = 0; i < num_redact_snaps; i++) {
	if (!redact_snaps_contains(origin_snaps, origin_num_snaps,
	redact_snaps[i])) {
	return (B_FALSE);
	}
	}
	return (B_TRUE);
	}

	static boolean_t
	redact_check(dmu_recv_begin_arg_t drba, dsl_dataset_t origin)
	{
	uint64_t *origin_snaps;
	uint64_t origin_num_snaps;
	dmu_recv_cookie_t *drc = drba->drba_cookie;
	struct drr_begin *drrb = drc->drc_drrb;
	int featureflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
	int err = 0;
	boolean_t ret = B_TRUE;
	uint64_t *redact_snaps;
	uint_t numredactsnaps;

	/*
	* If this is a full send stream, we're safe no matter what.
	*/
	if (drrb->drr_fromguid == 0)
	return (ret);

	VERIFY(dsl_dataset_get_uint64_array_feature(origin,
	SPA_FEATURE_REDACTED_DATASETS, &origin_num_snaps, &origin_snaps));

	if (nvlist_lookup_uint64_array(drc->drc_begin_nvl,
	BEGINNV_REDACT_FROM_SNAPS, &redact_snaps, &numredactsnaps) ==
	0) {
	/*
	* If the send stream was sent from the redaction bookmark or
	* the redacted version of the dataset, then we're safe. Verify
	* that this is from the a compatible redaction bookmark or
	* redacted dataset.
	*/
	if (!compatible_redact_snaps(origin_snaps, origin_num_snaps,
	redact_snaps, numredactsnaps)) {
	err = EINVAL;
	}
	} else if (featureflags & DMU_BACKUP_FEATURE_REDACTED) {
	/*
	* If the stream is redacted, it must be redacted with respect
	* to a subset of what the origin is redacted with respect to.
	* See case number 2 in the zfs man page section on redacted zfs
	* send.
	*/
	err = nvlist_lookup_uint64_array(drc->drc_begin_nvl,
	BEGINNV_REDACT_SNAPS, &redact_snaps, &numredactsnaps);

	if (err != 0 \|\| !compatible_redact_snaps(origin_snaps,
	origin_num_snaps, redact_snaps, numredactsnaps)) {
	err = EINVAL;
	}
	} else if (!redact_snaps_contains(origin_snaps, origin_num_snaps,
	drrb->drr_toguid)) {
	/*
	* If the stream isn't redacted but the origin is, this must be
	* one of the snapshots the origin is redacted with respect to.
	* See case number 1 in the zfs man page section on redacted zfs
	* send.
	*/
	err = EINVAL;
	}

	if (err != 0)
	ret = B_FALSE;
	return (ret);
	}

	/*
	* If we previously received a stream with --large-block, we don't support
	* receiving an incremental on top of it without --large-block. This avoids
	* forcing a read-modify-write or trying to re-aggregate a string of WRITE
	* records.
	*/
	static int
	recv_check_large_blocks(dsl_dataset_t *ds, uint64_t featureflags)
	{
	if (dsl_dataset_feature_is_active(ds, SPA_FEATURE_LARGE_BLOCKS) &&
	!(featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS))
	return (SET_ERROR(ZFS_ERR_STREAM_LARGE_BLOCK_MISMATCH));
	return (0);
	}

	static int
	recv_begin_check_existing_impl(dmu_recv_begin_arg_t drba, dsl_dataset_t ds,
	uint64_t fromguid, uint64_t featureflags)
	{
	uint64_t obj;
	uint64_t children;
	int error;
	dsl_dataset_t *snap;
	dsl_pool_t *dp = ds->ds_dir->dd_pool;
	boolean_t encrypted = ds->ds_dir->dd_crypto_obj != 0;
	boolean_t raw = (featureflags & DMU_BACKUP_FEATURE_RAW) != 0;
	boolean_t embed = (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) != 0;

	/* Temporary clone name must not exist. */
	error = zap_lookup(dp->dp_meta_objset,
	dsl_dir_phys(ds->ds_dir)->dd_child_dir_zapobj, recv_clone_name,
	8, 1, &obj);
	if (error != ENOENT)
	return (error == 0 ? SET_ERROR(EBUSY) : error);

	/* Resume state must not be set. */
	if (dsl_dataset_has_resume_receive_state(ds))
	return (SET_ERROR(EBUSY));

	/* New snapshot name must not exist if we're not healing it. */
	error = zap_lookup(dp->dp_meta_objset,
	dsl_dataset_phys(ds)->ds_snapnames_zapobj,
	drba->drba_cookie->drc_tosnap, 8, 1, &obj);
	if (drba->drba_cookie->drc_heal) {
	if (error != 0)
	return (error);
	} else if (error != ENOENT) {
	return (error == 0 ? SET_ERROR(EEXIST) : error);
	}

	/* Must not have children if receiving a ZVOL. */
	error = zap_count(dp->dp_meta_objset,
	dsl_dir_phys(ds->ds_dir)->dd_child_dir_zapobj, &children);
	if (error != 0)
	return (error);
	if (drba->drba_cookie->drc_drrb->drr_type != DMU_OST_ZFS &&
	children > 0)
	return (SET_ERROR(ZFS_ERR_WRONG_PARENT));

	/*
	* Check snapshot limit before receiving. We'll recheck again at the
	* end, but might as well abort before receiving if we're already over
	* the limit.
	*
	* Note that we do not check the file system limit with
	* dsl_dir_fscount_check because the temporary %clones don't count
	* against that limit.
	*/
	error = dsl_fs_ss_limit_check(ds->ds_dir, 1, ZFS_PROP_SNAPSHOT_LIMIT,
	NULL, drba->drba_cred, drba->drba_proc);
	if (error != 0)
	return (error);

	if (drba->drba_cookie->drc_heal) {
	/* Encryption is incompatible with embedded data. */
	if (encrypted && embed)
	return (SET_ERROR(EINVAL));

	/* Healing is not supported when in 'force' mode. */
	if (drba->drba_cookie->drc_force)
	return (SET_ERROR(EINVAL));

	/* Must have keys loaded if doing encrypted non-raw recv. */
	if (encrypted && !raw) {
	if (spa_keystore_lookup_key(dp->dp_spa, ds->ds_object,
	NULL, NULL) != 0)
	return (SET_ERROR(EACCES));
	}

	error = dsl_dataset_hold_obj(dp, obj, FTAG, &snap);
	if (error != 0)
	return (error);

	/*
	* When not doing best effort corrective recv healing can only
	* be done if the send stream is for the same snapshot as the
	* one we are trying to heal.
	*/
	if (zfs_recv_best_effort_corrective == 0 &&
	drba->drba_cookie->drc_drrb->drr_toguid !=
	dsl_dataset_phys(snap)->ds_guid) {
	dsl_dataset_rele(snap, FTAG);
	return (SET_ERROR(ENOTSUP));
	}
	dsl_dataset_rele(snap, FTAG);
	} else if (fromguid != 0) {
	/* Sanity check the incremental recv */
	uint64_t obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;

	/* Can't perform a raw receive on top of a non-raw receive */
	if (!encrypted && raw)
	return (SET_ERROR(EINVAL));

	/* Encryption is incompatible with embedded data */
	if (encrypted && embed)
	return (SET_ERROR(EINVAL));

	/* Find snapshot in this dir that matches fromguid. */
	while (obj != 0) {
	error = dsl_dataset_hold_obj(dp, obj, FTAG,
	&snap);
	if (error != 0)
	return (SET_ERROR(ENODEV));
	if (snap->ds_dir != ds->ds_dir) {
	dsl_dataset_rele(snap, FTAG);
	return (SET_ERROR(ENODEV));
	}
	if (dsl_dataset_phys(snap)->ds_guid == fromguid)
	break;
	obj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
	dsl_dataset_rele(snap, FTAG);
	}
	if (obj == 0)
	return (SET_ERROR(ENODEV));

	if (drba->drba_cookie->drc_force) {
	drba->drba_cookie->drc_fromsnapobj = obj;
	} else {
	/*
	* If we are not forcing, there must be no
	* changes since fromsnap. Raw sends have an
	* additional constraint that requires that
	* no "noop" snapshots exist between fromsnap
	* and tosnap for the IVset checking code to
	* work properly.
	*/
	if (dsl_dataset_modified_since_snap(ds, snap) \|\|
	(raw &&
	dsl_dataset_phys(ds)->ds_prev_snap_obj !=
	snap->ds_object)) {
	dsl_dataset_rele(snap, FTAG);
	return (SET_ERROR(ETXTBSY));
	}
	drba->drba_cookie->drc_fromsnapobj =
	ds->ds_prev->ds_object;
	}

	if (dsl_dataset_feature_is_active(snap,
	SPA_FEATURE_REDACTED_DATASETS) && !redact_check(drba,
	snap)) {
	dsl_dataset_rele(snap, FTAG);
	return (SET_ERROR(EINVAL));
	}

	error = recv_check_large_blocks(snap, featureflags);
	if (error != 0) {
	dsl_dataset_rele(snap, FTAG);
	return (error);
	}

	dsl_dataset_rele(snap, FTAG);
	} else {
	/* If full and not healing then must be forced. */
	if (!drba->drba_cookie->drc_force)
	return (SET_ERROR(EEXIST));

	/*
	* We don't support using zfs recv -F to blow away
	* encrypted filesystems. This would require the
	* dsl dir to point to the old encryption key and
	* the new one at the same time during the receive.
	*/
	if ((!encrypted && raw) \|\| encrypted)
	return (SET_ERROR(EINVAL));

	/*
	* Perform the same encryption checks we would if
	* we were creating a new dataset from scratch.
	*/
	if (!raw) {
	boolean_t will_encrypt;

	error = dmu_objset_create_crypt_check(
	ds->ds_dir->dd_parent, drba->drba_dcp,
	&will_encrypt);
	if (error != 0)
	return (error);

	if (will_encrypt && embed)
	return (SET_ERROR(EINVAL));
	}
	}

	return (0);
	}

	/*
	* Check that any feature flags used in the data stream we're receiving are
	* supported by the pool we are receiving into.
	*
	* Note that some of the features we explicitly check here have additional
	* (implicit) features they depend on, but those dependencies are enforced
	* through the zfeature_register() calls declaring the features that we
	* explicitly check.
	*/
	static int
	recv_begin_check_feature_flags_impl(uint64_t featureflags, spa_t *spa)
	{
	/*
	* Check if there are any unsupported feature flags.
	*/
	if (!DMU_STREAM_SUPPORTED(featureflags)) {
	return (SET_ERROR(ZFS_ERR_UNKNOWN_SEND_STREAM_FEATURE));
	}

	/* Verify pool version supports SA if SA_SPILL feature set */
	if ((featureflags & DMU_BACKUP_FEATURE_SA_SPILL) &&
	spa_version(spa) < SPA_VERSION_SA)
	return (SET_ERROR(ENOTSUP));

	/*
	* LZ4 compressed, ZSTD compressed, embedded, mooched, large blocks,
	* and large_dnodes in the stream can only be used if those pool
	* features are enabled because we don't attempt to decompress /
	* un-embed / un-mooch / split up the blocks / dnodes during the
	* receive process.
	*/
	if ((featureflags & DMU_BACKUP_FEATURE_LZ4) &&
	!spa_feature_is_enabled(spa, SPA_FEATURE_LZ4_COMPRESS))
	return (SET_ERROR(ENOTSUP));
	if ((featureflags & DMU_BACKUP_FEATURE_ZSTD) &&
	!spa_feature_is_enabled(spa, SPA_FEATURE_ZSTD_COMPRESS))
	return (SET_ERROR(ENOTSUP));
	if ((featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) &&
	!spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA))
	return (SET_ERROR(ENOTSUP));
	if ((featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS) &&
	!spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS))
	return (SET_ERROR(ENOTSUP));
	if ((featureflags & DMU_BACKUP_FEATURE_LARGE_DNODE) &&
	!spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE))
	return (SET_ERROR(ENOTSUP));

	/*
	* Receiving redacted streams requires that redacted datasets are
	* enabled.
	*/
	if ((featureflags & DMU_BACKUP_FEATURE_REDACTED) &&
	!spa_feature_is_enabled(spa, SPA_FEATURE_REDACTED_DATASETS))
	return (SET_ERROR(ENOTSUP));

	return (0);
	}

	static int
	dmu_recv_begin_check(void arg, dmu_tx_t tx)
	{
	dmu_recv_begin_arg_t *drba = arg;
	dsl_pool_t *dp = dmu_tx_pool(tx);
	struct drr_begin *drrb = drba->drba_cookie->drc_drrb;
	uint64_t fromguid = drrb->drr_fromguid;
	int flags = drrb->drr_flags;
	ds_hold_flags_t dsflags = DS_HOLD_FLAG_NONE;
	int error;
	uint64_t featureflags = drba->drba_cookie->drc_featureflags;
	dsl_dataset_t *ds;
	const char *tofs = drba->drba_cookie->drc_tofs;

	/* already checked */
	ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC);
	ASSERT(!(featureflags & DMU_BACKUP_FEATURE_RESUMING));

	if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
	DMU_COMPOUNDSTREAM \|\|
	drrb->drr_type >= DMU_OST_NUMTYPES \|\|
	((flags & DRR_FLAG_CLONE) && drba->drba_origin == NULL))
	return (SET_ERROR(EINVAL));

	error = recv_begin_check_feature_flags_impl(featureflags, dp->dp_spa);
	if (error != 0)
	return (error);

	/* Resumable receives require extensible datasets */
	if (drba->drba_cookie->drc_resumable &&
	!spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_EXTENSIBLE_DATASET))
	return (SET_ERROR(ENOTSUP));

	if (featureflags & DMU_BACKUP_FEATURE_RAW) {
	/* raw receives require the encryption feature */
	if (!spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_ENCRYPTION))
	return (SET_ERROR(ENOTSUP));

	/* embedded data is incompatible with encryption and raw recv */
	if (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)
	return (SET_ERROR(EINVAL));

	/* raw receives require spill block allocation flag */
	if (!(flags & DRR_FLAG_SPILL_BLOCK))
	return (SET_ERROR(ZFS_ERR_SPILL_BLOCK_FLAG_MISSING));
	} else {
	/*
	* We support unencrypted datasets below encrypted ones now,
	* so add the DS_HOLD_FLAG_DECRYPT flag only if we are dealing
	* with a dataset we may encrypt.
	*/
	if (drba->drba_dcp == NULL \|\|
	drba->drba_dcp->cp_crypt != ZIO_CRYPT_OFF) {
	dsflags \|= DS_HOLD_FLAG_DECRYPT;
	}
	}

	error = dsl_dataset_hold_flags(dp, tofs, dsflags, FTAG, &ds);
	if (error == 0) {
	/* target fs already exists; recv into temp clone */

	/* Can't recv a clone into an existing fs */
	if (flags & DRR_FLAG_CLONE \|\| drba->drba_origin) {
	dsl_dataset_rele_flags(ds, dsflags, FTAG);
	return (SET_ERROR(EINVAL));
	}

	error = recv_begin_check_existing_impl(drba, ds, fromguid,
	featureflags);
	dsl_dataset_rele_flags(ds, dsflags, FTAG);
	} else if (error == ENOENT) {
	/* target fs does not exist; must be a full backup or clone */
	char buf[ZFS_MAX_DATASET_NAME_LEN];
	objset_t *os;

	/* healing recv must be done "into" an existing snapshot */
	if (drba->drba_cookie->drc_heal == B_TRUE)
	return (SET_ERROR(ENOTSUP));

	/*
	* If it's a non-clone incremental, we are missing the
	* target fs, so fail the recv.
	*/
	if (fromguid != 0 && !((flags & DRR_FLAG_CLONE) \|\|
	drba->drba_origin))
	return (SET_ERROR(ENOENT));

	/*
	* If we're receiving a full send as a clone, and it doesn't
	* contain all the necessary free records and freeobject
	* records, reject it.
	*/
	if (fromguid == 0 && drba->drba_origin != NULL &&
	!(flags & DRR_FLAG_FREERECORDS))
	return (SET_ERROR(EINVAL));

	/* Open the parent of tofs */
	ASSERT3U(strlen(tofs), <, sizeof (buf));
	(void) strlcpy(buf, tofs, strrchr(tofs, '/') - tofs + 1);
	error = dsl_dataset_hold(dp, buf, FTAG, &ds);
	if (error != 0)
	return (error);

	if ((featureflags & DMU_BACKUP_FEATURE_RAW) == 0 &&
	drba->drba_origin == NULL) {
	boolean_t will_encrypt;

	/*
	* Check that we aren't breaking any encryption rules
	* and that we have all the parameters we need to
	* create an encrypted dataset if necessary. If we are
	* making an encrypted dataset the stream can't have
	* embedded data.
	*/
	error = dmu_objset_create_crypt_check(ds->ds_dir,
	drba->drba_dcp, &will_encrypt);
	if (error != 0) {
	dsl_dataset_rele(ds, FTAG);
	return (error);
	}

	if (will_encrypt &&
	(featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)) {
	dsl_dataset_rele(ds, FTAG);
	return (SET_ERROR(EINVAL));
	}
	}

	/*
	* Check filesystem and snapshot limits before receiving. We'll
	* recheck snapshot limits again at the end (we create the
	* filesystems and increment those counts during begin_sync).
	*/
	error = dsl_fs_ss_limit_check(ds->ds_dir, 1,
	ZFS_PROP_FILESYSTEM_LIMIT, NULL,
	drba->drba_cred, drba->drba_proc);
	if (error != 0) {
	dsl_dataset_rele(ds, FTAG);
	return (error);
	}

	error = dsl_fs_ss_limit_check(ds->ds_dir, 1,
	ZFS_PROP_SNAPSHOT_LIMIT, NULL,
	drba->drba_cred, drba->drba_proc);
	if (error != 0) {
	dsl_dataset_rele(ds, FTAG);
	return (error);
	}

	/* can't recv below anything but filesystems (eg. no ZVOLs) */
	error = dmu_objset_from_ds(ds, &os);
	if (error != 0) {
	dsl_dataset_rele(ds, FTAG);
	return (error);
	}
	if (dmu_objset_type(os) != DMU_OST_ZFS) {
	dsl_dataset_rele(ds, FTAG);
	return (SET_ERROR(ZFS_ERR_WRONG_PARENT));
	}

	if (drba->drba_origin != NULL) {
	dsl_dataset_t *origin;
	error = dsl_dataset_hold_flags(dp, drba->drba_origin,
	dsflags, FTAG, &origin);
	if (error != 0) {
	dsl_dataset_rele(ds, FTAG);
	return (error);
	}
	if (!origin->ds_is_snapshot) {
	dsl_dataset_rele_flags(origin, dsflags, FTAG);
	dsl_dataset_rele(ds, FTAG);
	return (SET_ERROR(EINVAL));
	}
	if (dsl_dataset_phys(origin)->ds_guid != fromguid &&
	fromguid != 0) {
	dsl_dataset_rele_flags(origin, dsflags, FTAG);
	dsl_dataset_rele(ds, FTAG);
	return (SET_ERROR(ENODEV));
	}

	if (origin->ds_dir->dd_crypto_obj != 0 &&
	(featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)) {
	dsl_dataset_rele_flags(origin, dsflags, FTAG);
	dsl_dataset_rele(ds, FTAG);
	return (SET_ERROR(EINVAL));
	}

	/*
	* If the origin is redacted we need to verify that this
	* send stream can safely be received on top of the
	* origin.
	*/
	if (dsl_dataset_feature_is_active(origin,
	SPA_FEATURE_REDACTED_DATASETS)) {
	if (!redact_check(drba, origin)) {
	dsl_dataset_rele_flags(origin, dsflags,
	FTAG);
	dsl_dataset_rele_flags(ds, dsflags,
	FTAG);
	return (SET_ERROR(EINVAL));
	}
	}

	error = recv_check_large_blocks(ds, featureflags);
	if (error != 0) {
	dsl_dataset_rele_flags(origin, dsflags, FTAG);
	dsl_dataset_rele_flags(ds, dsflags, FTAG);
	return (error);
	}

	dsl_dataset_rele_flags(origin, dsflags, FTAG);
	}

	dsl_dataset_rele(ds, FTAG);
	error = 0;
	}
	return (error);
	}

	static void
	dmu_recv_begin_sync(void arg, dmu_tx_t tx)
	{
	dmu_recv_begin_arg_t *drba = arg;
	dsl_pool_t *dp = dmu_tx_pool(tx);
	objset_t *mos = dp->dp_meta_objset;
	dmu_recv_cookie_t *drc = drba->drba_cookie;
	struct drr_begin *drrb = drc->drc_drrb;
	const char *tofs = drc->drc_tofs;
	uint64_t featureflags = drc->drc_featureflags;
	dsl_dataset_t ds, newds;
	objset_t *os;
	uint64_t dsobj;
	ds_hold_flags_t dsflags = DS_HOLD_FLAG_NONE;
	int error;
	uint64_t crflags = 0;
	dsl_crypto_params_t dummy_dcp = { 0 };
	dsl_crypto_params_t *dcp = drba->drba_dcp;

	if (drrb->drr_flags & DRR_FLAG_CI_DATA)
	crflags \|= DS_FLAG_CI_DATASET;

	if ((featureflags & DMU_BACKUP_FEATURE_RAW) == 0)
	dsflags \|= DS_HOLD_FLAG_DECRYPT;

	/*
	* Raw, non-incremental recvs always use a dummy dcp with
	* the raw cmd set. Raw incremental recvs do not use a dcp
	* since the encryption parameters are already set in stone.
	*/
	if (dcp == NULL && drrb->drr_fromguid == 0 &&
	drba->drba_origin == NULL) {
	ASSERT3P(dcp, ==, NULL);
	dcp = &dummy_dcp;

	if (featureflags & DMU_BACKUP_FEATURE_RAW)
	dcp->cp_cmd = DCP_CMD_RAW_RECV;
	}

	error = dsl_dataset_hold_flags(dp, tofs, dsflags, FTAG, &ds);
	if (error == 0) {
	/* Create temporary clone unless we're doing corrective recv */
	dsl_dataset_t *snap = NULL;

	if (drba->drba_cookie->drc_fromsnapobj != 0) {
	VERIFY0(dsl_dataset_hold_obj(dp,
	drba->drba_cookie->drc_fromsnapobj, FTAG, &snap));
	ASSERT3P(dcp, ==, NULL);
	}
	if (drc->drc_heal) {
	/* When healing we want to use the provided snapshot */
	VERIFY0(dsl_dataset_snap_lookup(ds, drc->drc_tosnap,
	&dsobj));
	} else {
	dsobj = dsl_dataset_create_sync(ds->ds_dir,
	recv_clone_name, snap, crflags, drba->drba_cred,
	dcp, tx);
	}
	if (drba->drba_cookie->drc_fromsnapobj != 0)
	dsl_dataset_rele(snap, FTAG);
	dsl_dataset_rele_flags(ds, dsflags, FTAG);
	} else {
	dsl_dir_t *dd;
	const char *tail;
	dsl_dataset_t *origin = NULL;

	VERIFY0(dsl_dir_hold(dp, tofs, FTAG, &dd, &tail));

	if (drba->drba_origin != NULL) {
	VERIFY0(dsl_dataset_hold(dp, drba->drba_origin,
	FTAG, &origin));
	ASSERT3P(dcp, ==, NULL);
	}

	/* Create new dataset. */
	dsobj = dsl_dataset_create_sync(dd, strrchr(tofs, '/') + 1,
	origin, crflags, drba->drba_cred, dcp, tx);
	if (origin != NULL)
	dsl_dataset_rele(origin, FTAG);
	dsl_dir_rele(dd, FTAG);
	drc->drc_newfs = B_TRUE;
	}
	VERIFY0(dsl_dataset_own_obj_force(dp, dsobj, dsflags, dmu_recv_tag,
	&newds));
	if (dsl_dataset_feature_is_active(newds,
	SPA_FEATURE_REDACTED_DATASETS)) {
	/*
	* If the origin dataset is redacted, the child will be redacted
	* when we create it. We clear the new dataset's
	* redaction info; if it should be redacted, we'll fill
	* in its information later.
	*/
	dsl_dataset_deactivate_feature(newds,
	SPA_FEATURE_REDACTED_DATASETS, tx);
	}
	VERIFY0(dmu_objset_from_ds(newds, &os));

	if (drc->drc_resumable) {
	dsl_dataset_zapify(newds, tx);
	if (drrb->drr_fromguid != 0) {
	VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_FROMGUID,
	8, 1, &drrb->drr_fromguid, tx));
	}
	VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_TOGUID,
	8, 1, &drrb->drr_toguid, tx));
	VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_TONAME,
	1, strlen(drrb->drr_toname) + 1, drrb->drr_toname, tx));
	uint64_t one = 1;
	uint64_t zero = 0;
	VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_OBJECT,
	8, 1, &one, tx));
	VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_OFFSET,
	8, 1, &zero, tx));
	VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_BYTES,
	8, 1, &zero, tx));
	if (featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS) {
	VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_LARGEBLOCK,
	8, 1, &one, tx));
	}
	if (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) {
	VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_EMBEDOK,
	8, 1, &one, tx));
	}
	if (featureflags & DMU_BACKUP_FEATURE_COMPRESSED) {
	VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_COMPRESSOK,
	8, 1, &one, tx));
	}
	if (featureflags & DMU_BACKUP_FEATURE_RAW) {
	VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_RAWOK,
	8, 1, &one, tx));
	}

	uint64_t *redact_snaps;
	uint_t numredactsnaps;
	if (nvlist_lookup_uint64_array(drc->drc_begin_nvl,
	BEGINNV_REDACT_FROM_SNAPS, &redact_snaps,
	&numredactsnaps) == 0) {
	VERIFY0(zap_add(mos, dsobj,
	DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS,
	sizeof (*redact_snaps), numredactsnaps,
	redact_snaps, tx));
	}
	}

	/*
	* Usually the os->os_encrypted value is tied to the presence of a
	* DSL Crypto Key object in the dd. However, that will not be received
	* until dmu_recv_stream(), so we set the value manually for now.
	*/
	if (featureflags & DMU_BACKUP_FEATURE_RAW) {
	os->os_encrypted = B_TRUE;
	drba->drba_cookie->drc_raw = B_TRUE;
	}

	if (featureflags & DMU_BACKUP_FEATURE_REDACTED) {
	uint64_t *redact_snaps;
	uint_t numredactsnaps;
	VERIFY0(nvlist_lookup_uint64_array(drc->drc_begin_nvl,
	BEGINNV_REDACT_SNAPS, &redact_snaps, &numredactsnaps));
	dsl_dataset_activate_redaction(newds, redact_snaps,
	numredactsnaps, tx);
	}

	dmu_buf_will_dirty(newds->ds_dbuf, tx);
	dsl_dataset_phys(newds)->ds_flags \|= DS_FLAG_INCONSISTENT;

	/*
	* If we actually created a non-clone, we need to create the objset
	* in our new dataset. If this is a raw send we postpone this until
	* dmu_recv_stream() so that we can allocate the metadnode with the
	* properties from the DRR_BEGIN payload.
	*/
	rrw_enter(&newds->ds_bp_rwlock, RW_READER, FTAG);
	if (BP_IS_HOLE(dsl_dataset_get_blkptr(newds)) &&
	(featureflags & DMU_BACKUP_FEATURE_RAW) == 0 &&
	!drc->drc_heal) {
	(void) dmu_objset_create_impl(dp->dp_spa,
	newds, dsl_dataset_get_blkptr(newds), drrb->drr_type, tx);
	}
	rrw_exit(&newds->ds_bp_rwlock, FTAG);

	drba->drba_cookie->drc_ds = newds;
	drba->drba_cookie->drc_os = os;

	spa_history_log_internal_ds(newds, "receive", tx, " ");
	}

	static int
	dmu_recv_resume_begin_check(void arg, dmu_tx_t tx)
	{
	dmu_recv_begin_arg_t *drba = arg;
	dmu_recv_cookie_t *drc = drba->drba_cookie;
	dsl_pool_t *dp = dmu_tx_pool(tx);
	struct drr_begin *drrb = drc->drc_drrb;
	int error;
	ds_hold_flags_t dsflags = DS_HOLD_FLAG_NONE;
	dsl_dataset_t *ds;
	const char *tofs = drc->drc_tofs;

	/* already checked */
	ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC);
	ASSERT(drc->drc_featureflags & DMU_BACKUP_FEATURE_RESUMING);

	if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
	DMU_COMPOUNDSTREAM \|\|
	drrb->drr_type >= DMU_OST_NUMTYPES)
	return (SET_ERROR(EINVAL));

	/*
	* This is mostly a sanity check since we should have already done these
	* checks during a previous attempt to receive the data.
	*/
	error = recv_begin_check_feature_flags_impl(drc->drc_featureflags,
	dp->dp_spa);
	if (error != 0)
	return (error);

	/* 6 extra bytes for /%recv */
	char recvname[ZFS_MAX_DATASET_NAME_LEN + 6];

	(void) snprintf(recvname, sizeof (recvname), "%s/%s",
	tofs, recv_clone_name);

	if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RAW) {
	/* raw receives require spill block allocation flag */
	if (!(drrb->drr_flags & DRR_FLAG_SPILL_BLOCK))
	return (SET_ERROR(ZFS_ERR_SPILL_BLOCK_FLAG_MISSING));
	} else {
	dsflags \|= DS_HOLD_FLAG_DECRYPT;
	}

	boolean_t recvexist = B_TRUE;
	if (dsl_dataset_hold_flags(dp, recvname, dsflags, FTAG, &ds) != 0) {
	/* %recv does not exist; continue in tofs */
	recvexist = B_FALSE;
	error = dsl_dataset_hold_flags(dp, tofs, dsflags, FTAG, &ds);
	if (error != 0)
	return (error);
	}

	/*
	* Resume of full/newfs recv on existing dataset should be done with
	* force flag
	*/
	if (recvexist && drrb->drr_fromguid == 0 && !drc->drc_force) {
	dsl_dataset_rele_flags(ds, dsflags, FTAG);
	return (SET_ERROR(ZFS_ERR_RESUME_EXISTS));
	}

	/* check that ds is marked inconsistent */
	if (!DS_IS_INCONSISTENT(ds)) {
	dsl_dataset_rele_flags(ds, dsflags, FTAG);
	return (SET_ERROR(EINVAL));
	}

	/* check that there is resuming data, and that the toguid matches */
	if (!dsl_dataset_is_zapified(ds)) {
	dsl_dataset_rele_flags(ds, dsflags, FTAG);
	return (SET_ERROR(EINVAL));
	}
	uint64_t val;
	error = zap_lookup(dp->dp_meta_objset, ds->ds_object,
	DS_FIELD_RESUME_TOGUID, sizeof (val), 1, &val);
	if (error != 0 \|\| drrb->drr_toguid != val) {
	dsl_dataset_rele_flags(ds, dsflags, FTAG);
	return (SET_ERROR(EINVAL));
	}

	/*
	* Check if the receive is still running. If so, it will be owned.
	* Note that nothing else can own the dataset (e.g. after the receive
	* fails) because it will be marked inconsistent.
	*/
	if (dsl_dataset_has_owner(ds)) {
	dsl_dataset_rele_flags(ds, dsflags, FTAG);
	return (SET_ERROR(EBUSY));
	}

	/* There should not be any snapshots of this fs yet. */
	if (ds->ds_prev != NULL && ds->ds_prev->ds_dir == ds->ds_dir) {
	dsl_dataset_rele_flags(ds, dsflags, FTAG);
	return (SET_ERROR(EINVAL));
	}

	/*
	* Note: resume point will be checked when we process the first WRITE
	* record.
	*/

	/* check that the origin matches */
	val = 0;
	(void) zap_lookup(dp->dp_meta_objset, ds->ds_object,
	DS_FIELD_RESUME_FROMGUID, sizeof (val), 1, &val);
	if (drrb->drr_fromguid != val) {
	dsl_dataset_rele_flags(ds, dsflags, FTAG);
	return (SET_ERROR(EINVAL));
	}

	if (ds->ds_prev != NULL && drrb->drr_fromguid != 0)
	drc->drc_fromsnapobj = ds->ds_prev->ds_object;

	/*
	* If we're resuming, and the send is redacted, then the original send
	* must have been redacted, and must have been redacted with respect to
	* the same snapshots.
	*/
	if (drc->drc_featureflags & DMU_BACKUP_FEATURE_REDACTED) {
	uint64_t num_ds_redact_snaps;
	uint64_t *ds_redact_snaps;

	uint_t num_stream_redact_snaps;
	uint64_t *stream_redact_snaps;

	if (nvlist_lookup_uint64_array(drc->drc_begin_nvl,
	BEGINNV_REDACT_SNAPS, &stream_redact_snaps,
	&num_stream_redact_snaps) != 0) {
	dsl_dataset_rele_flags(ds, dsflags, FTAG);
	return (SET_ERROR(EINVAL));
	}

	if (!dsl_dataset_get_uint64_array_feature(ds,
	SPA_FEATURE_REDACTED_DATASETS, &num_ds_redact_snaps,
	&ds_redact_snaps)) {
	dsl_dataset_rele_flags(ds, dsflags, FTAG);
	return (SET_ERROR(EINVAL));
	}

	for (int i = 0; i < num_ds_redact_snaps; i++) {
	if (!redact_snaps_contains(ds_redact_snaps,
	num_ds_redact_snaps, stream_redact_snaps[i])) {
	dsl_dataset_rele_flags(ds, dsflags, FTAG);
	return (SET_ERROR(EINVAL));
	}
	}
	}

	error = recv_check_large_blocks(ds, drc->drc_featureflags);
	if (error != 0) {
	dsl_dataset_rele_flags(ds, dsflags, FTAG);
	return (error);
	}

	dsl_dataset_rele_flags(ds, dsflags, FTAG);
	return (0);
	}

	static void
	dmu_recv_resume_begin_sync(void arg, dmu_tx_t tx)
	{
	dmu_recv_begin_arg_t *drba = arg;
	dsl_pool_t *dp = dmu_tx_pool(tx);
	const char *tofs = drba->drba_cookie->drc_tofs;
	uint64_t featureflags = drba->drba_cookie->drc_featureflags;
	dsl_dataset_t *ds;
	ds_hold_flags_t dsflags = DS_HOLD_FLAG_NONE;
	/* 6 extra bytes for /%recv */
	char recvname[ZFS_MAX_DATASET_NAME_LEN + 6];

	(void) snprintf(recvname, sizeof (recvname), "%s/%s", tofs,
	recv_clone_name);

	if (featureflags & DMU_BACKUP_FEATURE_RAW) {
	drba->drba_cookie->drc_raw = B_TRUE;
	} else {
	dsflags \|= DS_HOLD_FLAG_DECRYPT;
	}

	if (dsl_dataset_own_force(dp, recvname, dsflags, dmu_recv_tag, &ds)
	!= 0) {
	/* %recv does not exist; continue in tofs */
	VERIFY0(dsl_dataset_own_force(dp, tofs, dsflags, dmu_recv_tag,
	&ds));
	drba->drba_cookie->drc_newfs = B_TRUE;
	}

	ASSERT(DS_IS_INCONSISTENT(ds));
	rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
	ASSERT(!BP_IS_HOLE(dsl_dataset_get_blkptr(ds)) \|\|
	drba->drba_cookie->drc_raw);
	rrw_exit(&ds->ds_bp_rwlock, FTAG);

	drba->drba_cookie->drc_ds = ds;
	VERIFY0(dmu_objset_from_ds(ds, &drba->drba_cookie->drc_os));
	drba->drba_cookie->drc_should_save = B_TRUE;

	spa_history_log_internal_ds(ds, "resume receive", tx, " ");
	}

	/*
	* NB: callers MUST call dmu_recv_stream() if dmu_recv_begin()
	* succeeds; otherwise we will leak the holds on the datasets.
	*/
	int
	dmu_recv_begin(const char tofs, const char tosnap,
	dmu_replay_record_t *drr_begin, boolean_t force, boolean_t heal,
	boolean_t resumable, nvlist_t localprops, nvlist_t hidden_args,
	const char origin, dmu_recv_cookie_t drc, zfs_file_t *fp,
	offset_t *voffp)
	{
	dmu_recv_begin_arg_t drba = { 0 };
	int err = 0;

	memset(drc, 0, sizeof (dmu_recv_cookie_t));
	drc->drc_drr_begin = drr_begin;
	drc->drc_drrb = &drr_begin->drr_u.drr_begin;
	drc->drc_tosnap = tosnap;
	drc->drc_tofs = tofs;
	drc->drc_force = force;
	drc->drc_heal = heal;
	drc->drc_resumable = resumable;
	drc->drc_cred = CRED();
	drc->drc_proc = curproc;
	drc->drc_clone = (origin != NULL);

	if (drc->drc_drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC)) {
	drc->drc_byteswap = B_TRUE;
	(void) fletcher_4_incremental_byteswap(drr_begin,
	sizeof (dmu_replay_record_t), &drc->drc_cksum);
	byteswap_record(drr_begin);
	} else if (drc->drc_drrb->drr_magic == DMU_BACKUP_MAGIC) {
	(void) fletcher_4_incremental_native(drr_begin,
	sizeof (dmu_replay_record_t), &drc->drc_cksum);
	} else {
	return (SET_ERROR(EINVAL));
	}

	drc->drc_fp = fp;
	drc->drc_voff = *voffp;
	drc->drc_featureflags =
	DMU_GET_FEATUREFLAGS(drc->drc_drrb->drr_versioninfo);

	uint32_t payloadlen = drc->drc_drr_begin->drr_payloadlen;

	/*
	* Since OpenZFS 2.0.0, we have enforced a 64MB limit in userspace
	* configurable via ZFS_SENDRECV_MAX_NVLIST. We enforce 256MB as a hard
	* upper limit. Systems with less than 1GB of RAM will see a lower
	* limit from `arc_all_memory() / 4`.
	*/
	if (payloadlen > (MIN((1U << 28), arc_all_memory() / 4)))
	return (E2BIG);


	if (payloadlen != 0) {
	void *payload = vmem_alloc(payloadlen, KM_SLEEP);
	/*
	* For compatibility with recursive send streams, we don't do
	* this here if the stream could be part of a package. Instead,
	* we'll do it in dmu_recv_stream. If we pull the next header
	* too early, and it's the END record, we break the `recv_skip`
	* logic.
	*/

	err = receive_read_payload_and_next_header(drc, payloadlen,
	payload);
	if (err != 0) {
	vmem_free(payload, payloadlen);
	return (err);
	}
	err = nvlist_unpack(payload, payloadlen, &drc->drc_begin_nvl,
	KM_SLEEP);
	vmem_free(payload, payloadlen);
	if (err != 0) {
	kmem_free(drc->drc_next_rrd,
	sizeof (*drc->drc_next_rrd));
	return (err);
	}
	}

	if (drc->drc_drrb->drr_flags & DRR_FLAG_SPILL_BLOCK)
	drc->drc_spill = B_TRUE;

	drba.drba_origin = origin;
	drba.drba_cookie = drc;
	drba.drba_cred = CRED();
	drba.drba_proc = curproc;

	if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RESUMING) {
	err = dsl_sync_task(tofs,
	dmu_recv_resume_begin_check, dmu_recv_resume_begin_sync,
	&drba, 5, ZFS_SPACE_CHECK_NORMAL);
	} else {
	/*
	* For non-raw, non-incremental, non-resuming receives the
	* user can specify encryption parameters on the command line
	* with "zfs recv -o". For these receives we create a dcp and
	* pass it to the sync task. Creating the dcp will implicitly
	* remove the encryption params from the localprops nvlist,
	* which avoids errors when trying to set these normally
	* read-only properties. Any other kind of receive that
	* attempts to set these properties will fail as a result.
	*/
	if ((DMU_GET_FEATUREFLAGS(drc->drc_drrb->drr_versioninfo) &
	DMU_BACKUP_FEATURE_RAW) == 0 &&
	origin == NULL && drc->drc_drrb->drr_fromguid == 0) {
	err = dsl_crypto_params_create_nvlist(DCP_CMD_NONE,
	localprops, hidden_args, &drba.drba_dcp);
	}

	if (err == 0) {
	err = dsl_sync_task(tofs,
	dmu_recv_begin_check, dmu_recv_begin_sync,
	&drba, 5, ZFS_SPACE_CHECK_NORMAL);
	dsl_crypto_params_free(drba.drba_dcp, !!err);
	}
	}

	if (err != 0) {
	kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd));
	nvlist_free(drc->drc_begin_nvl);
	}
	return (err);
	}

	/*
	* Holds data need for corrective recv callback
	*/
	typedef struct cr_cb_data {
	uint64_t size;
	zbookmark_phys_t zb;
	spa_t *spa;
	} cr_cb_data_t;

	static void
	corrective_read_done(zio_t *zio)
	{
	cr_cb_data_t *data = zio->io_private;
	/* Corruption corrected; update error log if needed */
	if (zio->io_error == 0)
	spa_remove_error(data->spa, &data->zb, &zio->io_bp->blk_birth);
	kmem_free(data, sizeof (cr_cb_data_t));
	abd_free(zio->io_abd);
	}

	/*
	* zio_rewrite the data pointed to by bp with the data from the rrd's abd.
	*/
	static int
	do_corrective_recv(struct receive_writer_arg rwa, struct drr_write drrw,
	struct receive_record_arg rrd, blkptr_t bp)
	{
	int err;
	zio_t *io;
	zbookmark_phys_t zb;
	dnode_t *dn;
	abd_t *abd = rrd->abd;
	zio_cksum_t bp_cksum = bp->blk_cksum;
	zio_flag_t flags = ZIO_FLAG_SPECULATIVE \| ZIO_FLAG_DONT_RETRY \|
	ZIO_FLAG_CANFAIL;

	if (rwa->raw)
	flags \|= ZIO_FLAG_RAW;

	err = dnode_hold(rwa->os, drrw->drr_object, FTAG, &dn);
	if (err != 0)
	return (err);
	SET_BOOKMARK(&zb, dmu_objset_id(rwa->os), drrw->drr_object, 0,
	dbuf_whichblock(dn, 0, drrw->drr_offset));
	dnode_rele(dn, FTAG);

	if (!rwa->raw && DRR_WRITE_COMPRESSED(drrw)) {
	/* Decompress the stream data */
	abd_t *dabd = abd_alloc_linear(
	drrw->drr_logical_size, B_FALSE);
	err = zio_decompress_data(drrw->drr_compressiontype,
	abd, abd_to_buf(dabd), abd_get_size(abd),
	abd_get_size(dabd), NULL);

	if (err != 0) {
	abd_free(dabd);
	return (err);
	}
	/* Swap in the newly decompressed data into the abd */
	abd_free(abd);
	abd = dabd;
	}

	if (!rwa->raw && BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) {
	/* Recompress the data */
	abd_t *cabd = abd_alloc_linear(BP_GET_PSIZE(bp),
	B_FALSE);
	void *buf = abd_to_buf(cabd);
	uint64_t csize = zio_compress_data(BP_GET_COMPRESS(bp),
	abd, &buf, abd_get_size(abd),
	rwa->os->os_complevel);
	abd_zero_off(cabd, csize, BP_GET_PSIZE(bp) - csize);
	/* Swap in newly compressed data into the abd */
	abd_free(abd);
	abd = cabd;
	flags \|= ZIO_FLAG_RAW_COMPRESS;
	}

	/*
	* The stream is not encrypted but the data on-disk is.
	* We need to re-encrypt the buf using the same
	* encryption type, salt, iv, and mac that was used to encrypt
	* the block previosly.
	*/
	if (!rwa->raw && BP_USES_CRYPT(bp)) {
	dsl_dataset_t *ds;
	dsl_crypto_key_t *dck = NULL;
	uint8_t salt[ZIO_DATA_SALT_LEN];
	uint8_t iv[ZIO_DATA_IV_LEN];
	uint8_t mac[ZIO_DATA_MAC_LEN];
	boolean_t no_crypt = B_FALSE;
	dsl_pool_t *dp = dmu_objset_pool(rwa->os);
	abd_t *eabd = abd_alloc_linear(BP_GET_PSIZE(bp), B_FALSE);

	zio_crypt_decode_params_bp(bp, salt, iv);
	zio_crypt_decode_mac_bp(bp, mac);

	dsl_pool_config_enter(dp, FTAG);
	err = dsl_dataset_hold_flags(dp, rwa->tofs,
	DS_HOLD_FLAG_DECRYPT, FTAG, &ds);
	if (err != 0) {
	dsl_pool_config_exit(dp, FTAG);
	abd_free(eabd);
	return (SET_ERROR(EACCES));
	}

	/* Look up the key from the spa's keystore */
	err = spa_keystore_lookup_key(rwa->os->os_spa,
	zb.zb_objset, FTAG, &dck);
	if (err != 0) {
	dsl_dataset_rele_flags(ds, DS_HOLD_FLAG_DECRYPT,
	FTAG);
	dsl_pool_config_exit(dp, FTAG);
	abd_free(eabd);
	return (SET_ERROR(EACCES));
	}

	err = zio_do_crypt_abd(B_TRUE, &dck->dck_key,
	BP_GET_TYPE(bp), BP_SHOULD_BYTESWAP(bp), salt, iv,
	mac, abd_get_size(abd), abd, eabd, &no_crypt);

	spa_keystore_dsl_key_rele(rwa->os->os_spa, dck, FTAG);
	dsl_dataset_rele_flags(ds, DS_HOLD_FLAG_DECRYPT, FTAG);
	dsl_pool_config_exit(dp, FTAG);

	ASSERT0(no_crypt);
	if (err != 0) {
	abd_free(eabd);
	return (err);
	}
	/* Swap in the newly encrypted data into the abd */
	abd_free(abd);
	abd = eabd;

	/*
	* We want to prevent zio_rewrite() from trying to
	* encrypt the data again
	*/
	flags \|= ZIO_FLAG_RAW_ENCRYPT;
	}
	rrd->abd = abd;

	io = zio_rewrite(NULL, rwa->os->os_spa, bp->blk_birth, bp, abd,
	BP_GET_PSIZE(bp), NULL, NULL, ZIO_PRIORITY_SYNC_WRITE, flags, &zb);

	ASSERT(abd_get_size(abd) == BP_GET_LSIZE(bp) \|\|
	abd_get_size(abd) == BP_GET_PSIZE(bp));

	/* compute new bp checksum value and make sure it matches the old one */
	zio_checksum_compute(io, BP_GET_CHECKSUM(bp), abd, abd_get_size(abd));
	if (!ZIO_CHECKSUM_EQUAL(bp_cksum, io->io_bp->blk_cksum)) {
	zio_destroy(io);
	if (zfs_recv_best_effort_corrective != 0)
	return (0);
	return (SET_ERROR(ECKSUM));
	}

	/* Correct the corruption in place */
	err = zio_wait(io);
	if (err == 0) {
	cr_cb_data_t *cb_data =
	kmem_alloc(sizeof (cr_cb_data_t), KM_SLEEP);
	cb_data->spa = rwa->os->os_spa;
	cb_data->size = drrw->drr_logical_size;
	cb_data->zb = zb;
	/* Test if healing worked by re-reading the bp */
	err = zio_wait(zio_read(rwa->heal_pio, rwa->os->os_spa, bp,
	abd_alloc_for_io(drrw->drr_logical_size, B_FALSE),
	drrw->drr_logical_size, corrective_read_done,
	cb_data, ZIO_PRIORITY_ASYNC_READ, flags, NULL));
	}
	if (err != 0 && zfs_recv_best_effort_corrective != 0)
	err = 0;

	return (err);
	}

	static int
	receive_read(dmu_recv_cookie_t drc, int len, void buf)
	{
	int done = 0;

	/*
	* The code doesn't rely on this (lengths being multiples of 8). See
	* comment in dump_bytes.
	*/
	ASSERT(len % 8 == 0 \|\|
	(drc->drc_featureflags & DMU_BACKUP_FEATURE_RAW) != 0);

	while (done < len) {
	ssize_t resid = len - done;
	zfs_file_t *fp = drc->drc_fp;
	int err = zfs_file_read(fp, (char *)buf + done,
	len - done, &resid);
	if (err == 0 && resid == len - done) {
	/*
	* Note: ECKSUM or ZFS_ERR_STREAM_TRUNCATED indicates
	* that the receive was interrupted and can
	* potentially be resumed.
	*/
	err = SET_ERROR(ZFS_ERR_STREAM_TRUNCATED);
	}
	drc->drc_voff += len - done - resid;
	done = len - resid;
	if (err != 0)
	return (err);
	}

	drc->drc_bytes_read += len;

	ASSERT3U(done, ==, len);
	return (0);
	}

	static inline uint8_t
	deduce_nblkptr(dmu_object_type_t bonus_type, uint64_t bonus_size)
	{
	if (bonus_type == DMU_OT_SA) {
	return (1);
	} else {
	return (1 +
	((DN_OLD_MAX_BONUSLEN -
	MIN(DN_OLD_MAX_BONUSLEN, bonus_size)) >> SPA_BLKPTRSHIFT));
	}
	}

	static void
	save_resume_state(struct receive_writer_arg *rwa,
	uint64_t object, uint64_t offset, dmu_tx_t *tx)
	{
	int txgoff = dmu_tx_get_txg(tx) & TXG_MASK;

	if (!rwa->resumable)
	return;

	/*
	* We use ds_resume_bytes[] != 0 to indicate that we need to
	* update this on disk, so it must not be 0.
	*/
	ASSERT(rwa->bytes_read != 0);

	/*
	* We only resume from write records, which have a valid
	* (non-meta-dnode) object number.
	*/
	ASSERT(object != 0);

	/*
	* For resuming to work correctly, we must receive records in order,
	* sorted by object,offset. This is checked by the callers, but
	* assert it here for good measure.
	*/
	ASSERT3U(object, >=, rwa->os->os_dsl_dataset->ds_resume_object[txgoff]);
	ASSERT(object != rwa->os->os_dsl_dataset->ds_resume_object[txgoff] \|\|
	offset >= rwa->os->os_dsl_dataset->ds_resume_offset[txgoff]);
	ASSERT3U(rwa->bytes_read, >=,
	rwa->os->os_dsl_dataset->ds_resume_bytes[txgoff]);

	rwa->os->os_dsl_dataset->ds_resume_object[txgoff] = object;
	rwa->os->os_dsl_dataset->ds_resume_offset[txgoff] = offset;
	rwa->os->os_dsl_dataset->ds_resume_bytes[txgoff] = rwa->bytes_read;
	}

	static int
	receive_object_is_same_generation(objset_t *os, uint64_t object,
	dmu_object_type_t old_bonus_type, dmu_object_type_t new_bonus_type,
	const void new_bonus, boolean_t samegenp)
	{
	zfs_file_info_t zoi;
	int err;

	dmu_buf_t *old_bonus_dbuf;
	err = dmu_bonus_hold(os, object, FTAG, &old_bonus_dbuf);
	if (err != 0)
	return (err);
	err = dmu_get_file_info(os, old_bonus_type, old_bonus_dbuf->db_data,
	&zoi);
	dmu_buf_rele(old_bonus_dbuf, FTAG);
	if (err != 0)
	return (err);
	uint64_t old_gen = zoi.zfi_generation;

	err = dmu_get_file_info(os, new_bonus_type, new_bonus, &zoi);
	if (err != 0)
	return (err);
	uint64_t new_gen = zoi.zfi_generation;

	*samegenp = (old_gen == new_gen);
	return (0);
	}

	static int
	receive_handle_existing_object(const struct receive_writer_arg *rwa,
	const struct drr_object drro, const dmu_object_info_t doi,
	const void *bonus_data,
	uint64_t object_to_hold, uint32_t new_blksz)
	{
	uint32_t indblksz = drro->drr_indblkshift ?
	1ULL << drro->drr_indblkshift : 0;
	int nblkptr = deduce_nblkptr(drro->drr_bonustype,
	drro->drr_bonuslen);
	uint8_t dn_slots = drro->drr_dn_slots != 0 ?
	drro->drr_dn_slots : DNODE_MIN_SLOTS;
	boolean_t do_free_range = B_FALSE;
	int err;

	*object_to_hold = drro->drr_object;

	/* nblkptr should be bounded by the bonus size and type */
	if (rwa->raw && nblkptr != drro->drr_nblkptr)
	return (SET_ERROR(EINVAL));

	/*
	* After the previous send stream, the sending system may
	* have freed this object, and then happened to re-allocate
	* this object number in a later txg. In this case, we are
	* receiving a different logical file, and the block size may
	* appear to be different. i.e. we may have a different
	* block size for this object than what the send stream says.
	* In this case we need to remove the object's contents,
	* so that its structure can be changed and then its contents
	* entirely replaced by subsequent WRITE records.
	*
	* If this is a -L (--large-block) incremental stream, and
	* the previous stream was not -L, the block size may appear
	* to increase. i.e. we may have a smaller block size for
	* this object than what the send stream says. In this case
	* we need to keep the object's contents and block size
	* intact, so that we don't lose parts of the object's
	* contents that are not changed by this incremental send
	* stream.
	*
	* We can distinguish between the two above cases by using
	* the ZPL's generation number (see
	* receive_object_is_same_generation()). However, we only
	* want to rely on the generation number when absolutely
	* necessary, because with raw receives, the generation is
	* encrypted. We also want to minimize dependence on the
	* ZPL, so that other types of datasets can also be received
	* (e.g. ZVOLs, although note that ZVOLS currently do not
	* reallocate their objects or change their structure).
	* Therefore, we check a number of different cases where we
	* know it is safe to discard the object's contents, before
	* using the ZPL's generation number to make the above
	* distinction.
	*/
	if (drro->drr_blksz != doi->doi_data_block_size) {
	if (rwa->raw) {
	/*
	* RAW streams always have large blocks, so
	* we are sure that the data is not needed
	* due to changing --large-block to be on.
	* Which is fortunate since the bonus buffer
	* (which contains the ZPL generation) is
	* encrypted, and the key might not be
	* loaded.
	*/
	do_free_range = B_TRUE;
	} else if (rwa->full) {
	/*
	* This is a full send stream, so it always
	* replaces what we have. Even if the
	* generation numbers happen to match, this
	* can not actually be the same logical file.
	* This is relevant when receiving a full
	* send as a clone.
	*/
	do_free_range = B_TRUE;
	} else if (drro->drr_type !=
	DMU_OT_PLAIN_FILE_CONTENTS \|\|
	doi->doi_type != DMU_OT_PLAIN_FILE_CONTENTS) {
	/*
	* PLAIN_FILE_CONTENTS are the only type of
	* objects that have ever been stored with
	* large blocks, so we don't need the special
	* logic below. ZAP blocks can shrink (when
	* there's only one block), so we don't want
	* to hit the error below about block size
	* only increasing.
	*/
	do_free_range = B_TRUE;
	} else if (doi->doi_max_offset <=
	doi->doi_data_block_size) {
	/*
	* There is only one block. We can free it,
	* because its contents will be replaced by a
	* WRITE record. This can not be the no-L ->
	* -L case, because the no-L case would have
	* resulted in multiple blocks. If we
	* supported -L -> no-L, it would not be safe
	* to free the file's contents. Fortunately,
	* that is not allowed (see
	* recv_check_large_blocks()).
	*/
	do_free_range = B_TRUE;
	} else {
	boolean_t is_same_gen;
	err = receive_object_is_same_generation(rwa->os,
	drro->drr_object, doi->doi_bonus_type,
	drro->drr_bonustype, bonus_data, &is_same_gen);
	if (err != 0)
	return (SET_ERROR(EINVAL));

	if (is_same_gen) {
	/*
	* This is the same logical file, and
	* the block size must be increasing.
	* It could only decrease if
	* --large-block was changed to be
	* off, which is checked in
	* recv_check_large_blocks().
	*/
	if (drro->drr_blksz <=
	doi->doi_data_block_size)
	return (SET_ERROR(EINVAL));
	/*
	* We keep the existing blocksize and
	* contents.
	*/
	*new_blksz =
	doi->doi_data_block_size;
	} else {
	do_free_range = B_TRUE;
	}
	}
	}

	/* nblkptr can only decrease if the object was reallocated */
	if (nblkptr < doi->doi_nblkptr)
	do_free_range = B_TRUE;

	/* number of slots can only change on reallocation */
	if (dn_slots != doi->doi_dnodesize >> DNODE_SHIFT)
	do_free_range = B_TRUE;

	/*
	* For raw sends we also check a few other fields to
	* ensure we are preserving the objset structure exactly
	* as it was on the receive side:
	* - A changed indirect block size
	* - A smaller nlevels
	*/
	if (rwa->raw) {
	if (indblksz != doi->doi_metadata_block_size)
	do_free_range = B_TRUE;
	if (drro->drr_nlevels < doi->doi_indirection)
	do_free_range = B_TRUE;
	}

	if (do_free_range) {
	err = dmu_free_long_range(rwa->os, drro->drr_object,
	0, DMU_OBJECT_END);
	if (err != 0)
	return (SET_ERROR(EINVAL));
	}

	/*
	* The dmu does not currently support decreasing nlevels or changing
	* indirect block size if there is already one, same as changing the
	* number of of dnode slots on an object. For non-raw sends this
	* does not matter and the new object can just use the previous one's
	* parameters. For raw sends, however, the structure of the received
	* dnode (including indirects and dnode slots) must match that of the
	* send side. Therefore, instead of using dmu_object_reclaim(), we
	* must free the object completely and call dmu_object_claim_dnsize()
	* instead.
	*/
	if ((rwa->raw && ((doi->doi_indirection > 1 &&
	indblksz != doi->doi_metadata_block_size) \|\|
	drro->drr_nlevels < doi->doi_indirection)) \|\|
	dn_slots != doi->doi_dnodesize >> DNODE_SHIFT) {
	err = dmu_free_long_object(rwa->os, drro->drr_object);
	if (err != 0)
	return (SET_ERROR(EINVAL));

	txg_wait_synced(dmu_objset_pool(rwa->os), 0);
	*object_to_hold = DMU_NEW_OBJECT;
	}

	/*
	* For raw receives, free everything beyond the new incoming
	* maxblkid. Normally this would be done with a DRR_FREE
	* record that would come after this DRR_OBJECT record is
	* processed. However, for raw receives we manually set the
	* maxblkid from the drr_maxblkid and so we must first free
	* everything above that blkid to ensure the DMU is always
	* consistent with itself. We will never free the first block
	* of the object here because a maxblkid of 0 could indicate
	* an object with a single block or one with no blocks. This
	* free may be skipped when dmu_free_long_range() was called
	* above since it covers the entire object's contents.
	*/
	if (rwa->raw && *object_to_hold != DMU_NEW_OBJECT && !do_free_range) {
	err = dmu_free_long_range(rwa->os, drro->drr_object,
	(drro->drr_maxblkid + 1) * doi->doi_data_block_size,
	DMU_OBJECT_END);
	if (err != 0)
	return (SET_ERROR(EINVAL));
	}
	return (0);
	}

	noinline static int
	receive_object(struct receive_writer_arg rwa, struct drr_object drro,
	void *data)
	{
	dmu_object_info_t doi;
	dmu_tx_t *tx;
	int err;
	uint32_t new_blksz = drro->drr_blksz;
	uint8_t dn_slots = drro->drr_dn_slots != 0 ?
	drro->drr_dn_slots : DNODE_MIN_SLOTS;

	if (drro->drr_type == DMU_OT_NONE \|\|
	!DMU_OT_IS_VALID(drro->drr_type) \|\|
	!DMU_OT_IS_VALID(drro->drr_bonustype) \|\|
	drro->drr_checksumtype >= ZIO_CHECKSUM_FUNCTIONS \|\|
	drro->drr_compress >= ZIO_COMPRESS_FUNCTIONS \|\|
	P2PHASE(drro->drr_blksz, SPA_MINBLOCKSIZE) \|\|
	drro->drr_blksz < SPA_MINBLOCKSIZE \|\|
	drro->drr_blksz > spa_maxblocksize(dmu_objset_spa(rwa->os)) \|\|
	drro->drr_bonuslen >
	DN_BONUS_SIZE(spa_maxdnodesize(dmu_objset_spa(rwa->os))) \|\|
	dn_slots >
	(spa_maxdnodesize(dmu_objset_spa(rwa->os)) >> DNODE_SHIFT)) {
	return (SET_ERROR(EINVAL));
	}

	if (rwa->raw) {
	/*
	* We should have received a DRR_OBJECT_RANGE record
	* containing this block and stored it in rwa.
	*/
	if (drro->drr_object < rwa->or_firstobj \|\|
	drro->drr_object >= rwa->or_firstobj + rwa->or_numslots \|\|
	drro->drr_raw_bonuslen < drro->drr_bonuslen \|\|
	drro->drr_indblkshift > SPA_MAXBLOCKSHIFT \|\|
	drro->drr_nlevels > DN_MAX_LEVELS \|\|
	drro->drr_nblkptr > DN_MAX_NBLKPTR \|\|
	DN_SLOTS_TO_BONUSLEN(dn_slots) <
	drro->drr_raw_bonuslen)
	return (SET_ERROR(EINVAL));
	} else {
	/*
	* The DRR_OBJECT_SPILL flag is valid when the DRR_BEGIN
	* record indicates this by setting DRR_FLAG_SPILL_BLOCK.
	*/
	if (((drro->drr_flags & ~(DRR_OBJECT_SPILL))) \|\|
	(!rwa->spill && DRR_OBJECT_HAS_SPILL(drro->drr_flags))) {
	return (SET_ERROR(EINVAL));
	}

	if (drro->drr_raw_bonuslen != 0 \|\| drro->drr_nblkptr != 0 \|\|
	drro->drr_indblkshift != 0 \|\| drro->drr_nlevels != 0) {
	return (SET_ERROR(EINVAL));
	}
	}

	err = dmu_object_info(rwa->os, drro->drr_object, &doi);

	if (err != 0 && err != ENOENT && err != EEXIST)
	return (SET_ERROR(EINVAL));

	if (drro->drr_object > rwa->max_object)
	rwa->max_object = drro->drr_object;

	/*
	* If we are losing blkptrs or changing the block size this must
	* be a new file instance. We must clear out the previous file
	* contents before we can change this type of metadata in the dnode.
	* Raw receives will also check that the indirect structure of the
	* dnode hasn't changed.
	*/
	uint64_t object_to_hold;
	if (err == 0) {
	err = receive_handle_existing_object(rwa, drro, &doi, data,
	&object_to_hold, &new_blksz);
	if (err != 0)
	return (err);
	} else if (err == EEXIST) {
	/*
	* The object requested is currently an interior slot of a
	* multi-slot dnode. This will be resolved when the next txg
	* is synced out, since the send stream will have told us
	* to free this slot when we freed the associated dnode
	* earlier in the stream.
	*/
	txg_wait_synced(dmu_objset_pool(rwa->os), 0);

	if (dmu_object_info(rwa->os, drro->drr_object, NULL) != ENOENT)
	return (SET_ERROR(EINVAL));

	/* object was freed and we are about to allocate a new one */
	object_to_hold = DMU_NEW_OBJECT;
	} else {
	/*
	* If the only record in this range so far was DRR_FREEOBJECTS
	* with at least one actually freed object, it's possible that
	* the block will now be converted to a hole. We need to wait
	* for the txg to sync to prevent races.
	*/
	if (rwa->or_need_sync == ORNS_YES)
	txg_wait_synced(dmu_objset_pool(rwa->os), 0);

	/* object is free and we are about to allocate a new one */
	object_to_hold = DMU_NEW_OBJECT;
	}

	/* Only relevant for the first object in the range */
	rwa->or_need_sync = ORNS_NO;

	/*
	* If this is a multi-slot dnode there is a chance that this
	* object will expand into a slot that is already used by
	* another object from the previous snapshot. We must free
	* these objects before we attempt to allocate the new dnode.
	*/
	if (dn_slots > 1) {
	boolean_t need_sync = B_FALSE;

	for (uint64_t slot = drro->drr_object + 1;
	slot < drro->drr_object + dn_slots;
	slot++) {
	dmu_object_info_t slot_doi;

	err = dmu_object_info(rwa->os, slot, &slot_doi);
	if (err == ENOENT \|\| err == EEXIST)
	continue;
	else if (err != 0)
	return (err);

	err = dmu_free_long_object(rwa->os, slot);
	if (err != 0)
	return (err);

	need_sync = B_TRUE;
	}

	if (need_sync)
	txg_wait_synced(dmu_objset_pool(rwa->os), 0);
	}

	tx = dmu_tx_create(rwa->os);
	dmu_tx_hold_bonus(tx, object_to_hold);
	dmu_tx_hold_write(tx, object_to_hold, 0, 0);
	err = dmu_tx_assign(tx, TXG_WAIT);
	if (err != 0) {
	dmu_tx_abort(tx);
	return (err);
	}

	if (object_to_hold == DMU_NEW_OBJECT) {
	/* Currently free, wants to be allocated */
	err = dmu_object_claim_dnsize(rwa->os, drro->drr_object,
	drro->drr_type, new_blksz,
	drro->drr_bonustype, drro->drr_bonuslen,
	dn_slots << DNODE_SHIFT, tx);
	} else if (drro->drr_type != doi.doi_type \|\|
	new_blksz != doi.doi_data_block_size \|\|
	drro->drr_bonustype != doi.doi_bonus_type \|\|
	drro->drr_bonuslen != doi.doi_bonus_size) {
	/* Currently allocated, but with different properties */
	err = dmu_object_reclaim_dnsize(rwa->os, drro->drr_object,
	drro->drr_type, new_blksz,
	drro->drr_bonustype, drro->drr_bonuslen,
	dn_slots << DNODE_SHIFT, rwa->spill ?
	DRR_OBJECT_HAS_SPILL(drro->drr_flags) : B_FALSE, tx);
	} else if (rwa->spill && !DRR_OBJECT_HAS_SPILL(drro->drr_flags)) {
	/*
	* Currently allocated, the existing version of this object
	* may reference a spill block that is no longer allocated
	* at the source and needs to be freed.
	*/
	err = dmu_object_rm_spill(rwa->os, drro->drr_object, tx);
	}

	if (err != 0) {
	dmu_tx_commit(tx);
	return (SET_ERROR(EINVAL));
	}

	if (rwa->or_crypt_params_present) {
	/*
	* Set the crypt params for the buffer associated with this
	* range of dnodes. This causes the blkptr_t to have the
	* same crypt params (byteorder, salt, iv, mac) as on the
	* sending side.
	*
	* Since we are committing this tx now, it is possible for
	* the dnode block to end up on-disk with the incorrect MAC,
	* if subsequent objects in this block are received in a
	* different txg. However, since the dataset is marked as
	* inconsistent, no code paths will do a non-raw read (or
	* decrypt the block / verify the MAC). The receive code and
	* scrub code can safely do raw reads and verify the
	* checksum. They don't need to verify the MAC.
	*/
	dmu_buf_t *db = NULL;
	uint64_t offset = rwa->or_firstobj * DNODE_MIN_SIZE;

	err = dmu_buf_hold_by_dnode(DMU_META_DNODE(rwa->os),
	offset, FTAG, &db, DMU_READ_PREFETCH \| DMU_READ_NO_DECRYPT);
	if (err != 0) {
	dmu_tx_commit(tx);
	return (SET_ERROR(EINVAL));
	}

	dmu_buf_set_crypt_params(db, rwa->or_byteorder,
	rwa->or_salt, rwa->or_iv, rwa->or_mac, tx);

	dmu_buf_rele(db, FTAG);

	rwa->or_crypt_params_present = B_FALSE;
	}

	dmu_object_set_checksum(rwa->os, drro->drr_object,
	drro->drr_checksumtype, tx);
	dmu_object_set_compress(rwa->os, drro->drr_object,
	drro->drr_compress, tx);

	/* handle more restrictive dnode structuring for raw recvs */
	if (rwa->raw) {
	/*
	* Set the indirect block size, block shift, nlevels.
	* This will not fail because we ensured all of the
	* blocks were freed earlier if this is a new object.
	* For non-new objects block size and indirect block
	* shift cannot change and nlevels can only increase.
	*/
	ASSERT3U(new_blksz, ==, drro->drr_blksz);
	VERIFY0(dmu_object_set_blocksize(rwa->os, drro->drr_object,
	drro->drr_blksz, drro->drr_indblkshift, tx));
	VERIFY0(dmu_object_set_nlevels(rwa->os, drro->drr_object,
	drro->drr_nlevels, tx));

	/*
	* Set the maxblkid. This will always succeed because
	* we freed all blocks beyond the new maxblkid above.
	*/
	VERIFY0(dmu_object_set_maxblkid(rwa->os, drro->drr_object,
	drro->drr_maxblkid, tx));
	}

	if (data != NULL) {
	dmu_buf_t *db;
	dnode_t *dn;
	uint32_t flags = DMU_READ_NO_PREFETCH;

	if (rwa->raw)
	flags \|= DMU_READ_NO_DECRYPT;

	VERIFY0(dnode_hold(rwa->os, drro->drr_object, FTAG, &dn));
	VERIFY0(dmu_bonus_hold_by_dnode(dn, FTAG, &db, flags));

	dmu_buf_will_dirty(db, tx);

	ASSERT3U(db->db_size, >=, drro->drr_bonuslen);
	memcpy(db->db_data, data, DRR_OBJECT_PAYLOAD_SIZE(drro));

	/*
	* Raw bonus buffers have their byteorder determined by the
	* DRR_OBJECT_RANGE record.
	*/
	if (rwa->byteswap && !rwa->raw) {
	dmu_object_byteswap_t byteswap =
	DMU_OT_BYTESWAP(drro->drr_bonustype);
	dmu_ot_byteswap[byteswap].ob_func(db->db_data,
	DRR_OBJECT_PAYLOAD_SIZE(drro));
	}
	dmu_buf_rele(db, FTAG);
	dnode_rele(dn, FTAG);
	}
	+
	+ /*
	+ * If the receive fails, we want the resume stream to start with the
	+ * same record that we last successfully received. There is no way to
	+ * request resume from the object record, but we can benefit from the
	+ * fact that sender always sends object record before anything else,
	+ * after which it will "resend" data at offset 0 and resume normally.
	+ */
	+ save_resume_state(rwa, drro->drr_object, 0, tx);
	+
	dmu_tx_commit(tx);

	return (0);
	}

	noinline static int
	receive_freeobjects(struct receive_writer_arg *rwa,
	struct drr_freeobjects *drrfo)
	{
	uint64_t obj;
	int next_err = 0;

	if (drrfo->drr_firstobj + drrfo->drr_numobjs < drrfo->drr_firstobj)
	return (SET_ERROR(EINVAL));

	for (obj = drrfo->drr_firstobj == 0 ? 1 : drrfo->drr_firstobj;
	obj < drrfo->drr_firstobj + drrfo->drr_numobjs &&
	obj < DN_MAX_OBJECT && next_err == 0;
	next_err = dmu_object_next(rwa->os, &obj, FALSE, 0)) {
	dmu_object_info_t doi;
	int err;

	err = dmu_object_info(rwa->os, obj, &doi);
	if (err == ENOENT)
	continue;
	else if (err != 0)
	return (err);

	err = dmu_free_long_object(rwa->os, obj);

	if (err != 0)
	return (err);

	if (rwa->or_need_sync == ORNS_MAYBE)
	rwa->or_need_sync = ORNS_YES;
	}
	if (next_err != ESRCH)
	return (next_err);
	return (0);
	}

	/*
	* Note: if this fails, the caller will clean up any records left on the
	* rwa->write_batch list.
	*/
	static int
	flush_write_batch_impl(struct receive_writer_arg *rwa)
	{
	dnode_t *dn;
	int err;

	if (dnode_hold(rwa->os, rwa->last_object, FTAG, &dn) != 0)
	return (SET_ERROR(EINVAL));

	struct receive_record_arg *last_rrd = list_tail(&rwa->write_batch);
	struct drr_write *last_drrw = &last_rrd->header.drr_u.drr_write;

	struct receive_record_arg *first_rrd = list_head(&rwa->write_batch);
	struct drr_write *first_drrw = &first_rrd->header.drr_u.drr_write;

	ASSERT3U(rwa->last_object, ==, last_drrw->drr_object);
	ASSERT3U(rwa->last_offset, ==, last_drrw->drr_offset);

	dmu_tx_t *tx = dmu_tx_create(rwa->os);
	dmu_tx_hold_write_by_dnode(tx, dn, first_drrw->drr_offset,
	last_drrw->drr_offset - first_drrw->drr_offset +
	last_drrw->drr_logical_size);
	err = dmu_tx_assign(tx, TXG_WAIT);
	if (err != 0) {
	dmu_tx_abort(tx);
	dnode_rele(dn, FTAG);
	return (err);
	}

	struct receive_record_arg *rrd;
	while ((rrd = list_head(&rwa->write_batch)) != NULL) {
	struct drr_write *drrw = &rrd->header.drr_u.drr_write;
	abd_t *abd = rrd->abd;

	ASSERT3U(drrw->drr_object, ==, rwa->last_object);

	if (drrw->drr_logical_size != dn->dn_datablksz) {
	/*
	* The WRITE record is larger than the object's block
	* size. We must be receiving an incremental
	* large-block stream into a dataset that previously did
	* a non-large-block receive. Lightweight writes must
	* be exactly one block, so we need to decompress the
	* data (if compressed) and do a normal dmu_write().
	*/
	ASSERT3U(drrw->drr_logical_size, >, dn->dn_datablksz);
	if (DRR_WRITE_COMPRESSED(drrw)) {
	abd_t *decomp_abd =
	abd_alloc_linear(drrw->drr_logical_size,
	B_FALSE);

	err = zio_decompress_data(
	drrw->drr_compressiontype,
	abd, abd_to_buf(decomp_abd),
	abd_get_size(abd),
	abd_get_size(decomp_abd), NULL);

	if (err == 0) {
	dmu_write_by_dnode(dn,
	drrw->drr_offset,
	drrw->drr_logical_size,
	abd_to_buf(decomp_abd), tx);
	}
	abd_free(decomp_abd);
	} else {
	dmu_write_by_dnode(dn,
	drrw->drr_offset,
	drrw->drr_logical_size,
	abd_to_buf(abd), tx);
	}
	if (err == 0)
	abd_free(abd);
	} else {
	zio_prop_t zp = {0};
	dmu_write_policy(rwa->os, dn, 0, 0, &zp);

	zio_flag_t zio_flags = 0;

	if (rwa->raw) {
	zp.zp_encrypt = B_TRUE;
	zp.zp_compress = drrw->drr_compressiontype;
	zp.zp_byteorder = ZFS_HOST_BYTEORDER ^
	!!DRR_IS_RAW_BYTESWAPPED(drrw->drr_flags) ^
	rwa->byteswap;
	memcpy(zp.zp_salt, drrw->drr_salt,
	ZIO_DATA_SALT_LEN);
	memcpy(zp.zp_iv, drrw->drr_iv,
	ZIO_DATA_IV_LEN);
	memcpy(zp.zp_mac, drrw->drr_mac,
	ZIO_DATA_MAC_LEN);
	if (DMU_OT_IS_ENCRYPTED(zp.zp_type)) {
	zp.zp_nopwrite = B_FALSE;
	zp.zp_copies = MIN(zp.zp_copies,
	SPA_DVAS_PER_BP - 1);
	}
	zio_flags \|= ZIO_FLAG_RAW;
	} else if (DRR_WRITE_COMPRESSED(drrw)) {
	ASSERT3U(drrw->drr_compressed_size, >, 0);
	ASSERT3U(drrw->drr_logical_size, >=,
	drrw->drr_compressed_size);
	zp.zp_compress = drrw->drr_compressiontype;
	zio_flags \|= ZIO_FLAG_RAW_COMPRESS;
	} else if (rwa->byteswap) {
	/*
	* Note: compressed blocks never need to be
	* byteswapped, because WRITE records for
	* metadata blocks are never compressed. The
	* exception is raw streams, which are written
	* in the original byteorder, and the byteorder
	* bit is preserved in the BP by setting
	* zp_byteorder above.
	*/
	dmu_object_byteswap_t byteswap =
	DMU_OT_BYTESWAP(drrw->drr_type);
	dmu_ot_byteswap[byteswap].ob_func(
	abd_to_buf(abd),
	DRR_WRITE_PAYLOAD_SIZE(drrw));
	}

	/*
	* Since this data can't be read until the receive
	* completes, we can do a "lightweight" write for
	* improved performance.
	*/
	err = dmu_lightweight_write_by_dnode(dn,
	drrw->drr_offset, abd, &zp, zio_flags, tx);
	}

	if (err != 0) {
	/*
	* This rrd is left on the list, so the caller will
	* free it (and the abd).
	*/
	break;
	}

	/*
	* Note: If the receive fails, we want the resume stream to
	* start with the same record that we last successfully
	* received (as opposed to the next record), so that we can
	* verify that we are resuming from the correct location.
	*/
	save_resume_state(rwa, drrw->drr_object, drrw->drr_offset, tx);

	list_remove(&rwa->write_batch, rrd);
	kmem_free(rrd, sizeof (*rrd));
	}

	dmu_tx_commit(tx);
	dnode_rele(dn, FTAG);
	return (err);
	}

	noinline static int
	flush_write_batch(struct receive_writer_arg *rwa)
	{
	if (list_is_empty(&rwa->write_batch))
	return (0);
	int err = rwa->err;
	if (err == 0)
	err = flush_write_batch_impl(rwa);
	if (err != 0) {
	struct receive_record_arg *rrd;
	while ((rrd = list_remove_head(&rwa->write_batch)) != NULL) {
	abd_free(rrd->abd);
	kmem_free(rrd, sizeof (*rrd));
	}
	}
	ASSERT(list_is_empty(&rwa->write_batch));
	return (err);
	}

	noinline static int
	receive_process_write_record(struct receive_writer_arg *rwa,
	struct receive_record_arg *rrd)
	{
	int err = 0;

	ASSERT3U(rrd->header.drr_type, ==, DRR_WRITE);
	struct drr_write *drrw = &rrd->header.drr_u.drr_write;

	if (drrw->drr_offset + drrw->drr_logical_size < drrw->drr_offset \|\|
	!DMU_OT_IS_VALID(drrw->drr_type))
	return (SET_ERROR(EINVAL));

	if (rwa->heal) {
	blkptr_t *bp;
	dmu_buf_t *dbp;
	- dnode_t *dn;
	int flags = DB_RF_CANFAIL;

	if (rwa->raw)
	flags \|= DB_RF_NO_DECRYPT;

	if (rwa->byteswap) {
	dmu_object_byteswap_t byteswap =
	DMU_OT_BYTESWAP(drrw->drr_type);
	dmu_ot_byteswap[byteswap].ob_func(abd_to_buf(rrd->abd),
	DRR_WRITE_PAYLOAD_SIZE(drrw));
	}

	err = dmu_buf_hold_noread(rwa->os, drrw->drr_object,
	drrw->drr_offset, FTAG, &dbp);
	if (err != 0)
	return (err);

	/* Try to read the object to see if it needs healing */
	err = dbuf_read((dmu_buf_impl_t *)dbp, NULL, flags);
	/*
	* We only try to heal when dbuf_read() returns a ECKSUMs.
	* Other errors (even EIO) get returned to caller.
	* EIO indicates that the device is not present/accessible,
	* so writing to it will likely fail.
	* If the block is healthy, we don't want to overwrite it
	* unnecessarily.
	*/
	if (err != ECKSUM) {
	dmu_buf_rele(dbp, FTAG);
	return (err);
	}
	- dn = dmu_buf_dnode_enter(dbp);
	/* Make sure the on-disk block and recv record sizes match */
	- if (drrw->drr_logical_size !=
	- dn->dn_datablkszsec << SPA_MINBLOCKSHIFT) {
	+ if (drrw->drr_logical_size != dbp->db_size) {
	err = ENOTSUP;
	- dmu_buf_dnode_exit(dbp);
	dmu_buf_rele(dbp, FTAG);
	return (err);
	}
	/* Get the block pointer for the corrupted block */
	bp = dmu_buf_get_blkptr(dbp);
	err = do_corrective_recv(rwa, drrw, rrd, bp);
	- dmu_buf_dnode_exit(dbp);
	dmu_buf_rele(dbp, FTAG);
	return (err);
	}

	/*
	* For resuming to work, records must be in increasing order
	* by (object, offset).
	*/
	if (drrw->drr_object < rwa->last_object \|\|
	(drrw->drr_object == rwa->last_object &&
	drrw->drr_offset < rwa->last_offset)) {
	return (SET_ERROR(EINVAL));
	}

	struct receive_record_arg *first_rrd = list_head(&rwa->write_batch);
	struct drr_write *first_drrw = &first_rrd->header.drr_u.drr_write;
	uint64_t batch_size =
	MIN(zfs_recv_write_batch_size, DMU_MAX_ACCESS / 2);
	if (first_rrd != NULL &&
	(drrw->drr_object != first_drrw->drr_object \|\|
	drrw->drr_offset >= first_drrw->drr_offset + batch_size)) {
	err = flush_write_batch(rwa);
	if (err != 0)
	return (err);
	}

	rwa->last_object = drrw->drr_object;
	rwa->last_offset = drrw->drr_offset;

	if (rwa->last_object > rwa->max_object)
	rwa->max_object = rwa->last_object;

	list_insert_tail(&rwa->write_batch, rrd);
	/*
	* Return EAGAIN to indicate that we will use this rrd again,
	* so the caller should not free it
	*/
	return (EAGAIN);
	}

	static int
	receive_write_embedded(struct receive_writer_arg *rwa,
	struct drr_write_embedded drrwe, void data)
	{
	dmu_tx_t *tx;
	int err;

	if (drrwe->drr_offset + drrwe->drr_length < drrwe->drr_offset)
	return (SET_ERROR(EINVAL));

	if (drrwe->drr_psize > BPE_PAYLOAD_SIZE)
	return (SET_ERROR(EINVAL));

	if (drrwe->drr_etype >= NUM_BP_EMBEDDED_TYPES)
	return (SET_ERROR(EINVAL));
	if (drrwe->drr_compression >= ZIO_COMPRESS_FUNCTIONS)
	return (SET_ERROR(EINVAL));
	if (rwa->raw)
	return (SET_ERROR(EINVAL));

	if (drrwe->drr_object > rwa->max_object)
	rwa->max_object = drrwe->drr_object;

	tx = dmu_tx_create(rwa->os);

	dmu_tx_hold_write(tx, drrwe->drr_object,
	drrwe->drr_offset, drrwe->drr_length);
	err = dmu_tx_assign(tx, TXG_WAIT);
	if (err != 0) {
	dmu_tx_abort(tx);
	return (err);
	}

	dmu_write_embedded(rwa->os, drrwe->drr_object,
	drrwe->drr_offset, data, drrwe->drr_etype,
	drrwe->drr_compression, drrwe->drr_lsize, drrwe->drr_psize,
	rwa->byteswap ^ ZFS_HOST_BYTEORDER, tx);

	/* See comment in restore_write. */
	save_resume_state(rwa, drrwe->drr_object, drrwe->drr_offset, tx);
	dmu_tx_commit(tx);
	return (0);
	}

	static int
	receive_spill(struct receive_writer_arg rwa, struct drr_spill drrs,
	abd_t *abd)
	{
	dmu_buf_t db, db_spill;
	int err;

	if (drrs->drr_length < SPA_MINBLOCKSIZE \|\|
	drrs->drr_length > spa_maxblocksize(dmu_objset_spa(rwa->os)))
	return (SET_ERROR(EINVAL));

	/*
	* This is an unmodified spill block which was added to the stream
	* to resolve an issue with incorrectly removing spill blocks. It
	* should be ignored by current versions of the code which support
	* the DRR_FLAG_SPILL_BLOCK flag.
	*/
	if (rwa->spill && DRR_SPILL_IS_UNMODIFIED(drrs->drr_flags)) {
	abd_free(abd);
	return (0);
	}

	if (rwa->raw) {
	if (!DMU_OT_IS_VALID(drrs->drr_type) \|\|
	drrs->drr_compressiontype >= ZIO_COMPRESS_FUNCTIONS \|\|
	drrs->drr_compressed_size == 0)
	return (SET_ERROR(EINVAL));
	}

	if (dmu_object_info(rwa->os, drrs->drr_object, NULL) != 0)
	return (SET_ERROR(EINVAL));

	if (drrs->drr_object > rwa->max_object)
	rwa->max_object = drrs->drr_object;

	VERIFY0(dmu_bonus_hold(rwa->os, drrs->drr_object, FTAG, &db));
	if ((err = dmu_spill_hold_by_bonus(db, DMU_READ_NO_DECRYPT, FTAG,
	&db_spill)) != 0) {
	dmu_buf_rele(db, FTAG);
	return (err);
	}

	dmu_tx_t *tx = dmu_tx_create(rwa->os);

	dmu_tx_hold_spill(tx, db->db_object);

	err = dmu_tx_assign(tx, TXG_WAIT);
	if (err != 0) {
	dmu_buf_rele(db, FTAG);
	dmu_buf_rele(db_spill, FTAG);
	dmu_tx_abort(tx);
	return (err);
	}

	/*
	* Spill blocks may both grow and shrink. When a change in size
	* occurs any existing dbuf must be updated to match the logical
	* size of the provided arc_buf_t.
	*/
	if (db_spill->db_size != drrs->drr_length) {
	dmu_buf_will_fill(db_spill, tx, B_FALSE);
	VERIFY0(dbuf_spill_set_blksz(db_spill,
	drrs->drr_length, tx));
	}

	arc_buf_t *abuf;
	if (rwa->raw) {
	boolean_t byteorder = ZFS_HOST_BYTEORDER ^
	!!DRR_IS_RAW_BYTESWAPPED(drrs->drr_flags) ^
	rwa->byteswap;

	abuf = arc_loan_raw_buf(dmu_objset_spa(rwa->os),
	drrs->drr_object, byteorder, drrs->drr_salt,
	drrs->drr_iv, drrs->drr_mac, drrs->drr_type,
	drrs->drr_compressed_size, drrs->drr_length,
	drrs->drr_compressiontype, 0);
	} else {
	abuf = arc_loan_buf(dmu_objset_spa(rwa->os),
	DMU_OT_IS_METADATA(drrs->drr_type),
	drrs->drr_length);
	if (rwa->byteswap) {
	dmu_object_byteswap_t byteswap =
	DMU_OT_BYTESWAP(drrs->drr_type);
	dmu_ot_byteswap[byteswap].ob_func(abd_to_buf(abd),
	DRR_SPILL_PAYLOAD_SIZE(drrs));
	}
	}

	memcpy(abuf->b_data, abd_to_buf(abd), DRR_SPILL_PAYLOAD_SIZE(drrs));
	abd_free(abd);
	dbuf_assign_arcbuf((dmu_buf_impl_t *)db_spill, abuf, tx);

	dmu_buf_rele(db, FTAG);
	dmu_buf_rele(db_spill, FTAG);

	dmu_tx_commit(tx);
	return (0);
	}

	noinline static int
	receive_free(struct receive_writer_arg rwa, struct drr_free drrf)
	{
	int err;

	if (drrf->drr_length != -1ULL &&
	drrf->drr_offset + drrf->drr_length < drrf->drr_offset)
	return (SET_ERROR(EINVAL));

	if (dmu_object_info(rwa->os, drrf->drr_object, NULL) != 0)
	return (SET_ERROR(EINVAL));

	if (drrf->drr_object > rwa->max_object)
	rwa->max_object = drrf->drr_object;

	err = dmu_free_long_range(rwa->os, drrf->drr_object,
	drrf->drr_offset, drrf->drr_length);

	return (err);
	}

	static int
	receive_object_range(struct receive_writer_arg *rwa,
	struct drr_object_range *drror)
	{
	/*
	* By default, we assume this block is in our native format
	* (ZFS_HOST_BYTEORDER). We then take into account whether
	* the send stream is byteswapped (rwa->byteswap). Finally,
	* we need to byteswap again if this particular block was
	* in non-native format on the send side.
	*/
	boolean_t byteorder = ZFS_HOST_BYTEORDER ^ rwa->byteswap ^
	!!DRR_IS_RAW_BYTESWAPPED(drror->drr_flags);

	/*
	* Since dnode block sizes are constant, we should not need to worry
	* about making sure that the dnode block size is the same on the
	* sending and receiving sides for the time being. For non-raw sends,
	* this does not matter (and in fact we do not send a DRR_OBJECT_RANGE
	* record at all). Raw sends require this record type because the
	* encryption parameters are used to protect an entire block of bonus
	* buffers. If the size of dnode blocks ever becomes variable,
	* handling will need to be added to ensure that dnode block sizes
	* match on the sending and receiving side.
	*/
	if (drror->drr_numslots != DNODES_PER_BLOCK \|\|
	P2PHASE(drror->drr_firstobj, DNODES_PER_BLOCK) != 0 \|\|
	!rwa->raw)
	return (SET_ERROR(EINVAL));

	if (drror->drr_firstobj > rwa->max_object)
	rwa->max_object = drror->drr_firstobj;

	/*
	* The DRR_OBJECT_RANGE handling must be deferred to receive_object()
	* so that the block of dnodes is not written out when it's empty,
	* and converted to a HOLE BP.
	*/
	rwa->or_crypt_params_present = B_TRUE;
	rwa->or_firstobj = drror->drr_firstobj;
	rwa->or_numslots = drror->drr_numslots;
	memcpy(rwa->or_salt, drror->drr_salt, ZIO_DATA_SALT_LEN);
	memcpy(rwa->or_iv, drror->drr_iv, ZIO_DATA_IV_LEN);
	memcpy(rwa->or_mac, drror->drr_mac, ZIO_DATA_MAC_LEN);
	rwa->or_byteorder = byteorder;

	rwa->or_need_sync = ORNS_MAYBE;

	return (0);
	}

	/*
	* Until we have the ability to redact large ranges of data efficiently, we
	* process these records as frees.
	*/
	noinline static int
	receive_redact(struct receive_writer_arg rwa, struct drr_redact drrr)
	{
	struct drr_free drrf = {0};
	drrf.drr_length = drrr->drr_length;
	drrf.drr_object = drrr->drr_object;
	drrf.drr_offset = drrr->drr_offset;
	drrf.drr_toguid = drrr->drr_toguid;
	return (receive_free(rwa, &drrf));
	}

	/* used to destroy the drc_ds on error */
	static void
	dmu_recv_cleanup_ds(dmu_recv_cookie_t *drc)
	{
	dsl_dataset_t *ds = drc->drc_ds;
	ds_hold_flags_t dsflags;

	dsflags = (drc->drc_raw) ? DS_HOLD_FLAG_NONE : DS_HOLD_FLAG_DECRYPT;
	/*
	* Wait for the txg sync before cleaning up the receive. For
	* resumable receives, this ensures that our resume state has
	* been written out to disk. For raw receives, this ensures
	* that the user accounting code will not attempt to do anything
	* after we stopped receiving the dataset.
	*/
	txg_wait_synced(ds->ds_dir->dd_pool, 0);
	ds->ds_objset->os_raw_receive = B_FALSE;

	rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
	if (drc->drc_resumable && drc->drc_should_save &&
	!BP_IS_HOLE(dsl_dataset_get_blkptr(ds))) {
	rrw_exit(&ds->ds_bp_rwlock, FTAG);
	dsl_dataset_disown(ds, dsflags, dmu_recv_tag);
	} else {
	char name[ZFS_MAX_DATASET_NAME_LEN];
	rrw_exit(&ds->ds_bp_rwlock, FTAG);
	dsl_dataset_name(ds, name);
	dsl_dataset_disown(ds, dsflags, dmu_recv_tag);
	if (!drc->drc_heal)
	(void) dsl_destroy_head(name);
	}
	}

	static void
	receive_cksum(dmu_recv_cookie_t drc, int len, void buf)
	{
	if (drc->drc_byteswap) {
	(void) fletcher_4_incremental_byteswap(buf, len,
	&drc->drc_cksum);
	} else {
	(void) fletcher_4_incremental_native(buf, len, &drc->drc_cksum);
	}
	}

	/*
	* Read the payload into a buffer of size len, and update the current record's
	* payload field.
	* Allocate drc->drc_next_rrd and read the next record's header into
	* drc->drc_next_rrd->header.
	* Verify checksum of payload and next record.
	*/
	static int
	receive_read_payload_and_next_header(dmu_recv_cookie_t drc, int len, void buf)
	{
	int err;

	if (len != 0) {
	ASSERT3U(len, <=, SPA_MAXBLOCKSIZE);
	err = receive_read(drc, len, buf);
	if (err != 0)
	return (err);
	receive_cksum(drc, len, buf);

	/* note: rrd is NULL when reading the begin record's payload */
	if (drc->drc_rrd != NULL) {
	drc->drc_rrd->payload = buf;
	drc->drc_rrd->payload_size = len;
	drc->drc_rrd->bytes_read = drc->drc_bytes_read;
	}
	} else {
	ASSERT3P(buf, ==, NULL);
	}

	drc->drc_prev_cksum = drc->drc_cksum;

	drc->drc_next_rrd = kmem_zalloc(sizeof (*drc->drc_next_rrd), KM_SLEEP);
	err = receive_read(drc, sizeof (drc->drc_next_rrd->header),
	&drc->drc_next_rrd->header);
	drc->drc_next_rrd->bytes_read = drc->drc_bytes_read;

	if (err != 0) {
	kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd));
	drc->drc_next_rrd = NULL;
	return (err);
	}
	if (drc->drc_next_rrd->header.drr_type == DRR_BEGIN) {
	kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd));
	drc->drc_next_rrd = NULL;
	return (SET_ERROR(EINVAL));
	}

	/*
	* Note: checksum is of everything up to but not including the
	* checksum itself.
	*/
	ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
	==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t));
	receive_cksum(drc,
	offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
	&drc->drc_next_rrd->header);

	zio_cksum_t cksum_orig =
	drc->drc_next_rrd->header.drr_u.drr_checksum.drr_checksum;
	zio_cksum_t *cksump =
	&drc->drc_next_rrd->header.drr_u.drr_checksum.drr_checksum;

	if (drc->drc_byteswap)
	byteswap_record(&drc->drc_next_rrd->header);

	if ((!ZIO_CHECKSUM_IS_ZERO(cksump)) &&
	!ZIO_CHECKSUM_EQUAL(drc->drc_cksum, *cksump)) {
	kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd));
	drc->drc_next_rrd = NULL;
	return (SET_ERROR(ECKSUM));
	}

	receive_cksum(drc, sizeof (cksum_orig), &cksum_orig);

	return (0);
	}

	/*
	* Issue the prefetch reads for any necessary indirect blocks.
	*
	* We use the object ignore list to tell us whether or not to issue prefetches
	* for a given object. We do this for both correctness (in case the blocksize
	* of an object has changed) and performance (if the object doesn't exist, don't
	* needlessly try to issue prefetches). We also trim the list as we go through
	* the stream to prevent it from growing to an unbounded size.
	*
	* The object numbers within will always be in sorted order, and any write
	* records we see will also be in sorted order, but they're not sorted with
	* respect to each other (i.e. we can get several object records before
	* receiving each object's write records). As a result, once we've reached a
	* given object number, we can safely remove any reference to lower object
	* numbers in the ignore list. In practice, we receive up to 32 object records
	* before receiving write records, so the list can have up to 32 nodes in it.
	*/
	static void
	receive_read_prefetch(dmu_recv_cookie_t *drc, uint64_t object, uint64_t offset,
	uint64_t length)
	{
	if (!objlist_exists(drc->drc_ignore_objlist, object)) {
	dmu_prefetch(drc->drc_os, object, 1, offset, length,
	ZIO_PRIORITY_SYNC_READ);
	}
	}

	/*
	* Read records off the stream, issuing any necessary prefetches.
	*/
	static int
	receive_read_record(dmu_recv_cookie_t *drc)
	{
	int err;

	switch (drc->drc_rrd->header.drr_type) {
	case DRR_OBJECT:
	{
	struct drr_object *drro =
	&drc->drc_rrd->header.drr_u.drr_object;
	uint32_t size = DRR_OBJECT_PAYLOAD_SIZE(drro);
	void *buf = NULL;
	dmu_object_info_t doi;

	if (size != 0)
	buf = kmem_zalloc(size, KM_SLEEP);

	err = receive_read_payload_and_next_header(drc, size, buf);
	if (err != 0) {
	kmem_free(buf, size);
	return (err);
	}
	err = dmu_object_info(drc->drc_os, drro->drr_object, &doi);
	/*
	* See receive_read_prefetch for an explanation why we're
	* storing this object in the ignore_obj_list.
	*/
	if (err == ENOENT \|\| err == EEXIST \|\|
	(err == 0 && doi.doi_data_block_size != drro->drr_blksz)) {
	objlist_insert(drc->drc_ignore_objlist,
	drro->drr_object);
	err = 0;
	}
	return (err);
	}
	case DRR_FREEOBJECTS:
	{
	err = receive_read_payload_and_next_header(drc, 0, NULL);
	return (err);
	}
	case DRR_WRITE:
	{
	struct drr_write *drrw = &drc->drc_rrd->header.drr_u.drr_write;
	int size = DRR_WRITE_PAYLOAD_SIZE(drrw);
	abd_t *abd = abd_alloc_linear(size, B_FALSE);
	err = receive_read_payload_and_next_header(drc, size,
	abd_to_buf(abd));
	if (err != 0) {
	abd_free(abd);
	return (err);
	}
	drc->drc_rrd->abd = abd;
	receive_read_prefetch(drc, drrw->drr_object, drrw->drr_offset,
	drrw->drr_logical_size);
	return (err);
	}
	case DRR_WRITE_EMBEDDED:
	{
	struct drr_write_embedded *drrwe =
	&drc->drc_rrd->header.drr_u.drr_write_embedded;
	uint32_t size = P2ROUNDUP(drrwe->drr_psize, 8);
	void *buf = kmem_zalloc(size, KM_SLEEP);

	err = receive_read_payload_and_next_header(drc, size, buf);
	if (err != 0) {
	kmem_free(buf, size);
	return (err);
	}

	receive_read_prefetch(drc, drrwe->drr_object, drrwe->drr_offset,
	drrwe->drr_length);
	return (err);
	}
	case DRR_FREE:
	case DRR_REDACT:
	{
	/*
	* It might be beneficial to prefetch indirect blocks here, but
	* we don't really have the data to decide for sure.
	*/
	err = receive_read_payload_and_next_header(drc, 0, NULL);
	return (err);
	}
	case DRR_END:
	{
	struct drr_end *drre = &drc->drc_rrd->header.drr_u.drr_end;
	if (!ZIO_CHECKSUM_EQUAL(drc->drc_prev_cksum,
	drre->drr_checksum))
	return (SET_ERROR(ECKSUM));
	return (0);
	}
	case DRR_SPILL:
	{
	struct drr_spill *drrs = &drc->drc_rrd->header.drr_u.drr_spill;
	int size = DRR_SPILL_PAYLOAD_SIZE(drrs);
	abd_t *abd = abd_alloc_linear(size, B_FALSE);
	err = receive_read_payload_and_next_header(drc, size,
	abd_to_buf(abd));
	if (err != 0)
	abd_free(abd);
	else
	drc->drc_rrd->abd = abd;
	return (err);
	}
	case DRR_OBJECT_RANGE:
	{
	err = receive_read_payload_and_next_header(drc, 0, NULL);
	return (err);

	}
	default:
	return (SET_ERROR(EINVAL));
	}
	}



	static void
	dprintf_drr(struct receive_record_arg *rrd, int err)
	{
	#ifdef ZFS_DEBUG
	switch (rrd->header.drr_type) {
	case DRR_OBJECT:
	{
	struct drr_object *drro = &rrd->header.drr_u.drr_object;
	dprintf("drr_type = OBJECT obj = %llu type = %u "
	"bonustype = %u blksz = %u bonuslen = %u cksumtype = %u "
	"compress = %u dn_slots = %u err = %d\n",
	(u_longlong_t)drro->drr_object, drro->drr_type,
	drro->drr_bonustype, drro->drr_blksz, drro->drr_bonuslen,
	drro->drr_checksumtype, drro->drr_compress,
	drro->drr_dn_slots, err);
	break;
	}
	case DRR_FREEOBJECTS:
	{
	struct drr_freeobjects *drrfo =
	&rrd->header.drr_u.drr_freeobjects;
	dprintf("drr_type = FREEOBJECTS firstobj = %llu "
	"numobjs = %llu err = %d\n",
	(u_longlong_t)drrfo->drr_firstobj,
	(u_longlong_t)drrfo->drr_numobjs, err);
	break;
	}
	case DRR_WRITE:
	{
	struct drr_write *drrw = &rrd->header.drr_u.drr_write;
	dprintf("drr_type = WRITE obj = %llu type = %u offset = %llu "
	"lsize = %llu cksumtype = %u flags = %u "
	"compress = %u psize = %llu err = %d\n",
	(u_longlong_t)drrw->drr_object, drrw->drr_type,
	(u_longlong_t)drrw->drr_offset,
	(u_longlong_t)drrw->drr_logical_size,
	drrw->drr_checksumtype, drrw->drr_flags,
	drrw->drr_compressiontype,
	(u_longlong_t)drrw->drr_compressed_size, err);
	break;
	}
	case DRR_WRITE_BYREF:
	{
	struct drr_write_byref *drrwbr =
	&rrd->header.drr_u.drr_write_byref;
	dprintf("drr_type = WRITE_BYREF obj = %llu offset = %llu "
	"length = %llu toguid = %llx refguid = %llx "
	"refobject = %llu refoffset = %llu cksumtype = %u "
	"flags = %u err = %d\n",
	(u_longlong_t)drrwbr->drr_object,
	(u_longlong_t)drrwbr->drr_offset,
	(u_longlong_t)drrwbr->drr_length,
	(u_longlong_t)drrwbr->drr_toguid,
	(u_longlong_t)drrwbr->drr_refguid,
	(u_longlong_t)drrwbr->drr_refobject,
	(u_longlong_t)drrwbr->drr_refoffset,
	drrwbr->drr_checksumtype, drrwbr->drr_flags, err);
	break;
	}
	case DRR_WRITE_EMBEDDED:
	{
	struct drr_write_embedded *drrwe =
	&rrd->header.drr_u.drr_write_embedded;
	dprintf("drr_type = WRITE_EMBEDDED obj = %llu offset = %llu "
	"length = %llu compress = %u etype = %u lsize = %u "
	"psize = %u err = %d\n",
	(u_longlong_t)drrwe->drr_object,
	(u_longlong_t)drrwe->drr_offset,
	(u_longlong_t)drrwe->drr_length,
	drrwe->drr_compression, drrwe->drr_etype,
	drrwe->drr_lsize, drrwe->drr_psize, err);
	break;
	}
	case DRR_FREE:
	{
	struct drr_free *drrf = &rrd->header.drr_u.drr_free;
	dprintf("drr_type = FREE obj = %llu offset = %llu "
	"length = %lld err = %d\n",
	(u_longlong_t)drrf->drr_object,
	(u_longlong_t)drrf->drr_offset,
	(longlong_t)drrf->drr_length,
	err);
	break;
	}
	case DRR_SPILL:
	{
	struct drr_spill *drrs = &rrd->header.drr_u.drr_spill;
	dprintf("drr_type = SPILL obj = %llu length = %llu "
	"err = %d\n", (u_longlong_t)drrs->drr_object,
	(u_longlong_t)drrs->drr_length, err);
	break;
	}
	case DRR_OBJECT_RANGE:
	{
	struct drr_object_range *drror =
	&rrd->header.drr_u.drr_object_range;
	dprintf("drr_type = OBJECT_RANGE firstobj = %llu "
	"numslots = %llu flags = %u err = %d\n",
	(u_longlong_t)drror->drr_firstobj,
	(u_longlong_t)drror->drr_numslots,
	drror->drr_flags, err);
	break;
	}
	default:
	return;
	}
	#endif
	}

	/*
	* Commit the records to the pool.
	*/
	static int
	receive_process_record(struct receive_writer_arg *rwa,
	struct receive_record_arg *rrd)
	{
	int err;

	/* Processing in order, therefore bytes_read should be increasing. */
	ASSERT3U(rrd->bytes_read, >=, rwa->bytes_read);
	rwa->bytes_read = rrd->bytes_read;

	/* We can only heal write records; other ones get ignored */
	if (rwa->heal && rrd->header.drr_type != DRR_WRITE) {
	if (rrd->abd != NULL) {
	abd_free(rrd->abd);
	rrd->abd = NULL;
	} else if (rrd->payload != NULL) {
	kmem_free(rrd->payload, rrd->payload_size);
	rrd->payload = NULL;
	}
	return (0);
	}

	if (!rwa->heal && rrd->header.drr_type != DRR_WRITE) {
	err = flush_write_batch(rwa);
	if (err != 0) {
	if (rrd->abd != NULL) {
	abd_free(rrd->abd);
	rrd->abd = NULL;
	rrd->payload = NULL;
	} else if (rrd->payload != NULL) {
	kmem_free(rrd->payload, rrd->payload_size);
	rrd->payload = NULL;
	}

	return (err);
	}
	}

	switch (rrd->header.drr_type) {
	case DRR_OBJECT:
	{
	struct drr_object *drro = &rrd->header.drr_u.drr_object;
	err = receive_object(rwa, drro, rrd->payload);
	kmem_free(rrd->payload, rrd->payload_size);
	rrd->payload = NULL;
	break;
	}
	case DRR_FREEOBJECTS:
	{
	struct drr_freeobjects *drrfo =
	&rrd->header.drr_u.drr_freeobjects;
	err = receive_freeobjects(rwa, drrfo);
	break;
	}
	case DRR_WRITE:
	{
	err = receive_process_write_record(rwa, rrd);
	if (rwa->heal) {
	/*
	* If healing - always free the abd after processing
	*/
	abd_free(rrd->abd);
	rrd->abd = NULL;
	} else if (err != EAGAIN) {
	/*
	* On success, a non-healing
	* receive_process_write_record() returns
	* EAGAIN to indicate that we do not want to free
	* the rrd or arc_buf.
	*/
	ASSERT(err != 0);
	abd_free(rrd->abd);
	rrd->abd = NULL;
	}
	break;
	}
	case DRR_WRITE_EMBEDDED:
	{
	struct drr_write_embedded *drrwe =
	&rrd->header.drr_u.drr_write_embedded;
	err = receive_write_embedded(rwa, drrwe, rrd->payload);
	kmem_free(rrd->payload, rrd->payload_size);
	rrd->payload = NULL;
	break;
	}
	case DRR_FREE:
	{
	struct drr_free *drrf = &rrd->header.drr_u.drr_free;
	err = receive_free(rwa, drrf);
	break;
	}
	case DRR_SPILL:
	{
	struct drr_spill *drrs = &rrd->header.drr_u.drr_spill;
	err = receive_spill(rwa, drrs, rrd->abd);
	if (err != 0)
	abd_free(rrd->abd);
	rrd->abd = NULL;
	rrd->payload = NULL;
	break;
	}
	case DRR_OBJECT_RANGE:
	{
	struct drr_object_range *drror =
	&rrd->header.drr_u.drr_object_range;
	err = receive_object_range(rwa, drror);
	break;
	}
	case DRR_REDACT:
	{
	struct drr_redact *drrr = &rrd->header.drr_u.drr_redact;
	err = receive_redact(rwa, drrr);
	break;
	}
	default:
	err = (SET_ERROR(EINVAL));
	}

	if (err != 0)
	dprintf_drr(rrd, err);

	return (err);
	}

	/*
	* dmu_recv_stream's worker thread; pull records off the queue, and then call
	* receive_process_record When we're done, signal the main thread and exit.
	*/
	static __attribute__((noreturn)) void
	receive_writer_thread(void *arg)
	{
	struct receive_writer_arg *rwa = arg;
	struct receive_record_arg *rrd;
	fstrans_cookie_t cookie = spl_fstrans_mark();

	for (rrd = bqueue_dequeue(&rwa->q); !rrd->eos_marker;
	rrd = bqueue_dequeue(&rwa->q)) {
	/*
	* If there's an error, the main thread will stop putting things
	* on the queue, but we need to clear everything in it before we
	* can exit.
	*/
	int err = 0;
	if (rwa->err == 0) {
	err = receive_process_record(rwa, rrd);
	} else if (rrd->abd != NULL) {
	abd_free(rrd->abd);
	rrd->abd = NULL;
	rrd->payload = NULL;
	} else if (rrd->payload != NULL) {
	kmem_free(rrd->payload, rrd->payload_size);
	rrd->payload = NULL;
	}
	/*
	* EAGAIN indicates that this record has been saved (on
	* raw->write_batch), and will be used again, so we don't
	* free it.
	* When healing data we always need to free the record.
	*/
	if (err != EAGAIN \|\| rwa->heal) {
	if (rwa->err == 0)
	rwa->err = err;
	kmem_free(rrd, sizeof (*rrd));
	}
	}
	kmem_free(rrd, sizeof (*rrd));

	if (rwa->heal) {
	zio_wait(rwa->heal_pio);
	} else {
	int err = flush_write_batch(rwa);
	if (rwa->err == 0)
	rwa->err = err;
	}
	mutex_enter(&rwa->mutex);
	rwa->done = B_TRUE;
	cv_signal(&rwa->cv);
	mutex_exit(&rwa->mutex);
	spl_fstrans_unmark(cookie);
	thread_exit();
	}

	static int
	resume_check(dmu_recv_cookie_t drc, nvlist_t begin_nvl)
	{
	uint64_t val;
	objset_t *mos = dmu_objset_pool(drc->drc_os)->dp_meta_objset;
	uint64_t dsobj = dmu_objset_id(drc->drc_os);
	uint64_t resume_obj, resume_off;

	if (nvlist_lookup_uint64(begin_nvl,
	"resume_object", &resume_obj) != 0 \|\|
	nvlist_lookup_uint64(begin_nvl,
	"resume_offset", &resume_off) != 0) {
	return (SET_ERROR(EINVAL));
	}
	VERIFY0(zap_lookup(mos, dsobj,
	DS_FIELD_RESUME_OBJECT, sizeof (val), 1, &val));
	if (resume_obj != val)
	return (SET_ERROR(EINVAL));
	VERIFY0(zap_lookup(mos, dsobj,
	DS_FIELD_RESUME_OFFSET, sizeof (val), 1, &val));
	if (resume_off != val)
	return (SET_ERROR(EINVAL));

	return (0);
	}

	/*
	* Read in the stream's records, one by one, and apply them to the pool. There
	* are two threads involved; the thread that calls this function will spin up a
	* worker thread, read the records off the stream one by one, and issue
	* prefetches for any necessary indirect blocks. It will then push the records
	* onto an internal blocking queue. The worker thread will pull the records off
	* the queue, and actually write the data into the DMU. This way, the worker
	* thread doesn't have to wait for reads to complete, since everything it needs
	* (the indirect blocks) will be prefetched.
	*
	* NB: callers must call dmu_recv_end() if this succeeds.
	*/
	int
	dmu_recv_stream(dmu_recv_cookie_t drc, offset_t voffp)
	{
	int err = 0;
	struct receive_writer_arg rwa = kmem_zalloc(sizeof (rwa), KM_SLEEP);

	if (dsl_dataset_has_resume_receive_state(drc->drc_ds)) {
	uint64_t bytes = 0;
	(void) zap_lookup(drc->drc_ds->ds_dir->dd_pool->dp_meta_objset,
	drc->drc_ds->ds_object, DS_FIELD_RESUME_BYTES,
	sizeof (bytes), 1, &bytes);
	drc->drc_bytes_read += bytes;
	}

	drc->drc_ignore_objlist = objlist_create();

	/* these were verified in dmu_recv_begin */
	ASSERT3U(DMU_GET_STREAM_HDRTYPE(drc->drc_drrb->drr_versioninfo), ==,
	DMU_SUBSTREAM);
	ASSERT3U(drc->drc_drrb->drr_type, <, DMU_OST_NUMTYPES);

	ASSERT(dsl_dataset_phys(drc->drc_ds)->ds_flags & DS_FLAG_INCONSISTENT);
	ASSERT0(drc->drc_os->os_encrypted &&
	(drc->drc_featureflags & DMU_BACKUP_FEATURE_EMBED_DATA));

	/* handle DSL encryption key payload */
	if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RAW) {
	nvlist_t *keynvl = NULL;

	ASSERT(drc->drc_os->os_encrypted);
	ASSERT(drc->drc_raw);

	err = nvlist_lookup_nvlist(drc->drc_begin_nvl, "crypt_keydata",
	&keynvl);
	if (err != 0)
	goto out;

	if (!drc->drc_heal) {
	/*
	* If this is a new dataset we set the key immediately.
	* Otherwise we don't want to change the key until we
	* are sure the rest of the receive succeeded so we
	* stash the keynvl away until then.
	*/
	err = dsl_crypto_recv_raw(spa_name(drc->drc_os->os_spa),
	drc->drc_ds->ds_object, drc->drc_fromsnapobj,
	drc->drc_drrb->drr_type, keynvl, drc->drc_newfs);
	if (err != 0)
	goto out;
	}

	/* see comment in dmu_recv_end_sync() */
	drc->drc_ivset_guid = 0;
	(void) nvlist_lookup_uint64(keynvl, "to_ivset_guid",
	&drc->drc_ivset_guid);

	if (!drc->drc_newfs)
	drc->drc_keynvl = fnvlist_dup(keynvl);
	}

	if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RESUMING) {
	err = resume_check(drc, drc->drc_begin_nvl);
	if (err != 0)
	goto out;
	}

	/*
	* For compatibility with recursive send streams, we do this here,
	* rather than in dmu_recv_begin. If we pull the next header too
	* early, and it's the END record, we break the `recv_skip` logic.
	*/
	if (drc->drc_drr_begin->drr_payloadlen == 0) {
	err = receive_read_payload_and_next_header(drc, 0, NULL);
	if (err != 0)
	goto out;
	}

	/*
	* If we failed before this point we will clean up any new resume
	* state that was created. Now that we've gotten past the initial
	* checks we are ok to retain that resume state.
	*/
	drc->drc_should_save = B_TRUE;

	(void) bqueue_init(&rwa->q, zfs_recv_queue_ff,
	MAX(zfs_recv_queue_length, 2 * zfs_max_recordsize),
	offsetof(struct receive_record_arg, node));
	cv_init(&rwa->cv, NULL, CV_DEFAULT, NULL);
	mutex_init(&rwa->mutex, NULL, MUTEX_DEFAULT, NULL);
	rwa->os = drc->drc_os;
	rwa->byteswap = drc->drc_byteswap;
	rwa->heal = drc->drc_heal;
	rwa->tofs = drc->drc_tofs;
	rwa->resumable = drc->drc_resumable;
	rwa->raw = drc->drc_raw;
	rwa->spill = drc->drc_spill;
	rwa->full = (drc->drc_drr_begin->drr_u.drr_begin.drr_fromguid == 0);
	rwa->os->os_raw_receive = drc->drc_raw;
	if (drc->drc_heal) {
	rwa->heal_pio = zio_root(drc->drc_os->os_spa, NULL, NULL,
	ZIO_FLAG_GODFATHER);
	}
	list_create(&rwa->write_batch, sizeof (struct receive_record_arg),
	offsetof(struct receive_record_arg, node.bqn_node));

	(void) thread_create(NULL, 0, receive_writer_thread, rwa, 0, curproc,
	TS_RUN, minclsyspri);
	/*
	* We're reading rwa->err without locks, which is safe since we are the
	* only reader, and the worker thread is the only writer. It's ok if we
	* miss a write for an iteration or two of the loop, since the writer
	* thread will keep freeing records we send it until we send it an eos
	* marker.
	*
	* We can leave this loop in 3 ways: First, if rwa->err is
	* non-zero. In that case, the writer thread will free the rrd we just
	* pushed. Second, if we're interrupted; in that case, either it's the
	* first loop and drc->drc_rrd was never allocated, or it's later, and
	* drc->drc_rrd has been handed off to the writer thread who will free
	* it. Finally, if receive_read_record fails or we're at the end of the
	* stream, then we free drc->drc_rrd and exit.
	*/
	while (rwa->err == 0) {
	if (issig(JUSTLOOKING) && issig(FORREAL)) {
	err = SET_ERROR(EINTR);
	break;
	}

	ASSERT3P(drc->drc_rrd, ==, NULL);
	drc->drc_rrd = drc->drc_next_rrd;
	drc->drc_next_rrd = NULL;
	/* Allocates and loads header into drc->drc_next_rrd */
	err = receive_read_record(drc);

	if (drc->drc_rrd->header.drr_type == DRR_END \|\| err != 0) {
	kmem_free(drc->drc_rrd, sizeof (*drc->drc_rrd));
	drc->drc_rrd = NULL;
	break;
	}

	bqueue_enqueue(&rwa->q, drc->drc_rrd,
	sizeof (struct receive_record_arg) +
	drc->drc_rrd->payload_size);
	drc->drc_rrd = NULL;
	}

	ASSERT3P(drc->drc_rrd, ==, NULL);
	drc->drc_rrd = kmem_zalloc(sizeof (*drc->drc_rrd), KM_SLEEP);
	drc->drc_rrd->eos_marker = B_TRUE;
	bqueue_enqueue_flush(&rwa->q, drc->drc_rrd, 1);

	mutex_enter(&rwa->mutex);
	while (!rwa->done) {
	/*
	* We need to use cv_wait_sig() so that any process that may
	* be sleeping here can still fork.
	*/
	(void) cv_wait_sig(&rwa->cv, &rwa->mutex);
	}
	mutex_exit(&rwa->mutex);

	/*
	* If we are receiving a full stream as a clone, all object IDs which
	* are greater than the maximum ID referenced in the stream are
	* by definition unused and must be freed.
	*/
	if (drc->drc_clone && drc->drc_drrb->drr_fromguid == 0) {
	uint64_t obj = rwa->max_object + 1;
	int free_err = 0;
	int next_err = 0;

	while (next_err == 0) {
	free_err = dmu_free_long_object(rwa->os, obj);
	if (free_err != 0 && free_err != ENOENT)
	break;

	next_err = dmu_object_next(rwa->os, &obj, FALSE, 0);
	}

	if (err == 0) {
	if (free_err != 0 && free_err != ENOENT)
	err = free_err;
	else if (next_err != ESRCH)
	err = next_err;
	}
	}

	cv_destroy(&rwa->cv);
	mutex_destroy(&rwa->mutex);
	bqueue_destroy(&rwa->q);
	list_destroy(&rwa->write_batch);
	if (err == 0)
	err = rwa->err;

	out:
	/*
	* If we hit an error before we started the receive_writer_thread
	* we need to clean up the next_rrd we create by processing the
	* DRR_BEGIN record.
	*/
	if (drc->drc_next_rrd != NULL)
	kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd));

	/*
	* The objset will be invalidated by dmu_recv_end() when we do
	* dsl_dataset_clone_swap_sync_impl().
	*/
	drc->drc_os = NULL;

	kmem_free(rwa, sizeof (*rwa));
	nvlist_free(drc->drc_begin_nvl);

	if (err != 0) {
	/*
	* Clean up references. If receive is not resumable,
	* destroy what we created, so we don't leave it in
	* the inconsistent state.
	*/
	dmu_recv_cleanup_ds(drc);
	nvlist_free(drc->drc_keynvl);
	}

	objlist_destroy(drc->drc_ignore_objlist);
	drc->drc_ignore_objlist = NULL;
	*voffp = drc->drc_voff;
	return (err);
	}

	static int
	dmu_recv_end_check(void arg, dmu_tx_t tx)
	{
	dmu_recv_cookie_t *drc = arg;
	dsl_pool_t *dp = dmu_tx_pool(tx);
	int error;

	ASSERT3P(drc->drc_ds->ds_owner, ==, dmu_recv_tag);

	if (drc->drc_heal) {
	error = 0;
	} else if (!drc->drc_newfs) {
	dsl_dataset_t *origin_head;

	error = dsl_dataset_hold(dp, drc->drc_tofs, FTAG, &origin_head);
	if (error != 0)
	return (error);
	if (drc->drc_force) {
	/*
	* We will destroy any snapshots in tofs (i.e. before
	* origin_head) that are after the origin (which is
	* the snap before drc_ds, because drc_ds can not
	* have any snaps of its own).
	*/
	uint64_t obj;

	obj = dsl_dataset_phys(origin_head)->ds_prev_snap_obj;
	while (obj !=
	dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj) {
	dsl_dataset_t *snap;
	error = dsl_dataset_hold_obj(dp, obj, FTAG,
	&snap);
	if (error != 0)
	break;
	if (snap->ds_dir != origin_head->ds_dir)
	error = SET_ERROR(EINVAL);
	if (error == 0) {
	error = dsl_destroy_snapshot_check_impl(
	snap, B_FALSE);
	}
	obj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
	dsl_dataset_rele(snap, FTAG);
	if (error != 0)
	break;
	}
	if (error != 0) {
	dsl_dataset_rele(origin_head, FTAG);
	return (error);
	}
	}
	if (drc->drc_keynvl != NULL) {
	error = dsl_crypto_recv_raw_key_check(drc->drc_ds,
	drc->drc_keynvl, tx);
	if (error != 0) {
	dsl_dataset_rele(origin_head, FTAG);
	return (error);
	}
	}

	error = dsl_dataset_clone_swap_check_impl(drc->drc_ds,
	origin_head, drc->drc_force, drc->drc_owner, tx);
	if (error != 0) {
	dsl_dataset_rele(origin_head, FTAG);
	return (error);
	}
	error = dsl_dataset_snapshot_check_impl(origin_head,
	drc->drc_tosnap, tx, B_TRUE, 1,
	drc->drc_cred, drc->drc_proc);
	dsl_dataset_rele(origin_head, FTAG);
	if (error != 0)
	return (error);

	error = dsl_destroy_head_check_impl(drc->drc_ds, 1);
	} else {
	error = dsl_dataset_snapshot_check_impl(drc->drc_ds,
	drc->drc_tosnap, tx, B_TRUE, 1,
	drc->drc_cred, drc->drc_proc);
	}
	return (error);
	}

	static void
	dmu_recv_end_sync(void arg, dmu_tx_t tx)
	{
	dmu_recv_cookie_t *drc = arg;
	dsl_pool_t *dp = dmu_tx_pool(tx);
	boolean_t encrypted = drc->drc_ds->ds_dir->dd_crypto_obj != 0;
	uint64_t newsnapobj = 0;

	spa_history_log_internal_ds(drc->drc_ds, "finish receiving",
	tx, "snap=%s", drc->drc_tosnap);
	drc->drc_ds->ds_objset->os_raw_receive = B_FALSE;

	if (drc->drc_heal) {
	if (drc->drc_keynvl != NULL) {
	nvlist_free(drc->drc_keynvl);
	drc->drc_keynvl = NULL;
	}
	} else if (!drc->drc_newfs) {
	dsl_dataset_t *origin_head;

	VERIFY0(dsl_dataset_hold(dp, drc->drc_tofs, FTAG,
	&origin_head));

	if (drc->drc_force) {
	/*
	* Destroy any snapshots of drc_tofs (origin_head)
	* after the origin (the snap before drc_ds).
	*/
	uint64_t obj;

	obj = dsl_dataset_phys(origin_head)->ds_prev_snap_obj;
	while (obj !=
	dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj) {
	dsl_dataset_t *snap;
	VERIFY0(dsl_dataset_hold_obj(dp, obj, FTAG,
	&snap));
	ASSERT3P(snap->ds_dir, ==, origin_head->ds_dir);
	obj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
	dsl_destroy_snapshot_sync_impl(snap,
	B_FALSE, tx);
	dsl_dataset_rele(snap, FTAG);
	}
	}
	if (drc->drc_keynvl != NULL) {
	dsl_crypto_recv_raw_key_sync(drc->drc_ds,
	drc->drc_keynvl, tx);
	nvlist_free(drc->drc_keynvl);
	drc->drc_keynvl = NULL;
	}

	VERIFY3P(drc->drc_ds->ds_prev, ==,
	origin_head->ds_prev);

	dsl_dataset_clone_swap_sync_impl(drc->drc_ds,
	origin_head, tx);
	/*
	* The objset was evicted by dsl_dataset_clone_swap_sync_impl,
	* so drc_os is no longer valid.
	*/
	drc->drc_os = NULL;

	dsl_dataset_snapshot_sync_impl(origin_head,
	drc->drc_tosnap, tx);

	/* set snapshot's creation time and guid */
	dmu_buf_will_dirty(origin_head->ds_prev->ds_dbuf, tx);
	dsl_dataset_phys(origin_head->ds_prev)->ds_creation_time =
	drc->drc_drrb->drr_creation_time;
	dsl_dataset_phys(origin_head->ds_prev)->ds_guid =
	drc->drc_drrb->drr_toguid;
	dsl_dataset_phys(origin_head->ds_prev)->ds_flags &=
	~DS_FLAG_INCONSISTENT;

	dmu_buf_will_dirty(origin_head->ds_dbuf, tx);
	dsl_dataset_phys(origin_head)->ds_flags &=
	~DS_FLAG_INCONSISTENT;

	newsnapobj =
	dsl_dataset_phys(origin_head)->ds_prev_snap_obj;

	dsl_dataset_rele(origin_head, FTAG);
	dsl_destroy_head_sync_impl(drc->drc_ds, tx);

	if (drc->drc_owner != NULL)
	VERIFY3P(origin_head->ds_owner, ==, drc->drc_owner);
	} else {
	dsl_dataset_t *ds = drc->drc_ds;

	dsl_dataset_snapshot_sync_impl(ds, drc->drc_tosnap, tx);

	/* set snapshot's creation time and guid */
	dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx);
	dsl_dataset_phys(ds->ds_prev)->ds_creation_time =
	drc->drc_drrb->drr_creation_time;
	dsl_dataset_phys(ds->ds_prev)->ds_guid =
	drc->drc_drrb->drr_toguid;
	dsl_dataset_phys(ds->ds_prev)->ds_flags &=
	~DS_FLAG_INCONSISTENT;

	dmu_buf_will_dirty(ds->ds_dbuf, tx);
	dsl_dataset_phys(ds)->ds_flags &= ~DS_FLAG_INCONSISTENT;
	if (dsl_dataset_has_resume_receive_state(ds)) {
	(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
	DS_FIELD_RESUME_FROMGUID, tx);
	(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
	DS_FIELD_RESUME_OBJECT, tx);
	(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
	DS_FIELD_RESUME_OFFSET, tx);
	(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
	DS_FIELD_RESUME_BYTES, tx);
	(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
	DS_FIELD_RESUME_TOGUID, tx);
	(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
	DS_FIELD_RESUME_TONAME, tx);
	(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
	DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS, tx);
	}
	newsnapobj =
	dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj;
	}

	/*
	* If this is a raw receive, the crypt_keydata nvlist will include
	* a to_ivset_guid for us to set on the new snapshot. This value
	* will override the value generated by the snapshot code. However,
	* this value may not be present, because older implementations of
	* the raw send code did not include this value, and we are still
	* allowed to receive them if the zfs_disable_ivset_guid_check
	* tunable is set, in which case we will leave the newly-generated
	* value.
	*/
	if (!drc->drc_heal && drc->drc_raw && drc->drc_ivset_guid != 0) {
	dmu_object_zapify(dp->dp_meta_objset, newsnapobj,
	DMU_OT_DSL_DATASET, tx);
	VERIFY0(zap_update(dp->dp_meta_objset, newsnapobj,
	DS_FIELD_IVSET_GUID, sizeof (uint64_t), 1,
	&drc->drc_ivset_guid, tx));
	}

	/*
	* Release the hold from dmu_recv_begin. This must be done before
	* we return to open context, so that when we free the dataset's dnode
	* we can evict its bonus buffer. Since the dataset may be destroyed
	* at this point (and therefore won't have a valid pointer to the spa)
	* we release the key mapping manually here while we do have a valid
	* pointer, if it exists.
	*/
	if (!drc->drc_raw && encrypted) {
	(void) spa_keystore_remove_mapping(dmu_tx_pool(tx)->dp_spa,
	drc->drc_ds->ds_object, drc->drc_ds);
	}
	dsl_dataset_disown(drc->drc_ds, 0, dmu_recv_tag);
	drc->drc_ds = NULL;
	}

	static int dmu_recv_end_modified_blocks = 3;

	static int
	dmu_recv_existing_end(dmu_recv_cookie_t *drc)
	{
	#ifdef _KERNEL
	/*
	* We will be destroying the ds; make sure its origin is unmounted if
	* necessary.
	*/
	char name[ZFS_MAX_DATASET_NAME_LEN];
	dsl_dataset_name(drc->drc_ds, name);
	zfs_destroy_unmount_origin(name);
	#endif

	return (dsl_sync_task(drc->drc_tofs,
	dmu_recv_end_check, dmu_recv_end_sync, drc,
	dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL));
	}

	static int
	dmu_recv_new_end(dmu_recv_cookie_t *drc)
	{
	return (dsl_sync_task(drc->drc_tofs,
	dmu_recv_end_check, dmu_recv_end_sync, drc,
	dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL));
	}

	int
	dmu_recv_end(dmu_recv_cookie_t drc, void owner)
	{
	int error;

	drc->drc_owner = owner;

	if (drc->drc_newfs)
	error = dmu_recv_new_end(drc);
	else
	error = dmu_recv_existing_end(drc);

	if (error != 0) {
	dmu_recv_cleanup_ds(drc);
	nvlist_free(drc->drc_keynvl);
	} else if (!drc->drc_heal) {
	if (drc->drc_newfs) {
	zvol_create_minor(drc->drc_tofs);
	}
	char *snapname = kmem_asprintf("%s@%s",
	drc->drc_tofs, drc->drc_tosnap);
	zvol_create_minor(snapname);
	kmem_strfree(snapname);
	}
	return (error);
	}

	/*
	* Return TRUE if this objset is currently being received into.
	*/
	boolean_t
	dmu_objset_is_receiving(objset_t *os)
	{
	return (os->os_dsl_dataset != NULL &&
	os->os_dsl_dataset->ds_owner == dmu_recv_tag);
	}

	ZFS_MODULE_PARAM(zfs_recv, zfs_recv_, queue_length, UINT, ZMOD_RW,
	"Maximum receive queue length");

	ZFS_MODULE_PARAM(zfs_recv, zfs_recv_, queue_ff, UINT, ZMOD_RW,
	"Receive queue fill fraction");

	ZFS_MODULE_PARAM(zfs_recv, zfs_recv_, write_batch_size, UINT, ZMOD_RW,
	"Maximum amount of writes to batch into one transaction");

	ZFS_MODULE_PARAM(zfs_recv, zfs_recv_, best_effort_corrective, INT, ZMOD_RW,
	"Ignore errors during corrective receive");
	/* END CSTYLED */
	diff --git a/sys/contrib/openzfs/module/zfs/dmu_zfetch.c b/sys/contrib/openzfs/module/zfs/dmu_zfetch.c
	index d0acaf502066..ed50f1889b59 100644
	--- a/sys/contrib/openzfs/module/zfs/dmu_zfetch.c
	+++ b/sys/contrib/openzfs/module/zfs/dmu_zfetch.c
	@@ -1,587 +1,773 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
	* Use is subject to license terms.
	*/

	/*
	* Copyright (c) 2013, 2017 by Delphix. All rights reserved.
	*/

	#include <sys/zfs_context.h>
	#include <sys/arc_impl.h>
	#include <sys/dnode.h>
	#include <sys/dmu_objset.h>
	#include <sys/dmu_zfetch.h>
	#include <sys/dmu.h>
	#include <sys/dbuf.h>
	#include <sys/kstat.h>
	#include <sys/wmsum.h>

	/*
	* This tunable disables predictive prefetch. Note that it leaves "prescient"
	* prefetch (e.g. prefetch for zfs send) intact. Unlike predictive prefetch,
	* prescient prefetch never issues i/os that end up not being needed,
	* so it can't hurt performance.
	*/

	static int zfs_prefetch_disable = B_FALSE;

	/* max # of streams per zfetch */
	static unsigned int zfetch_max_streams = 8;
	/* min time before stream reclaim */
	static unsigned int zfetch_min_sec_reap = 1;
	/* max time before stream delete */
	static unsigned int zfetch_max_sec_reap = 2;
	#ifdef _ILP32
	/* min bytes to prefetch per stream (default 2MB) */
	static unsigned int zfetch_min_distance = 2 * 1024 * 1024;
	/* max bytes to prefetch per stream (default 8MB) */
	unsigned int zfetch_max_distance = 8 * 1024 * 1024;
	#else
	/* min bytes to prefetch per stream (default 4MB) */
	static unsigned int zfetch_min_distance = 4 * 1024 * 1024;
	/* max bytes to prefetch per stream (default 64MB) */
	unsigned int zfetch_max_distance = 64 * 1024 * 1024;
	#endif
	/* max bytes to prefetch indirects for per stream (default 64MB) */
	unsigned int zfetch_max_idistance = 64 * 1024 * 1024;
	+/* max request reorder distance within a stream (default 16MB) */
	+unsigned int zfetch_max_reorder = 16 * 1024 * 1024;
	+/* Max log2 fraction of holes in a stream */
	+unsigned int zfetch_hole_shift = 2;

	typedef struct zfetch_stats {
	kstat_named_t zfetchstat_hits;
	+ kstat_named_t zfetchstat_future;
	+ kstat_named_t zfetchstat_stride;
	+ kstat_named_t zfetchstat_past;
	kstat_named_t zfetchstat_misses;
	kstat_named_t zfetchstat_max_streams;
	kstat_named_t zfetchstat_io_issued;
	kstat_named_t zfetchstat_io_active;
	} zfetch_stats_t;

	static zfetch_stats_t zfetch_stats = {
	{ "hits", KSTAT_DATA_UINT64 },
	+ { "future", KSTAT_DATA_UINT64 },
	+ { "stride", KSTAT_DATA_UINT64 },
	+ { "past", KSTAT_DATA_UINT64 },
	{ "misses", KSTAT_DATA_UINT64 },
	{ "max_streams", KSTAT_DATA_UINT64 },
	{ "io_issued", KSTAT_DATA_UINT64 },
	{ "io_active", KSTAT_DATA_UINT64 },
	};

	struct {
	wmsum_t zfetchstat_hits;
	+ wmsum_t zfetchstat_future;
	+ wmsum_t zfetchstat_stride;
	+ wmsum_t zfetchstat_past;
	wmsum_t zfetchstat_misses;
	wmsum_t zfetchstat_max_streams;
	wmsum_t zfetchstat_io_issued;
	aggsum_t zfetchstat_io_active;
	} zfetch_sums;

	#define ZFETCHSTAT_BUMP(stat) \
	wmsum_add(&zfetch_sums.stat, 1)
	#define ZFETCHSTAT_ADD(stat, val) \
	wmsum_add(&zfetch_sums.stat, val)


	static kstat_t *zfetch_ksp;

	static int
	zfetch_kstats_update(kstat_t *ksp, int rw)
	{
	zfetch_stats_t *zs = ksp->ks_data;

	if (rw == KSTAT_WRITE)
	return (EACCES);
	zs->zfetchstat_hits.value.ui64 =
	wmsum_value(&zfetch_sums.zfetchstat_hits);
	+ zs->zfetchstat_future.value.ui64 =
	+ wmsum_value(&zfetch_sums.zfetchstat_future);
	+ zs->zfetchstat_stride.value.ui64 =
	+ wmsum_value(&zfetch_sums.zfetchstat_stride);
	+ zs->zfetchstat_past.value.ui64 =
	+ wmsum_value(&zfetch_sums.zfetchstat_past);
	zs->zfetchstat_misses.value.ui64 =
	wmsum_value(&zfetch_sums.zfetchstat_misses);
	zs->zfetchstat_max_streams.value.ui64 =
	wmsum_value(&zfetch_sums.zfetchstat_max_streams);
	zs->zfetchstat_io_issued.value.ui64 =
	wmsum_value(&zfetch_sums.zfetchstat_io_issued);
	zs->zfetchstat_io_active.value.ui64 =
	aggsum_value(&zfetch_sums.zfetchstat_io_active);
	return (0);
	}

	void
	zfetch_init(void)
	{
	wmsum_init(&zfetch_sums.zfetchstat_hits, 0);
	+ wmsum_init(&zfetch_sums.zfetchstat_future, 0);
	+ wmsum_init(&zfetch_sums.zfetchstat_stride, 0);
	+ wmsum_init(&zfetch_sums.zfetchstat_past, 0);
	wmsum_init(&zfetch_sums.zfetchstat_misses, 0);
	wmsum_init(&zfetch_sums.zfetchstat_max_streams, 0);
	wmsum_init(&zfetch_sums.zfetchstat_io_issued, 0);
	aggsum_init(&zfetch_sums.zfetchstat_io_active, 0);

	zfetch_ksp = kstat_create("zfs", 0, "zfetchstats", "misc",
	KSTAT_TYPE_NAMED, sizeof (zfetch_stats) / sizeof (kstat_named_t),
	KSTAT_FLAG_VIRTUAL);

	if (zfetch_ksp != NULL) {
	zfetch_ksp->ks_data = &zfetch_stats;
	zfetch_ksp->ks_update = zfetch_kstats_update;
	kstat_install(zfetch_ksp);
	}
	}

	void
	zfetch_fini(void)
	{
	if (zfetch_ksp != NULL) {
	kstat_delete(zfetch_ksp);
	zfetch_ksp = NULL;
	}

	wmsum_fini(&zfetch_sums.zfetchstat_hits);
	+ wmsum_fini(&zfetch_sums.zfetchstat_future);
	+ wmsum_fini(&zfetch_sums.zfetchstat_stride);
	+ wmsum_fini(&zfetch_sums.zfetchstat_past);
	wmsum_fini(&zfetch_sums.zfetchstat_misses);
	wmsum_fini(&zfetch_sums.zfetchstat_max_streams);
	wmsum_fini(&zfetch_sums.zfetchstat_io_issued);
	ASSERT0(aggsum_value(&zfetch_sums.zfetchstat_io_active));
	aggsum_fini(&zfetch_sums.zfetchstat_io_active);
	}

	/*
	* This takes a pointer to a zfetch structure and a dnode. It performs the
	* necessary setup for the zfetch structure, grokking data from the
	* associated dnode.
	*/
	void
	dmu_zfetch_init(zfetch_t zf, dnode_t dno)
	{
	if (zf == NULL)
	return;
	zf->zf_dnode = dno;
	zf->zf_numstreams = 0;

	list_create(&zf->zf_stream, sizeof (zstream_t),
	offsetof(zstream_t, zs_node));

	mutex_init(&zf->zf_lock, NULL, MUTEX_DEFAULT, NULL);
	}

	static void
	dmu_zfetch_stream_fini(zstream_t *zs)
	{
	ASSERT(!list_link_active(&zs->zs_node));
	zfs_refcount_destroy(&zs->zs_callers);
	zfs_refcount_destroy(&zs->zs_refs);
	kmem_free(zs, sizeof (*zs));
	}

	static void
	dmu_zfetch_stream_remove(zfetch_t zf, zstream_t zs)
	{
	ASSERT(MUTEX_HELD(&zf->zf_lock));
	list_remove(&zf->zf_stream, zs);
	zf->zf_numstreams--;
	membar_producer();
	if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0)
	dmu_zfetch_stream_fini(zs);
	}

	/*
	* Clean-up state associated with a zfetch structure (e.g. destroy the
	* streams). This doesn't free the zfetch_t itself, that's left to the caller.
	*/
	void
	dmu_zfetch_fini(zfetch_t *zf)
	{
	zstream_t *zs;

	mutex_enter(&zf->zf_lock);
	while ((zs = list_head(&zf->zf_stream)) != NULL)
	dmu_zfetch_stream_remove(zf, zs);
	mutex_exit(&zf->zf_lock);
	list_destroy(&zf->zf_stream);
	mutex_destroy(&zf->zf_lock);

	zf->zf_dnode = NULL;
	}

	/*
	* If there aren't too many active streams already, create one more.
	* In process delete/reuse all streams without hits for zfetch_max_sec_reap.
	* If needed, reuse oldest stream without hits for zfetch_min_sec_reap or ever.
	* The "blkid" argument is the next block that we expect this stream to access.
	*/
	static void
	dmu_zfetch_stream_create(zfetch_t *zf, uint64_t blkid)
	{
	zstream_t zs, zs_next, *zs_old = NULL;
	- hrtime_t now = gethrtime(), t;
	+ uint_t now = gethrestime_sec(), t;

	ASSERT(MUTEX_HELD(&zf->zf_lock));

	/*
	* Delete too old streams, reusing the first found one.
	*/
	- t = now - SEC2NSEC(zfetch_max_sec_reap);
	+ t = now - zfetch_max_sec_reap;
	for (zs = list_head(&zf->zf_stream); zs != NULL; zs = zs_next) {
	zs_next = list_next(&zf->zf_stream, zs);
	/*
	* Skip if still active. 1 -- zf_stream reference.
	*/
	- if (zfs_refcount_count(&zs->zs_refs) != 1)
	+ if ((int)(zs->zs_atime - t) >= 0)
	continue;
	- if (zs->zs_atime > t)
	+ if (zfs_refcount_count(&zs->zs_refs) != 1)
	continue;
	if (zs_old)
	dmu_zfetch_stream_remove(zf, zs);
	else
	zs_old = zs;
	}
	if (zs_old) {
	zs = zs_old;
	+ list_remove(&zf->zf_stream, zs);
	goto reuse;
	}

	/*
	* The maximum number of streams is normally zfetch_max_streams,
	* but for small files we lower it such that it's at least possible
	* for all the streams to be non-overlapping.
	*/
	uint32_t max_streams = MAX(1, MIN(zfetch_max_streams,
	- zf->zf_dnode->dn_maxblkid * zf->zf_dnode->dn_datablksz /
	+ (zf->zf_dnode->dn_maxblkid << zf->zf_dnode->dn_datablkshift) /
	zfetch_max_distance));
	if (zf->zf_numstreams >= max_streams) {
	- t = now - SEC2NSEC(zfetch_min_sec_reap);
	+ t = now - zfetch_min_sec_reap;
	for (zs = list_head(&zf->zf_stream); zs != NULL;
	zs = list_next(&zf->zf_stream, zs)) {
	- if (zfs_refcount_count(&zs->zs_refs) != 1)
	+ if ((int)(zs->zs_atime - t) >= 0)
	continue;
	- if (zs->zs_atime > t)
	+ if (zfs_refcount_count(&zs->zs_refs) != 1)
	continue;
	- if (zs_old == NULL \|\| zs->zs_atime < zs_old->zs_atime)
	+ if (zs_old == NULL \|\|
	+ (int)(zs_old->zs_atime - zs->zs_atime) >= 0)
	zs_old = zs;
	}
	if (zs_old) {
	zs = zs_old;
	+ list_remove(&zf->zf_stream, zs);
	goto reuse;
	}
	ZFETCHSTAT_BUMP(zfetchstat_max_streams);
	return;
	}

	zs = kmem_zalloc(sizeof (*zs), KM_SLEEP);
	- zs->zs_fetch = zf;
	zfs_refcount_create(&zs->zs_callers);
	zfs_refcount_create(&zs->zs_refs);
	/* One reference for zf_stream. */
	zfs_refcount_add(&zs->zs_refs, NULL);
	zf->zf_numstreams++;
	- list_insert_head(&zf->zf_stream, zs);

	reuse:
	+ list_insert_head(&zf->zf_stream, zs);
	zs->zs_blkid = blkid;
	+ /* Allow immediate stream reuse until first hit. */
	+ zs->zs_atime = now - zfetch_min_sec_reap;
	+ memset(zs->zs_ranges, 0, sizeof (zs->zs_ranges));
	zs->zs_pf_dist = 0;
	+ zs->zs_ipf_dist = 0;
	zs->zs_pf_start = blkid;
	zs->zs_pf_end = blkid;
	- zs->zs_ipf_dist = 0;
	zs->zs_ipf_start = blkid;
	zs->zs_ipf_end = blkid;
	- /* Allow immediate stream reuse until first hit. */
	- zs->zs_atime = now - SEC2NSEC(zfetch_min_sec_reap);
	zs->zs_missed = B_FALSE;
	zs->zs_more = B_FALSE;
	}

	static void
	dmu_zfetch_done(void *arg, uint64_t level, uint64_t blkid, boolean_t io_issued)
	{
	zstream_t *zs = arg;

	if (io_issued && level == 0 && blkid < zs->zs_blkid)
	zs->zs_more = B_TRUE;
	if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0)
	dmu_zfetch_stream_fini(zs);
	aggsum_add(&zfetch_sums.zfetchstat_io_active, -1);
	}

	+/*
	+ * Process stream hit access for nblks blocks starting at zs_blkid. Return
	+ * number of blocks to proceed for after aggregation with future ranges.
	+ */
	+static uint64_t
	+dmu_zfetch_hit(zstream_t *zs, uint64_t nblks)
	+{
	+ uint_t i, j;
	+
	+ /* Optimize sequential accesses (no future ranges). */
	+ if (zs->zs_ranges[0].start == 0)
	+ goto done;
	+
	+ /* Look for intersections with further ranges. */
	+ for (i = 0; i < ZFETCH_RANGES; i++) {
	+ zsrange_t *r = &zs->zs_ranges[i];
	+ if (r->start == 0 \|\| r->start > nblks)
	+ break;
	+ if (r->end >= nblks) {
	+ nblks = r->end;
	+ i++;
	+ break;
	+ }
	+ }
	+
	+ /* Delete all found intersecting ranges, updates remaining. */
	+ for (j = 0; i < ZFETCH_RANGES; i++, j++) {
	+ if (zs->zs_ranges[i].start == 0)
	+ break;
	+ ASSERT3U(zs->zs_ranges[i].start, >, nblks);
	+ ASSERT3U(zs->zs_ranges[i].end, >, nblks);
	+ zs->zs_ranges[j].start = zs->zs_ranges[i].start - nblks;
	+ zs->zs_ranges[j].end = zs->zs_ranges[i].end - nblks;
	+ }
	+ if (j < ZFETCH_RANGES) {
	+ zs->zs_ranges[j].start = 0;
	+ zs->zs_ranges[j].end = 0;
	+ }
	+
	+done:
	+ zs->zs_blkid += nblks;
	+ return (nblks);
	+}
	+
	+/*
	+ * Process future stream access for nblks blocks starting at blkid. Return
	+ * number of blocks to proceed for if future ranges reach fill threshold.
	+ */
	+static uint64_t
	+dmu_zfetch_future(zstream_t *zs, uint64_t blkid, uint64_t nblks)
	+{
	+ ASSERT3U(blkid, >, zs->zs_blkid);
	+ blkid -= zs->zs_blkid;
	+ ASSERT3U(blkid + nblks, <=, UINT16_MAX);
	+
	+ /* Search for first and last intersection or insert point. */
	+ uint_t f = ZFETCH_RANGES, l = 0, i;
	+ for (i = 0; i < ZFETCH_RANGES; i++) {
	+ zsrange_t *r = &zs->zs_ranges[i];
	+ if (r->start == 0 \|\| r->start > blkid + nblks)
	+ break;
	+ if (r->end < blkid)
	+ continue;
	+ if (f > i)
	+ f = i;
	+ if (l < i)
	+ l = i;
	+ }
	+ if (f <= l) {
	+ /* Got some intersecting range, expand it if needed. */
	+ if (zs->zs_ranges[f].start > blkid)
	+ zs->zs_ranges[f].start = blkid;
	+ zs->zs_ranges[f].end = MAX(zs->zs_ranges[l].end, blkid + nblks);
	+ if (f < l) {
	+ /* Got more than one intersection, remove others. */
	+ for (f++, l++; l < ZFETCH_RANGES; f++, l++) {
	+ zs->zs_ranges[f].start = zs->zs_ranges[l].start;
	+ zs->zs_ranges[f].end = zs->zs_ranges[l].end;
	+ }
	+ zs->zs_ranges[f].start = 0;
	+ zs->zs_ranges[f].end = 0;
	+ }
	+ } else if (i < ZFETCH_RANGES) {
	+ /* Got no intersecting ranges, insert new one. */
	+ for (l = ZFETCH_RANGES - 1; l > i; l--) {
	+ zs->zs_ranges[l].start = zs->zs_ranges[l - 1].start;
	+ zs->zs_ranges[l].end = zs->zs_ranges[l - 1].end;
	+ }
	+ zs->zs_ranges[i].start = blkid;
	+ zs->zs_ranges[i].end = blkid + nblks;
	+ } else {
	+ /* No space left to insert. Drop the range. */
	+ return (0);
	+ }
	+
	+ /* Check if with the new access addition we reached fill threshold. */
	+ if (zfetch_hole_shift >= 16)
	+ return (0);
	+ uint_t hole = 0;
	+ for (i = f = l = 0; i < ZFETCH_RANGES; i++) {
	+ zsrange_t *r = &zs->zs_ranges[i];
	+ if (r->start == 0)
	+ break;
	+ hole += r->start - f;
	+ f = r->end;
	+ if (hole <= r->end >> zfetch_hole_shift)
	+ l = r->end;
	+ }
	+ if (l > 0)
	+ return (dmu_zfetch_hit(zs, l));
	+
	+ return (0);
	+}
	+
	/*
	* This is the predictive prefetch entry point. dmu_zfetch_prepare()
	* associates dnode access specified with blkid and nblks arguments with
	* prefetch stream, predicts further accesses based on that stats and returns
	* the stream pointer on success. That pointer must later be passed to
	* dmu_zfetch_run() to initiate the speculative prefetch for the stream and
	* release it. dmu_zfetch() is a wrapper for simple cases when window between
	* prediction and prefetch initiation is not needed.
	* fetch_data argument specifies whether actual data blocks should be fetched:
	* FALSE -- prefetch only indirect blocks for predicted data blocks;
	* TRUE -- prefetch predicted data blocks plus following indirect blocks.
	*/
	zstream_t *
	dmu_zfetch_prepare(zfetch_t *zf, uint64_t blkid, uint64_t nblks,
	boolean_t fetch_data, boolean_t have_lock)
	{
	zstream_t *zs;
	spa_t *spa = zf->zf_dnode->dn_objset->os_spa;
	+ zfs_prefetch_type_t os_prefetch = zf->zf_dnode->dn_objset->os_prefetch;

	- if (zfs_prefetch_disable)
	+ if (zfs_prefetch_disable \|\| os_prefetch == ZFS_PREFETCH_NONE)
	return (NULL);
	+
	+ if (os_prefetch == ZFS_PREFETCH_METADATA)
	+ fetch_data = B_FALSE;
	+
	/*
	* If we haven't yet loaded the indirect vdevs' mappings, we
	* can only read from blocks that we carefully ensure are on
	* concrete vdevs (or previously-loaded indirect vdevs). So we
	* can't allow the predictive prefetcher to attempt reads of other
	* blocks (e.g. of the MOS's dnode object).
	*/
	if (!spa_indirect_vdevs_loaded(spa))
	return (NULL);

	/*
	* As a fast path for small (single-block) files, ignore access
	* to the first block.
	*/
	if (!have_lock && blkid == 0)
	return (NULL);

	if (!have_lock)
	rw_enter(&zf->zf_dnode->dn_struct_rwlock, RW_READER);

	/*
	* A fast path for small files for which no prefetch will
	* happen.
	*/
	uint64_t maxblkid = zf->zf_dnode->dn_maxblkid;
	if (maxblkid < 2) {
	if (!have_lock)
	rw_exit(&zf->zf_dnode->dn_struct_rwlock);
	return (NULL);
	}
	mutex_enter(&zf->zf_lock);

	/*
	- * Find matching prefetch stream. Depending on whether the accesses
	+ * Find perfect prefetch stream. Depending on whether the accesses
	* are block-aligned, first block of the new access may either follow
	* the last block of the previous access, or be equal to it.
	*/
	+ unsigned int dbs = zf->zf_dnode->dn_datablkshift;
	+ uint64_t end_blkid = blkid + nblks;
	for (zs = list_head(&zf->zf_stream); zs != NULL;
	zs = list_next(&zf->zf_stream, zs)) {
	if (blkid == zs->zs_blkid) {
	- break;
	+ goto hit;
	} else if (blkid + 1 == zs->zs_blkid) {
	blkid++;
	nblks--;
	- break;
	+ goto hit;
	}
	}

	/*
	- * If the file is ending, remove the matching stream if found.
	- * If not found then it is too late to create a new one now.
	+ * Find close enough prefetch stream. Access crossing stream position
	+ * is a hit in its new part. Access ahead of stream position considered
	+ * a hit for metadata prefetch, since we do not care about fill percent,
	+ * or stored for future otherwise. Access behind stream position is
	+ * silently ignored, since we already skipped it reaching fill percent.
	*/
	- uint64_t end_of_access_blkid = blkid + nblks;
	- if (end_of_access_blkid >= maxblkid) {
	- if (zs != NULL)
	- dmu_zfetch_stream_remove(zf, zs);
	- mutex_exit(&zf->zf_lock);
	- if (!have_lock)
	- rw_exit(&zf->zf_dnode->dn_struct_rwlock);
	- return (NULL);
	+ uint_t max_reorder = MIN((zfetch_max_reorder >> dbs) + 1, UINT16_MAX);
	+ uint_t t = gethrestime_sec() - zfetch_max_sec_reap;
	+ for (zs = list_head(&zf->zf_stream); zs != NULL;
	+ zs = list_next(&zf->zf_stream, zs)) {
	+ if (blkid > zs->zs_blkid) {
	+ if (end_blkid <= zs->zs_blkid + max_reorder) {
	+ if (!fetch_data) {
	+ nblks = dmu_zfetch_hit(zs,
	+ end_blkid - zs->zs_blkid);
	+ ZFETCHSTAT_BUMP(zfetchstat_stride);
	+ goto future;
	+ }
	+ nblks = dmu_zfetch_future(zs, blkid, nblks);
	+ if (nblks > 0)
	+ ZFETCHSTAT_BUMP(zfetchstat_stride);
	+ else
	+ ZFETCHSTAT_BUMP(zfetchstat_future);
	+ goto future;
	+ }
	+ } else if (end_blkid >= zs->zs_blkid) {
	+ nblks -= zs->zs_blkid - blkid;
	+ blkid += zs->zs_blkid - blkid;
	+ goto hit;
	+ } else if (end_blkid + max_reorder > zs->zs_blkid &&
	+ (int)(zs->zs_atime - t) >= 0) {
	+ ZFETCHSTAT_BUMP(zfetchstat_past);
	+ zs->zs_atime = gethrestime_sec();
	+ goto out;
	+ }
	}

	- /* Exit if we already prefetched this block before. */
	- if (nblks == 0) {
	- mutex_exit(&zf->zf_lock);
	- if (!have_lock)
	- rw_exit(&zf->zf_dnode->dn_struct_rwlock);
	- return (NULL);
	- }
	+ /*
	+ * This access is not part of any existing stream. Create a new
	+ * stream for it unless we are at the end of file.
	+ */
	+ if (end_blkid < maxblkid)
	+ dmu_zfetch_stream_create(zf, end_blkid);
	+ mutex_exit(&zf->zf_lock);
	+ if (!have_lock)
	+ rw_exit(&zf->zf_dnode->dn_struct_rwlock);
	+ ZFETCHSTAT_BUMP(zfetchstat_misses);
	+ return (NULL);

	- if (zs == NULL) {
	- /*
	- * This access is not part of any existing stream. Create
	- * a new stream for it.
	- */
	- dmu_zfetch_stream_create(zf, end_of_access_blkid);
	+hit:
	+ nblks = dmu_zfetch_hit(zs, nblks);
	+ ZFETCHSTAT_BUMP(zfetchstat_hits);
	+
	+future:
	+ zs->zs_atime = gethrestime_sec();
	+
	+ /* Exit if we already prefetched for this position before. */
	+ if (nblks == 0)
	+ goto out;
	+
	+ /* If the file is ending, remove the stream. */
	+ end_blkid = zs->zs_blkid;
	+ if (end_blkid >= maxblkid) {
	+ dmu_zfetch_stream_remove(zf, zs);
	+out:
	mutex_exit(&zf->zf_lock);
	if (!have_lock)
	rw_exit(&zf->zf_dnode->dn_struct_rwlock);
	- ZFETCHSTAT_BUMP(zfetchstat_misses);
	return (NULL);
	}

	/*
	* This access was to a block that we issued a prefetch for on
	* behalf of this stream. Calculate further prefetch distances.
	*
	* Start prefetch from the demand access size (nblks). Double the
	* distance every access up to zfetch_min_distance. After that only
	* if needed increase the distance by 1/8 up to zfetch_max_distance.
	*
	* Don't double the distance beyond single block if we have more
	* than ~6% of ARC held by active prefetches. It should help with
	* getting out of RAM on some badly mispredicted read patterns.
	*/
	- unsigned int dbs = zf->zf_dnode->dn_datablkshift;
	unsigned int nbytes = nblks << dbs;
	unsigned int pf_nblks;
	if (fetch_data) {
	if (unlikely(zs->zs_pf_dist < nbytes))
	zs->zs_pf_dist = nbytes;
	else if (zs->zs_pf_dist < zfetch_min_distance &&
	(zs->zs_pf_dist < (1 << dbs) \|\|
	aggsum_compare(&zfetch_sums.zfetchstat_io_active,
	arc_c_max >> (4 + dbs)) < 0))
	zs->zs_pf_dist *= 2;
	else if (zs->zs_more)
	zs->zs_pf_dist += zs->zs_pf_dist / 8;
	zs->zs_more = B_FALSE;
	if (zs->zs_pf_dist > zfetch_max_distance)
	zs->zs_pf_dist = zfetch_max_distance;
	pf_nblks = zs->zs_pf_dist >> dbs;
	} else {
	pf_nblks = 0;
	}
	- if (zs->zs_pf_start < end_of_access_blkid)
	- zs->zs_pf_start = end_of_access_blkid;
	- if (zs->zs_pf_end < end_of_access_blkid + pf_nblks)
	- zs->zs_pf_end = end_of_access_blkid + pf_nblks;
	+ if (zs->zs_pf_start < end_blkid)
	+ zs->zs_pf_start = end_blkid;
	+ if (zs->zs_pf_end < end_blkid + pf_nblks)
	+ zs->zs_pf_end = end_blkid + pf_nblks;

	/*
	* Do the same for indirects, starting where we will stop reading
	* data blocks (and the indirects that point to them).
	*/
	if (unlikely(zs->zs_ipf_dist < nbytes))
	zs->zs_ipf_dist = nbytes;
	else
	zs->zs_ipf_dist *= 2;
	if (zs->zs_ipf_dist > zfetch_max_idistance)
	zs->zs_ipf_dist = zfetch_max_idistance;
	pf_nblks = zs->zs_ipf_dist >> dbs;
	if (zs->zs_ipf_start < zs->zs_pf_end)
	zs->zs_ipf_start = zs->zs_pf_end;
	if (zs->zs_ipf_end < zs->zs_pf_end + pf_nblks)
	zs->zs_ipf_end = zs->zs_pf_end + pf_nblks;

	- zs->zs_blkid = end_of_access_blkid;
	- /* Protect the stream from reclamation. */
	- zs->zs_atime = gethrtime();
	zfs_refcount_add(&zs->zs_refs, NULL);
	/* Count concurrent callers. */
	zfs_refcount_add(&zs->zs_callers, NULL);
	mutex_exit(&zf->zf_lock);

	if (!have_lock)
	rw_exit(&zf->zf_dnode->dn_struct_rwlock);
	-
	- ZFETCHSTAT_BUMP(zfetchstat_hits);
	return (zs);
	}

	void
	-dmu_zfetch_run(zstream_t *zs, boolean_t missed, boolean_t have_lock)
	+dmu_zfetch_run(zfetch_t zf, zstream_t zs, boolean_t missed,
	+ boolean_t have_lock)
	{
	- zfetch_t *zf = zs->zs_fetch;
	int64_t pf_start, pf_end, ipf_start, ipf_end;
	int epbs, issued;

	if (missed)
	zs->zs_missed = missed;

	/*
	* Postpone the prefetch if there are more concurrent callers.
	* It happens when multiple requests are waiting for the same
	* indirect block. The last one will run the prefetch for all.
	*/
	if (zfs_refcount_remove(&zs->zs_callers, NULL) != 0) {
	/* Drop reference taken in dmu_zfetch_prepare(). */
	if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0)
	dmu_zfetch_stream_fini(zs);
	return;
	}

	mutex_enter(&zf->zf_lock);
	if (zs->zs_missed) {
	pf_start = zs->zs_pf_start;
	pf_end = zs->zs_pf_start = zs->zs_pf_end;
	} else {
	pf_start = pf_end = 0;
	}
	ipf_start = zs->zs_ipf_start;
	ipf_end = zs->zs_ipf_start = zs->zs_ipf_end;
	mutex_exit(&zf->zf_lock);
	ASSERT3S(pf_start, <=, pf_end);
	ASSERT3S(ipf_start, <=, ipf_end);

	epbs = zf->zf_dnode->dn_indblkshift - SPA_BLKPTRSHIFT;
	ipf_start = P2ROUNDUP(ipf_start, 1 << epbs) >> epbs;
	ipf_end = P2ROUNDUP(ipf_end, 1 << epbs) >> epbs;
	ASSERT3S(ipf_start, <=, ipf_end);
	issued = pf_end - pf_start + ipf_end - ipf_start;
	if (issued > 1) {
	/* More references on top of taken in dmu_zfetch_prepare(). */
	zfs_refcount_add_few(&zs->zs_refs, issued - 1, NULL);
	} else if (issued == 0) {
	/* Some other thread has done our work, so drop the ref. */
	if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0)
	dmu_zfetch_stream_fini(zs);
	return;
	}
	aggsum_add(&zfetch_sums.zfetchstat_io_active, issued);

	if (!have_lock)
	rw_enter(&zf->zf_dnode->dn_struct_rwlock, RW_READER);

	issued = 0;
	for (int64_t blk = pf_start; blk < pf_end; blk++) {
	issued += dbuf_prefetch_impl(zf->zf_dnode, 0, blk,
	ZIO_PRIORITY_ASYNC_READ, 0, dmu_zfetch_done, zs);
	}
	for (int64_t iblk = ipf_start; iblk < ipf_end; iblk++) {
	issued += dbuf_prefetch_impl(zf->zf_dnode, 1, iblk,
	ZIO_PRIORITY_ASYNC_READ, 0, dmu_zfetch_done, zs);
	}

	if (!have_lock)
	rw_exit(&zf->zf_dnode->dn_struct_rwlock);

	if (issued)
	ZFETCHSTAT_ADD(zfetchstat_io_issued, issued);
	}

	void
	dmu_zfetch(zfetch_t *zf, uint64_t blkid, uint64_t nblks, boolean_t fetch_data,
	boolean_t missed, boolean_t have_lock)
	{
	zstream_t *zs;

	zs = dmu_zfetch_prepare(zf, blkid, nblks, fetch_data, have_lock);
	if (zs)
	- dmu_zfetch_run(zs, missed, have_lock);
	+ dmu_zfetch_run(zf, zs, missed, have_lock);
	}

	ZFS_MODULE_PARAM(zfs_prefetch, zfs_prefetch_, disable, INT, ZMOD_RW,
	"Disable all ZFS prefetching");

	ZFS_MODULE_PARAM(zfs_prefetch, zfetch_, max_streams, UINT, ZMOD_RW,
	"Max number of streams per zfetch");

	ZFS_MODULE_PARAM(zfs_prefetch, zfetch_, min_sec_reap, UINT, ZMOD_RW,
	"Min time before stream reclaim");

	ZFS_MODULE_PARAM(zfs_prefetch, zfetch_, max_sec_reap, UINT, ZMOD_RW,
	"Max time before stream delete");

	ZFS_MODULE_PARAM(zfs_prefetch, zfetch_, min_distance, UINT, ZMOD_RW,
	"Min bytes to prefetch per stream");

	ZFS_MODULE_PARAM(zfs_prefetch, zfetch_, max_distance, UINT, ZMOD_RW,
	"Max bytes to prefetch per stream");

	ZFS_MODULE_PARAM(zfs_prefetch, zfetch_, max_idistance, UINT, ZMOD_RW,
	"Max bytes to prefetch indirects for per stream");
	+
	+ZFS_MODULE_PARAM(zfs_prefetch, zfetch_, max_reorder, UINT, ZMOD_RW,
	+ "Max request reorder distance within a stream");
	+
	+ZFS_MODULE_PARAM(zfs_prefetch, zfetch_, hole_shift, UINT, ZMOD_RW,
	+ "Max log2 fraction of holes in a stream");
	diff --git a/sys/contrib/openzfs/module/zfs/dsl_deadlist.c b/sys/contrib/openzfs/module/zfs/dsl_deadlist.c
	index ac30a370813f..e6c8d4be13b4 100644
	--- a/sys/contrib/openzfs/module/zfs/dsl_deadlist.c
	+++ b/sys/contrib/openzfs/module/zfs/dsl_deadlist.c
	@@ -1,1113 +1,1113 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2012, 2019 by Delphix. All rights reserved.
	* Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
	*/

	#include <sys/dmu.h>
	#include <sys/zap.h>
	#include <sys/zfs_context.h>
	#include <sys/dsl_pool.h>
	#include <sys/dsl_dataset.h>

	/*
	* Deadlist concurrency:
	*
	* Deadlists can only be modified from the syncing thread.
	*
	* Except for dsl_deadlist_insert(), it can only be modified with the
	* dp_config_rwlock held with RW_WRITER.
	*
	* The accessors (dsl_deadlist_space() and dsl_deadlist_space_range()) can
	* be called concurrently, from open context, with the dl_config_rwlock held
	* with RW_READER.
	*
	* Therefore, we only need to provide locking between dsl_deadlist_insert() and
	* the accessors, protecting:
	* dl_phys->dl_used,comp,uncomp
	* and protecting the dl_tree from being loaded.
	* The locking is provided by dl_lock. Note that locking on the bpobj_t
	* provides its own locking, and dl_oldfmt is immutable.
	*/

	/*
	* Livelist Overview
	* ================
	*
	* Livelists use the same 'deadlist_t' struct as deadlists and are also used
	* to track blkptrs over the lifetime of a dataset. Livelists however, belong
	* to clones and track the blkptrs that are clone-specific (were born after
	* the clone's creation). The exception is embedded block pointers which are
	* not included in livelists because they do not need to be freed.
	*
	* When it comes time to delete the clone, the livelist provides a quick
	* reference as to what needs to be freed. For this reason, livelists also track
	* when clone-specific blkptrs are freed before deletion to prevent double
	* frees. Each blkptr in a livelist is marked as a FREE or an ALLOC and the
	* deletion algorithm iterates backwards over the livelist, matching
	* FREE/ALLOC pairs and then freeing those ALLOCs which remain. livelists
	* are also updated in the case when blkptrs are remapped: the old version
	* of the blkptr is cancelled out with a FREE and the new version is tracked
	* with an ALLOC.
	*
	* To bound the amount of memory required for deletion, livelists over a
	* certain size are spread over multiple entries. Entries are grouped by
	* birth txg so we can be sure the ALLOC/FREE pair for a given blkptr will
	* be in the same entry. This allows us to delete livelists incrementally
	* over multiple syncs, one entry at a time.
	*
	* During the lifetime of the clone, livelists can get extremely large.
	* Their size is managed by periodic condensing (preemptively cancelling out
	* FREE/ALLOC pairs). Livelists are disabled when a clone is promoted or when
	* the shared space between the clone and its origin is so small that it
	* doesn't make sense to use livelists anymore.
	*/

	/*
	* The threshold sublist size at which we create a new sub-livelist for the
	* next txg. However, since blkptrs of the same transaction group must be in
	* the same sub-list, the actual sublist size may exceed this. When picking the
	* size we had to balance the fact that larger sublists mean fewer sublists
	* (decreasing the cost of insertion) against the consideration that sublists
	* will be loaded into memory and shouldn't take up an inordinate amount of
	* space. We settled on ~500000 entries, corresponding to roughly 128M.
	*/
	uint64_t zfs_livelist_max_entries = 500000;

	/*
	* We can approximate how much of a performance gain a livelist will give us
	* based on the percentage of blocks shared between the clone and its origin.
	* 0 percent shared means that the clone has completely diverged and that the
	* old method is maximally effective: every read from the block tree will
	* result in lots of frees. Livelists give us gains when they track blocks
	* scattered across the tree, when one read in the old method might only
	* result in a few frees. Once the clone has been overwritten enough,
	* writes are no longer sparse and we'll no longer get much of a benefit from
	* tracking them with a livelist. We chose a lower limit of 75 percent shared
	* (25 percent overwritten). This means that 1/4 of all block pointers will be
	* freed (e.g. each read frees 256, out of a max of 1024) so we expect livelists
	* to make deletion 4x faster. Once the amount of shared space drops below this
	* threshold, the clone will revert to the old deletion method.
	*/
	int zfs_livelist_min_percent_shared = 75;

	static int
	dsl_deadlist_compare(const void arg1, const void arg2)
	{
	const dsl_deadlist_entry_t *dle1 = arg1;
	const dsl_deadlist_entry_t *dle2 = arg2;

	return (TREE_CMP(dle1->dle_mintxg, dle2->dle_mintxg));
	}

	static int
	dsl_deadlist_cache_compare(const void arg1, const void arg2)
	{
	const dsl_deadlist_cache_entry_t *dlce1 = arg1;
	const dsl_deadlist_cache_entry_t *dlce2 = arg2;

	return (TREE_CMP(dlce1->dlce_mintxg, dlce2->dlce_mintxg));
	}

	static void
	dsl_deadlist_load_tree(dsl_deadlist_t *dl)
	{
	zap_cursor_t zc;
	zap_attribute_t za;
	int error;

	ASSERT(MUTEX_HELD(&dl->dl_lock));

	ASSERT(!dl->dl_oldfmt);
	if (dl->dl_havecache) {
	/*
	* After loading the tree, the caller may modify the tree,
	* e.g. to add or remove nodes, or to make a node no longer
	* refer to the empty_bpobj. These changes would make the
	* dl_cache incorrect. Therefore we discard the cache here,
	* so that it can't become incorrect.
	*/
	dsl_deadlist_cache_entry_t *dlce;
	void *cookie = NULL;
	while ((dlce = avl_destroy_nodes(&dl->dl_cache, &cookie))
	!= NULL) {
	kmem_free(dlce, sizeof (*dlce));
	}
	avl_destroy(&dl->dl_cache);
	dl->dl_havecache = B_FALSE;
	}
	if (dl->dl_havetree)
	return;

	avl_create(&dl->dl_tree, dsl_deadlist_compare,
	sizeof (dsl_deadlist_entry_t),
	offsetof(dsl_deadlist_entry_t, dle_node));
	for (zap_cursor_init(&zc, dl->dl_os, dl->dl_object);
	(error = zap_cursor_retrieve(&zc, &za)) == 0;
	zap_cursor_advance(&zc)) {
	dsl_deadlist_entry_t dle = kmem_alloc(sizeof (dle), KM_SLEEP);
	dle->dle_mintxg = zfs_strtonum(za.za_name, NULL);

	/*
	* Prefetch all the bpobj's so that we do that i/o
	* in parallel. Then open them all in a second pass.
	*/
	dle->dle_bpobj.bpo_object = za.za_first_integer;
	- dmu_prefetch(dl->dl_os, dle->dle_bpobj.bpo_object,
	- 0, 0, 0, ZIO_PRIORITY_SYNC_READ);
	+ dmu_prefetch_dnode(dl->dl_os, dle->dle_bpobj.bpo_object,
	+ ZIO_PRIORITY_SYNC_READ);

	avl_add(&dl->dl_tree, dle);
	}
	VERIFY3U(error, ==, ENOENT);
	zap_cursor_fini(&zc);

	for (dsl_deadlist_entry_t *dle = avl_first(&dl->dl_tree);
	dle != NULL; dle = AVL_NEXT(&dl->dl_tree, dle)) {
	VERIFY0(bpobj_open(&dle->dle_bpobj, dl->dl_os,
	dle->dle_bpobj.bpo_object));
	}
	dl->dl_havetree = B_TRUE;
	}

	/*
	* Load only the non-empty bpobj's into the dl_cache. The cache is an analog
	* of the dl_tree, but contains only non-empty_bpobj nodes from the ZAP. It
	* is used only for gathering space statistics. The dl_cache has two
	* advantages over the dl_tree:
	*
	* 1. Loading the dl_cache is ~5x faster than loading the dl_tree (if it's
	* mostly empty_bpobj's), due to less CPU overhead to open the empty_bpobj
	* many times and to inquire about its (zero) space stats many times.
	*
	* 2. The dl_cache uses less memory than the dl_tree. We only need to load
	* the dl_tree of snapshots when deleting a snapshot, after which we free the
	* dl_tree with dsl_deadlist_discard_tree
	*/
	static void
	dsl_deadlist_load_cache(dsl_deadlist_t *dl)
	{
	zap_cursor_t zc;
	zap_attribute_t za;
	int error;

	ASSERT(MUTEX_HELD(&dl->dl_lock));

	ASSERT(!dl->dl_oldfmt);
	if (dl->dl_havecache)
	return;

	uint64_t empty_bpobj = dmu_objset_pool(dl->dl_os)->dp_empty_bpobj;

	avl_create(&dl->dl_cache, dsl_deadlist_cache_compare,
	sizeof (dsl_deadlist_cache_entry_t),
	offsetof(dsl_deadlist_cache_entry_t, dlce_node));
	for (zap_cursor_init(&zc, dl->dl_os, dl->dl_object);
	(error = zap_cursor_retrieve(&zc, &za)) == 0;
	zap_cursor_advance(&zc)) {
	if (za.za_first_integer == empty_bpobj)
	continue;
	dsl_deadlist_cache_entry_t *dlce =
	kmem_zalloc(sizeof (*dlce), KM_SLEEP);
	dlce->dlce_mintxg = zfs_strtonum(za.za_name, NULL);

	/*
	* Prefetch all the bpobj's so that we do that i/o
	* in parallel. Then open them all in a second pass.
	*/
	dlce->dlce_bpobj = za.za_first_integer;
	- dmu_prefetch(dl->dl_os, dlce->dlce_bpobj,
	- 0, 0, 0, ZIO_PRIORITY_SYNC_READ);
	+ dmu_prefetch_dnode(dl->dl_os, dlce->dlce_bpobj,
	+ ZIO_PRIORITY_SYNC_READ);
	avl_add(&dl->dl_cache, dlce);
	}
	VERIFY3U(error, ==, ENOENT);
	zap_cursor_fini(&zc);

	for (dsl_deadlist_cache_entry_t *dlce = avl_first(&dl->dl_cache);
	dlce != NULL; dlce = AVL_NEXT(&dl->dl_cache, dlce)) {
	bpobj_t bpo;
	VERIFY0(bpobj_open(&bpo, dl->dl_os, dlce->dlce_bpobj));

	VERIFY0(bpobj_space(&bpo,
	&dlce->dlce_bytes, &dlce->dlce_comp, &dlce->dlce_uncomp));
	bpobj_close(&bpo);
	}
	dl->dl_havecache = B_TRUE;
	}

	/*
	* Discard the tree to save memory.
	*/
	void
	dsl_deadlist_discard_tree(dsl_deadlist_t *dl)
	{
	mutex_enter(&dl->dl_lock);

	if (!dl->dl_havetree) {
	mutex_exit(&dl->dl_lock);
	return;
	}
	dsl_deadlist_entry_t *dle;
	void *cookie = NULL;
	while ((dle = avl_destroy_nodes(&dl->dl_tree, &cookie)) != NULL) {
	bpobj_close(&dle->dle_bpobj);
	kmem_free(dle, sizeof (*dle));
	}
	avl_destroy(&dl->dl_tree);

	dl->dl_havetree = B_FALSE;
	mutex_exit(&dl->dl_lock);
	}

	void
	dsl_deadlist_iterate(dsl_deadlist_t dl, deadlist_iter_t func, void args)
	{
	dsl_deadlist_entry_t *dle;

	ASSERT(dsl_deadlist_is_open(dl));

	mutex_enter(&dl->dl_lock);
	dsl_deadlist_load_tree(dl);
	mutex_exit(&dl->dl_lock);
	for (dle = avl_first(&dl->dl_tree); dle != NULL;
	dle = AVL_NEXT(&dl->dl_tree, dle)) {
	if (func(args, dle) != 0)
	break;
	}
	}

	void
	dsl_deadlist_open(dsl_deadlist_t dl, objset_t os, uint64_t object)
	{
	dmu_object_info_t doi;

	ASSERT(!dsl_deadlist_is_open(dl));

	mutex_init(&dl->dl_lock, NULL, MUTEX_DEFAULT, NULL);
	dl->dl_os = os;
	dl->dl_object = object;
	VERIFY0(dmu_bonus_hold(os, object, dl, &dl->dl_dbuf));
	dmu_object_info_from_db(dl->dl_dbuf, &doi);
	if (doi.doi_type == DMU_OT_BPOBJ) {
	dmu_buf_rele(dl->dl_dbuf, dl);
	dl->dl_dbuf = NULL;
	dl->dl_oldfmt = B_TRUE;
	VERIFY0(bpobj_open(&dl->dl_bpobj, os, object));
	return;
	}

	dl->dl_oldfmt = B_FALSE;
	dl->dl_phys = dl->dl_dbuf->db_data;
	dl->dl_havetree = B_FALSE;
	dl->dl_havecache = B_FALSE;
	}

	boolean_t
	dsl_deadlist_is_open(dsl_deadlist_t *dl)
	{
	return (dl->dl_os != NULL);
	}

	void
	dsl_deadlist_close(dsl_deadlist_t *dl)
	{
	ASSERT(dsl_deadlist_is_open(dl));
	mutex_destroy(&dl->dl_lock);

	if (dl->dl_oldfmt) {
	dl->dl_oldfmt = B_FALSE;
	bpobj_close(&dl->dl_bpobj);
	dl->dl_os = NULL;
	dl->dl_object = 0;
	return;
	}

	if (dl->dl_havetree) {
	dsl_deadlist_entry_t *dle;
	void *cookie = NULL;
	while ((dle = avl_destroy_nodes(&dl->dl_tree, &cookie))
	!= NULL) {
	bpobj_close(&dle->dle_bpobj);
	kmem_free(dle, sizeof (*dle));
	}
	avl_destroy(&dl->dl_tree);
	}
	if (dl->dl_havecache) {
	dsl_deadlist_cache_entry_t *dlce;
	void *cookie = NULL;
	while ((dlce = avl_destroy_nodes(&dl->dl_cache, &cookie))
	!= NULL) {
	kmem_free(dlce, sizeof (*dlce));
	}
	avl_destroy(&dl->dl_cache);
	}
	dmu_buf_rele(dl->dl_dbuf, dl);
	dl->dl_dbuf = NULL;
	dl->dl_phys = NULL;
	dl->dl_os = NULL;
	dl->dl_object = 0;
	}

	uint64_t
	dsl_deadlist_alloc(objset_t os, dmu_tx_t tx)
	{
	if (spa_version(dmu_objset_spa(os)) < SPA_VERSION_DEADLISTS)
	return (bpobj_alloc(os, SPA_OLD_MAXBLOCKSIZE, tx));
	return (zap_create(os, DMU_OT_DEADLIST, DMU_OT_DEADLIST_HDR,
	sizeof (dsl_deadlist_phys_t), tx));
	}

	void
	dsl_deadlist_free(objset_t os, uint64_t dlobj, dmu_tx_t tx)
	{
	dmu_object_info_t doi;
	zap_cursor_t zc;
	zap_attribute_t za;
	int error;

	VERIFY0(dmu_object_info(os, dlobj, &doi));
	if (doi.doi_type == DMU_OT_BPOBJ) {
	bpobj_free(os, dlobj, tx);
	return;
	}

	for (zap_cursor_init(&zc, os, dlobj);
	(error = zap_cursor_retrieve(&zc, &za)) == 0;
	zap_cursor_advance(&zc)) {
	uint64_t obj = za.za_first_integer;
	if (obj == dmu_objset_pool(os)->dp_empty_bpobj)
	bpobj_decr_empty(os, tx);
	else
	bpobj_free(os, obj, tx);
	}
	VERIFY3U(error, ==, ENOENT);
	zap_cursor_fini(&zc);
	VERIFY0(dmu_object_free(os, dlobj, tx));
	}

	static void
	dle_enqueue(dsl_deadlist_t dl, dsl_deadlist_entry_t dle,
	const blkptr_t bp, boolean_t bp_freed, dmu_tx_t tx)
	{
	ASSERT(MUTEX_HELD(&dl->dl_lock));
	if (dle->dle_bpobj.bpo_object ==
	dmu_objset_pool(dl->dl_os)->dp_empty_bpobj) {
	uint64_t obj = bpobj_alloc(dl->dl_os, SPA_OLD_MAXBLOCKSIZE, tx);
	bpobj_close(&dle->dle_bpobj);
	bpobj_decr_empty(dl->dl_os, tx);
	VERIFY0(bpobj_open(&dle->dle_bpobj, dl->dl_os, obj));
	VERIFY0(zap_update_int_key(dl->dl_os, dl->dl_object,
	dle->dle_mintxg, obj, tx));
	}
	bpobj_enqueue(&dle->dle_bpobj, bp, bp_freed, tx);
	}

	static void
	dle_enqueue_subobj(dsl_deadlist_t dl, dsl_deadlist_entry_t dle,
	uint64_t obj, dmu_tx_t *tx)
	{
	ASSERT(MUTEX_HELD(&dl->dl_lock));
	if (dle->dle_bpobj.bpo_object !=
	dmu_objset_pool(dl->dl_os)->dp_empty_bpobj) {
	bpobj_enqueue_subobj(&dle->dle_bpobj, obj, tx);
	} else {
	bpobj_close(&dle->dle_bpobj);
	bpobj_decr_empty(dl->dl_os, tx);
	VERIFY0(bpobj_open(&dle->dle_bpobj, dl->dl_os, obj));
	VERIFY0(zap_update_int_key(dl->dl_os, dl->dl_object,
	dle->dle_mintxg, obj, tx));
	}
	}

	/*
	* Prefetch metadata required for dle_enqueue_subobj().
	*/
	static void
	dle_prefetch_subobj(dsl_deadlist_t dl, dsl_deadlist_entry_t dle,
	uint64_t obj)
	{
	if (dle->dle_bpobj.bpo_object !=
	dmu_objset_pool(dl->dl_os)->dp_empty_bpobj)
	bpobj_prefetch_subobj(&dle->dle_bpobj, obj);
	}

	void
	dsl_deadlist_insert(dsl_deadlist_t dl, const blkptr_t bp, boolean_t bp_freed,
	dmu_tx_t *tx)
	{
	dsl_deadlist_entry_t dle_tofind;
	dsl_deadlist_entry_t *dle;
	avl_index_t where;

	if (dl->dl_oldfmt) {
	bpobj_enqueue(&dl->dl_bpobj, bp, bp_freed, tx);
	return;
	}

	mutex_enter(&dl->dl_lock);
	dsl_deadlist_load_tree(dl);

	dmu_buf_will_dirty(dl->dl_dbuf, tx);

	int sign = bp_freed ? -1 : +1;
	dl->dl_phys->dl_used +=
	sign * bp_get_dsize_sync(dmu_objset_spa(dl->dl_os), bp);
	dl->dl_phys->dl_comp += sign * BP_GET_PSIZE(bp);
	dl->dl_phys->dl_uncomp += sign * BP_GET_UCSIZE(bp);

	dle_tofind.dle_mintxg = bp->blk_birth;
	dle = avl_find(&dl->dl_tree, &dle_tofind, &where);
	if (dle == NULL)
	dle = avl_nearest(&dl->dl_tree, where, AVL_BEFORE);
	else
	dle = AVL_PREV(&dl->dl_tree, dle);

	if (dle == NULL) {
	zfs_panic_recover("blkptr at %p has invalid BLK_BIRTH %llu",
	bp, (longlong_t)bp->blk_birth);
	dle = avl_first(&dl->dl_tree);
	}

	ASSERT3P(dle, !=, NULL);
	dle_enqueue(dl, dle, bp, bp_freed, tx);
	mutex_exit(&dl->dl_lock);
	}

	int
	dsl_deadlist_insert_alloc_cb(void arg, const blkptr_t bp, dmu_tx_t *tx)
	{
	dsl_deadlist_t *dl = arg;
	dsl_deadlist_insert(dl, bp, B_FALSE, tx);
	return (0);
	}

	int
	dsl_deadlist_insert_free_cb(void arg, const blkptr_t bp, dmu_tx_t *tx)
	{
	dsl_deadlist_t *dl = arg;
	dsl_deadlist_insert(dl, bp, B_TRUE, tx);
	return (0);
	}

	/*
	* Insert new key in deadlist, which must be > all current entries.
	* mintxg is not inclusive.
	*/
	void
	dsl_deadlist_add_key(dsl_deadlist_t dl, uint64_t mintxg, dmu_tx_t tx)
	{
	uint64_t obj;
	dsl_deadlist_entry_t *dle;

	if (dl->dl_oldfmt)
	return;

	dle = kmem_alloc(sizeof (*dle), KM_SLEEP);
	dle->dle_mintxg = mintxg;

	mutex_enter(&dl->dl_lock);
	dsl_deadlist_load_tree(dl);

	obj = bpobj_alloc_empty(dl->dl_os, SPA_OLD_MAXBLOCKSIZE, tx);
	VERIFY0(bpobj_open(&dle->dle_bpobj, dl->dl_os, obj));
	avl_add(&dl->dl_tree, dle);

	VERIFY0(zap_add_int_key(dl->dl_os, dl->dl_object,
	mintxg, obj, tx));
	mutex_exit(&dl->dl_lock);
	}

	/*
	* Remove this key, merging its entries into the previous key.
	*/
	void
	dsl_deadlist_remove_key(dsl_deadlist_t dl, uint64_t mintxg, dmu_tx_t tx)
	{
	dsl_deadlist_entry_t dle_tofind;
	dsl_deadlist_entry_t dle, dle_prev;

	if (dl->dl_oldfmt)
	return;
	mutex_enter(&dl->dl_lock);
	dsl_deadlist_load_tree(dl);

	dle_tofind.dle_mintxg = mintxg;
	dle = avl_find(&dl->dl_tree, &dle_tofind, NULL);
	ASSERT3P(dle, !=, NULL);
	dle_prev = AVL_PREV(&dl->dl_tree, dle);
	ASSERT3P(dle_prev, !=, NULL);

	dle_enqueue_subobj(dl, dle_prev, dle->dle_bpobj.bpo_object, tx);

	avl_remove(&dl->dl_tree, dle);
	bpobj_close(&dle->dle_bpobj);
	kmem_free(dle, sizeof (*dle));

	VERIFY0(zap_remove_int(dl->dl_os, dl->dl_object, mintxg, tx));
	mutex_exit(&dl->dl_lock);
	}

	/*
	* Remove a deadlist entry and all of its contents by removing the entry from
	* the deadlist's avl tree, freeing the entry's bpobj and adjusting the
	* deadlist's space accounting accordingly.
	*/
	void
	dsl_deadlist_remove_entry(dsl_deadlist_t dl, uint64_t mintxg, dmu_tx_t tx)
	{
	uint64_t used, comp, uncomp;
	dsl_deadlist_entry_t dle_tofind;
	dsl_deadlist_entry_t *dle;
	objset_t *os = dl->dl_os;

	if (dl->dl_oldfmt)
	return;

	mutex_enter(&dl->dl_lock);
	dsl_deadlist_load_tree(dl);

	dle_tofind.dle_mintxg = mintxg;
	dle = avl_find(&dl->dl_tree, &dle_tofind, NULL);
	VERIFY3P(dle, !=, NULL);

	avl_remove(&dl->dl_tree, dle);
	VERIFY0(zap_remove_int(os, dl->dl_object, mintxg, tx));
	VERIFY0(bpobj_space(&dle->dle_bpobj, &used, &comp, &uncomp));
	dmu_buf_will_dirty(dl->dl_dbuf, tx);
	dl->dl_phys->dl_used -= used;
	dl->dl_phys->dl_comp -= comp;
	dl->dl_phys->dl_uncomp -= uncomp;
	if (dle->dle_bpobj.bpo_object == dmu_objset_pool(os)->dp_empty_bpobj) {
	bpobj_decr_empty(os, tx);
	} else {
	bpobj_free(os, dle->dle_bpobj.bpo_object, tx);
	}
	bpobj_close(&dle->dle_bpobj);
	kmem_free(dle, sizeof (*dle));
	mutex_exit(&dl->dl_lock);
	}

	/*
	* Clear out the contents of a deadlist_entry by freeing its bpobj,
	* replacing it with an empty bpobj and adjusting the deadlist's
	* space accounting
	*/
	void
	dsl_deadlist_clear_entry(dsl_deadlist_entry_t dle, dsl_deadlist_t dl,
	dmu_tx_t *tx)
	{
	uint64_t new_obj, used, comp, uncomp;
	objset_t *os = dl->dl_os;

	mutex_enter(&dl->dl_lock);
	VERIFY0(zap_remove_int(os, dl->dl_object, dle->dle_mintxg, tx));
	VERIFY0(bpobj_space(&dle->dle_bpobj, &used, &comp, &uncomp));
	dmu_buf_will_dirty(dl->dl_dbuf, tx);
	dl->dl_phys->dl_used -= used;
	dl->dl_phys->dl_comp -= comp;
	dl->dl_phys->dl_uncomp -= uncomp;
	if (dle->dle_bpobj.bpo_object == dmu_objset_pool(os)->dp_empty_bpobj)
	bpobj_decr_empty(os, tx);
	else
	bpobj_free(os, dle->dle_bpobj.bpo_object, tx);
	bpobj_close(&dle->dle_bpobj);
	new_obj = bpobj_alloc_empty(os, SPA_OLD_MAXBLOCKSIZE, tx);
	VERIFY0(bpobj_open(&dle->dle_bpobj, os, new_obj));
	VERIFY0(zap_add_int_key(os, dl->dl_object, dle->dle_mintxg,
	new_obj, tx));
	ASSERT(bpobj_is_empty(&dle->dle_bpobj));
	mutex_exit(&dl->dl_lock);
	}

	/*
	* Return the first entry in deadlist's avl tree
	*/
	dsl_deadlist_entry_t *
	dsl_deadlist_first(dsl_deadlist_t *dl)
	{
	dsl_deadlist_entry_t *dle;

	mutex_enter(&dl->dl_lock);
	dsl_deadlist_load_tree(dl);
	dle = avl_first(&dl->dl_tree);
	mutex_exit(&dl->dl_lock);

	return (dle);
	}

	/*
	* Return the last entry in deadlist's avl tree
	*/
	dsl_deadlist_entry_t *
	dsl_deadlist_last(dsl_deadlist_t *dl)
	{
	dsl_deadlist_entry_t *dle;

	mutex_enter(&dl->dl_lock);
	dsl_deadlist_load_tree(dl);
	dle = avl_last(&dl->dl_tree);
	mutex_exit(&dl->dl_lock);

	return (dle);
	}

	/*
	* Walk ds's snapshots to regenerate generate ZAP & AVL.
	*/
	static void
	dsl_deadlist_regenerate(objset_t *os, uint64_t dlobj,
	uint64_t mrs_obj, dmu_tx_t *tx)
	{
	dsl_deadlist_t dl = { 0 };
	dsl_pool_t *dp = dmu_objset_pool(os);

	dsl_deadlist_open(&dl, os, dlobj);
	if (dl.dl_oldfmt) {
	dsl_deadlist_close(&dl);
	return;
	}

	while (mrs_obj != 0) {
	dsl_dataset_t *ds;
	VERIFY0(dsl_dataset_hold_obj(dp, mrs_obj, FTAG, &ds));
	dsl_deadlist_add_key(&dl,
	dsl_dataset_phys(ds)->ds_prev_snap_txg, tx);
	mrs_obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
	dsl_dataset_rele(ds, FTAG);
	}
	dsl_deadlist_close(&dl);
	}

	uint64_t
	dsl_deadlist_clone(dsl_deadlist_t *dl, uint64_t maxtxg,
	uint64_t mrs_obj, dmu_tx_t *tx)
	{
	dsl_deadlist_entry_t *dle;
	uint64_t newobj;

	newobj = dsl_deadlist_alloc(dl->dl_os, tx);

	if (dl->dl_oldfmt) {
	dsl_deadlist_regenerate(dl->dl_os, newobj, mrs_obj, tx);
	return (newobj);
	}

	mutex_enter(&dl->dl_lock);
	dsl_deadlist_load_tree(dl);

	for (dle = avl_first(&dl->dl_tree); dle;
	dle = AVL_NEXT(&dl->dl_tree, dle)) {
	uint64_t obj;

	if (dle->dle_mintxg >= maxtxg)
	break;

	obj = bpobj_alloc_empty(dl->dl_os, SPA_OLD_MAXBLOCKSIZE, tx);
	VERIFY0(zap_add_int_key(dl->dl_os, newobj,
	dle->dle_mintxg, obj, tx));
	}
	mutex_exit(&dl->dl_lock);
	return (newobj);
	}

	void
	dsl_deadlist_space(dsl_deadlist_t *dl,
	uint64_t usedp, uint64_t compp, uint64_t *uncompp)
	{
	ASSERT(dsl_deadlist_is_open(dl));
	if (dl->dl_oldfmt) {
	VERIFY0(bpobj_space(&dl->dl_bpobj,
	usedp, compp, uncompp));
	return;
	}

	mutex_enter(&dl->dl_lock);
	*usedp = dl->dl_phys->dl_used;
	*compp = dl->dl_phys->dl_comp;
	*uncompp = dl->dl_phys->dl_uncomp;
	mutex_exit(&dl->dl_lock);
	}

	/*
	* return space used in the range (mintxg, maxtxg].
	* Includes maxtxg, does not include mintxg.
	* mintxg and maxtxg must both be keys in the deadlist (unless maxtxg is
	* UINT64_MAX).
	*/
	void
	dsl_deadlist_space_range(dsl_deadlist_t *dl, uint64_t mintxg, uint64_t maxtxg,
	uint64_t usedp, uint64_t compp, uint64_t *uncompp)
	{
	dsl_deadlist_cache_entry_t *dlce;
	dsl_deadlist_cache_entry_t dlce_tofind;
	avl_index_t where;

	if (dl->dl_oldfmt) {
	VERIFY0(bpobj_space_range(&dl->dl_bpobj,
	mintxg, maxtxg, usedp, compp, uncompp));
	return;
	}

	usedp = compp = *uncompp = 0;

	mutex_enter(&dl->dl_lock);
	dsl_deadlist_load_cache(dl);
	dlce_tofind.dlce_mintxg = mintxg;
	dlce = avl_find(&dl->dl_cache, &dlce_tofind, &where);

	/*
	* If this mintxg doesn't exist, it may be an empty_bpobj which
	* is omitted from the sparse tree. Start at the next non-empty
	* entry.
	*/
	if (dlce == NULL)
	dlce = avl_nearest(&dl->dl_cache, where, AVL_AFTER);

	for (; dlce && dlce->dlce_mintxg < maxtxg;
	dlce = AVL_NEXT(&dl->dl_tree, dlce)) {
	*usedp += dlce->dlce_bytes;
	*compp += dlce->dlce_comp;
	*uncompp += dlce->dlce_uncomp;
	}

	mutex_exit(&dl->dl_lock);
	}

	static void
	dsl_deadlist_insert_bpobj(dsl_deadlist_t *dl, uint64_t obj, uint64_t birth,
	dmu_tx_t *tx)
	{
	dsl_deadlist_entry_t dle_tofind;
	dsl_deadlist_entry_t *dle;
	avl_index_t where;
	uint64_t used, comp, uncomp;
	bpobj_t bpo;

	ASSERT(MUTEX_HELD(&dl->dl_lock));

	VERIFY0(bpobj_open(&bpo, dl->dl_os, obj));
	VERIFY0(bpobj_space(&bpo, &used, &comp, &uncomp));
	bpobj_close(&bpo);

	dsl_deadlist_load_tree(dl);

	dmu_buf_will_dirty(dl->dl_dbuf, tx);
	dl->dl_phys->dl_used += used;
	dl->dl_phys->dl_comp += comp;
	dl->dl_phys->dl_uncomp += uncomp;

	dle_tofind.dle_mintxg = birth;
	dle = avl_find(&dl->dl_tree, &dle_tofind, &where);
	if (dle == NULL)
	dle = avl_nearest(&dl->dl_tree, where, AVL_BEFORE);
	dle_enqueue_subobj(dl, dle, obj, tx);
	}

	/*
	* Prefetch metadata required for dsl_deadlist_insert_bpobj().
	*/
	static void
	dsl_deadlist_prefetch_bpobj(dsl_deadlist_t *dl, uint64_t obj, uint64_t birth)
	{
	dsl_deadlist_entry_t dle_tofind;
	dsl_deadlist_entry_t *dle;
	avl_index_t where;

	ASSERT(MUTEX_HELD(&dl->dl_lock));

	dsl_deadlist_load_tree(dl);

	dle_tofind.dle_mintxg = birth;
	dle = avl_find(&dl->dl_tree, &dle_tofind, &where);
	if (dle == NULL)
	dle = avl_nearest(&dl->dl_tree, where, AVL_BEFORE);
	dle_prefetch_subobj(dl, dle, obj);
	}

	static int
	dsl_deadlist_insert_cb(void arg, const blkptr_t bp, boolean_t bp_freed,
	dmu_tx_t *tx)
	{
	dsl_deadlist_t *dl = arg;
	dsl_deadlist_insert(dl, bp, bp_freed, tx);
	return (0);
	}

	/*
	* Merge the deadlist pointed to by 'obj' into dl. obj will be left as
	* an empty deadlist.
	*/
	void
	dsl_deadlist_merge(dsl_deadlist_t dl, uint64_t obj, dmu_tx_t tx)
	{
	zap_cursor_t zc, pzc;
	zap_attribute_t za, pza;
	dmu_buf_t *bonus;
	dsl_deadlist_phys_t *dlp;
	dmu_object_info_t doi;
	int error, perror, i;

	VERIFY0(dmu_object_info(dl->dl_os, obj, &doi));
	if (doi.doi_type == DMU_OT_BPOBJ) {
	bpobj_t bpo;
	VERIFY0(bpobj_open(&bpo, dl->dl_os, obj));
	VERIFY0(bpobj_iterate(&bpo, dsl_deadlist_insert_cb, dl, tx));
	bpobj_close(&bpo);
	return;
	}

	za = kmem_alloc(sizeof (*za), KM_SLEEP);
	pza = kmem_alloc(sizeof (*pza), KM_SLEEP);

	mutex_enter(&dl->dl_lock);
	/*
	* Prefetch up to 128 deadlists first and then more as we progress.
	* The limit is a balance between ARC use and diminishing returns.
	*/
	for (zap_cursor_init(&pzc, dl->dl_os, obj), i = 0;
	(perror = zap_cursor_retrieve(&pzc, pza)) == 0 && i < 128;
	zap_cursor_advance(&pzc), i++) {
	dsl_deadlist_prefetch_bpobj(dl, pza->za_first_integer,
	zfs_strtonum(pza->za_name, NULL));
	}
	for (zap_cursor_init(&zc, dl->dl_os, obj);
	(error = zap_cursor_retrieve(&zc, za)) == 0;
	zap_cursor_advance(&zc)) {
	dsl_deadlist_insert_bpobj(dl, za->za_first_integer,
	zfs_strtonum(za->za_name, NULL), tx);
	VERIFY0(zap_remove(dl->dl_os, obj, za->za_name, tx));
	if (perror == 0) {
	dsl_deadlist_prefetch_bpobj(dl, pza->za_first_integer,
	zfs_strtonum(pza->za_name, NULL));
	zap_cursor_advance(&pzc);
	perror = zap_cursor_retrieve(&pzc, pza);
	}
	}
	VERIFY3U(error, ==, ENOENT);
	zap_cursor_fini(&zc);
	zap_cursor_fini(&pzc);

	VERIFY0(dmu_bonus_hold(dl->dl_os, obj, FTAG, &bonus));
	dlp = bonus->db_data;
	dmu_buf_will_dirty(bonus, tx);
	memset(dlp, 0, sizeof (*dlp));
	dmu_buf_rele(bonus, FTAG);
	mutex_exit(&dl->dl_lock);

	kmem_free(za, sizeof (*za));
	kmem_free(pza, sizeof (*pza));
	}

	/*
	* Remove entries on dl that are born > mintxg, and put them on the bpobj.
	*/
	void
	dsl_deadlist_move_bpobj(dsl_deadlist_t dl, bpobj_t bpo, uint64_t mintxg,
	dmu_tx_t *tx)
	{
	dsl_deadlist_entry_t dle_tofind;
	dsl_deadlist_entry_t dle, pdle;
	avl_index_t where;
	int i;

	ASSERT(!dl->dl_oldfmt);

	mutex_enter(&dl->dl_lock);
	dmu_buf_will_dirty(dl->dl_dbuf, tx);
	dsl_deadlist_load_tree(dl);

	dle_tofind.dle_mintxg = mintxg;
	dle = avl_find(&dl->dl_tree, &dle_tofind, &where);
	if (dle == NULL)
	dle = avl_nearest(&dl->dl_tree, where, AVL_AFTER);
	/*
	* Prefetch up to 128 deadlists first and then more as we progress.
	* The limit is a balance between ARC use and diminishing returns.
	*/
	for (pdle = dle, i = 0; pdle && i < 128; i++) {
	bpobj_prefetch_subobj(bpo, pdle->dle_bpobj.bpo_object);
	pdle = AVL_NEXT(&dl->dl_tree, pdle);
	}
	while (dle) {
	uint64_t used, comp, uncomp;
	dsl_deadlist_entry_t *dle_next;

	bpobj_enqueue_subobj(bpo, dle->dle_bpobj.bpo_object, tx);
	if (pdle) {
	bpobj_prefetch_subobj(bpo, pdle->dle_bpobj.bpo_object);
	pdle = AVL_NEXT(&dl->dl_tree, pdle);
	}

	VERIFY0(bpobj_space(&dle->dle_bpobj,
	&used, &comp, &uncomp));
	ASSERT3U(dl->dl_phys->dl_used, >=, used);
	ASSERT3U(dl->dl_phys->dl_comp, >=, comp);
	ASSERT3U(dl->dl_phys->dl_uncomp, >=, uncomp);
	dl->dl_phys->dl_used -= used;
	dl->dl_phys->dl_comp -= comp;
	dl->dl_phys->dl_uncomp -= uncomp;

	VERIFY0(zap_remove_int(dl->dl_os, dl->dl_object,
	dle->dle_mintxg, tx));

	dle_next = AVL_NEXT(&dl->dl_tree, dle);
	avl_remove(&dl->dl_tree, dle);
	bpobj_close(&dle->dle_bpobj);
	kmem_free(dle, sizeof (*dle));
	dle = dle_next;
	}
	mutex_exit(&dl->dl_lock);
	}

	typedef struct livelist_entry {
	blkptr_t le_bp;
	uint32_t le_refcnt;
	avl_node_t le_node;
	} livelist_entry_t;

	static int
	livelist_compare(const void larg, const void rarg)
	{
	const blkptr_t l = &((livelist_entry_t )larg)->le_bp;
	const blkptr_t r = &((livelist_entry_t )rarg)->le_bp;

	/* Sort them according to dva[0] */
	uint64_t l_dva0_vdev = DVA_GET_VDEV(&l->blk_dva[0]);
	uint64_t r_dva0_vdev = DVA_GET_VDEV(&r->blk_dva[0]);

	if (l_dva0_vdev != r_dva0_vdev)
	return (TREE_CMP(l_dva0_vdev, r_dva0_vdev));

	/* if vdevs are equal, sort by offsets. */
	uint64_t l_dva0_offset = DVA_GET_OFFSET(&l->blk_dva[0]);
	uint64_t r_dva0_offset = DVA_GET_OFFSET(&r->blk_dva[0]);
	return (TREE_CMP(l_dva0_offset, r_dva0_offset));
	}

	struct livelist_iter_arg {
	avl_tree_t *avl;
	bplist_t *to_free;
	zthr_t *t;
	};

	/*
	* Expects an AVL tree which is incrementally filled will FREE blkptrs
	* and used to match up ALLOC/FREE pairs. ALLOC'd blkptrs without a
	* corresponding FREE are stored in the supplied bplist.
	*
	* Note that multiple FREE and ALLOC entries for the same blkptr may be
	* encountered when dedup or block cloning is involved. For this reason we
	* keep a refcount for all the FREE entries of each blkptr and ensure that
	* each of those FREE entries has a corresponding ALLOC preceding it.
	*/
	static int
	dsl_livelist_iterate(void arg, const blkptr_t bp, boolean_t bp_freed,
	dmu_tx_t *tx)
	{
	struct livelist_iter_arg *lia = arg;
	avl_tree_t *avl = lia->avl;
	bplist_t *to_free = lia->to_free;
	zthr_t *t = lia->t;
	ASSERT(tx == NULL);

	if ((t != NULL) && (zthr_has_waiters(t) \|\| zthr_iscancelled(t)))
	return (SET_ERROR(EINTR));

	livelist_entry_t node;
	node.le_bp = *bp;
	livelist_entry_t *found = avl_find(avl, &node, NULL);
	if (found) {
	ASSERT3U(BP_GET_PSIZE(bp), ==, BP_GET_PSIZE(&found->le_bp));
	ASSERT3U(BP_GET_CHECKSUM(bp), ==,
	BP_GET_CHECKSUM(&found->le_bp));
	ASSERT3U(BP_PHYSICAL_BIRTH(bp), ==,
	BP_PHYSICAL_BIRTH(&found->le_bp));
	}
	if (bp_freed) {
	if (found == NULL) {
	/* first free entry for this blkptr */
	livelist_entry_t *e =
	kmem_alloc(sizeof (livelist_entry_t), KM_SLEEP);
	e->le_bp = *bp;
	e->le_refcnt = 1;
	avl_add(avl, e);
	} else {
	/*
	* Deduped or cloned block free. We could assert D bit
	* for dedup, but there is no such one for cloning.
	*/
	ASSERT3U(found->le_refcnt + 1, >, found->le_refcnt);
	found->le_refcnt++;
	}
	} else {
	if (found == NULL) {
	/* block is currently marked as allocated */
	bplist_append(to_free, bp);
	} else {
	/* alloc matches a free entry */
	ASSERT3U(found->le_refcnt, !=, 0);
	found->le_refcnt--;
	if (found->le_refcnt == 0) {
	/* all tracked free pairs have been matched */
	avl_remove(avl, found);
	kmem_free(found, sizeof (livelist_entry_t));
	}
	}
	}
	return (0);
	}

	/*
	* Accepts a bpobj and a bplist. Will insert into the bplist the blkptrs
	* which have an ALLOC entry but no matching FREE
	*/
	int
	dsl_process_sub_livelist(bpobj_t bpobj, bplist_t to_free, zthr_t *t,
	uint64_t *size)
	{
	avl_tree_t avl;
	avl_create(&avl, livelist_compare, sizeof (livelist_entry_t),
	offsetof(livelist_entry_t, le_node));

	/* process the sublist */
	struct livelist_iter_arg arg = {
	.avl = &avl,
	.to_free = to_free,
	.t = t
	};
	int err = bpobj_iterate_nofree(bpobj, dsl_livelist_iterate, &arg, size);
	VERIFY(err != 0 \|\| avl_numnodes(&avl) == 0);

	void *cookie = NULL;
	livelist_entry_t *le = NULL;
	while ((le = avl_destroy_nodes(&avl, &cookie)) != NULL) {
	kmem_free(le, sizeof (livelist_entry_t));
	}
	avl_destroy(&avl);
	return (err);
	}

	ZFS_MODULE_PARAM(zfs_livelist, zfs_livelist_, max_entries, U64, ZMOD_RW,
	"Size to start the next sub-livelist in a livelist");

	ZFS_MODULE_PARAM(zfs_livelist, zfs_livelist_, min_percent_shared, INT, ZMOD_RW,
	"Threshold at which livelist is disabled");
	diff --git a/sys/contrib/openzfs/module/zfs/metaslab.c b/sys/contrib/openzfs/module/zfs/metaslab.c
	index 5809a832bcb0..dbfc00362ff8 100644
	--- a/sys/contrib/openzfs/module/zfs/metaslab.c
	+++ b/sys/contrib/openzfs/module/zfs/metaslab.c
	@@ -1,6233 +1,6233 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2011, 2019 by Delphix. All rights reserved.
	* Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
	* Copyright (c) 2015, Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2017, Intel Corporation.
	*/

	#include <sys/zfs_context.h>
	#include <sys/dmu.h>
	#include <sys/dmu_tx.h>
	#include <sys/space_map.h>
	#include <sys/metaslab_impl.h>
	#include <sys/vdev_impl.h>
	#include <sys/vdev_draid.h>
	#include <sys/zio.h>
	#include <sys/spa_impl.h>
	#include <sys/zfeature.h>
	#include <sys/vdev_indirect_mapping.h>
	#include <sys/zap.h>
	#include <sys/btree.h>

	#define WITH_DF_BLOCK_ALLOCATOR

	#define GANG_ALLOCATION(flags) \
	((flags) & (METASLAB_GANG_CHILD \| METASLAB_GANG_HEADER))

	/*
	* Metaslab granularity, in bytes. This is roughly similar to what would be
	* referred to as the "stripe size" in traditional RAID arrays. In normal
	* operation, we will try to write this amount of data to each disk before
	* moving on to the next top-level vdev.
	*/
	static uint64_t metaslab_aliquot = 1024 * 1024;

	/*
	* For testing, make some blocks above a certain size be gang blocks.
	*/
	uint64_t metaslab_force_ganging = SPA_MAXBLOCKSIZE + 1;

	/*
	* Of blocks of size >= metaslab_force_ganging, actually gang them this often.
	*/
	uint_t metaslab_force_ganging_pct = 3;

	/*
	* In pools where the log space map feature is not enabled we touch
	* multiple metaslabs (and their respective space maps) with each
	* transaction group. Thus, we benefit from having a small space map
	* block size since it allows us to issue more I/O operations scattered
	* around the disk. So a sane default for the space map block size
	* is 8~16K.
	*/
	int zfs_metaslab_sm_blksz_no_log = (1 << 14);

	/*
	* When the log space map feature is enabled, we accumulate a lot of
	* changes per metaslab that are flushed once in a while so we benefit
	* from a bigger block size like 128K for the metaslab space maps.
	*/
	int zfs_metaslab_sm_blksz_with_log = (1 << 17);

	/*
	* The in-core space map representation is more compact than its on-disk form.
	* The zfs_condense_pct determines how much more compact the in-core
	* space map representation must be before we compact it on-disk.
	* Values should be greater than or equal to 100.
	*/
	uint_t zfs_condense_pct = 200;

	/*
	* Condensing a metaslab is not guaranteed to actually reduce the amount of
	* space used on disk. In particular, a space map uses data in increments of
	* MAX(1 << ashift, space_map_blksz), so a metaslab might use the
	* same number of blocks after condensing. Since the goal of condensing is to
	* reduce the number of IOPs required to read the space map, we only want to
	* condense when we can be sure we will reduce the number of blocks used by the
	* space map. Unfortunately, we cannot precisely compute whether or not this is
	* the case in metaslab_should_condense since we are holding ms_lock. Instead,
	* we apply the following heuristic: do not condense a spacemap unless the
	* uncondensed size consumes greater than zfs_metaslab_condense_block_threshold
	* blocks.
	*/
	static const int zfs_metaslab_condense_block_threshold = 4;

	/*
	* The zfs_mg_noalloc_threshold defines which metaslab groups should
	* be eligible for allocation. The value is defined as a percentage of
	* free space. Metaslab groups that have more free space than
	* zfs_mg_noalloc_threshold are always eligible for allocations. Once
	* a metaslab group's free space is less than or equal to the
	* zfs_mg_noalloc_threshold the allocator will avoid allocating to that
	* group unless all groups in the pool have reached zfs_mg_noalloc_threshold.
	* Once all groups in the pool reach zfs_mg_noalloc_threshold then all
	* groups are allowed to accept allocations. Gang blocks are always
	* eligible to allocate on any metaslab group. The default value of 0 means
	* no metaslab group will be excluded based on this criterion.
	*/
	static uint_t zfs_mg_noalloc_threshold = 0;

	/*
	* Metaslab groups are considered eligible for allocations if their
	* fragmentation metric (measured as a percentage) is less than or
	* equal to zfs_mg_fragmentation_threshold. If a metaslab group
	* exceeds this threshold then it will be skipped unless all metaslab
	* groups within the metaslab class have also crossed this threshold.
	*
	* This tunable was introduced to avoid edge cases where we continue
	* allocating from very fragmented disks in our pool while other, less
	* fragmented disks, exists. On the other hand, if all disks in the
	* pool are uniformly approaching the threshold, the threshold can
	* be a speed bump in performance, where we keep switching the disks
	* that we allocate from (e.g. we allocate some segments from disk A
	* making it bypassing the threshold while freeing segments from disk
	* B getting its fragmentation below the threshold).
	*
	* Empirically, we've seen that our vdev selection for allocations is
	* good enough that fragmentation increases uniformly across all vdevs
	* the majority of the time. Thus we set the threshold percentage high
	* enough to avoid hitting the speed bump on pools that are being pushed
	* to the edge.
	*/
	static uint_t zfs_mg_fragmentation_threshold = 95;

	/*
	* Allow metaslabs to keep their active state as long as their fragmentation
	* percentage is less than or equal to zfs_metaslab_fragmentation_threshold. An
	* active metaslab that exceeds this threshold will no longer keep its active
	* status allowing better metaslabs to be selected.
	*/
	static uint_t zfs_metaslab_fragmentation_threshold = 70;

	/*
	* When set will load all metaslabs when pool is first opened.
	*/
	int metaslab_debug_load = B_FALSE;

	/*
	* When set will prevent metaslabs from being unloaded.
	*/
	static int metaslab_debug_unload = B_FALSE;

	/*
	* Minimum size which forces the dynamic allocator to change
	* it's allocation strategy. Once the space map cannot satisfy
	* an allocation of this size then it switches to using more
	* aggressive strategy (i.e search by size rather than offset).
	*/
	uint64_t metaslab_df_alloc_threshold = SPA_OLD_MAXBLOCKSIZE;

	/*
	* The minimum free space, in percent, which must be available
	* in a space map to continue allocations in a first-fit fashion.
	* Once the space map's free space drops below this level we dynamically
	* switch to using best-fit allocations.
	*/
	uint_t metaslab_df_free_pct = 4;

	/*
	* Maximum distance to search forward from the last offset. Without this
	* limit, fragmented pools can see >100,000 iterations and
	* metaslab_block_picker() becomes the performance limiting factor on
	* high-performance storage.
	*
	* With the default setting of 16MB, we typically see less than 500
	* iterations, even with very fragmented, ashift=9 pools. The maximum number
	* of iterations possible is:
	* metaslab_df_max_search / (2 * (1<<ashift))
	* With the default setting of 16MB this is 16*1024 (with ashift=9) or
	* 2048 (with ashift=12).
	*/
	static uint_t metaslab_df_max_search = 16 * 1024 * 1024;

	/*
	* Forces the metaslab_block_picker function to search for at least this many
	* segments forwards until giving up on finding a segment that the allocation
	* will fit into.
	*/
	static const uint32_t metaslab_min_search_count = 100;

	/*
	* If we are not searching forward (due to metaslab_df_max_search,
	* metaslab_df_free_pct, or metaslab_df_alloc_threshold), this tunable
	* controls what segment is used. If it is set, we will use the largest free
	* segment. If it is not set, we will use a segment of exactly the requested
	* size (or larger).
	*/
	static int metaslab_df_use_largest_segment = B_FALSE;

	/*
	* These tunables control how long a metaslab will remain loaded after the
	* last allocation from it. A metaslab can't be unloaded until at least
	* metaslab_unload_delay TXG's and metaslab_unload_delay_ms milliseconds
	* have elapsed. However, zfs_metaslab_mem_limit may cause it to be
	* unloaded sooner. These settings are intended to be generous -- to keep
	* metaslabs loaded for a long time, reducing the rate of metaslab loading.
	*/
	static uint_t metaslab_unload_delay = 32;
	static uint_t metaslab_unload_delay_ms = 10 * 60 * 1000; /* ten minutes */

	/*
	* Max number of metaslabs per group to preload.
	*/
	uint_t metaslab_preload_limit = 10;

	/*
	* Enable/disable preloading of metaslab.
	*/
	static int metaslab_preload_enabled = B_TRUE;

	/*
	* Enable/disable fragmentation weighting on metaslabs.
	*/
	static int metaslab_fragmentation_factor_enabled = B_TRUE;

	/*
	* Enable/disable lba weighting (i.e. outer tracks are given preference).
	*/
	static int metaslab_lba_weighting_enabled = B_TRUE;

	/*
	* Enable/disable metaslab group biasing.
	*/
	static int metaslab_bias_enabled = B_TRUE;

	/*
	* Enable/disable remapping of indirect DVAs to their concrete vdevs.
	*/
	static const boolean_t zfs_remap_blkptr_enable = B_TRUE;

	/*
	* Enable/disable segment-based metaslab selection.
	*/
	static int zfs_metaslab_segment_weight_enabled = B_TRUE;

	/*
	* When using segment-based metaslab selection, we will continue
	* allocating from the active metaslab until we have exhausted
	* zfs_metaslab_switch_threshold of its buckets.
	*/
	static int zfs_metaslab_switch_threshold = 2;

	/*
	* Internal switch to enable/disable the metaslab allocation tracing
	* facility.
	*/
	static const boolean_t metaslab_trace_enabled = B_FALSE;

	/*
	* Maximum entries that the metaslab allocation tracing facility will keep
	* in a given list when running in non-debug mode. We limit the number
	* of entries in non-debug mode to prevent us from using up too much memory.
	* The limit should be sufficiently large that we don't expect any allocation
	* to every exceed this value. In debug mode, the system will panic if this
	* limit is ever reached allowing for further investigation.
	*/
	static const uint64_t metaslab_trace_max_entries = 5000;

	/*
	* Maximum number of metaslabs per group that can be disabled
	* simultaneously.
	*/
	static const int max_disabled_ms = 3;

	/*
	* Time (in seconds) to respect ms_max_size when the metaslab is not loaded.
	* To avoid 64-bit overflow, don't set above UINT32_MAX.
	*/
	static uint64_t zfs_metaslab_max_size_cache_sec = 1 * 60 * 60; /* 1 hour */

	/*
	* Maximum percentage of memory to use on storing loaded metaslabs. If loading
	* a metaslab would take it over this percentage, the oldest selected metaslab
	* is automatically unloaded.
	*/
	static uint_t zfs_metaslab_mem_limit = 25;

	/*
	* Force the per-metaslab range trees to use 64-bit integers to store
	* segments. Used for debugging purposes.
	*/
	static const boolean_t zfs_metaslab_force_large_segs = B_FALSE;

	/*
	* By default we only store segments over a certain size in the size-sorted
	* metaslab trees (ms_allocatable_by_size and
	* ms_unflushed_frees_by_size). This dramatically reduces memory usage and
	* improves load and unload times at the cost of causing us to use slightly
	* larger segments than we would otherwise in some cases.
	*/
	static const uint32_t metaslab_by_size_min_shift = 14;

	/*
	* If not set, we will first try normal allocation. If that fails then
	* we will do a gang allocation. If that fails then we will do a "try hard"
	* gang allocation. If that fails then we will have a multi-layer gang
	* block.
	*
	* If set, we will first try normal allocation. If that fails then
	* we will do a "try hard" allocation. If that fails we will do a gang
	* allocation. If that fails we will do a "try hard" gang allocation. If
	* that fails then we will have a multi-layer gang block.
	*/
	static int zfs_metaslab_try_hard_before_gang = B_FALSE;

	/*
	* When not trying hard, we only consider the best zfs_metaslab_find_max_tries
	* metaslabs. This improves performance, especially when there are many
	* metaslabs per vdev and the allocation can't actually be satisfied (so we
	* would otherwise iterate all the metaslabs). If there is a metaslab with a
	* worse weight but it can actually satisfy the allocation, we won't find it
	* until trying hard. This may happen if the worse metaslab is not loaded
	* (and the true weight is better than we have calculated), or due to weight
	* bucketization. E.g. we are looking for a 60K segment, and the best
	* metaslabs all have free segments in the 32-63K bucket, but the best
	* zfs_metaslab_find_max_tries metaslabs have ms_max_size <60KB, and a
	* subsequent metaslab has ms_max_size >60KB (but fewer segments in this
	* bucket, and therefore a lower weight).
	*/
	static uint_t zfs_metaslab_find_max_tries = 100;

	static uint64_t metaslab_weight(metaslab_t *, boolean_t);
	static void metaslab_set_fragmentation(metaslab_t *, boolean_t);
	static void metaslab_free_impl(vdev_t *, uint64_t, uint64_t, boolean_t);
	static void metaslab_check_free_impl(vdev_t *, uint64_t, uint64_t);

	static void metaslab_passivate(metaslab_t *msp, uint64_t weight);
	static uint64_t metaslab_weight_from_range_tree(metaslab_t *msp);
	static void metaslab_flush_update(metaslab_t , dmu_tx_t );
	static unsigned int metaslab_idx_func(multilist_t , void );
	static void metaslab_evict(metaslab_t *, uint64_t);
	static void metaslab_rt_add(range_tree_t rt, range_seg_t rs, void *arg);
	kmem_cache_t *metaslab_alloc_trace_cache;

	typedef struct metaslab_stats {
	kstat_named_t metaslabstat_trace_over_limit;
	kstat_named_t metaslabstat_reload_tree;
	kstat_named_t metaslabstat_too_many_tries;
	kstat_named_t metaslabstat_try_hard;
	} metaslab_stats_t;

	static metaslab_stats_t metaslab_stats = {
	{ "trace_over_limit", KSTAT_DATA_UINT64 },
	{ "reload_tree", KSTAT_DATA_UINT64 },
	{ "too_many_tries", KSTAT_DATA_UINT64 },
	{ "try_hard", KSTAT_DATA_UINT64 },
	};

	#define METASLABSTAT_BUMP(stat) \
	atomic_inc_64(&metaslab_stats.stat.value.ui64);


	static kstat_t *metaslab_ksp;

	void
	metaslab_stat_init(void)
	{
	ASSERT(metaslab_alloc_trace_cache == NULL);
	metaslab_alloc_trace_cache = kmem_cache_create(
	"metaslab_alloc_trace_cache", sizeof (metaslab_alloc_trace_t),
	0, NULL, NULL, NULL, NULL, NULL, 0);
	metaslab_ksp = kstat_create("zfs", 0, "metaslab_stats",
	"misc", KSTAT_TYPE_NAMED, sizeof (metaslab_stats) /
	sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
	if (metaslab_ksp != NULL) {
	metaslab_ksp->ks_data = &metaslab_stats;
	kstat_install(metaslab_ksp);
	}
	}

	void
	metaslab_stat_fini(void)
	{
	if (metaslab_ksp != NULL) {
	kstat_delete(metaslab_ksp);
	metaslab_ksp = NULL;
	}

	kmem_cache_destroy(metaslab_alloc_trace_cache);
	metaslab_alloc_trace_cache = NULL;
	}

	/*
	* ==========================================================================
	* Metaslab classes
	* ==========================================================================
	*/
	metaslab_class_t *
	metaslab_class_create(spa_t spa, const metaslab_ops_t ops)
	{
	metaslab_class_t *mc;

	mc = kmem_zalloc(offsetof(metaslab_class_t,
	mc_allocator[spa->spa_alloc_count]), KM_SLEEP);

	mc->mc_spa = spa;
	mc->mc_ops = ops;
	mutex_init(&mc->mc_lock, NULL, MUTEX_DEFAULT, NULL);
	multilist_create(&mc->mc_metaslab_txg_list, sizeof (metaslab_t),
	offsetof(metaslab_t, ms_class_txg_node), metaslab_idx_func);
	for (int i = 0; i < spa->spa_alloc_count; i++) {
	metaslab_class_allocator_t *mca = &mc->mc_allocator[i];
	mca->mca_rotor = NULL;
	zfs_refcount_create_tracked(&mca->mca_alloc_slots);
	}

	return (mc);
	}

	void
	metaslab_class_destroy(metaslab_class_t *mc)
	{
	spa_t *spa = mc->mc_spa;

	ASSERT(mc->mc_alloc == 0);
	ASSERT(mc->mc_deferred == 0);
	ASSERT(mc->mc_space == 0);
	ASSERT(mc->mc_dspace == 0);

	for (int i = 0; i < spa->spa_alloc_count; i++) {
	metaslab_class_allocator_t *mca = &mc->mc_allocator[i];
	ASSERT(mca->mca_rotor == NULL);
	zfs_refcount_destroy(&mca->mca_alloc_slots);
	}
	mutex_destroy(&mc->mc_lock);
	multilist_destroy(&mc->mc_metaslab_txg_list);
	kmem_free(mc, offsetof(metaslab_class_t,
	mc_allocator[spa->spa_alloc_count]));
	}

	int
	metaslab_class_validate(metaslab_class_t *mc)
	{
	metaslab_group_t *mg;
	vdev_t *vd;

	/*
	* Must hold one of the spa_config locks.
	*/
	ASSERT(spa_config_held(mc->mc_spa, SCL_ALL, RW_READER) \|\|
	spa_config_held(mc->mc_spa, SCL_ALL, RW_WRITER));

	if ((mg = mc->mc_allocator[0].mca_rotor) == NULL)
	return (0);

	do {
	vd = mg->mg_vd;
	ASSERT(vd->vdev_mg != NULL);
	ASSERT3P(vd->vdev_top, ==, vd);
	ASSERT3P(mg->mg_class, ==, mc);
	ASSERT3P(vd->vdev_ops, !=, &vdev_hole_ops);
	} while ((mg = mg->mg_next) != mc->mc_allocator[0].mca_rotor);

	return (0);
	}

	static void
	metaslab_class_space_update(metaslab_class_t *mc, int64_t alloc_delta,
	int64_t defer_delta, int64_t space_delta, int64_t dspace_delta)
	{
	atomic_add_64(&mc->mc_alloc, alloc_delta);
	atomic_add_64(&mc->mc_deferred, defer_delta);
	atomic_add_64(&mc->mc_space, space_delta);
	atomic_add_64(&mc->mc_dspace, dspace_delta);
	}

	uint64_t
	metaslab_class_get_alloc(metaslab_class_t *mc)
	{
	return (mc->mc_alloc);
	}

	uint64_t
	metaslab_class_get_deferred(metaslab_class_t *mc)
	{
	return (mc->mc_deferred);
	}

	uint64_t
	metaslab_class_get_space(metaslab_class_t *mc)
	{
	return (mc->mc_space);
	}

	uint64_t
	metaslab_class_get_dspace(metaslab_class_t *mc)
	{
	return (spa_deflate(mc->mc_spa) ? mc->mc_dspace : mc->mc_space);
	}

	void
	metaslab_class_histogram_verify(metaslab_class_t *mc)
	{
	spa_t *spa = mc->mc_spa;
	vdev_t *rvd = spa->spa_root_vdev;
	uint64_t *mc_hist;
	int i;

	if ((zfs_flags & ZFS_DEBUG_HISTOGRAM_VERIFY) == 0)
	return;

	mc_hist = kmem_zalloc(sizeof (uint64_t) * RANGE_TREE_HISTOGRAM_SIZE,
	KM_SLEEP);

	mutex_enter(&mc->mc_lock);
	for (int c = 0; c < rvd->vdev_children; c++) {
	vdev_t *tvd = rvd->vdev_child[c];
	metaslab_group_t *mg = vdev_get_mg(tvd, mc);

	/*
	* Skip any holes, uninitialized top-levels, or
	* vdevs that are not in this metalab class.
	*/
	if (!vdev_is_concrete(tvd) \|\| tvd->vdev_ms_shift == 0 \|\|
	mg->mg_class != mc) {
	continue;
	}

	IMPLY(mg == mg->mg_vd->vdev_log_mg,
	mc == spa_embedded_log_class(mg->mg_vd->vdev_spa));

	for (i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++)
	mc_hist[i] += mg->mg_histogram[i];
	}

	for (i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++) {
	VERIFY3U(mc_hist[i], ==, mc->mc_histogram[i]);
	}

	mutex_exit(&mc->mc_lock);
	kmem_free(mc_hist, sizeof (uint64_t) * RANGE_TREE_HISTOGRAM_SIZE);
	}

	/*
	* Calculate the metaslab class's fragmentation metric. The metric
	* is weighted based on the space contribution of each metaslab group.
	* The return value will be a number between 0 and 100 (inclusive), or
	* ZFS_FRAG_INVALID if the metric has not been set. See comment above the
	* zfs_frag_table for more information about the metric.
	*/
	uint64_t
	metaslab_class_fragmentation(metaslab_class_t *mc)
	{
	vdev_t *rvd = mc->mc_spa->spa_root_vdev;
	uint64_t fragmentation = 0;

	spa_config_enter(mc->mc_spa, SCL_VDEV, FTAG, RW_READER);

	for (int c = 0; c < rvd->vdev_children; c++) {
	vdev_t *tvd = rvd->vdev_child[c];
	metaslab_group_t *mg = tvd->vdev_mg;

	/*
	* Skip any holes, uninitialized top-levels,
	* or vdevs that are not in this metalab class.
	*/
	if (!vdev_is_concrete(tvd) \|\| tvd->vdev_ms_shift == 0 \|\|
	mg->mg_class != mc) {
	continue;
	}

	/*
	* If a metaslab group does not contain a fragmentation
	* metric then just bail out.
	*/
	if (mg->mg_fragmentation == ZFS_FRAG_INVALID) {
	spa_config_exit(mc->mc_spa, SCL_VDEV, FTAG);
	return (ZFS_FRAG_INVALID);
	}

	/*
	* Determine how much this metaslab_group is contributing
	* to the overall pool fragmentation metric.
	*/
	fragmentation += mg->mg_fragmentation *
	metaslab_group_get_space(mg);
	}
	fragmentation /= metaslab_class_get_space(mc);

	ASSERT3U(fragmentation, <=, 100);
	spa_config_exit(mc->mc_spa, SCL_VDEV, FTAG);
	return (fragmentation);
	}

	/*
	* Calculate the amount of expandable space that is available in
	* this metaslab class. If a device is expanded then its expandable
	* space will be the amount of allocatable space that is currently not
	* part of this metaslab class.
	*/
	uint64_t
	metaslab_class_expandable_space(metaslab_class_t *mc)
	{
	vdev_t *rvd = mc->mc_spa->spa_root_vdev;
	uint64_t space = 0;

	spa_config_enter(mc->mc_spa, SCL_VDEV, FTAG, RW_READER);
	for (int c = 0; c < rvd->vdev_children; c++) {
	vdev_t *tvd = rvd->vdev_child[c];
	metaslab_group_t *mg = tvd->vdev_mg;

	if (!vdev_is_concrete(tvd) \|\| tvd->vdev_ms_shift == 0 \|\|
	mg->mg_class != mc) {
	continue;
	}

	/*
	* Calculate if we have enough space to add additional
	* metaslabs. We report the expandable space in terms
	* of the metaslab size since that's the unit of expansion.
	*/
	space += P2ALIGN(tvd->vdev_max_asize - tvd->vdev_asize,
	1ULL << tvd->vdev_ms_shift);
	}
	spa_config_exit(mc->mc_spa, SCL_VDEV, FTAG);
	return (space);
	}

	void
	metaslab_class_evict_old(metaslab_class_t *mc, uint64_t txg)
	{
	multilist_t *ml = &mc->mc_metaslab_txg_list;
	for (int i = 0; i < multilist_get_num_sublists(ml); i++) {
	- multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
	+ multilist_sublist_t *mls = multilist_sublist_lock_idx(ml, i);
	metaslab_t *msp = multilist_sublist_head(mls);
	multilist_sublist_unlock(mls);
	while (msp != NULL) {
	mutex_enter(&msp->ms_lock);

	/*
	* If the metaslab has been removed from the list
	* (which could happen if we were at the memory limit
	* and it was evicted during this loop), then we can't
	* proceed and we should restart the sublist.
	*/
	if (!multilist_link_active(&msp->ms_class_txg_node)) {
	mutex_exit(&msp->ms_lock);
	i--;
	break;
	}
	- mls = multilist_sublist_lock(ml, i);
	+ mls = multilist_sublist_lock_idx(ml, i);
	metaslab_t *next_msp = multilist_sublist_next(mls, msp);
	multilist_sublist_unlock(mls);
	if (txg >
	msp->ms_selected_txg + metaslab_unload_delay &&
	gethrtime() > msp->ms_selected_time +
	(uint64_t)MSEC2NSEC(metaslab_unload_delay_ms)) {
	metaslab_evict(msp, txg);
	} else {
	/*
	* Once we've hit a metaslab selected too
	* recently to evict, we're done evicting for
	* now.
	*/
	mutex_exit(&msp->ms_lock);
	break;
	}
	mutex_exit(&msp->ms_lock);
	msp = next_msp;
	}
	}
	}

	static int
	metaslab_compare(const void x1, const void x2)
	{
	const metaslab_t m1 = (const metaslab_t )x1;
	const metaslab_t m2 = (const metaslab_t )x2;

	int sort1 = 0;
	int sort2 = 0;
	if (m1->ms_allocator != -1 && m1->ms_primary)
	sort1 = 1;
	else if (m1->ms_allocator != -1 && !m1->ms_primary)
	sort1 = 2;
	if (m2->ms_allocator != -1 && m2->ms_primary)
	sort2 = 1;
	else if (m2->ms_allocator != -1 && !m2->ms_primary)
	sort2 = 2;

	/*
	* Sort inactive metaslabs first, then primaries, then secondaries. When
	* selecting a metaslab to allocate from, an allocator first tries its
	* primary, then secondary active metaslab. If it doesn't have active
	* metaslabs, or can't allocate from them, it searches for an inactive
	* metaslab to activate. If it can't find a suitable one, it will steal
	* a primary or secondary metaslab from another allocator.
	*/
	if (sort1 < sort2)
	return (-1);
	if (sort1 > sort2)
	return (1);

	int cmp = TREE_CMP(m2->ms_weight, m1->ms_weight);
	if (likely(cmp))
	return (cmp);

	IMPLY(TREE_CMP(m1->ms_start, m2->ms_start) == 0, m1 == m2);

	return (TREE_CMP(m1->ms_start, m2->ms_start));
	}

	/*
	* ==========================================================================
	* Metaslab groups
	* ==========================================================================
	*/
	/*
	* Update the allocatable flag and the metaslab group's capacity.
	* The allocatable flag is set to true if the capacity is below
	* the zfs_mg_noalloc_threshold or has a fragmentation value that is
	* greater than zfs_mg_fragmentation_threshold. If a metaslab group
	* transitions from allocatable to non-allocatable or vice versa then the
	* metaslab group's class is updated to reflect the transition.
	*/
	static void
	metaslab_group_alloc_update(metaslab_group_t *mg)
	{
	vdev_t *vd = mg->mg_vd;
	metaslab_class_t *mc = mg->mg_class;
	vdev_stat_t *vs = &vd->vdev_stat;
	boolean_t was_allocatable;
	boolean_t was_initialized;

	ASSERT(vd == vd->vdev_top);
	ASSERT3U(spa_config_held(mc->mc_spa, SCL_ALLOC, RW_READER), ==,
	SCL_ALLOC);

	mutex_enter(&mg->mg_lock);
	was_allocatable = mg->mg_allocatable;
	was_initialized = mg->mg_initialized;

	mg->mg_free_capacity = ((vs->vs_space - vs->vs_alloc) * 100) /
	(vs->vs_space + 1);

	mutex_enter(&mc->mc_lock);

	/*
	* If the metaslab group was just added then it won't
	* have any space until we finish syncing out this txg.
	* At that point we will consider it initialized and available
	* for allocations. We also don't consider non-activated
	* metaslab groups (e.g. vdevs that are in the middle of being removed)
	* to be initialized, because they can't be used for allocation.
	*/
	mg->mg_initialized = metaslab_group_initialized(mg);
	if (!was_initialized && mg->mg_initialized) {
	mc->mc_groups++;
	} else if (was_initialized && !mg->mg_initialized) {
	ASSERT3U(mc->mc_groups, >, 0);
	mc->mc_groups--;
	}
	if (mg->mg_initialized)
	mg->mg_no_free_space = B_FALSE;

	/*
	* A metaslab group is considered allocatable if it has plenty
	* of free space or is not heavily fragmented. We only take
	* fragmentation into account if the metaslab group has a valid
	* fragmentation metric (i.e. a value between 0 and 100).
	*/
	mg->mg_allocatable = (mg->mg_activation_count > 0 &&
	mg->mg_free_capacity > zfs_mg_noalloc_threshold &&
	(mg->mg_fragmentation == ZFS_FRAG_INVALID \|\|
	mg->mg_fragmentation <= zfs_mg_fragmentation_threshold));

	/*
	* The mc_alloc_groups maintains a count of the number of
	* groups in this metaslab class that are still above the
	* zfs_mg_noalloc_threshold. This is used by the allocating
	* threads to determine if they should avoid allocations to
	* a given group. The allocator will avoid allocations to a group
	* if that group has reached or is below the zfs_mg_noalloc_threshold
	* and there are still other groups that are above the threshold.
	* When a group transitions from allocatable to non-allocatable or
	* vice versa we update the metaslab class to reflect that change.
	* When the mc_alloc_groups value drops to 0 that means that all
	* groups have reached the zfs_mg_noalloc_threshold making all groups
	* eligible for allocations. This effectively means that all devices
	* are balanced again.
	*/
	if (was_allocatable && !mg->mg_allocatable)
	mc->mc_alloc_groups--;
	else if (!was_allocatable && mg->mg_allocatable)
	mc->mc_alloc_groups++;
	mutex_exit(&mc->mc_lock);

	mutex_exit(&mg->mg_lock);
	}

	int
	metaslab_sort_by_flushed(const void va, const void vb)
	{
	const metaslab_t *a = va;
	const metaslab_t *b = vb;

	int cmp = TREE_CMP(a->ms_unflushed_txg, b->ms_unflushed_txg);
	if (likely(cmp))
	return (cmp);

	uint64_t a_vdev_id = a->ms_group->mg_vd->vdev_id;
	uint64_t b_vdev_id = b->ms_group->mg_vd->vdev_id;
	cmp = TREE_CMP(a_vdev_id, b_vdev_id);
	if (cmp)
	return (cmp);

	return (TREE_CMP(a->ms_id, b->ms_id));
	}

	metaslab_group_t *
	metaslab_group_create(metaslab_class_t mc, vdev_t vd, int allocators)
	{
	metaslab_group_t *mg;

	mg = kmem_zalloc(offsetof(metaslab_group_t,
	mg_allocator[allocators]), KM_SLEEP);
	mutex_init(&mg->mg_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&mg->mg_ms_disabled_lock, NULL, MUTEX_DEFAULT, NULL);
	cv_init(&mg->mg_ms_disabled_cv, NULL, CV_DEFAULT, NULL);
	avl_create(&mg->mg_metaslab_tree, metaslab_compare,
	sizeof (metaslab_t), offsetof(metaslab_t, ms_group_node));
	mg->mg_vd = vd;
	mg->mg_class = mc;
	mg->mg_activation_count = 0;
	mg->mg_initialized = B_FALSE;
	mg->mg_no_free_space = B_TRUE;
	mg->mg_allocators = allocators;

	for (int i = 0; i < allocators; i++) {
	metaslab_group_allocator_t *mga = &mg->mg_allocator[i];
	zfs_refcount_create_tracked(&mga->mga_alloc_queue_depth);
	}

	return (mg);
	}

	void
	metaslab_group_destroy(metaslab_group_t *mg)
	{
	ASSERT(mg->mg_prev == NULL);
	ASSERT(mg->mg_next == NULL);
	/*
	* We may have gone below zero with the activation count
	* either because we never activated in the first place or
	* because we're done, and possibly removing the vdev.
	*/
	ASSERT(mg->mg_activation_count <= 0);

	avl_destroy(&mg->mg_metaslab_tree);
	mutex_destroy(&mg->mg_lock);
	mutex_destroy(&mg->mg_ms_disabled_lock);
	cv_destroy(&mg->mg_ms_disabled_cv);

	for (int i = 0; i < mg->mg_allocators; i++) {
	metaslab_group_allocator_t *mga = &mg->mg_allocator[i];
	zfs_refcount_destroy(&mga->mga_alloc_queue_depth);
	}
	kmem_free(mg, offsetof(metaslab_group_t,
	mg_allocator[mg->mg_allocators]));
	}

	void
	metaslab_group_activate(metaslab_group_t *mg)
	{
	metaslab_class_t *mc = mg->mg_class;
	spa_t *spa = mc->mc_spa;
	metaslab_group_t mgprev, mgnext;

	ASSERT3U(spa_config_held(spa, SCL_ALLOC, RW_WRITER), !=, 0);

	ASSERT(mg->mg_prev == NULL);
	ASSERT(mg->mg_next == NULL);
	ASSERT(mg->mg_activation_count <= 0);

	if (++mg->mg_activation_count <= 0)
	return;

	mg->mg_aliquot = metaslab_aliquot * MAX(1,
	vdev_get_ndisks(mg->mg_vd) - vdev_get_nparity(mg->mg_vd));
	metaslab_group_alloc_update(mg);

	if ((mgprev = mc->mc_allocator[0].mca_rotor) == NULL) {
	mg->mg_prev = mg;
	mg->mg_next = mg;
	} else {
	mgnext = mgprev->mg_next;
	mg->mg_prev = mgprev;
	mg->mg_next = mgnext;
	mgprev->mg_next = mg;
	mgnext->mg_prev = mg;
	}
	for (int i = 0; i < spa->spa_alloc_count; i++) {
	mc->mc_allocator[i].mca_rotor = mg;
	mg = mg->mg_next;
	}
	}

	/*
	* Passivate a metaslab group and remove it from the allocation rotor.
	* Callers must hold both the SCL_ALLOC and SCL_ZIO lock prior to passivating
	* a metaslab group. This function will momentarily drop spa_config_locks
	* that are lower than the SCL_ALLOC lock (see comment below).
	*/
	void
	metaslab_group_passivate(metaslab_group_t *mg)
	{
	metaslab_class_t *mc = mg->mg_class;
	spa_t *spa = mc->mc_spa;
	metaslab_group_t mgprev, mgnext;
	int locks = spa_config_held(spa, SCL_ALL, RW_WRITER);

	ASSERT3U(spa_config_held(spa, SCL_ALLOC \| SCL_ZIO, RW_WRITER), ==,
	(SCL_ALLOC \| SCL_ZIO));

	if (--mg->mg_activation_count != 0) {
	for (int i = 0; i < spa->spa_alloc_count; i++)
	ASSERT(mc->mc_allocator[i].mca_rotor != mg);
	ASSERT(mg->mg_prev == NULL);
	ASSERT(mg->mg_next == NULL);
	ASSERT(mg->mg_activation_count < 0);
	return;
	}

	/*
	* The spa_config_lock is an array of rwlocks, ordered as
	* follows (from highest to lowest):
	* SCL_CONFIG > SCL_STATE > SCL_L2ARC > SCL_ALLOC >
	* SCL_ZIO > SCL_FREE > SCL_VDEV
	* (For more information about the spa_config_lock see spa_misc.c)
	* The higher the lock, the broader its coverage. When we passivate
	* a metaslab group, we must hold both the SCL_ALLOC and the SCL_ZIO
	* config locks. However, the metaslab group's taskq might be trying
	* to preload metaslabs so we must drop the SCL_ZIO lock and any
	* lower locks to allow the I/O to complete. At a minimum,
	* we continue to hold the SCL_ALLOC lock, which prevents any future
	* allocations from taking place and any changes to the vdev tree.
	*/
	spa_config_exit(spa, locks & ~(SCL_ZIO - 1), spa);
	taskq_wait_outstanding(spa->spa_metaslab_taskq, 0);
	spa_config_enter(spa, locks & ~(SCL_ZIO - 1), spa, RW_WRITER);
	metaslab_group_alloc_update(mg);
	for (int i = 0; i < mg->mg_allocators; i++) {
	metaslab_group_allocator_t *mga = &mg->mg_allocator[i];
	metaslab_t *msp = mga->mga_primary;
	if (msp != NULL) {
	mutex_enter(&msp->ms_lock);
	metaslab_passivate(msp,
	metaslab_weight_from_range_tree(msp));
	mutex_exit(&msp->ms_lock);
	}
	msp = mga->mga_secondary;
	if (msp != NULL) {
	mutex_enter(&msp->ms_lock);
	metaslab_passivate(msp,
	metaslab_weight_from_range_tree(msp));
	mutex_exit(&msp->ms_lock);
	}
	}

	mgprev = mg->mg_prev;
	mgnext = mg->mg_next;

	if (mg == mgnext) {
	mgnext = NULL;
	} else {
	mgprev->mg_next = mgnext;
	mgnext->mg_prev = mgprev;
	}
	for (int i = 0; i < spa->spa_alloc_count; i++) {
	if (mc->mc_allocator[i].mca_rotor == mg)
	mc->mc_allocator[i].mca_rotor = mgnext;
	}

	mg->mg_prev = NULL;
	mg->mg_next = NULL;
	}

	boolean_t
	metaslab_group_initialized(metaslab_group_t *mg)
	{
	vdev_t *vd = mg->mg_vd;
	vdev_stat_t *vs = &vd->vdev_stat;

	return (vs->vs_space != 0 && mg->mg_activation_count > 0);
	}

	uint64_t
	metaslab_group_get_space(metaslab_group_t *mg)
	{
	/*
	* Note that the number of nodes in mg_metaslab_tree may be one less
	* than vdev_ms_count, due to the embedded log metaslab.
	*/
	mutex_enter(&mg->mg_lock);
	uint64_t ms_count = avl_numnodes(&mg->mg_metaslab_tree);
	mutex_exit(&mg->mg_lock);
	return ((1ULL << mg->mg_vd->vdev_ms_shift) * ms_count);
	}

	void
	metaslab_group_histogram_verify(metaslab_group_t *mg)
	{
	uint64_t *mg_hist;
	avl_tree_t *t = &mg->mg_metaslab_tree;
	uint64_t ashift = mg->mg_vd->vdev_ashift;

	if ((zfs_flags & ZFS_DEBUG_HISTOGRAM_VERIFY) == 0)
	return;

	mg_hist = kmem_zalloc(sizeof (uint64_t) * RANGE_TREE_HISTOGRAM_SIZE,
	KM_SLEEP);

	ASSERT3U(RANGE_TREE_HISTOGRAM_SIZE, >=,
	SPACE_MAP_HISTOGRAM_SIZE + ashift);

	mutex_enter(&mg->mg_lock);
	for (metaslab_t *msp = avl_first(t);
	msp != NULL; msp = AVL_NEXT(t, msp)) {
	VERIFY3P(msp->ms_group, ==, mg);
	/* skip if not active */
	if (msp->ms_sm == NULL)
	continue;

	for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) {
	mg_hist[i + ashift] +=
	msp->ms_sm->sm_phys->smp_histogram[i];
	}
	}

	for (int i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i ++)
	VERIFY3U(mg_hist[i], ==, mg->mg_histogram[i]);

	mutex_exit(&mg->mg_lock);

	kmem_free(mg_hist, sizeof (uint64_t) * RANGE_TREE_HISTOGRAM_SIZE);
	}

	static void
	metaslab_group_histogram_add(metaslab_group_t mg, metaslab_t msp)
	{
	metaslab_class_t *mc = mg->mg_class;
	uint64_t ashift = mg->mg_vd->vdev_ashift;

	ASSERT(MUTEX_HELD(&msp->ms_lock));
	if (msp->ms_sm == NULL)
	return;

	mutex_enter(&mg->mg_lock);
	mutex_enter(&mc->mc_lock);
	for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) {
	IMPLY(mg == mg->mg_vd->vdev_log_mg,
	mc == spa_embedded_log_class(mg->mg_vd->vdev_spa));
	mg->mg_histogram[i + ashift] +=
	msp->ms_sm->sm_phys->smp_histogram[i];
	mc->mc_histogram[i + ashift] +=
	msp->ms_sm->sm_phys->smp_histogram[i];
	}
	mutex_exit(&mc->mc_lock);
	mutex_exit(&mg->mg_lock);
	}

	void
	metaslab_group_histogram_remove(metaslab_group_t mg, metaslab_t msp)
	{
	metaslab_class_t *mc = mg->mg_class;
	uint64_t ashift = mg->mg_vd->vdev_ashift;

	ASSERT(MUTEX_HELD(&msp->ms_lock));
	if (msp->ms_sm == NULL)
	return;

	mutex_enter(&mg->mg_lock);
	mutex_enter(&mc->mc_lock);
	for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) {
	ASSERT3U(mg->mg_histogram[i + ashift], >=,
	msp->ms_sm->sm_phys->smp_histogram[i]);
	ASSERT3U(mc->mc_histogram[i + ashift], >=,
	msp->ms_sm->sm_phys->smp_histogram[i]);
	IMPLY(mg == mg->mg_vd->vdev_log_mg,
	mc == spa_embedded_log_class(mg->mg_vd->vdev_spa));

	mg->mg_histogram[i + ashift] -=
	msp->ms_sm->sm_phys->smp_histogram[i];
	mc->mc_histogram[i + ashift] -=
	msp->ms_sm->sm_phys->smp_histogram[i];
	}
	mutex_exit(&mc->mc_lock);
	mutex_exit(&mg->mg_lock);
	}

	static void
	metaslab_group_add(metaslab_group_t mg, metaslab_t msp)
	{
	ASSERT(msp->ms_group == NULL);
	mutex_enter(&mg->mg_lock);
	msp->ms_group = mg;
	msp->ms_weight = 0;
	avl_add(&mg->mg_metaslab_tree, msp);
	mutex_exit(&mg->mg_lock);

	mutex_enter(&msp->ms_lock);
	metaslab_group_histogram_add(mg, msp);
	mutex_exit(&msp->ms_lock);
	}

	static void
	metaslab_group_remove(metaslab_group_t mg, metaslab_t msp)
	{
	mutex_enter(&msp->ms_lock);
	metaslab_group_histogram_remove(mg, msp);
	mutex_exit(&msp->ms_lock);

	mutex_enter(&mg->mg_lock);
	ASSERT(msp->ms_group == mg);
	avl_remove(&mg->mg_metaslab_tree, msp);

	metaslab_class_t *mc = msp->ms_group->mg_class;
	multilist_sublist_t *mls =
	multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp);
	if (multilist_link_active(&msp->ms_class_txg_node))
	multilist_sublist_remove(mls, msp);
	multilist_sublist_unlock(mls);

	msp->ms_group = NULL;
	mutex_exit(&mg->mg_lock);
	}

	static void
	metaslab_group_sort_impl(metaslab_group_t mg, metaslab_t msp, uint64_t weight)
	{
	ASSERT(MUTEX_HELD(&msp->ms_lock));
	ASSERT(MUTEX_HELD(&mg->mg_lock));
	ASSERT(msp->ms_group == mg);

	avl_remove(&mg->mg_metaslab_tree, msp);
	msp->ms_weight = weight;
	avl_add(&mg->mg_metaslab_tree, msp);

	}

	static void
	metaslab_group_sort(metaslab_group_t mg, metaslab_t msp, uint64_t weight)
	{
	/*
	* Although in principle the weight can be any value, in
	* practice we do not use values in the range [1, 511].
	*/
	ASSERT(weight >= SPA_MINBLOCKSIZE \|\| weight == 0);
	ASSERT(MUTEX_HELD(&msp->ms_lock));

	mutex_enter(&mg->mg_lock);
	metaslab_group_sort_impl(mg, msp, weight);
	mutex_exit(&mg->mg_lock);
	}

	/*
	* Calculate the fragmentation for a given metaslab group. We can use
	* a simple average here since all metaslabs within the group must have
	* the same size. The return value will be a value between 0 and 100
	* (inclusive), or ZFS_FRAG_INVALID if less than half of the metaslab in this
	* group have a fragmentation metric.
	*/
	uint64_t
	metaslab_group_fragmentation(metaslab_group_t *mg)
	{
	vdev_t *vd = mg->mg_vd;
	uint64_t fragmentation = 0;
	uint64_t valid_ms = 0;

	for (int m = 0; m < vd->vdev_ms_count; m++) {
	metaslab_t *msp = vd->vdev_ms[m];

	if (msp->ms_fragmentation == ZFS_FRAG_INVALID)
	continue;
	if (msp->ms_group != mg)
	continue;

	valid_ms++;
	fragmentation += msp->ms_fragmentation;
	}

	if (valid_ms <= mg->mg_vd->vdev_ms_count / 2)
	return (ZFS_FRAG_INVALID);

	fragmentation /= valid_ms;
	ASSERT3U(fragmentation, <=, 100);
	return (fragmentation);
	}

	/*
	* Determine if a given metaslab group should skip allocations. A metaslab
	* group should avoid allocations if its free capacity is less than the
	* zfs_mg_noalloc_threshold or its fragmentation metric is greater than
	* zfs_mg_fragmentation_threshold and there is at least one metaslab group
	* that can still handle allocations. If the allocation throttle is enabled
	* then we skip allocations to devices that have reached their maximum
	* allocation queue depth unless the selected metaslab group is the only
	* eligible group remaining.
	*/
	static boolean_t
	metaslab_group_allocatable(metaslab_group_t mg, metaslab_group_t rotor,
	int flags, uint64_t psize, int allocator, int d)
	{
	spa_t *spa = mg->mg_vd->vdev_spa;
	metaslab_class_t *mc = mg->mg_class;

	/*
	* We can only consider skipping this metaslab group if it's
	* in the normal metaslab class and there are other metaslab
	* groups to select from. Otherwise, we always consider it eligible
	* for allocations.
	*/
	if ((mc != spa_normal_class(spa) &&
	mc != spa_special_class(spa) &&
	mc != spa_dedup_class(spa)) \|\|
	mc->mc_groups <= 1)
	return (B_TRUE);

	/*
	* If the metaslab group's mg_allocatable flag is set (see comments
	* in metaslab_group_alloc_update() for more information) and
	* the allocation throttle is disabled then allow allocations to this
	* device. However, if the allocation throttle is enabled then
	* check if we have reached our allocation limit (mga_alloc_queue_depth)
	* to determine if we should allow allocations to this metaslab group.
	* If all metaslab groups are no longer considered allocatable
	* (mc_alloc_groups == 0) or we're trying to allocate the smallest
	* gang block size then we allow allocations on this metaslab group
	* regardless of the mg_allocatable or throttle settings.
	*/
	if (mg->mg_allocatable) {
	metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator];
	int64_t qdepth;
	uint64_t qmax = mga->mga_cur_max_alloc_queue_depth;

	if (!mc->mc_alloc_throttle_enabled)
	return (B_TRUE);

	/*
	* If this metaslab group does not have any free space, then
	* there is no point in looking further.
	*/
	if (mg->mg_no_free_space)
	return (B_FALSE);

	/*
	* Some allocations (e.g., those coming from device removal
	* where the * allocations are not even counted in the
	* metaslab * allocation queues) are allowed to bypass
	* the throttle.
	*/
	if (flags & METASLAB_DONT_THROTTLE)
	return (B_TRUE);

	/*
	* Relax allocation throttling for ditto blocks. Due to
	* random imbalances in allocation it tends to push copies
	* to one vdev, that looks a bit better at the moment.
	*/
	qmax = qmax * (4 + d) / 4;

	qdepth = zfs_refcount_count(&mga->mga_alloc_queue_depth);

	/*
	* If this metaslab group is below its qmax or it's
	* the only allocatable metaslab group, then attempt
	* to allocate from it.
	*/
	if (qdepth < qmax \|\| mc->mc_alloc_groups == 1)
	return (B_TRUE);
	ASSERT3U(mc->mc_alloc_groups, >, 1);

	/*
	* Since this metaslab group is at or over its qmax, we
	* need to determine if there are metaslab groups after this
	* one that might be able to handle this allocation. This is
	* racy since we can't hold the locks for all metaslab
	* groups at the same time when we make this check.
	*/
	for (metaslab_group_t *mgp = mg->mg_next;
	mgp != rotor; mgp = mgp->mg_next) {
	metaslab_group_allocator_t *mgap =
	&mgp->mg_allocator[allocator];
	qmax = mgap->mga_cur_max_alloc_queue_depth;
	qmax = qmax * (4 + d) / 4;
	qdepth =
	zfs_refcount_count(&mgap->mga_alloc_queue_depth);

	/*
	* If there is another metaslab group that
	* might be able to handle the allocation, then
	* we return false so that we skip this group.
	*/
	if (qdepth < qmax && !mgp->mg_no_free_space)
	return (B_FALSE);
	}

	/*
	* We didn't find another group to handle the allocation
	* so we can't skip this metaslab group even though
	* we are at or over our qmax.
	*/
	return (B_TRUE);

	} else if (mc->mc_alloc_groups == 0 \|\| psize == SPA_MINBLOCKSIZE) {
	return (B_TRUE);
	}
	return (B_FALSE);
	}

	/*
	* ==========================================================================
	* Range tree callbacks
	* ==========================================================================
	*/

	/*
	* Comparison function for the private size-ordered tree using 32-bit
	* ranges. Tree is sorted by size, larger sizes at the end of the tree.
	*/
	__attribute__((always_inline)) inline
	static int
	metaslab_rangesize32_compare(const void x1, const void x2)
	{
	const range_seg32_t *r1 = x1;
	const range_seg32_t *r2 = x2;

	uint64_t rs_size1 = r1->rs_end - r1->rs_start;
	uint64_t rs_size2 = r2->rs_end - r2->rs_start;

	int cmp = TREE_CMP(rs_size1, rs_size2);

	return (cmp + !cmp * TREE_CMP(r1->rs_start, r2->rs_start));
	}

	/*
	* Comparison function for the private size-ordered tree using 64-bit
	* ranges. Tree is sorted by size, larger sizes at the end of the tree.
	*/
	__attribute__((always_inline)) inline
	static int
	metaslab_rangesize64_compare(const void x1, const void x2)
	{
	const range_seg64_t *r1 = x1;
	const range_seg64_t *r2 = x2;

	uint64_t rs_size1 = r1->rs_end - r1->rs_start;
	uint64_t rs_size2 = r2->rs_end - r2->rs_start;

	int cmp = TREE_CMP(rs_size1, rs_size2);

	return (cmp + !cmp * TREE_CMP(r1->rs_start, r2->rs_start));
	}

	typedef struct metaslab_rt_arg {
	zfs_btree_t *mra_bt;
	uint32_t mra_floor_shift;
	} metaslab_rt_arg_t;

	struct mssa_arg {
	range_tree_t *rt;
	metaslab_rt_arg_t *mra;
	};

	static void
	metaslab_size_sorted_add(void *arg, uint64_t start, uint64_t size)
	{
	struct mssa_arg *mssap = arg;
	range_tree_t *rt = mssap->rt;
	metaslab_rt_arg_t *mrap = mssap->mra;
	range_seg_max_t seg = {0};
	rs_set_start(&seg, rt, start);
	rs_set_end(&seg, rt, start + size);
	metaslab_rt_add(rt, &seg, mrap);
	}

	static void
	metaslab_size_tree_full_load(range_tree_t *rt)
	{
	metaslab_rt_arg_t *mrap = rt->rt_arg;
	METASLABSTAT_BUMP(metaslabstat_reload_tree);
	ASSERT0(zfs_btree_numnodes(mrap->mra_bt));
	mrap->mra_floor_shift = 0;
	struct mssa_arg arg = {0};
	arg.rt = rt;
	arg.mra = mrap;
	range_tree_walk(rt, metaslab_size_sorted_add, &arg);
	}


	ZFS_BTREE_FIND_IN_BUF_FUNC(metaslab_rt_find_rangesize32_in_buf,
	range_seg32_t, metaslab_rangesize32_compare)

	ZFS_BTREE_FIND_IN_BUF_FUNC(metaslab_rt_find_rangesize64_in_buf,
	range_seg64_t, metaslab_rangesize64_compare)

	/*
	* Create any block allocator specific components. The current allocators
	* rely on using both a size-ordered range_tree_t and an array of uint64_t's.
	*/
	static void
	metaslab_rt_create(range_tree_t rt, void arg)
	{
	metaslab_rt_arg_t *mrap = arg;
	zfs_btree_t *size_tree = mrap->mra_bt;

	size_t size;
	int (compare) (const void , const void *);
	bt_find_in_buf_f bt_find;
	switch (rt->rt_type) {
	case RANGE_SEG32:
	size = sizeof (range_seg32_t);
	compare = metaslab_rangesize32_compare;
	bt_find = metaslab_rt_find_rangesize32_in_buf;
	break;
	case RANGE_SEG64:
	size = sizeof (range_seg64_t);
	compare = metaslab_rangesize64_compare;
	bt_find = metaslab_rt_find_rangesize64_in_buf;
	break;
	default:
	panic("Invalid range seg type %d", rt->rt_type);
	}
	zfs_btree_create(size_tree, compare, bt_find, size);
	mrap->mra_floor_shift = metaslab_by_size_min_shift;
	}

	static void
	metaslab_rt_destroy(range_tree_t rt, void arg)
	{
	(void) rt;
	metaslab_rt_arg_t *mrap = arg;
	zfs_btree_t *size_tree = mrap->mra_bt;

	zfs_btree_destroy(size_tree);
	kmem_free(mrap, sizeof (*mrap));
	}

	static void
	metaslab_rt_add(range_tree_t rt, range_seg_t rs, void *arg)
	{
	metaslab_rt_arg_t *mrap = arg;
	zfs_btree_t *size_tree = mrap->mra_bt;

	if (rs_get_end(rs, rt) - rs_get_start(rs, rt) <
	(1ULL << mrap->mra_floor_shift))
	return;

	zfs_btree_add(size_tree, rs);
	}

	static void
	metaslab_rt_remove(range_tree_t rt, range_seg_t rs, void *arg)
	{
	metaslab_rt_arg_t *mrap = arg;
	zfs_btree_t *size_tree = mrap->mra_bt;

	if (rs_get_end(rs, rt) - rs_get_start(rs, rt) < (1ULL <<
	mrap->mra_floor_shift))
	return;

	zfs_btree_remove(size_tree, rs);
	}

	static void
	metaslab_rt_vacate(range_tree_t rt, void arg)
	{
	metaslab_rt_arg_t *mrap = arg;
	zfs_btree_t *size_tree = mrap->mra_bt;
	zfs_btree_clear(size_tree);
	zfs_btree_destroy(size_tree);

	metaslab_rt_create(rt, arg);
	}

	static const range_tree_ops_t metaslab_rt_ops = {
	.rtop_create = metaslab_rt_create,
	.rtop_destroy = metaslab_rt_destroy,
	.rtop_add = metaslab_rt_add,
	.rtop_remove = metaslab_rt_remove,
	.rtop_vacate = metaslab_rt_vacate
	};

	/*
	* ==========================================================================
	* Common allocator routines
	* ==========================================================================
	*/

	/*
	* Return the maximum contiguous segment within the metaslab.
	*/
	uint64_t
	metaslab_largest_allocatable(metaslab_t *msp)
	{
	zfs_btree_t *t = &msp->ms_allocatable_by_size;
	range_seg_t *rs;

	if (t == NULL)
	return (0);
	if (zfs_btree_numnodes(t) == 0)
	metaslab_size_tree_full_load(msp->ms_allocatable);

	rs = zfs_btree_last(t, NULL);
	if (rs == NULL)
	return (0);

	return (rs_get_end(rs, msp->ms_allocatable) - rs_get_start(rs,
	msp->ms_allocatable));
	}

	/*
	* Return the maximum contiguous segment within the unflushed frees of this
	* metaslab.
	*/
	static uint64_t
	metaslab_largest_unflushed_free(metaslab_t *msp)
	{
	ASSERT(MUTEX_HELD(&msp->ms_lock));

	if (msp->ms_unflushed_frees == NULL)
	return (0);

	if (zfs_btree_numnodes(&msp->ms_unflushed_frees_by_size) == 0)
	metaslab_size_tree_full_load(msp->ms_unflushed_frees);
	range_seg_t *rs = zfs_btree_last(&msp->ms_unflushed_frees_by_size,
	NULL);
	if (rs == NULL)
	return (0);

	/*
	* When a range is freed from the metaslab, that range is added to
	* both the unflushed frees and the deferred frees. While the block
	* will eventually be usable, if the metaslab were loaded the range
	* would not be added to the ms_allocatable tree until TXG_DEFER_SIZE
	* txgs had passed. As a result, when attempting to estimate an upper
	* bound for the largest currently-usable free segment in the
	* metaslab, we need to not consider any ranges currently in the defer
	* trees. This algorithm approximates the largest available chunk in
	* the largest range in the unflushed_frees tree by taking the first
	* chunk. While this may be a poor estimate, it should only remain so
	* briefly and should eventually self-correct as frees are no longer
	* deferred. Similar logic applies to the ms_freed tree. See
	* metaslab_load() for more details.
	*
	* There are two primary sources of inaccuracy in this estimate. Both
	* are tolerated for performance reasons. The first source is that we
	* only check the largest segment for overlaps. Smaller segments may
	* have more favorable overlaps with the other trees, resulting in
	* larger usable chunks. Second, we only look at the first chunk in
	* the largest segment; there may be other usable chunks in the
	* largest segment, but we ignore them.
	*/
	uint64_t rstart = rs_get_start(rs, msp->ms_unflushed_frees);
	uint64_t rsize = rs_get_end(rs, msp->ms_unflushed_frees) - rstart;
	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
	uint64_t start = 0;
	uint64_t size = 0;
	boolean_t found = range_tree_find_in(msp->ms_defer[t], rstart,
	rsize, &start, &size);
	if (found) {
	if (rstart == start)
	return (0);
	rsize = start - rstart;
	}
	}

	uint64_t start = 0;
	uint64_t size = 0;
	boolean_t found = range_tree_find_in(msp->ms_freed, rstart,
	rsize, &start, &size);
	if (found)
	rsize = start - rstart;

	return (rsize);
	}

	static range_seg_t *
	metaslab_block_find(zfs_btree_t t, range_tree_t rt, uint64_t start,
	uint64_t size, zfs_btree_index_t *where)
	{
	range_seg_t *rs;
	range_seg_max_t rsearch;

	rs_set_start(&rsearch, rt, start);
	rs_set_end(&rsearch, rt, start + size);

	rs = zfs_btree_find(t, &rsearch, where);
	if (rs == NULL) {
	rs = zfs_btree_next(t, where, where);
	}

	return (rs);
	}

	#if defined(WITH_DF_BLOCK_ALLOCATOR) \|\| \
	defined(WITH_CF_BLOCK_ALLOCATOR)

	/*
	* This is a helper function that can be used by the allocator to find a
	* suitable block to allocate. This will search the specified B-tree looking
	* for a block that matches the specified criteria.
	*/
	static uint64_t
	metaslab_block_picker(range_tree_t rt, uint64_t cursor, uint64_t size,
	uint64_t max_search)
	{
	if (*cursor == 0)
	*cursor = rt->rt_start;
	zfs_btree_t *bt = &rt->rt_root;
	zfs_btree_index_t where;
	range_seg_t rs = metaslab_block_find(bt, rt, cursor, size, &where);
	uint64_t first_found;
	int count_searched = 0;

	if (rs != NULL)
	first_found = rs_get_start(rs, rt);

	while (rs != NULL && (rs_get_start(rs, rt) - first_found <=
	max_search \|\| count_searched < metaslab_min_search_count)) {
	uint64_t offset = rs_get_start(rs, rt);
	if (offset + size <= rs_get_end(rs, rt)) {
	*cursor = offset + size;
	return (offset);
	}
	rs = zfs_btree_next(bt, &where, &where);
	count_searched++;
	}

	*cursor = 0;
	return (-1ULL);
	}
	#endif /* WITH_DF/CF_BLOCK_ALLOCATOR */

	#if defined(WITH_DF_BLOCK_ALLOCATOR)
	/*
	* ==========================================================================
	* Dynamic Fit (df) block allocator
	*
	* Search for a free chunk of at least this size, starting from the last
	* offset (for this alignment of block) looking for up to
	* metaslab_df_max_search bytes (16MB). If a large enough free chunk is not
	* found within 16MB, then return a free chunk of exactly the requested size (or
	* larger).
	*
	* If it seems like searching from the last offset will be unproductive, skip
	* that and just return a free chunk of exactly the requested size (or larger).
	* This is based on metaslab_df_alloc_threshold and metaslab_df_free_pct. This
	* mechanism is probably not very useful and may be removed in the future.
	*
	* The behavior when not searching can be changed to return the largest free
	* chunk, instead of a free chunk of exactly the requested size, by setting
	* metaslab_df_use_largest_segment.
	* ==========================================================================
	*/
	static uint64_t
	metaslab_df_alloc(metaslab_t *msp, uint64_t size)
	{
	/*
	* Find the largest power of 2 block size that evenly divides the
	* requested size. This is used to try to allocate blocks with similar
	* alignment from the same area of the metaslab (i.e. same cursor
	* bucket) but it does not guarantee that other allocations sizes
	* may exist in the same region.
	*/
	uint64_t align = size & -size;
	uint64_t *cursor = &msp->ms_lbas[highbit64(align) - 1];
	range_tree_t *rt = msp->ms_allocatable;
	uint_t free_pct = range_tree_space(rt) * 100 / msp->ms_size;
	uint64_t offset;

	ASSERT(MUTEX_HELD(&msp->ms_lock));

	/*
	* If we're running low on space, find a segment based on size,
	* rather than iterating based on offset.
	*/
	if (metaslab_largest_allocatable(msp) < metaslab_df_alloc_threshold \|\|
	free_pct < metaslab_df_free_pct) {
	offset = -1;
	} else {
	offset = metaslab_block_picker(rt,
	cursor, size, metaslab_df_max_search);
	}

	if (offset == -1) {
	range_seg_t *rs;
	if (zfs_btree_numnodes(&msp->ms_allocatable_by_size) == 0)
	metaslab_size_tree_full_load(msp->ms_allocatable);

	if (metaslab_df_use_largest_segment) {
	/* use largest free segment */
	rs = zfs_btree_last(&msp->ms_allocatable_by_size, NULL);
	} else {
	zfs_btree_index_t where;
	/* use segment of this size, or next largest */
	rs = metaslab_block_find(&msp->ms_allocatable_by_size,
	rt, msp->ms_start, size, &where);
	}
	if (rs != NULL && rs_get_start(rs, rt) + size <= rs_get_end(rs,
	rt)) {
	offset = rs_get_start(rs, rt);
	*cursor = offset + size;
	}
	}

	return (offset);
	}

	const metaslab_ops_t zfs_metaslab_ops = {
	metaslab_df_alloc
	};
	#endif /* WITH_DF_BLOCK_ALLOCATOR */

	#if defined(WITH_CF_BLOCK_ALLOCATOR)
	/*
	* ==========================================================================
	* Cursor fit block allocator -
	* Select the largest region in the metaslab, set the cursor to the beginning
	* of the range and the cursor_end to the end of the range. As allocations
	* are made advance the cursor. Continue allocating from the cursor until
	* the range is exhausted and then find a new range.
	* ==========================================================================
	*/
	static uint64_t
	metaslab_cf_alloc(metaslab_t *msp, uint64_t size)
	{
	range_tree_t *rt = msp->ms_allocatable;
	zfs_btree_t *t = &msp->ms_allocatable_by_size;
	uint64_t *cursor = &msp->ms_lbas[0];
	uint64_t *cursor_end = &msp->ms_lbas[1];
	uint64_t offset = 0;

	ASSERT(MUTEX_HELD(&msp->ms_lock));

	ASSERT3U(cursor_end, >=, cursor);

	if ((cursor + size) > cursor_end) {
	range_seg_t *rs;

	if (zfs_btree_numnodes(t) == 0)
	metaslab_size_tree_full_load(msp->ms_allocatable);
	rs = zfs_btree_last(t, NULL);
	if (rs == NULL \|\| (rs_get_end(rs, rt) - rs_get_start(rs, rt)) <
	size)
	return (-1ULL);

	*cursor = rs_get_start(rs, rt);
	*cursor_end = rs_get_end(rs, rt);
	}

	offset = *cursor;
	*cursor += size;

	return (offset);
	}

	const metaslab_ops_t zfs_metaslab_ops = {
	metaslab_cf_alloc
	};
	#endif /* WITH_CF_BLOCK_ALLOCATOR */

	#if defined(WITH_NDF_BLOCK_ALLOCATOR)
	/*
	* ==========================================================================
	* New dynamic fit allocator -
	* Select a region that is large enough to allocate 2^metaslab_ndf_clump_shift
	* contiguous blocks. If no region is found then just use the largest segment
	* that remains.
	* ==========================================================================
	*/

	/*
	* Determines desired number of contiguous blocks (2^metaslab_ndf_clump_shift)
	* to request from the allocator.
	*/
	uint64_t metaslab_ndf_clump_shift = 4;

	static uint64_t
	metaslab_ndf_alloc(metaslab_t *msp, uint64_t size)
	{
	zfs_btree_t *t = &msp->ms_allocatable->rt_root;
	range_tree_t *rt = msp->ms_allocatable;
	zfs_btree_index_t where;
	range_seg_t *rs;
	range_seg_max_t rsearch;
	uint64_t hbit = highbit64(size);
	uint64_t *cursor = &msp->ms_lbas[hbit - 1];
	uint64_t max_size = metaslab_largest_allocatable(msp);

	ASSERT(MUTEX_HELD(&msp->ms_lock));

	if (max_size < size)
	return (-1ULL);

	rs_set_start(&rsearch, rt, *cursor);
	rs_set_end(&rsearch, rt, *cursor + size);

	rs = zfs_btree_find(t, &rsearch, &where);
	if (rs == NULL \|\| (rs_get_end(rs, rt) - rs_get_start(rs, rt)) < size) {
	t = &msp->ms_allocatable_by_size;

	rs_set_start(&rsearch, rt, 0);
	rs_set_end(&rsearch, rt, MIN(max_size, 1ULL << (hbit +
	metaslab_ndf_clump_shift)));

	rs = zfs_btree_find(t, &rsearch, &where);
	if (rs == NULL)
	rs = zfs_btree_next(t, &where, &where);
	ASSERT(rs != NULL);
	}

	if ((rs_get_end(rs, rt) - rs_get_start(rs, rt)) >= size) {
	*cursor = rs_get_start(rs, rt) + size;
	return (rs_get_start(rs, rt));
	}
	return (-1ULL);
	}

	const metaslab_ops_t zfs_metaslab_ops = {
	metaslab_ndf_alloc
	};
	#endif /* WITH_NDF_BLOCK_ALLOCATOR */


	/*
	* ==========================================================================
	* Metaslabs
	* ==========================================================================
	*/

	/*
	* Wait for any in-progress metaslab loads to complete.
	*/
	static void
	metaslab_load_wait(metaslab_t *msp)
	{
	ASSERT(MUTEX_HELD(&msp->ms_lock));

	while (msp->ms_loading) {
	ASSERT(!msp->ms_loaded);
	cv_wait(&msp->ms_load_cv, &msp->ms_lock);
	}
	}

	/*
	* Wait for any in-progress flushing to complete.
	*/
	static void
	metaslab_flush_wait(metaslab_t *msp)
	{
	ASSERT(MUTEX_HELD(&msp->ms_lock));

	while (msp->ms_flushing)
	cv_wait(&msp->ms_flush_cv, &msp->ms_lock);
	}

	static unsigned int
	metaslab_idx_func(multilist_t ml, void arg)
	{
	metaslab_t *msp = arg;

	/*
	* ms_id values are allocated sequentially, so full 64bit
	* division would be a waste of time, so limit it to 32 bits.
	*/
	return ((unsigned int)msp->ms_id % multilist_get_num_sublists(ml));
	}

	uint64_t
	metaslab_allocated_space(metaslab_t *msp)
	{
	return (msp->ms_allocated_space);
	}

	/*
	* Verify that the space accounting on disk matches the in-core range_trees.
	*/
	static void
	metaslab_verify_space(metaslab_t *msp, uint64_t txg)
	{
	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
	uint64_t allocating = 0;
	uint64_t sm_free_space, msp_free_space;

	ASSERT(MUTEX_HELD(&msp->ms_lock));
	ASSERT(!msp->ms_condensing);

	if ((zfs_flags & ZFS_DEBUG_METASLAB_VERIFY) == 0)
	return;

	/*
	* We can only verify the metaslab space when we're called
	* from syncing context with a loaded metaslab that has an
	* allocated space map. Calling this in non-syncing context
	* does not provide a consistent view of the metaslab since
	* we're performing allocations in the future.
	*/
	if (txg != spa_syncing_txg(spa) \|\| msp->ms_sm == NULL \|\|
	!msp->ms_loaded)
	return;

	/*
	* Even though the smp_alloc field can get negative,
	* when it comes to a metaslab's space map, that should
	* never be the case.
	*/
	ASSERT3S(space_map_allocated(msp->ms_sm), >=, 0);

	ASSERT3U(space_map_allocated(msp->ms_sm), >=,
	range_tree_space(msp->ms_unflushed_frees));

	ASSERT3U(metaslab_allocated_space(msp), ==,
	space_map_allocated(msp->ms_sm) +
	range_tree_space(msp->ms_unflushed_allocs) -
	range_tree_space(msp->ms_unflushed_frees));

	sm_free_space = msp->ms_size - metaslab_allocated_space(msp);

	/*
	* Account for future allocations since we would have
	* already deducted that space from the ms_allocatable.
	*/
	for (int t = 0; t < TXG_CONCURRENT_STATES; t++) {
	allocating +=
	range_tree_space(msp->ms_allocating[(txg + t) & TXG_MASK]);
	}
	ASSERT3U(allocating + msp->ms_allocated_this_txg, ==,
	msp->ms_allocating_total);

	ASSERT3U(msp->ms_deferspace, ==,
	range_tree_space(msp->ms_defer[0]) +
	range_tree_space(msp->ms_defer[1]));

	msp_free_space = range_tree_space(msp->ms_allocatable) + allocating +
	msp->ms_deferspace + range_tree_space(msp->ms_freed);

	VERIFY3U(sm_free_space, ==, msp_free_space);
	}

	static void
	metaslab_aux_histograms_clear(metaslab_t *msp)
	{
	/*
	* Auxiliary histograms are only cleared when resetting them,
	* which can only happen while the metaslab is loaded.
	*/
	ASSERT(msp->ms_loaded);

	memset(msp->ms_synchist, 0, sizeof (msp->ms_synchist));
	for (int t = 0; t < TXG_DEFER_SIZE; t++)
	memset(msp->ms_deferhist[t], 0, sizeof (msp->ms_deferhist[t]));
	}

	static void
	metaslab_aux_histogram_add(uint64_t *histogram, uint64_t shift,
	range_tree_t *rt)
	{
	/*
	* This is modeled after space_map_histogram_add(), so refer to that
	* function for implementation details. We want this to work like
	* the space map histogram, and not the range tree histogram, as we
	* are essentially constructing a delta that will be later subtracted
	* from the space map histogram.
	*/
	int idx = 0;
	for (int i = shift; i < RANGE_TREE_HISTOGRAM_SIZE; i++) {
	ASSERT3U(i, >=, idx + shift);
	histogram[idx] += rt->rt_histogram[i] << (i - idx - shift);

	if (idx < SPACE_MAP_HISTOGRAM_SIZE - 1) {
	ASSERT3U(idx + shift, ==, i);
	idx++;
	ASSERT3U(idx, <, SPACE_MAP_HISTOGRAM_SIZE);
	}
	}
	}

	/*
	* Called at every sync pass that the metaslab gets synced.
	*
	* The reason is that we want our auxiliary histograms to be updated
	* wherever the metaslab's space map histogram is updated. This way
	* we stay consistent on which parts of the metaslab space map's
	* histogram are currently not available for allocations (e.g because
	* they are in the defer, freed, and freeing trees).
	*/
	static void
	metaslab_aux_histograms_update(metaslab_t *msp)
	{
	space_map_t *sm = msp->ms_sm;
	ASSERT(sm != NULL);

	/*
	* This is similar to the metaslab's space map histogram updates
	* that take place in metaslab_sync(). The only difference is that
	* we only care about segments that haven't made it into the
	* ms_allocatable tree yet.
	*/
	if (msp->ms_loaded) {
	metaslab_aux_histograms_clear(msp);

	metaslab_aux_histogram_add(msp->ms_synchist,
	sm->sm_shift, msp->ms_freed);

	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
	metaslab_aux_histogram_add(msp->ms_deferhist[t],
	sm->sm_shift, msp->ms_defer[t]);
	}
	}

	metaslab_aux_histogram_add(msp->ms_synchist,
	sm->sm_shift, msp->ms_freeing);
	}

	/*
	* Called every time we are done syncing (writing to) the metaslab,
	* i.e. at the end of each sync pass.
	* [see the comment in metaslab_impl.h for ms_synchist, ms_deferhist]
	*/
	static void
	metaslab_aux_histograms_update_done(metaslab_t *msp, boolean_t defer_allowed)
	{
	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
	space_map_t *sm = msp->ms_sm;

	if (sm == NULL) {
	/*
	* We came here from metaslab_init() when creating/opening a
	* pool, looking at a metaslab that hasn't had any allocations
	* yet.
	*/
	return;
	}

	/*
	* This is similar to the actions that we take for the ms_freed
	* and ms_defer trees in metaslab_sync_done().
	*/
	uint64_t hist_index = spa_syncing_txg(spa) % TXG_DEFER_SIZE;
	if (defer_allowed) {
	memcpy(msp->ms_deferhist[hist_index], msp->ms_synchist,
	sizeof (msp->ms_synchist));
	} else {
	memset(msp->ms_deferhist[hist_index], 0,
	sizeof (msp->ms_deferhist[hist_index]));
	}
	memset(msp->ms_synchist, 0, sizeof (msp->ms_synchist));
	}

	/*
	* Ensure that the metaslab's weight and fragmentation are consistent
	* with the contents of the histogram (either the range tree's histogram
	* or the space map's depending whether the metaslab is loaded).
	*/
	static void
	metaslab_verify_weight_and_frag(metaslab_t *msp)
	{
	ASSERT(MUTEX_HELD(&msp->ms_lock));

	if ((zfs_flags & ZFS_DEBUG_METASLAB_VERIFY) == 0)
	return;

	/*
	* We can end up here from vdev_remove_complete(), in which case we
	* cannot do these assertions because we hold spa config locks and
	* thus we are not allowed to read from the DMU.
	*
	* We check if the metaslab group has been removed and if that's
	* the case we return immediately as that would mean that we are
	* here from the aforementioned code path.
	*/
	if (msp->ms_group == NULL)
	return;

	/*
	* Devices being removed always return a weight of 0 and leave
	* fragmentation and ms_max_size as is - there is nothing for
	* us to verify here.
	*/
	vdev_t *vd = msp->ms_group->mg_vd;
	if (vd->vdev_removing)
	return;

	/*
	* If the metaslab is dirty it probably means that we've done
	* some allocations or frees that have changed our histograms
	* and thus the weight.
	*/
	for (int t = 0; t < TXG_SIZE; t++) {
	if (txg_list_member(&vd->vdev_ms_list, msp, t))
	return;
	}

	/*
	* This verification checks that our in-memory state is consistent
	* with what's on disk. If the pool is read-only then there aren't
	* any changes and we just have the initially-loaded state.
	*/
	if (!spa_writeable(msp->ms_group->mg_vd->vdev_spa))
	return;

	/* some extra verification for in-core tree if you can */
	if (msp->ms_loaded) {
	range_tree_stat_verify(msp->ms_allocatable);
	VERIFY(space_map_histogram_verify(msp->ms_sm,
	msp->ms_allocatable));
	}

	uint64_t weight = msp->ms_weight;
	uint64_t was_active = msp->ms_weight & METASLAB_ACTIVE_MASK;
	boolean_t space_based = WEIGHT_IS_SPACEBASED(msp->ms_weight);
	uint64_t frag = msp->ms_fragmentation;
	uint64_t max_segsize = msp->ms_max_size;

	msp->ms_weight = 0;
	msp->ms_fragmentation = 0;

	/*
	* This function is used for verification purposes and thus should
	* not introduce any side-effects/mutations on the system's state.
	*
	* Regardless of whether metaslab_weight() thinks this metaslab
	* should be active or not, we want to ensure that the actual weight
	* (and therefore the value of ms_weight) would be the same if it
	* was to be recalculated at this point.
	*
	* In addition we set the nodirty flag so metaslab_weight() does
	* not dirty the metaslab for future TXGs (e.g. when trying to
	* force condensing to upgrade the metaslab spacemaps).
	*/
	msp->ms_weight = metaslab_weight(msp, B_TRUE) \| was_active;

	VERIFY3U(max_segsize, ==, msp->ms_max_size);

	/*
	* If the weight type changed then there is no point in doing
	* verification. Revert fields to their original values.
	*/
	if ((space_based && !WEIGHT_IS_SPACEBASED(msp->ms_weight)) \|\|
	(!space_based && WEIGHT_IS_SPACEBASED(msp->ms_weight))) {
	msp->ms_fragmentation = frag;
	msp->ms_weight = weight;
	return;
	}

	VERIFY3U(msp->ms_fragmentation, ==, frag);
	VERIFY3U(msp->ms_weight, ==, weight);
	}

	/*
	* If we're over the zfs_metaslab_mem_limit, select the loaded metaslab from
	* this class that was used longest ago, and attempt to unload it. We don't
	* want to spend too much time in this loop to prevent performance
	* degradation, and we expect that most of the time this operation will
	* succeed. Between that and the normal unloading processing during txg sync,
	* we expect this to keep the metaslab memory usage under control.
	*/
	static void
	metaslab_potentially_evict(metaslab_class_t *mc)
	{
	#ifdef _KERNEL
	uint64_t allmem = arc_all_memory();
	uint64_t inuse = spl_kmem_cache_inuse(zfs_btree_leaf_cache);
	uint64_t size = spl_kmem_cache_entry_size(zfs_btree_leaf_cache);
	uint_t tries = 0;
	for (; allmem * zfs_metaslab_mem_limit / 100 < inuse * size &&
	tries < multilist_get_num_sublists(&mc->mc_metaslab_txg_list) * 2;
	tries++) {
	unsigned int idx = multilist_get_random_index(
	&mc->mc_metaslab_txg_list);
	multilist_sublist_t *mls =
	- multilist_sublist_lock(&mc->mc_metaslab_txg_list, idx);
	+ multilist_sublist_lock_idx(&mc->mc_metaslab_txg_list, idx);
	metaslab_t *msp = multilist_sublist_head(mls);
	multilist_sublist_unlock(mls);
	while (msp != NULL && allmem * zfs_metaslab_mem_limit / 100 <
	inuse * size) {
	- VERIFY3P(mls, ==, multilist_sublist_lock(
	+ VERIFY3P(mls, ==, multilist_sublist_lock_idx(
	&mc->mc_metaslab_txg_list, idx));
	ASSERT3U(idx, ==,
	metaslab_idx_func(&mc->mc_metaslab_txg_list, msp));

	if (!multilist_link_active(&msp->ms_class_txg_node)) {
	multilist_sublist_unlock(mls);
	break;
	}
	metaslab_t *next_msp = multilist_sublist_next(mls, msp);
	multilist_sublist_unlock(mls);
	/*
	* If the metaslab is currently loading there are two
	* cases. If it's the metaslab we're evicting, we
	* can't continue on or we'll panic when we attempt to
	* recursively lock the mutex. If it's another
	* metaslab that's loading, it can be safely skipped,
	* since we know it's very new and therefore not a
	* good eviction candidate. We check later once the
	* lock is held that the metaslab is fully loaded
	* before actually unloading it.
	*/
	if (msp->ms_loading) {
	msp = next_msp;
	inuse =
	spl_kmem_cache_inuse(zfs_btree_leaf_cache);
	continue;
	}
	/*
	* We can't unload metaslabs with no spacemap because
	* they're not ready to be unloaded yet. We can't
	* unload metaslabs with outstanding allocations
	* because doing so could cause the metaslab's weight
	* to decrease while it's unloaded, which violates an
	* invariant that we use to prevent unnecessary
	* loading. We also don't unload metaslabs that are
	* currently active because they are high-weight
	* metaslabs that are likely to be used in the near
	* future.
	*/
	mutex_enter(&msp->ms_lock);
	if (msp->ms_allocator == -1 && msp->ms_sm != NULL &&
	msp->ms_allocating_total == 0) {
	metaslab_unload(msp);
	}
	mutex_exit(&msp->ms_lock);
	msp = next_msp;
	inuse = spl_kmem_cache_inuse(zfs_btree_leaf_cache);
	}
	}
	#else
	(void) mc, (void) zfs_metaslab_mem_limit;
	#endif
	}

	static int
	metaslab_load_impl(metaslab_t *msp)
	{
	int error = 0;

	ASSERT(MUTEX_HELD(&msp->ms_lock));
	ASSERT(msp->ms_loading);
	ASSERT(!msp->ms_condensing);

	/*
	* We temporarily drop the lock to unblock other operations while we
	* are reading the space map. Therefore, metaslab_sync() and
	* metaslab_sync_done() can run at the same time as we do.
	*
	* If we are using the log space maps, metaslab_sync() can't write to
	* the metaslab's space map while we are loading as we only write to
	* it when we are flushing the metaslab, and that can't happen while
	* we are loading it.
	*
	* If we are not using log space maps though, metaslab_sync() can
	* append to the space map while we are loading. Therefore we load
	* only entries that existed when we started the load. Additionally,
	* metaslab_sync_done() has to wait for the load to complete because
	* there are potential races like metaslab_load() loading parts of the
	* space map that are currently being appended by metaslab_sync(). If
	* we didn't, the ms_allocatable would have entries that
	* metaslab_sync_done() would try to re-add later.
	*
	* That's why before dropping the lock we remember the synced length
	* of the metaslab and read up to that point of the space map,
	* ignoring entries appended by metaslab_sync() that happen after we
	* drop the lock.
	*/
	uint64_t length = msp->ms_synced_length;
	mutex_exit(&msp->ms_lock);

	hrtime_t load_start = gethrtime();
	metaslab_rt_arg_t *mrap;
	if (msp->ms_allocatable->rt_arg == NULL) {
	mrap = kmem_zalloc(sizeof (*mrap), KM_SLEEP);
	} else {
	mrap = msp->ms_allocatable->rt_arg;
	msp->ms_allocatable->rt_ops = NULL;
	msp->ms_allocatable->rt_arg = NULL;
	}
	mrap->mra_bt = &msp->ms_allocatable_by_size;
	mrap->mra_floor_shift = metaslab_by_size_min_shift;

	if (msp->ms_sm != NULL) {
	error = space_map_load_length(msp->ms_sm, msp->ms_allocatable,
	SM_FREE, length);

	/* Now, populate the size-sorted tree. */
	metaslab_rt_create(msp->ms_allocatable, mrap);
	msp->ms_allocatable->rt_ops = &metaslab_rt_ops;
	msp->ms_allocatable->rt_arg = mrap;

	struct mssa_arg arg = {0};
	arg.rt = msp->ms_allocatable;
	arg.mra = mrap;
	range_tree_walk(msp->ms_allocatable, metaslab_size_sorted_add,
	&arg);
	} else {
	/*
	* Add the size-sorted tree first, since we don't need to load
	* the metaslab from the spacemap.
	*/
	metaslab_rt_create(msp->ms_allocatable, mrap);
	msp->ms_allocatable->rt_ops = &metaslab_rt_ops;
	msp->ms_allocatable->rt_arg = mrap;
	/*
	* The space map has not been allocated yet, so treat
	* all the space in the metaslab as free and add it to the
	* ms_allocatable tree.
	*/
	range_tree_add(msp->ms_allocatable,
	msp->ms_start, msp->ms_size);

	if (msp->ms_new) {
	/*
	* If the ms_sm doesn't exist, this means that this
	* metaslab hasn't gone through metaslab_sync() and
	* thus has never been dirtied. So we shouldn't
	* expect any unflushed allocs or frees from previous
	* TXGs.
	*/
	ASSERT(range_tree_is_empty(msp->ms_unflushed_allocs));
	ASSERT(range_tree_is_empty(msp->ms_unflushed_frees));
	}
	}

	/*
	* We need to grab the ms_sync_lock to prevent metaslab_sync() from
	* changing the ms_sm (or log_sm) and the metaslab's range trees
	* while we are about to use them and populate the ms_allocatable.
	* The ms_lock is insufficient for this because metaslab_sync() doesn't
	* hold the ms_lock while writing the ms_checkpointing tree to disk.
	*/
	mutex_enter(&msp->ms_sync_lock);
	mutex_enter(&msp->ms_lock);

	ASSERT(!msp->ms_condensing);
	ASSERT(!msp->ms_flushing);

	if (error != 0) {
	mutex_exit(&msp->ms_sync_lock);
	return (error);
	}

	ASSERT3P(msp->ms_group, !=, NULL);
	msp->ms_loaded = B_TRUE;

	/*
	* Apply all the unflushed changes to ms_allocatable right
	* away so any manipulations we do below have a clear view
	* of what is allocated and what is free.
	*/
	range_tree_walk(msp->ms_unflushed_allocs,
	range_tree_remove, msp->ms_allocatable);
	range_tree_walk(msp->ms_unflushed_frees,
	range_tree_add, msp->ms_allocatable);

	ASSERT3P(msp->ms_group, !=, NULL);
	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
	if (spa_syncing_log_sm(spa) != NULL) {
	ASSERT(spa_feature_is_enabled(spa,
	SPA_FEATURE_LOG_SPACEMAP));

	/*
	* If we use a log space map we add all the segments
	* that are in ms_unflushed_frees so they are available
	* for allocation.
	*
	* ms_allocatable needs to contain all free segments
	* that are ready for allocations (thus not segments
	* from ms_freeing, ms_freed, and the ms_defer trees).
	* But if we grab the lock in this code path at a sync
	* pass later that 1, then it also contains the
	* segments of ms_freed (they were added to it earlier
	* in this path through ms_unflushed_frees). So we
	* need to remove all the segments that exist in
	* ms_freed from ms_allocatable as they will be added
	* later in metaslab_sync_done().
	*
	* When there's no log space map, the ms_allocatable
	* correctly doesn't contain any segments that exist
	* in ms_freed [see ms_synced_length].
	*/
	range_tree_walk(msp->ms_freed,
	range_tree_remove, msp->ms_allocatable);
	}

	/*
	* If we are not using the log space map, ms_allocatable
	* contains the segments that exist in the ms_defer trees
	* [see ms_synced_length]. Thus we need to remove them
	* from ms_allocatable as they will be added again in
	* metaslab_sync_done().
	*
	* If we are using the log space map, ms_allocatable still
	* contains the segments that exist in the ms_defer trees.
	* Not because it read them through the ms_sm though. But
	* because these segments are part of ms_unflushed_frees
	* whose segments we add to ms_allocatable earlier in this
	* code path.
	*/
	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
	range_tree_walk(msp->ms_defer[t],
	range_tree_remove, msp->ms_allocatable);
	}

	/*
	* Call metaslab_recalculate_weight_and_sort() now that the
	* metaslab is loaded so we get the metaslab's real weight.
	*
	* Unless this metaslab was created with older software and
	* has not yet been converted to use segment-based weight, we
	* expect the new weight to be better or equal to the weight
	* that the metaslab had while it was not loaded. This is
	* because the old weight does not take into account the
	* consolidation of adjacent segments between TXGs. [see
	* comment for ms_synchist and ms_deferhist[] for more info]
	*/
	uint64_t weight = msp->ms_weight;
	uint64_t max_size = msp->ms_max_size;
	metaslab_recalculate_weight_and_sort(msp);
	if (!WEIGHT_IS_SPACEBASED(weight))
	ASSERT3U(weight, <=, msp->ms_weight);
	msp->ms_max_size = metaslab_largest_allocatable(msp);
	ASSERT3U(max_size, <=, msp->ms_max_size);
	hrtime_t load_end = gethrtime();
	msp->ms_load_time = load_end;
	zfs_dbgmsg("metaslab_load: txg %llu, spa %s, vdev_id %llu, "
	"ms_id %llu, smp_length %llu, "
	"unflushed_allocs %llu, unflushed_frees %llu, "
	"freed %llu, defer %llu + %llu, unloaded time %llu ms, "
	"loading_time %lld ms, ms_max_size %llu, "
	"max size error %lld, "
	"old_weight %llx, new_weight %llx",
	(u_longlong_t)spa_syncing_txg(spa), spa_name(spa),
	(u_longlong_t)msp->ms_group->mg_vd->vdev_id,
	(u_longlong_t)msp->ms_id,
	(u_longlong_t)space_map_length(msp->ms_sm),
	(u_longlong_t)range_tree_space(msp->ms_unflushed_allocs),
	(u_longlong_t)range_tree_space(msp->ms_unflushed_frees),
	(u_longlong_t)range_tree_space(msp->ms_freed),
	(u_longlong_t)range_tree_space(msp->ms_defer[0]),
	(u_longlong_t)range_tree_space(msp->ms_defer[1]),
	(longlong_t)((load_start - msp->ms_unload_time) / 1000000),
	(longlong_t)((load_end - load_start) / 1000000),
	(u_longlong_t)msp->ms_max_size,
	(u_longlong_t)msp->ms_max_size - max_size,
	(u_longlong_t)weight, (u_longlong_t)msp->ms_weight);

	metaslab_verify_space(msp, spa_syncing_txg(spa));
	mutex_exit(&msp->ms_sync_lock);
	return (0);
	}

	int
	metaslab_load(metaslab_t *msp)
	{
	ASSERT(MUTEX_HELD(&msp->ms_lock));

	/*
	* There may be another thread loading the same metaslab, if that's
	* the case just wait until the other thread is done and return.
	*/
	metaslab_load_wait(msp);
	if (msp->ms_loaded)
	return (0);
	VERIFY(!msp->ms_loading);
	ASSERT(!msp->ms_condensing);

	/*
	* We set the loading flag BEFORE potentially dropping the lock to
	* wait for an ongoing flush (see ms_flushing below). This way other
	* threads know that there is already a thread that is loading this
	* metaslab.
	*/
	msp->ms_loading = B_TRUE;

	/*
	* Wait for any in-progress flushing to finish as we drop the ms_lock
	* both here (during space_map_load()) and in metaslab_flush() (when
	* we flush our changes to the ms_sm).
	*/
	if (msp->ms_flushing)
	metaslab_flush_wait(msp);

	/*
	* In the possibility that we were waiting for the metaslab to be
	* flushed (where we temporarily dropped the ms_lock), ensure that
	* no one else loaded the metaslab somehow.
	*/
	ASSERT(!msp->ms_loaded);

	/*
	* If we're loading a metaslab in the normal class, consider evicting
	* another one to keep our memory usage under the limit defined by the
	* zfs_metaslab_mem_limit tunable.
	*/
	if (spa_normal_class(msp->ms_group->mg_class->mc_spa) ==
	msp->ms_group->mg_class) {
	metaslab_potentially_evict(msp->ms_group->mg_class);
	}

	int error = metaslab_load_impl(msp);

	ASSERT(MUTEX_HELD(&msp->ms_lock));
	msp->ms_loading = B_FALSE;
	cv_broadcast(&msp->ms_load_cv);

	return (error);
	}

	void
	metaslab_unload(metaslab_t *msp)
	{
	ASSERT(MUTEX_HELD(&msp->ms_lock));

	/*
	* This can happen if a metaslab is selected for eviction (in
	* metaslab_potentially_evict) and then unloaded during spa_sync (via
	* metaslab_class_evict_old).
	*/
	if (!msp->ms_loaded)
	return;

	range_tree_vacate(msp->ms_allocatable, NULL, NULL);
	msp->ms_loaded = B_FALSE;
	msp->ms_unload_time = gethrtime();

	msp->ms_activation_weight = 0;
	msp->ms_weight &= ~METASLAB_ACTIVE_MASK;

	if (msp->ms_group != NULL) {
	metaslab_class_t *mc = msp->ms_group->mg_class;
	multilist_sublist_t *mls =
	multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp);
	if (multilist_link_active(&msp->ms_class_txg_node))
	multilist_sublist_remove(mls, msp);
	multilist_sublist_unlock(mls);

	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
	zfs_dbgmsg("metaslab_unload: txg %llu, spa %s, vdev_id %llu, "
	"ms_id %llu, weight %llx, "
	"selected txg %llu (%llu ms ago), alloc_txg %llu, "
	"loaded %llu ms ago, max_size %llu",
	(u_longlong_t)spa_syncing_txg(spa), spa_name(spa),
	(u_longlong_t)msp->ms_group->mg_vd->vdev_id,
	(u_longlong_t)msp->ms_id,
	(u_longlong_t)msp->ms_weight,
	(u_longlong_t)msp->ms_selected_txg,
	(u_longlong_t)(msp->ms_unload_time -
	msp->ms_selected_time) / 1000 / 1000,
	(u_longlong_t)msp->ms_alloc_txg,
	(u_longlong_t)(msp->ms_unload_time -
	msp->ms_load_time) / 1000 / 1000,
	(u_longlong_t)msp->ms_max_size);
	}

	/*
	* We explicitly recalculate the metaslab's weight based on its space
	* map (as it is now not loaded). We want unload metaslabs to always
	* have their weights calculated from the space map histograms, while
	* loaded ones have it calculated from their in-core range tree
	* [see metaslab_load()]. This way, the weight reflects the information
	* available in-core, whether it is loaded or not.
	*
	* If ms_group == NULL means that we came here from metaslab_fini(),
	* at which point it doesn't make sense for us to do the recalculation
	* and the sorting.
	*/
	if (msp->ms_group != NULL)
	metaslab_recalculate_weight_and_sort(msp);
	}

	/*
	* We want to optimize the memory use of the per-metaslab range
	* trees. To do this, we store the segments in the range trees in
	* units of sectors, zero-indexing from the start of the metaslab. If
	* the vdev_ms_shift - the vdev_ashift is less than 32, we can store
	* the ranges using two uint32_ts, rather than two uint64_ts.
	*/
	range_seg_type_t
	metaslab_calculate_range_tree_type(vdev_t vdev, metaslab_t msp,
	uint64_t start, uint64_t shift)
	{
	if (vdev->vdev_ms_shift - vdev->vdev_ashift < 32 &&
	!zfs_metaslab_force_large_segs) {
	*shift = vdev->vdev_ashift;
	*start = msp->ms_start;
	return (RANGE_SEG32);
	} else {
	*shift = 0;
	*start = 0;
	return (RANGE_SEG64);
	}
	}

	void
	metaslab_set_selected_txg(metaslab_t *msp, uint64_t txg)
	{
	ASSERT(MUTEX_HELD(&msp->ms_lock));
	metaslab_class_t *mc = msp->ms_group->mg_class;
	multilist_sublist_t *mls =
	multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp);
	if (multilist_link_active(&msp->ms_class_txg_node))
	multilist_sublist_remove(mls, msp);
	msp->ms_selected_txg = txg;
	msp->ms_selected_time = gethrtime();
	multilist_sublist_insert_tail(mls, msp);
	multilist_sublist_unlock(mls);
	}

	void
	metaslab_space_update(vdev_t vd, metaslab_class_t mc, int64_t alloc_delta,
	int64_t defer_delta, int64_t space_delta)
	{
	vdev_space_update(vd, alloc_delta, defer_delta, space_delta);

	ASSERT3P(vd->vdev_spa->spa_root_vdev, ==, vd->vdev_parent);
	ASSERT(vd->vdev_ms_count != 0);

	metaslab_class_space_update(mc, alloc_delta, defer_delta, space_delta,
	vdev_deflated_space(vd, space_delta));
	}

	int
	metaslab_init(metaslab_group_t *mg, uint64_t id, uint64_t object,
	uint64_t txg, metaslab_t **msp)
	{
	vdev_t *vd = mg->mg_vd;
	spa_t *spa = vd->vdev_spa;
	objset_t *mos = spa->spa_meta_objset;
	metaslab_t *ms;
	int error;

	ms = kmem_zalloc(sizeof (metaslab_t), KM_SLEEP);
	mutex_init(&ms->ms_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&ms->ms_sync_lock, NULL, MUTEX_DEFAULT, NULL);
	cv_init(&ms->ms_load_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&ms->ms_flush_cv, NULL, CV_DEFAULT, NULL);
	multilist_link_init(&ms->ms_class_txg_node);

	ms->ms_id = id;
	ms->ms_start = id << vd->vdev_ms_shift;
	ms->ms_size = 1ULL << vd->vdev_ms_shift;
	ms->ms_allocator = -1;
	ms->ms_new = B_TRUE;

	vdev_ops_t *ops = vd->vdev_ops;
	if (ops->vdev_op_metaslab_init != NULL)
	ops->vdev_op_metaslab_init(vd, &ms->ms_start, &ms->ms_size);

	/*
	* We only open space map objects that already exist. All others
	* will be opened when we finally allocate an object for it. For
	* readonly pools there is no need to open the space map object.
	*
	* Note:
	* When called from vdev_expand(), we can't call into the DMU as
	* we are holding the spa_config_lock as a writer and we would
	* deadlock [see relevant comment in vdev_metaslab_init()]. in
	* that case, the object parameter is zero though, so we won't
	* call into the DMU.
	*/
	if (object != 0 && !(spa->spa_mode == SPA_MODE_READ &&
	!spa->spa_read_spacemaps)) {
	error = space_map_open(&ms->ms_sm, mos, object, ms->ms_start,
	ms->ms_size, vd->vdev_ashift);

	if (error != 0) {
	kmem_free(ms, sizeof (metaslab_t));
	return (error);
	}

	ASSERT(ms->ms_sm != NULL);
	ms->ms_allocated_space = space_map_allocated(ms->ms_sm);
	}

	uint64_t shift, start;
	range_seg_type_t type =
	metaslab_calculate_range_tree_type(vd, ms, &start, &shift);

	ms->ms_allocatable = range_tree_create(NULL, type, NULL, start, shift);
	for (int t = 0; t < TXG_SIZE; t++) {
	ms->ms_allocating[t] = range_tree_create(NULL, type,
	NULL, start, shift);
	}
	ms->ms_freeing = range_tree_create(NULL, type, NULL, start, shift);
	ms->ms_freed = range_tree_create(NULL, type, NULL, start, shift);
	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
	ms->ms_defer[t] = range_tree_create(NULL, type, NULL,
	start, shift);
	}
	ms->ms_checkpointing =
	range_tree_create(NULL, type, NULL, start, shift);
	ms->ms_unflushed_allocs =
	range_tree_create(NULL, type, NULL, start, shift);

	metaslab_rt_arg_t mrap = kmem_zalloc(sizeof (mrap), KM_SLEEP);
	mrap->mra_bt = &ms->ms_unflushed_frees_by_size;
	mrap->mra_floor_shift = metaslab_by_size_min_shift;
	ms->ms_unflushed_frees = range_tree_create(&metaslab_rt_ops,
	type, mrap, start, shift);

	ms->ms_trim = range_tree_create(NULL, type, NULL, start, shift);

	metaslab_group_add(mg, ms);
	metaslab_set_fragmentation(ms, B_FALSE);

	/*
	* If we're opening an existing pool (txg == 0) or creating
	* a new one (txg == TXG_INITIAL), all space is available now.
	* If we're adding space to an existing pool, the new space
	* does not become available until after this txg has synced.
	* The metaslab's weight will also be initialized when we sync
	* out this txg. This ensures that we don't attempt to allocate
	* from it before we have initialized it completely.
	*/
	if (txg <= TXG_INITIAL) {
	metaslab_sync_done(ms, 0);
	metaslab_space_update(vd, mg->mg_class,
	metaslab_allocated_space(ms), 0, 0);
	}

	if (txg != 0) {
	vdev_dirty(vd, 0, NULL, txg);
	vdev_dirty(vd, VDD_METASLAB, ms, txg);
	}

	*msp = ms;

	return (0);
	}

	static void
	metaslab_fini_flush_data(metaslab_t *msp)
	{
	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;

	if (metaslab_unflushed_txg(msp) == 0) {
	ASSERT3P(avl_find(&spa->spa_metaslabs_by_flushed, msp, NULL),
	==, NULL);
	return;
	}
	ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP));

	mutex_enter(&spa->spa_flushed_ms_lock);
	avl_remove(&spa->spa_metaslabs_by_flushed, msp);
	mutex_exit(&spa->spa_flushed_ms_lock);

	spa_log_sm_decrement_mscount(spa, metaslab_unflushed_txg(msp));
	spa_log_summary_decrement_mscount(spa, metaslab_unflushed_txg(msp),
	metaslab_unflushed_dirty(msp));
	}

	uint64_t
	metaslab_unflushed_changes_memused(metaslab_t *ms)
	{
	return ((range_tree_numsegs(ms->ms_unflushed_allocs) +
	range_tree_numsegs(ms->ms_unflushed_frees)) *
	ms->ms_unflushed_allocs->rt_root.bt_elem_size);
	}

	void
	metaslab_fini(metaslab_t *msp)
	{
	metaslab_group_t *mg = msp->ms_group;
	vdev_t *vd = mg->mg_vd;
	spa_t *spa = vd->vdev_spa;

	metaslab_fini_flush_data(msp);

	metaslab_group_remove(mg, msp);

	mutex_enter(&msp->ms_lock);
	VERIFY(msp->ms_group == NULL);

	/*
	* If this metaslab hasn't been through metaslab_sync_done() yet its
	* space hasn't been accounted for in its vdev and doesn't need to be
	* subtracted.
	*/
	if (!msp->ms_new) {
	metaslab_space_update(vd, mg->mg_class,
	-metaslab_allocated_space(msp), 0, -msp->ms_size);

	}
	space_map_close(msp->ms_sm);
	msp->ms_sm = NULL;

	metaslab_unload(msp);

	range_tree_destroy(msp->ms_allocatable);
	range_tree_destroy(msp->ms_freeing);
	range_tree_destroy(msp->ms_freed);

	ASSERT3U(spa->spa_unflushed_stats.sus_memused, >=,
	metaslab_unflushed_changes_memused(msp));
	spa->spa_unflushed_stats.sus_memused -=
	metaslab_unflushed_changes_memused(msp);
	range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL);
	range_tree_destroy(msp->ms_unflushed_allocs);
	range_tree_destroy(msp->ms_checkpointing);
	range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL);
	range_tree_destroy(msp->ms_unflushed_frees);

	for (int t = 0; t < TXG_SIZE; t++) {
	range_tree_destroy(msp->ms_allocating[t]);
	}
	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
	range_tree_destroy(msp->ms_defer[t]);
	}
	ASSERT0(msp->ms_deferspace);

	for (int t = 0; t < TXG_SIZE; t++)
	ASSERT(!txg_list_member(&vd->vdev_ms_list, msp, t));

	range_tree_vacate(msp->ms_trim, NULL, NULL);
	range_tree_destroy(msp->ms_trim);

	mutex_exit(&msp->ms_lock);
	cv_destroy(&msp->ms_load_cv);
	cv_destroy(&msp->ms_flush_cv);
	mutex_destroy(&msp->ms_lock);
	mutex_destroy(&msp->ms_sync_lock);
	ASSERT3U(msp->ms_allocator, ==, -1);

	kmem_free(msp, sizeof (metaslab_t));
	}

	#define FRAGMENTATION_TABLE_SIZE 17

	/*
	* This table defines a segment size based fragmentation metric that will
	* allow each metaslab to derive its own fragmentation value. This is done
	* by calculating the space in each bucket of the spacemap histogram and
	* multiplying that by the fragmentation metric in this table. Doing
	* this for all buckets and dividing it by the total amount of free
	* space in this metaslab (i.e. the total free space in all buckets) gives
	* us the fragmentation metric. This means that a high fragmentation metric
	* equates to most of the free space being comprised of small segments.
	* Conversely, if the metric is low, then most of the free space is in
	* large segments. A 10% change in fragmentation equates to approximately
	* double the number of segments.
	*
	* This table defines 0% fragmented space using 16MB segments. Testing has
	* shown that segments that are greater than or equal to 16MB do not suffer
	* from drastic performance problems. Using this value, we derive the rest
	* of the table. Since the fragmentation value is never stored on disk, it
	* is possible to change these calculations in the future.
	*/
	static const int zfs_frag_table[FRAGMENTATION_TABLE_SIZE] = {
	100, /* 512B */
	100, /* 1K */
	98, /* 2K */
	95, /* 4K */
	90, /* 8K */
	80, /* 16K */
	70, /* 32K */
	60, /* 64K */
	50, /* 128K */
	40, /* 256K */
	30, /* 512K */
	20, /* 1M */
	15, /* 2M */
	10, /* 4M */
	5, /* 8M */
	0 /* 16M */
	};

	/*
	* Calculate the metaslab's fragmentation metric and set ms_fragmentation.
	* Setting this value to ZFS_FRAG_INVALID means that the metaslab has not
	* been upgraded and does not support this metric. Otherwise, the return
	* value should be in the range [0, 100].
	*/
	static void
	metaslab_set_fragmentation(metaslab_t *msp, boolean_t nodirty)
	{
	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
	uint64_t fragmentation = 0;
	uint64_t total = 0;
	boolean_t feature_enabled = spa_feature_is_enabled(spa,
	SPA_FEATURE_SPACEMAP_HISTOGRAM);

	if (!feature_enabled) {
	msp->ms_fragmentation = ZFS_FRAG_INVALID;
	return;
	}

	/*
	* A null space map means that the entire metaslab is free
	* and thus is not fragmented.
	*/
	if (msp->ms_sm == NULL) {
	msp->ms_fragmentation = 0;
	return;
	}

	/*
	* If this metaslab's space map has not been upgraded, flag it
	* so that we upgrade next time we encounter it.
	*/
	if (msp->ms_sm->sm_dbuf->db_size != sizeof (space_map_phys_t)) {
	uint64_t txg = spa_syncing_txg(spa);
	vdev_t *vd = msp->ms_group->mg_vd;

	/*
	* If we've reached the final dirty txg, then we must
	* be shutting down the pool. We don't want to dirty
	* any data past this point so skip setting the condense
	* flag. We can retry this action the next time the pool
	* is imported. We also skip marking this metaslab for
	* condensing if the caller has explicitly set nodirty.
	*/
	if (!nodirty &&
	spa_writeable(spa) && txg < spa_final_dirty_txg(spa)) {
	msp->ms_condense_wanted = B_TRUE;
	vdev_dirty(vd, VDD_METASLAB, msp, txg + 1);
	zfs_dbgmsg("txg %llu, requesting force condense: "
	"ms_id %llu, vdev_id %llu", (u_longlong_t)txg,
	(u_longlong_t)msp->ms_id,
	(u_longlong_t)vd->vdev_id);
	}
	msp->ms_fragmentation = ZFS_FRAG_INVALID;
	return;
	}

	for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) {
	uint64_t space = 0;
	uint8_t shift = msp->ms_sm->sm_shift;

	int idx = MIN(shift - SPA_MINBLOCKSHIFT + i,
	FRAGMENTATION_TABLE_SIZE - 1);

	if (msp->ms_sm->sm_phys->smp_histogram[i] == 0)
	continue;

	space = msp->ms_sm->sm_phys->smp_histogram[i] << (i + shift);
	total += space;

	ASSERT3U(idx, <, FRAGMENTATION_TABLE_SIZE);
	fragmentation += space * zfs_frag_table[idx];
	}

	if (total > 0)
	fragmentation /= total;
	ASSERT3U(fragmentation, <=, 100);

	msp->ms_fragmentation = fragmentation;
	}

	/*
	* Compute a weight -- a selection preference value -- for the given metaslab.
	* This is based on the amount of free space, the level of fragmentation,
	* the LBA range, and whether the metaslab is loaded.
	*/
	static uint64_t
	metaslab_space_weight(metaslab_t *msp)
	{
	metaslab_group_t *mg = msp->ms_group;
	vdev_t *vd = mg->mg_vd;
	uint64_t weight, space;

	ASSERT(MUTEX_HELD(&msp->ms_lock));

	/*
	* The baseline weight is the metaslab's free space.
	*/
	space = msp->ms_size - metaslab_allocated_space(msp);

	if (metaslab_fragmentation_factor_enabled &&
	msp->ms_fragmentation != ZFS_FRAG_INVALID) {
	/*
	* Use the fragmentation information to inversely scale
	* down the baseline weight. We need to ensure that we
	* don't exclude this metaslab completely when it's 100%
	* fragmented. To avoid this we reduce the fragmented value
	* by 1.
	*/
	space = (space * (100 - (msp->ms_fragmentation - 1))) / 100;

	/*
	* If space < SPA_MINBLOCKSIZE, then we will not allocate from
	* this metaslab again. The fragmentation metric may have
	* decreased the space to something smaller than
	* SPA_MINBLOCKSIZE, so reset the space to SPA_MINBLOCKSIZE
	* so that we can consume any remaining space.
	*/
	if (space > 0 && space < SPA_MINBLOCKSIZE)
	space = SPA_MINBLOCKSIZE;
	}
	weight = space;

	/*
	* Modern disks have uniform bit density and constant angular velocity.
	* Therefore, the outer recording zones are faster (higher bandwidth)
	* than the inner zones by the ratio of outer to inner track diameter,
	* which is typically around 2:1. We account for this by assigning
	* higher weight to lower metaslabs (multiplier ranging from 2x to 1x).
	* In effect, this means that we'll select the metaslab with the most
	* free bandwidth rather than simply the one with the most free space.
	*/
	if (!vd->vdev_nonrot && metaslab_lba_weighting_enabled) {
	weight = 2 * weight - (msp->ms_id * weight) / vd->vdev_ms_count;
	ASSERT(weight >= space && weight <= 2 * space);
	}

	/*
	* If this metaslab is one we're actively using, adjust its
	* weight to make it preferable to any inactive metaslab so
	* we'll polish it off. If the fragmentation on this metaslab
	* has exceed our threshold, then don't mark it active.
	*/
	if (msp->ms_loaded && msp->ms_fragmentation != ZFS_FRAG_INVALID &&
	msp->ms_fragmentation <= zfs_metaslab_fragmentation_threshold) {
	weight \|= (msp->ms_weight & METASLAB_ACTIVE_MASK);
	}

	WEIGHT_SET_SPACEBASED(weight);
	return (weight);
	}

	/*
	* Return the weight of the specified metaslab, according to the segment-based
	* weighting algorithm. The metaslab must be loaded. This function can
	* be called within a sync pass since it relies only on the metaslab's
	* range tree which is always accurate when the metaslab is loaded.
	*/
	static uint64_t
	metaslab_weight_from_range_tree(metaslab_t *msp)
	{
	uint64_t weight = 0;
	uint32_t segments = 0;

	ASSERT(msp->ms_loaded);

	for (int i = RANGE_TREE_HISTOGRAM_SIZE - 1; i >= SPA_MINBLOCKSHIFT;
	i--) {
	uint8_t shift = msp->ms_group->mg_vd->vdev_ashift;
	int max_idx = SPACE_MAP_HISTOGRAM_SIZE + shift - 1;

	segments <<= 1;
	segments += msp->ms_allocatable->rt_histogram[i];

	/*
	* The range tree provides more precision than the space map
	* and must be downgraded so that all values fit within the
	* space map's histogram. This allows us to compare loaded
	* vs. unloaded metaslabs to determine which metaslab is
	* considered "best".
	*/
	if (i > max_idx)
	continue;

	if (segments != 0) {
	WEIGHT_SET_COUNT(weight, segments);
	WEIGHT_SET_INDEX(weight, i);
	WEIGHT_SET_ACTIVE(weight, 0);
	break;
	}
	}
	return (weight);
	}

	/*
	* Calculate the weight based on the on-disk histogram. Should be applied
	* only to unloaded metaslabs (i.e no incoming allocations) in-order to
	* give results consistent with the on-disk state
	*/
	static uint64_t
	metaslab_weight_from_spacemap(metaslab_t *msp)
	{
	space_map_t *sm = msp->ms_sm;
	ASSERT(!msp->ms_loaded);
	ASSERT(sm != NULL);
	ASSERT3U(space_map_object(sm), !=, 0);
	ASSERT3U(sm->sm_dbuf->db_size, ==, sizeof (space_map_phys_t));

	/*
	* Create a joint histogram from all the segments that have made
	* it to the metaslab's space map histogram, that are not yet
	* available for allocation because they are still in the freeing
	* pipeline (e.g. freeing, freed, and defer trees). Then subtract
	* these segments from the space map's histogram to get a more
	* accurate weight.
	*/
	uint64_t deferspace_histogram[SPACE_MAP_HISTOGRAM_SIZE] = {0};
	for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++)
	deferspace_histogram[i] += msp->ms_synchist[i];
	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
	for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) {
	deferspace_histogram[i] += msp->ms_deferhist[t][i];
	}
	}

	uint64_t weight = 0;
	for (int i = SPACE_MAP_HISTOGRAM_SIZE - 1; i >= 0; i--) {
	ASSERT3U(sm->sm_phys->smp_histogram[i], >=,
	deferspace_histogram[i]);
	uint64_t count =
	sm->sm_phys->smp_histogram[i] - deferspace_histogram[i];
	if (count != 0) {
	WEIGHT_SET_COUNT(weight, count);
	WEIGHT_SET_INDEX(weight, i + sm->sm_shift);
	WEIGHT_SET_ACTIVE(weight, 0);
	break;
	}
	}
	return (weight);
	}

	/*
	* Compute a segment-based weight for the specified metaslab. The weight
	* is determined by highest bucket in the histogram. The information
	* for the highest bucket is encoded into the weight value.
	*/
	static uint64_t
	metaslab_segment_weight(metaslab_t *msp)
	{
	metaslab_group_t *mg = msp->ms_group;
	uint64_t weight = 0;
	uint8_t shift = mg->mg_vd->vdev_ashift;

	ASSERT(MUTEX_HELD(&msp->ms_lock));

	/*
	* The metaslab is completely free.
	*/
	if (metaslab_allocated_space(msp) == 0) {
	int idx = highbit64(msp->ms_size) - 1;
	int max_idx = SPACE_MAP_HISTOGRAM_SIZE + shift - 1;

	if (idx < max_idx) {
	WEIGHT_SET_COUNT(weight, 1ULL);
	WEIGHT_SET_INDEX(weight, idx);
	} else {
	WEIGHT_SET_COUNT(weight, 1ULL << (idx - max_idx));
	WEIGHT_SET_INDEX(weight, max_idx);
	}
	WEIGHT_SET_ACTIVE(weight, 0);
	ASSERT(!WEIGHT_IS_SPACEBASED(weight));
	return (weight);
	}

	ASSERT3U(msp->ms_sm->sm_dbuf->db_size, ==, sizeof (space_map_phys_t));

	/*
	* If the metaslab is fully allocated then just make the weight 0.
	*/
	if (metaslab_allocated_space(msp) == msp->ms_size)
	return (0);
	/*
	* If the metaslab is already loaded, then use the range tree to
	* determine the weight. Otherwise, we rely on the space map information
	* to generate the weight.
	*/
	if (msp->ms_loaded) {
	weight = metaslab_weight_from_range_tree(msp);
	} else {
	weight = metaslab_weight_from_spacemap(msp);
	}

	/*
	* If the metaslab was active the last time we calculated its weight
	* then keep it active. We want to consume the entire region that
	* is associated with this weight.
	*/
	if (msp->ms_activation_weight != 0 && weight != 0)
	WEIGHT_SET_ACTIVE(weight, WEIGHT_GET_ACTIVE(msp->ms_weight));
	return (weight);
	}

	/*
	* Determine if we should attempt to allocate from this metaslab. If the
	* metaslab is loaded, then we can determine if the desired allocation
	* can be satisfied by looking at the size of the maximum free segment
	* on that metaslab. Otherwise, we make our decision based on the metaslab's
	* weight. For segment-based weighting we can determine the maximum
	* allocation based on the index encoded in its value. For space-based
	* weights we rely on the entire weight (excluding the weight-type bit).
	*/
	static boolean_t
	metaslab_should_allocate(metaslab_t *msp, uint64_t asize, boolean_t try_hard)
	{
	/*
	* This case will usually but not always get caught by the checks below;
	* metaslabs can be loaded by various means, including the trim and
	* initialize code. Once that happens, without this check they are
	* allocatable even before they finish their first txg sync.
	*/
	if (unlikely(msp->ms_new))
	return (B_FALSE);

	/*
	* If the metaslab is loaded, ms_max_size is definitive and we can use
	* the fast check. If it's not, the ms_max_size is a lower bound (once
	* set), and we should use the fast check as long as we're not in
	* try_hard and it's been less than zfs_metaslab_max_size_cache_sec
	* seconds since the metaslab was unloaded.
	*/
	if (msp->ms_loaded \|\|
	(msp->ms_max_size != 0 && !try_hard && gethrtime() <
	msp->ms_unload_time + SEC2NSEC(zfs_metaslab_max_size_cache_sec)))
	return (msp->ms_max_size >= asize);

	boolean_t should_allocate;
	if (!WEIGHT_IS_SPACEBASED(msp->ms_weight)) {
	/*
	* The metaslab segment weight indicates segments in the
	* range [2^i, 2^(i+1)), where i is the index in the weight.
	* Since the asize might be in the middle of the range, we
	* should attempt the allocation if asize < 2^(i+1).
	*/
	should_allocate = (asize <
	1ULL << (WEIGHT_GET_INDEX(msp->ms_weight) + 1));
	} else {
	should_allocate = (asize <=
	(msp->ms_weight & ~METASLAB_WEIGHT_TYPE));
	}

	return (should_allocate);
	}

	static uint64_t
	metaslab_weight(metaslab_t *msp, boolean_t nodirty)
	{
	vdev_t *vd = msp->ms_group->mg_vd;
	spa_t *spa = vd->vdev_spa;
	uint64_t weight;

	ASSERT(MUTEX_HELD(&msp->ms_lock));

	metaslab_set_fragmentation(msp, nodirty);

	/*
	* Update the maximum size. If the metaslab is loaded, this will
	* ensure that we get an accurate maximum size if newly freed space
	* has been added back into the free tree. If the metaslab is
	* unloaded, we check if there's a larger free segment in the
	* unflushed frees. This is a lower bound on the largest allocatable
	* segment size. Coalescing of adjacent entries may reveal larger
	* allocatable segments, but we aren't aware of those until loading
	* the space map into a range tree.
	*/
	if (msp->ms_loaded) {
	msp->ms_max_size = metaslab_largest_allocatable(msp);
	} else {
	msp->ms_max_size = MAX(msp->ms_max_size,
	metaslab_largest_unflushed_free(msp));
	}

	/*
	* Segment-based weighting requires space map histogram support.
	*/
	if (zfs_metaslab_segment_weight_enabled &&
	spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_HISTOGRAM) &&
	(msp->ms_sm == NULL \|\| msp->ms_sm->sm_dbuf->db_size ==
	sizeof (space_map_phys_t))) {
	weight = metaslab_segment_weight(msp);
	} else {
	weight = metaslab_space_weight(msp);
	}
	return (weight);
	}

	void
	metaslab_recalculate_weight_and_sort(metaslab_t *msp)
	{
	ASSERT(MUTEX_HELD(&msp->ms_lock));

	/* note: we preserve the mask (e.g. indication of primary, etc..) */
	uint64_t was_active = msp->ms_weight & METASLAB_ACTIVE_MASK;
	metaslab_group_sort(msp->ms_group, msp,
	metaslab_weight(msp, B_FALSE) \| was_active);
	}

	static int
	metaslab_activate_allocator(metaslab_group_t mg, metaslab_t msp,
	int allocator, uint64_t activation_weight)
	{
	metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator];
	ASSERT(MUTEX_HELD(&msp->ms_lock));

	/*
	* If we're activating for the claim code, we don't want to actually
	* set the metaslab up for a specific allocator.
	*/
	if (activation_weight == METASLAB_WEIGHT_CLAIM) {
	ASSERT0(msp->ms_activation_weight);
	msp->ms_activation_weight = msp->ms_weight;
	metaslab_group_sort(mg, msp, msp->ms_weight \|
	activation_weight);
	return (0);
	}

	metaslab_t **mspp = (activation_weight == METASLAB_WEIGHT_PRIMARY ?
	&mga->mga_primary : &mga->mga_secondary);

	mutex_enter(&mg->mg_lock);
	if (*mspp != NULL) {
	mutex_exit(&mg->mg_lock);
	return (EEXIST);
	}

	*mspp = msp;
	ASSERT3S(msp->ms_allocator, ==, -1);
	msp->ms_allocator = allocator;
	msp->ms_primary = (activation_weight == METASLAB_WEIGHT_PRIMARY);

	ASSERT0(msp->ms_activation_weight);
	msp->ms_activation_weight = msp->ms_weight;
	metaslab_group_sort_impl(mg, msp,
	msp->ms_weight \| activation_weight);
	mutex_exit(&mg->mg_lock);

	return (0);
	}

	static int
	metaslab_activate(metaslab_t *msp, int allocator, uint64_t activation_weight)
	{
	ASSERT(MUTEX_HELD(&msp->ms_lock));

	/*
	* The current metaslab is already activated for us so there
	* is nothing to do. Already activated though, doesn't mean
	* that this metaslab is activated for our allocator nor our
	* requested activation weight. The metaslab could have started
	* as an active one for our allocator but changed allocators
	* while we were waiting to grab its ms_lock or we stole it
	* [see find_valid_metaslab()]. This means that there is a
	* possibility of passivating a metaslab of another allocator
	* or from a different activation mask, from this thread.
	*/
	if ((msp->ms_weight & METASLAB_ACTIVE_MASK) != 0) {
	ASSERT(msp->ms_loaded);
	return (0);
	}

	int error = metaslab_load(msp);
	if (error != 0) {
	metaslab_group_sort(msp->ms_group, msp, 0);
	return (error);
	}

	/*
	* When entering metaslab_load() we may have dropped the
	* ms_lock because we were loading this metaslab, or we
	* were waiting for another thread to load it for us. In
	* that scenario, we recheck the weight of the metaslab
	* to see if it was activated by another thread.
	*
	* If the metaslab was activated for another allocator or
	* it was activated with a different activation weight (e.g.
	* we wanted to make it a primary but it was activated as
	* secondary) we return error (EBUSY).
	*
	* If the metaslab was activated for the same allocator
	* and requested activation mask, skip activating it.
	*/
	if ((msp->ms_weight & METASLAB_ACTIVE_MASK) != 0) {
	if (msp->ms_allocator != allocator)
	return (EBUSY);

	if ((msp->ms_weight & activation_weight) == 0)
	return (SET_ERROR(EBUSY));

	EQUIV((activation_weight == METASLAB_WEIGHT_PRIMARY),
	msp->ms_primary);
	return (0);
	}

	/*
	* If the metaslab has literally 0 space, it will have weight 0. In
	* that case, don't bother activating it. This can happen if the
	* metaslab had space during find_valid_metaslab, but another thread
	* loaded it and used all that space while we were waiting to grab the
	* lock.
	*/
	if (msp->ms_weight == 0) {
	ASSERT0(range_tree_space(msp->ms_allocatable));
	return (SET_ERROR(ENOSPC));
	}

	if ((error = metaslab_activate_allocator(msp->ms_group, msp,
	allocator, activation_weight)) != 0) {
	return (error);
	}

	ASSERT(msp->ms_loaded);
	ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK);

	return (0);
	}

	static void
	metaslab_passivate_allocator(metaslab_group_t mg, metaslab_t msp,
	uint64_t weight)
	{
	ASSERT(MUTEX_HELD(&msp->ms_lock));
	ASSERT(msp->ms_loaded);

	if (msp->ms_weight & METASLAB_WEIGHT_CLAIM) {
	metaslab_group_sort(mg, msp, weight);
	return;
	}

	mutex_enter(&mg->mg_lock);
	ASSERT3P(msp->ms_group, ==, mg);
	ASSERT3S(0, <=, msp->ms_allocator);
	ASSERT3U(msp->ms_allocator, <, mg->mg_allocators);

	metaslab_group_allocator_t *mga = &mg->mg_allocator[msp->ms_allocator];
	if (msp->ms_primary) {
	ASSERT3P(mga->mga_primary, ==, msp);
	ASSERT(msp->ms_weight & METASLAB_WEIGHT_PRIMARY);
	mga->mga_primary = NULL;
	} else {
	ASSERT3P(mga->mga_secondary, ==, msp);
	ASSERT(msp->ms_weight & METASLAB_WEIGHT_SECONDARY);
	mga->mga_secondary = NULL;
	}
	msp->ms_allocator = -1;
	metaslab_group_sort_impl(mg, msp, weight);
	mutex_exit(&mg->mg_lock);
	}

	static void
	metaslab_passivate(metaslab_t *msp, uint64_t weight)
	{
	uint64_t size __maybe_unused = weight & ~METASLAB_WEIGHT_TYPE;

	/*
	* If size < SPA_MINBLOCKSIZE, then we will not allocate from
	* this metaslab again. In that case, it had better be empty,
	* or we would be leaving space on the table.
	*/
	ASSERT(!WEIGHT_IS_SPACEBASED(msp->ms_weight) \|\|
	size >= SPA_MINBLOCKSIZE \|\|
	range_tree_space(msp->ms_allocatable) == 0);
	ASSERT0(weight & METASLAB_ACTIVE_MASK);

	ASSERT(msp->ms_activation_weight != 0);
	msp->ms_activation_weight = 0;
	metaslab_passivate_allocator(msp->ms_group, msp, weight);
	ASSERT0(msp->ms_weight & METASLAB_ACTIVE_MASK);
	}

	/*
	* Segment-based metaslabs are activated once and remain active until
	* we either fail an allocation attempt (similar to space-based metaslabs)
	* or have exhausted the free space in zfs_metaslab_switch_threshold
	* buckets since the metaslab was activated. This function checks to see
	* if we've exhausted the zfs_metaslab_switch_threshold buckets in the
	* metaslab and passivates it proactively. This will allow us to select a
	* metaslab with a larger contiguous region, if any, remaining within this
	* metaslab group. If we're in sync pass > 1, then we continue using this
	* metaslab so that we don't dirty more block and cause more sync passes.
	*/
	static void
	metaslab_segment_may_passivate(metaslab_t *msp)
	{
	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;

	if (WEIGHT_IS_SPACEBASED(msp->ms_weight) \|\| spa_sync_pass(spa) > 1)
	return;

	/*
	* Since we are in the middle of a sync pass, the most accurate
	* information that is accessible to us is the in-core range tree
	* histogram; calculate the new weight based on that information.
	*/
	uint64_t weight = metaslab_weight_from_range_tree(msp);
	int activation_idx = WEIGHT_GET_INDEX(msp->ms_activation_weight);
	int current_idx = WEIGHT_GET_INDEX(weight);

	if (current_idx <= activation_idx - zfs_metaslab_switch_threshold)
	metaslab_passivate(msp, weight);
	}

	static void
	metaslab_preload(void *arg)
	{
	metaslab_t *msp = arg;
	metaslab_class_t *mc = msp->ms_group->mg_class;
	spa_t *spa = mc->mc_spa;
	fstrans_cookie_t cookie = spl_fstrans_mark();

	ASSERT(!MUTEX_HELD(&msp->ms_group->mg_lock));

	mutex_enter(&msp->ms_lock);
	(void) metaslab_load(msp);
	metaslab_set_selected_txg(msp, spa_syncing_txg(spa));
	mutex_exit(&msp->ms_lock);
	spl_fstrans_unmark(cookie);
	}

	static void
	metaslab_group_preload(metaslab_group_t *mg)
	{
	spa_t *spa = mg->mg_vd->vdev_spa;
	metaslab_t *msp;
	avl_tree_t *t = &mg->mg_metaslab_tree;
	int m = 0;

	if (spa_shutting_down(spa) \|\| !metaslab_preload_enabled)
	return;

	mutex_enter(&mg->mg_lock);

	/*
	* Load the next potential metaslabs
	*/
	for (msp = avl_first(t); msp != NULL; msp = AVL_NEXT(t, msp)) {
	ASSERT3P(msp->ms_group, ==, mg);

	/*
	* We preload only the maximum number of metaslabs specified
	* by metaslab_preload_limit. If a metaslab is being forced
	* to condense then we preload it too. This will ensure
	* that force condensing happens in the next txg.
	*/
	if (++m > metaslab_preload_limit && !msp->ms_condense_wanted) {
	continue;
	}

	VERIFY(taskq_dispatch(spa->spa_metaslab_taskq, metaslab_preload,
	msp, TQ_SLEEP \| (m <= mg->mg_allocators ? TQ_FRONT : 0))
	!= TASKQID_INVALID);
	}
	mutex_exit(&mg->mg_lock);
	}

	/*
	* Determine if the space map's on-disk footprint is past our tolerance for
	* inefficiency. We would like to use the following criteria to make our
	* decision:
	*
	* 1. Do not condense if the size of the space map object would dramatically
	* increase as a result of writing out the free space range tree.
	*
	* 2. Condense if the on on-disk space map representation is at least
	* zfs_condense_pct/100 times the size of the optimal representation
	* (i.e. zfs_condense_pct = 110 and in-core = 1MB, optimal = 1.1MB).
	*
	* 3. Do not condense if the on-disk size of the space map does not actually
	* decrease.
	*
	* Unfortunately, we cannot compute the on-disk size of the space map in this
	* context because we cannot accurately compute the effects of compression, etc.
	* Instead, we apply the heuristic described in the block comment for
	* zfs_metaslab_condense_block_threshold - we only condense if the space used
	* is greater than a threshold number of blocks.
	*/
	static boolean_t
	metaslab_should_condense(metaslab_t *msp)
	{
	space_map_t *sm = msp->ms_sm;
	vdev_t *vd = msp->ms_group->mg_vd;
	uint64_t vdev_blocksize = 1ULL << vd->vdev_ashift;

	ASSERT(MUTEX_HELD(&msp->ms_lock));
	ASSERT(msp->ms_loaded);
	ASSERT(sm != NULL);
	ASSERT3U(spa_sync_pass(vd->vdev_spa), ==, 1);

	/*
	* We always condense metaslabs that are empty and metaslabs for
	* which a condense request has been made.
	*/
	if (range_tree_numsegs(msp->ms_allocatable) == 0 \|\|
	msp->ms_condense_wanted)
	return (B_TRUE);

	uint64_t record_size = MAX(sm->sm_blksz, vdev_blocksize);
	uint64_t object_size = space_map_length(sm);
	uint64_t optimal_size = space_map_estimate_optimal_size(sm,
	msp->ms_allocatable, SM_NO_VDEVID);

	return (object_size >= (optimal_size * zfs_condense_pct / 100) &&
	object_size > zfs_metaslab_condense_block_threshold * record_size);
	}

	/*
	* Condense the on-disk space map representation to its minimized form.
	* The minimized form consists of a small number of allocations followed
	* by the entries of the free range tree (ms_allocatable). The condensed
	* spacemap contains all the entries of previous TXGs (including those in
	* the pool-wide log spacemaps; thus this is effectively a superset of
	* metaslab_flush()), but this TXG's entries still need to be written.
	*/
	static void
	metaslab_condense(metaslab_t msp, dmu_tx_t tx)
	{
	range_tree_t *condense_tree;
	space_map_t *sm = msp->ms_sm;
	uint64_t txg = dmu_tx_get_txg(tx);
	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;

	ASSERT(MUTEX_HELD(&msp->ms_lock));
	ASSERT(msp->ms_loaded);
	ASSERT(msp->ms_sm != NULL);

	/*
	* In order to condense the space map, we need to change it so it
	* only describes which segments are currently allocated and free.
	*
	* All the current free space resides in the ms_allocatable, all
	* the ms_defer trees, and all the ms_allocating trees. We ignore
	* ms_freed because it is empty because we're in sync pass 1. We
	* ignore ms_freeing because these changes are not yet reflected
	* in the spacemap (they will be written later this txg).
	*
	* So to truncate the space map to represent all the entries of
	* previous TXGs we do the following:
	*
	* 1] We create a range tree (condense tree) that is 100% empty.
	* 2] We add to it all segments found in the ms_defer trees
	* as those segments are marked as free in the original space
	* map. We do the same with the ms_allocating trees for the same
	* reason. Adding these segments should be a relatively
	* inexpensive operation since we expect these trees to have a
	* small number of nodes.
	* 3] We vacate any unflushed allocs, since they are not frees we
	* need to add to the condense tree. Then we vacate any
	* unflushed frees as they should already be part of ms_allocatable.
	* 4] At this point, we would ideally like to add all segments
	* in the ms_allocatable tree from the condense tree. This way
	* we would write all the entries of the condense tree as the
	* condensed space map, which would only contain freed
	* segments with everything else assumed to be allocated.
	*
	* Doing so can be prohibitively expensive as ms_allocatable can
	* be large, and therefore computationally expensive to add to
	* the condense_tree. Instead we first sync out an entry marking
	* everything as allocated, then the condense_tree and then the
	* ms_allocatable, in the condensed space map. While this is not
	* optimal, it is typically close to optimal and more importantly
	* much cheaper to compute.
	*
	* 5] Finally, as both of the unflushed trees were written to our
	* new and condensed metaslab space map, we basically flushed
	* all the unflushed changes to disk, thus we call
	* metaslab_flush_update().
	*/
	ASSERT3U(spa_sync_pass(spa), ==, 1);
	ASSERT(range_tree_is_empty(msp->ms_freed)); /* since it is pass 1 */

	zfs_dbgmsg("condensing: txg %llu, msp[%llu] %px, vdev id %llu, "
	"spa %s, smp size %llu, segments %llu, forcing condense=%s",
	(u_longlong_t)txg, (u_longlong_t)msp->ms_id, msp,
	(u_longlong_t)msp->ms_group->mg_vd->vdev_id,
	spa->spa_name, (u_longlong_t)space_map_length(msp->ms_sm),
	(u_longlong_t)range_tree_numsegs(msp->ms_allocatable),
	msp->ms_condense_wanted ? "TRUE" : "FALSE");

	msp->ms_condense_wanted = B_FALSE;

	range_seg_type_t type;
	uint64_t shift, start;
	type = metaslab_calculate_range_tree_type(msp->ms_group->mg_vd, msp,
	&start, &shift);

	condense_tree = range_tree_create(NULL, type, NULL, start, shift);

	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
	range_tree_walk(msp->ms_defer[t],
	range_tree_add, condense_tree);
	}

	for (int t = 0; t < TXG_CONCURRENT_STATES; t++) {
	range_tree_walk(msp->ms_allocating[(txg + t) & TXG_MASK],
	range_tree_add, condense_tree);
	}

	ASSERT3U(spa->spa_unflushed_stats.sus_memused, >=,
	metaslab_unflushed_changes_memused(msp));
	spa->spa_unflushed_stats.sus_memused -=
	metaslab_unflushed_changes_memused(msp);
	range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL);
	range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL);

	/*
	* We're about to drop the metaslab's lock thus allowing other
	* consumers to change it's content. Set the metaslab's ms_condensing
	* flag to ensure that allocations on this metaslab do not occur
	* while we're in the middle of committing it to disk. This is only
	* critical for ms_allocatable as all other range trees use per TXG
	* views of their content.
	*/
	msp->ms_condensing = B_TRUE;

	mutex_exit(&msp->ms_lock);
	uint64_t object = space_map_object(msp->ms_sm);
	space_map_truncate(sm,
	spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP) ?
	zfs_metaslab_sm_blksz_with_log : zfs_metaslab_sm_blksz_no_log, tx);

	/*
	* space_map_truncate() may have reallocated the spacemap object.
	* If so, update the vdev_ms_array.
	*/
	if (space_map_object(msp->ms_sm) != object) {
	object = space_map_object(msp->ms_sm);
	dmu_write(spa->spa_meta_objset,
	msp->ms_group->mg_vd->vdev_ms_array, sizeof (uint64_t) *
	msp->ms_id, sizeof (uint64_t), &object, tx);
	}

	/*
	* Note:
	* When the log space map feature is enabled, each space map will
	* always have ALLOCS followed by FREES for each sync pass. This is
	* typically true even when the log space map feature is disabled,
	* except from the case where a metaslab goes through metaslab_sync()
	* and gets condensed. In that case the metaslab's space map will have
	* ALLOCS followed by FREES (due to condensing) followed by ALLOCS
	* followed by FREES (due to space_map_write() in metaslab_sync()) for
	* sync pass 1.
	*/
	range_tree_t *tmp_tree = range_tree_create(NULL, type, NULL, start,
	shift);
	range_tree_add(tmp_tree, msp->ms_start, msp->ms_size);
	space_map_write(sm, tmp_tree, SM_ALLOC, SM_NO_VDEVID, tx);
	space_map_write(sm, msp->ms_allocatable, SM_FREE, SM_NO_VDEVID, tx);
	space_map_write(sm, condense_tree, SM_FREE, SM_NO_VDEVID, tx);

	range_tree_vacate(condense_tree, NULL, NULL);
	range_tree_destroy(condense_tree);
	range_tree_vacate(tmp_tree, NULL, NULL);
	range_tree_destroy(tmp_tree);
	mutex_enter(&msp->ms_lock);

	msp->ms_condensing = B_FALSE;
	metaslab_flush_update(msp, tx);
	}

	static void
	metaslab_unflushed_add(metaslab_t msp, dmu_tx_t tx)
	{
	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
	ASSERT(spa_syncing_log_sm(spa) != NULL);
	ASSERT(msp->ms_sm != NULL);
	ASSERT(range_tree_is_empty(msp->ms_unflushed_allocs));
	ASSERT(range_tree_is_empty(msp->ms_unflushed_frees));

	mutex_enter(&spa->spa_flushed_ms_lock);
	metaslab_set_unflushed_txg(msp, spa_syncing_txg(spa), tx);
	metaslab_set_unflushed_dirty(msp, B_TRUE);
	avl_add(&spa->spa_metaslabs_by_flushed, msp);
	mutex_exit(&spa->spa_flushed_ms_lock);

	spa_log_sm_increment_current_mscount(spa);
	spa_log_summary_add_flushed_metaslab(spa, B_TRUE);
	}

	void
	metaslab_unflushed_bump(metaslab_t msp, dmu_tx_t tx, boolean_t dirty)
	{
	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
	ASSERT(spa_syncing_log_sm(spa) != NULL);
	ASSERT(msp->ms_sm != NULL);
	ASSERT(metaslab_unflushed_txg(msp) != 0);
	ASSERT3P(avl_find(&spa->spa_metaslabs_by_flushed, msp, NULL), ==, msp);
	ASSERT(range_tree_is_empty(msp->ms_unflushed_allocs));
	ASSERT(range_tree_is_empty(msp->ms_unflushed_frees));

	VERIFY3U(tx->tx_txg, <=, spa_final_dirty_txg(spa));

	/* update metaslab's position in our flushing tree */
	uint64_t ms_prev_flushed_txg = metaslab_unflushed_txg(msp);
	boolean_t ms_prev_flushed_dirty = metaslab_unflushed_dirty(msp);
	mutex_enter(&spa->spa_flushed_ms_lock);
	avl_remove(&spa->spa_metaslabs_by_flushed, msp);
	metaslab_set_unflushed_txg(msp, spa_syncing_txg(spa), tx);
	metaslab_set_unflushed_dirty(msp, dirty);
	avl_add(&spa->spa_metaslabs_by_flushed, msp);
	mutex_exit(&spa->spa_flushed_ms_lock);

	/* update metaslab counts of spa_log_sm_t nodes */
	spa_log_sm_decrement_mscount(spa, ms_prev_flushed_txg);
	spa_log_sm_increment_current_mscount(spa);

	/* update log space map summary */
	spa_log_summary_decrement_mscount(spa, ms_prev_flushed_txg,
	ms_prev_flushed_dirty);
	spa_log_summary_add_flushed_metaslab(spa, dirty);

	/* cleanup obsolete logs if any */
	spa_cleanup_old_sm_logs(spa, tx);
	}

	/*
	* Called when the metaslab has been flushed (its own spacemap now reflects
	* all the contents of the pool-wide spacemap log). Updates the metaslab's
	* metadata and any pool-wide related log space map data (e.g. summary,
	* obsolete logs, etc..) to reflect that.
	*/
	static void
	metaslab_flush_update(metaslab_t msp, dmu_tx_t tx)
	{
	metaslab_group_t *mg = msp->ms_group;
	spa_t *spa = mg->mg_vd->vdev_spa;

	ASSERT(MUTEX_HELD(&msp->ms_lock));

	ASSERT3U(spa_sync_pass(spa), ==, 1);

	/*
	* Just because a metaslab got flushed, that doesn't mean that
	* it will pass through metaslab_sync_done(). Thus, make sure to
	* update ms_synced_length here in case it doesn't.
	*/
	msp->ms_synced_length = space_map_length(msp->ms_sm);

	/*
	* We may end up here from metaslab_condense() without the
	* feature being active. In that case this is a no-op.
	*/
	if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP) \|\|
	metaslab_unflushed_txg(msp) == 0)
	return;

	metaslab_unflushed_bump(msp, tx, B_FALSE);
	}

	boolean_t
	metaslab_flush(metaslab_t msp, dmu_tx_t tx)
	{
	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;

	ASSERT(MUTEX_HELD(&msp->ms_lock));
	ASSERT3U(spa_sync_pass(spa), ==, 1);
	ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP));

	ASSERT(msp->ms_sm != NULL);
	ASSERT(metaslab_unflushed_txg(msp) != 0);
	ASSERT(avl_find(&spa->spa_metaslabs_by_flushed, msp, NULL) != NULL);

	/*
	* There is nothing wrong with flushing the same metaslab twice, as
	* this codepath should work on that case. However, the current
	* flushing scheme makes sure to avoid this situation as we would be
	* making all these calls without having anything meaningful to write
	* to disk. We assert this behavior here.
	*/
	ASSERT3U(metaslab_unflushed_txg(msp), <, dmu_tx_get_txg(tx));

	/*
	* We can not flush while loading, because then we would
	* not load the ms_unflushed_{allocs,frees}.
	*/
	if (msp->ms_loading)
	return (B_FALSE);

	metaslab_verify_space(msp, dmu_tx_get_txg(tx));
	metaslab_verify_weight_and_frag(msp);

	/*
	* Metaslab condensing is effectively flushing. Therefore if the
	* metaslab can be condensed we can just condense it instead of
	* flushing it.
	*
	* Note that metaslab_condense() does call metaslab_flush_update()
	* so we can just return immediately after condensing. We also
	* don't need to care about setting ms_flushing or broadcasting
	* ms_flush_cv, even if we temporarily drop the ms_lock in
	* metaslab_condense(), as the metaslab is already loaded.
	*/
	if (msp->ms_loaded && metaslab_should_condense(msp)) {
	metaslab_group_t *mg = msp->ms_group;

	/*
	* For all histogram operations below refer to the
	* comments of metaslab_sync() where we follow a
	* similar procedure.
	*/
	metaslab_group_histogram_verify(mg);
	metaslab_class_histogram_verify(mg->mg_class);
	metaslab_group_histogram_remove(mg, msp);

	metaslab_condense(msp, tx);

	space_map_histogram_clear(msp->ms_sm);
	space_map_histogram_add(msp->ms_sm, msp->ms_allocatable, tx);
	ASSERT(range_tree_is_empty(msp->ms_freed));
	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
	space_map_histogram_add(msp->ms_sm,
	msp->ms_defer[t], tx);
	}
	metaslab_aux_histograms_update(msp);

	metaslab_group_histogram_add(mg, msp);
	metaslab_group_histogram_verify(mg);
	metaslab_class_histogram_verify(mg->mg_class);

	metaslab_verify_space(msp, dmu_tx_get_txg(tx));

	/*
	* Since we recreated the histogram (and potentially
	* the ms_sm too while condensing) ensure that the
	* weight is updated too because we are not guaranteed
	* that this metaslab is dirty and will go through
	* metaslab_sync_done().
	*/
	metaslab_recalculate_weight_and_sort(msp);
	return (B_TRUE);
	}

	msp->ms_flushing = B_TRUE;
	uint64_t sm_len_before = space_map_length(msp->ms_sm);

	mutex_exit(&msp->ms_lock);
	space_map_write(msp->ms_sm, msp->ms_unflushed_allocs, SM_ALLOC,
	SM_NO_VDEVID, tx);
	space_map_write(msp->ms_sm, msp->ms_unflushed_frees, SM_FREE,
	SM_NO_VDEVID, tx);
	mutex_enter(&msp->ms_lock);

	uint64_t sm_len_after = space_map_length(msp->ms_sm);
	if (zfs_flags & ZFS_DEBUG_LOG_SPACEMAP) {
	zfs_dbgmsg("flushing: txg %llu, spa %s, vdev_id %llu, "
	"ms_id %llu, unflushed_allocs %llu, unflushed_frees %llu, "
	"appended %llu bytes", (u_longlong_t)dmu_tx_get_txg(tx),
	spa_name(spa),
	(u_longlong_t)msp->ms_group->mg_vd->vdev_id,
	(u_longlong_t)msp->ms_id,
	(u_longlong_t)range_tree_space(msp->ms_unflushed_allocs),
	(u_longlong_t)range_tree_space(msp->ms_unflushed_frees),
	(u_longlong_t)(sm_len_after - sm_len_before));
	}

	ASSERT3U(spa->spa_unflushed_stats.sus_memused, >=,
	metaslab_unflushed_changes_memused(msp));
	spa->spa_unflushed_stats.sus_memused -=
	metaslab_unflushed_changes_memused(msp);
	range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL);
	range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL);

	metaslab_verify_space(msp, dmu_tx_get_txg(tx));
	metaslab_verify_weight_and_frag(msp);

	metaslab_flush_update(msp, tx);

	metaslab_verify_space(msp, dmu_tx_get_txg(tx));
	metaslab_verify_weight_and_frag(msp);

	msp->ms_flushing = B_FALSE;
	cv_broadcast(&msp->ms_flush_cv);
	return (B_TRUE);
	}

	/*
	* Write a metaslab to disk in the context of the specified transaction group.
	*/
	void
	metaslab_sync(metaslab_t *msp, uint64_t txg)
	{
	metaslab_group_t *mg = msp->ms_group;
	vdev_t *vd = mg->mg_vd;
	spa_t *spa = vd->vdev_spa;
	objset_t *mos = spa_meta_objset(spa);
	range_tree_t *alloctree = msp->ms_allocating[txg & TXG_MASK];
	dmu_tx_t *tx;

	ASSERT(!vd->vdev_ishole);

	/*
	* This metaslab has just been added so there's no work to do now.
	*/
	if (msp->ms_new) {
	ASSERT0(range_tree_space(alloctree));
	ASSERT0(range_tree_space(msp->ms_freeing));
	ASSERT0(range_tree_space(msp->ms_freed));
	ASSERT0(range_tree_space(msp->ms_checkpointing));
	ASSERT0(range_tree_space(msp->ms_trim));
	return;
	}

	/*
	* Normally, we don't want to process a metaslab if there are no
	* allocations or frees to perform. However, if the metaslab is being
	* forced to condense, it's loaded and we're not beyond the final
	* dirty txg, we need to let it through. Not condensing beyond the
	* final dirty txg prevents an issue where metaslabs that need to be
	* condensed but were loaded for other reasons could cause a panic
	* here. By only checking the txg in that branch of the conditional,
	* we preserve the utility of the VERIFY statements in all other
	* cases.
	*/
	if (range_tree_is_empty(alloctree) &&
	range_tree_is_empty(msp->ms_freeing) &&
	range_tree_is_empty(msp->ms_checkpointing) &&
	!(msp->ms_loaded && msp->ms_condense_wanted &&
	txg <= spa_final_dirty_txg(spa)))
	return;


	VERIFY3U(txg, <=, spa_final_dirty_txg(spa));

	/*
	* The only state that can actually be changing concurrently
	* with metaslab_sync() is the metaslab's ms_allocatable. No
	* other thread can be modifying this txg's alloc, freeing,
	* freed, or space_map_phys_t. We drop ms_lock whenever we
	* could call into the DMU, because the DMU can call down to
	* us (e.g. via zio_free()) at any time.
	*
	* The spa_vdev_remove_thread() can be reading metaslab state
	* concurrently, and it is locked out by the ms_sync_lock.
	* Note that the ms_lock is insufficient for this, because it
	* is dropped by space_map_write().
	*/
	tx = dmu_tx_create_assigned(spa_get_dsl(spa), txg);

	/*
	* Generate a log space map if one doesn't exist already.
	*/
	spa_generate_syncing_log_sm(spa, tx);

	if (msp->ms_sm == NULL) {
	uint64_t new_object = space_map_alloc(mos,
	spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP) ?
	zfs_metaslab_sm_blksz_with_log :
	zfs_metaslab_sm_blksz_no_log, tx);
	VERIFY3U(new_object, !=, 0);

	dmu_write(mos, vd->vdev_ms_array, sizeof (uint64_t) *
	msp->ms_id, sizeof (uint64_t), &new_object, tx);

	VERIFY0(space_map_open(&msp->ms_sm, mos, new_object,
	msp->ms_start, msp->ms_size, vd->vdev_ashift));
	ASSERT(msp->ms_sm != NULL);

	ASSERT(range_tree_is_empty(msp->ms_unflushed_allocs));
	ASSERT(range_tree_is_empty(msp->ms_unflushed_frees));
	ASSERT0(metaslab_allocated_space(msp));
	}

	if (!range_tree_is_empty(msp->ms_checkpointing) &&
	vd->vdev_checkpoint_sm == NULL) {
	ASSERT(spa_has_checkpoint(spa));

	uint64_t new_object = space_map_alloc(mos,
	zfs_vdev_standard_sm_blksz, tx);
	VERIFY3U(new_object, !=, 0);

	VERIFY0(space_map_open(&vd->vdev_checkpoint_sm,
	mos, new_object, 0, vd->vdev_asize, vd->vdev_ashift));
	ASSERT3P(vd->vdev_checkpoint_sm, !=, NULL);

	/*
	* We save the space map object as an entry in vdev_top_zap
	* so it can be retrieved when the pool is reopened after an
	* export or through zdb.
	*/
	VERIFY0(zap_add(vd->vdev_spa->spa_meta_objset,
	vd->vdev_top_zap, VDEV_TOP_ZAP_POOL_CHECKPOINT_SM,
	sizeof (new_object), 1, &new_object, tx));
	}

	mutex_enter(&msp->ms_sync_lock);
	mutex_enter(&msp->ms_lock);

	/*
	* Note: metaslab_condense() clears the space map's histogram.
	* Therefore we must verify and remove this histogram before
	* condensing.
	*/
	metaslab_group_histogram_verify(mg);
	metaslab_class_histogram_verify(mg->mg_class);
	metaslab_group_histogram_remove(mg, msp);

	if (spa->spa_sync_pass == 1 && msp->ms_loaded &&
	metaslab_should_condense(msp))
	metaslab_condense(msp, tx);

	/*
	* We'll be going to disk to sync our space accounting, thus we
	* drop the ms_lock during that time so allocations coming from
	* open-context (ZIL) for future TXGs do not block.
	*/
	mutex_exit(&msp->ms_lock);
	space_map_t *log_sm = spa_syncing_log_sm(spa);
	if (log_sm != NULL) {
	ASSERT(spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP));
	if (metaslab_unflushed_txg(msp) == 0)
	metaslab_unflushed_add(msp, tx);
	else if (!metaslab_unflushed_dirty(msp))
	metaslab_unflushed_bump(msp, tx, B_TRUE);

	space_map_write(log_sm, alloctree, SM_ALLOC,
	vd->vdev_id, tx);
	space_map_write(log_sm, msp->ms_freeing, SM_FREE,
	vd->vdev_id, tx);
	mutex_enter(&msp->ms_lock);

	ASSERT3U(spa->spa_unflushed_stats.sus_memused, >=,
	metaslab_unflushed_changes_memused(msp));
	spa->spa_unflushed_stats.sus_memused -=
	metaslab_unflushed_changes_memused(msp);
	range_tree_remove_xor_add(alloctree,
	msp->ms_unflushed_frees, msp->ms_unflushed_allocs);
	range_tree_remove_xor_add(msp->ms_freeing,
	msp->ms_unflushed_allocs, msp->ms_unflushed_frees);
	spa->spa_unflushed_stats.sus_memused +=
	metaslab_unflushed_changes_memused(msp);
	} else {
	ASSERT(!spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP));

	space_map_write(msp->ms_sm, alloctree, SM_ALLOC,
	SM_NO_VDEVID, tx);
	space_map_write(msp->ms_sm, msp->ms_freeing, SM_FREE,
	SM_NO_VDEVID, tx);
	mutex_enter(&msp->ms_lock);
	}

	msp->ms_allocated_space += range_tree_space(alloctree);
	ASSERT3U(msp->ms_allocated_space, >=,
	range_tree_space(msp->ms_freeing));
	msp->ms_allocated_space -= range_tree_space(msp->ms_freeing);

	if (!range_tree_is_empty(msp->ms_checkpointing)) {
	ASSERT(spa_has_checkpoint(spa));
	ASSERT3P(vd->vdev_checkpoint_sm, !=, NULL);

	/*
	* Since we are doing writes to disk and the ms_checkpointing
	* tree won't be changing during that time, we drop the
	* ms_lock while writing to the checkpoint space map, for the
	* same reason mentioned above.
	*/
	mutex_exit(&msp->ms_lock);
	space_map_write(vd->vdev_checkpoint_sm,
	msp->ms_checkpointing, SM_FREE, SM_NO_VDEVID, tx);
	mutex_enter(&msp->ms_lock);

	spa->spa_checkpoint_info.sci_dspace +=
	range_tree_space(msp->ms_checkpointing);
	vd->vdev_stat.vs_checkpoint_space +=
	range_tree_space(msp->ms_checkpointing);
	ASSERT3U(vd->vdev_stat.vs_checkpoint_space, ==,
	-space_map_allocated(vd->vdev_checkpoint_sm));

	range_tree_vacate(msp->ms_checkpointing, NULL, NULL);
	}

	if (msp->ms_loaded) {
	/*
	* When the space map is loaded, we have an accurate
	* histogram in the range tree. This gives us an opportunity
	* to bring the space map's histogram up-to-date so we clear
	* it first before updating it.
	*/
	space_map_histogram_clear(msp->ms_sm);
	space_map_histogram_add(msp->ms_sm, msp->ms_allocatable, tx);

	/*
	* Since we've cleared the histogram we need to add back
	* any free space that has already been processed, plus
	* any deferred space. This allows the on-disk histogram
	* to accurately reflect all free space even if some space
	* is not yet available for allocation (i.e. deferred).
	*/
	space_map_histogram_add(msp->ms_sm, msp->ms_freed, tx);

	/*
	* Add back any deferred free space that has not been
	* added back into the in-core free tree yet. This will
	* ensure that we don't end up with a space map histogram
	* that is completely empty unless the metaslab is fully
	* allocated.
	*/
	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
	space_map_histogram_add(msp->ms_sm,
	msp->ms_defer[t], tx);
	}
	}

	/*
	* Always add the free space from this sync pass to the space
	* map histogram. We want to make sure that the on-disk histogram
	* accounts for all free space. If the space map is not loaded,
	* then we will lose some accuracy but will correct it the next
	* time we load the space map.
	*/
	space_map_histogram_add(msp->ms_sm, msp->ms_freeing, tx);
	metaslab_aux_histograms_update(msp);

	metaslab_group_histogram_add(mg, msp);
	metaslab_group_histogram_verify(mg);
	metaslab_class_histogram_verify(mg->mg_class);

	/*
	* For sync pass 1, we avoid traversing this txg's free range tree
	* and instead will just swap the pointers for freeing and freed.
	* We can safely do this since the freed_tree is guaranteed to be
	* empty on the initial pass.
	*
	* Keep in mind that even if we are currently using a log spacemap
	* we want current frees to end up in the ms_allocatable (but not
	* get appended to the ms_sm) so their ranges can be reused as usual.
	*/
	if (spa_sync_pass(spa) == 1) {
	range_tree_swap(&msp->ms_freeing, &msp->ms_freed);
	ASSERT0(msp->ms_allocated_this_txg);
	} else {
	range_tree_vacate(msp->ms_freeing,
	range_tree_add, msp->ms_freed);
	}
	msp->ms_allocated_this_txg += range_tree_space(alloctree);
	range_tree_vacate(alloctree, NULL, NULL);

	ASSERT0(range_tree_space(msp->ms_allocating[txg & TXG_MASK]));
	ASSERT0(range_tree_space(msp->ms_allocating[TXG_CLEAN(txg)
	& TXG_MASK]));
	ASSERT0(range_tree_space(msp->ms_freeing));
	ASSERT0(range_tree_space(msp->ms_checkpointing));

	mutex_exit(&msp->ms_lock);

	/*
	* Verify that the space map object ID has been recorded in the
	* vdev_ms_array.
	*/
	uint64_t object;
	VERIFY0(dmu_read(mos, vd->vdev_ms_array,
	msp->ms_id * sizeof (uint64_t), sizeof (uint64_t), &object, 0));
	VERIFY3U(object, ==, space_map_object(msp->ms_sm));

	mutex_exit(&msp->ms_sync_lock);
	dmu_tx_commit(tx);
	}

	static void
	metaslab_evict(metaslab_t *msp, uint64_t txg)
	{
	if (!msp->ms_loaded \|\| msp->ms_disabled != 0)
	return;

	for (int t = 1; t < TXG_CONCURRENT_STATES; t++) {
	VERIFY0(range_tree_space(
	msp->ms_allocating[(txg + t) & TXG_MASK]));
	}
	if (msp->ms_allocator != -1)
	metaslab_passivate(msp, msp->ms_weight & ~METASLAB_ACTIVE_MASK);

	if (!metaslab_debug_unload)
	metaslab_unload(msp);
	}

	/*
	* Called after a transaction group has completely synced to mark
	* all of the metaslab's free space as usable.
	*/
	void
	metaslab_sync_done(metaslab_t *msp, uint64_t txg)
	{
	metaslab_group_t *mg = msp->ms_group;
	vdev_t *vd = mg->mg_vd;
	spa_t *spa = vd->vdev_spa;
	range_tree_t **defer_tree;
	int64_t alloc_delta, defer_delta;
	boolean_t defer_allowed = B_TRUE;

	ASSERT(!vd->vdev_ishole);

	mutex_enter(&msp->ms_lock);

	if (msp->ms_new) {
	/* this is a new metaslab, add its capacity to the vdev */
	metaslab_space_update(vd, mg->mg_class, 0, 0, msp->ms_size);

	/* there should be no allocations nor frees at this point */
	VERIFY0(msp->ms_allocated_this_txg);
	VERIFY0(range_tree_space(msp->ms_freed));
	}

	ASSERT0(range_tree_space(msp->ms_freeing));
	ASSERT0(range_tree_space(msp->ms_checkpointing));

	defer_tree = &msp->ms_defer[txg % TXG_DEFER_SIZE];

	uint64_t free_space = metaslab_class_get_space(spa_normal_class(spa)) -
	metaslab_class_get_alloc(spa_normal_class(spa));
	if (free_space <= spa_get_slop_space(spa) \|\| vd->vdev_removing) {
	defer_allowed = B_FALSE;
	}

	defer_delta = 0;
	alloc_delta = msp->ms_allocated_this_txg -
	range_tree_space(msp->ms_freed);

	if (defer_allowed) {
	defer_delta = range_tree_space(msp->ms_freed) -
	range_tree_space(*defer_tree);
	} else {
	defer_delta -= range_tree_space(*defer_tree);
	}
	metaslab_space_update(vd, mg->mg_class, alloc_delta + defer_delta,
	defer_delta, 0);

	if (spa_syncing_log_sm(spa) == NULL) {
	/*
	* If there's a metaslab_load() in progress and we don't have
	* a log space map, it means that we probably wrote to the
	* metaslab's space map. If this is the case, we need to
	* make sure that we wait for the load to complete so that we
	* have a consistent view at the in-core side of the metaslab.
	*/
	metaslab_load_wait(msp);
	} else {
	ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP));
	}

	/*
	* When auto-trimming is enabled, free ranges which are added to
	* ms_allocatable are also be added to ms_trim. The ms_trim tree is
	* periodically consumed by the vdev_autotrim_thread() which issues
	* trims for all ranges and then vacates the tree. The ms_trim tree
	* can be discarded at any time with the sole consequence of recent
	* frees not being trimmed.
	*/
	if (spa_get_autotrim(spa) == SPA_AUTOTRIM_ON) {
	range_tree_walk(*defer_tree, range_tree_add, msp->ms_trim);
	if (!defer_allowed) {
	range_tree_walk(msp->ms_freed, range_tree_add,
	msp->ms_trim);
	}
	} else {
	range_tree_vacate(msp->ms_trim, NULL, NULL);
	}

	/*
	* Move the frees from the defer_tree back to the free
	* range tree (if it's loaded). Swap the freed_tree and
	* the defer_tree -- this is safe to do because we've
	* just emptied out the defer_tree.
	*/
	range_tree_vacate(*defer_tree,
	msp->ms_loaded ? range_tree_add : NULL, msp->ms_allocatable);
	if (defer_allowed) {
	range_tree_swap(&msp->ms_freed, defer_tree);
	} else {
	range_tree_vacate(msp->ms_freed,
	msp->ms_loaded ? range_tree_add : NULL,
	msp->ms_allocatable);
	}

	msp->ms_synced_length = space_map_length(msp->ms_sm);

	msp->ms_deferspace += defer_delta;
	ASSERT3S(msp->ms_deferspace, >=, 0);
	ASSERT3S(msp->ms_deferspace, <=, msp->ms_size);
	if (msp->ms_deferspace != 0) {
	/*
	* Keep syncing this metaslab until all deferred frees
	* are back in circulation.
	*/
	vdev_dirty(vd, VDD_METASLAB, msp, txg + 1);
	}
	metaslab_aux_histograms_update_done(msp, defer_allowed);

	if (msp->ms_new) {
	msp->ms_new = B_FALSE;
	mutex_enter(&mg->mg_lock);
	mg->mg_ms_ready++;
	mutex_exit(&mg->mg_lock);
	}

	/*
	* Re-sort metaslab within its group now that we've adjusted
	* its allocatable space.
	*/
	metaslab_recalculate_weight_and_sort(msp);

	ASSERT0(range_tree_space(msp->ms_allocating[txg & TXG_MASK]));
	ASSERT0(range_tree_space(msp->ms_freeing));
	ASSERT0(range_tree_space(msp->ms_freed));
	ASSERT0(range_tree_space(msp->ms_checkpointing));
	msp->ms_allocating_total -= msp->ms_allocated_this_txg;
	msp->ms_allocated_this_txg = 0;
	mutex_exit(&msp->ms_lock);
	}

	void
	metaslab_sync_reassess(metaslab_group_t *mg)
	{
	spa_t *spa = mg->mg_class->mc_spa;

	spa_config_enter(spa, SCL_ALLOC, FTAG, RW_READER);
	metaslab_group_alloc_update(mg);
	mg->mg_fragmentation = metaslab_group_fragmentation(mg);

	/*
	* Preload the next potential metaslabs but only on active
	* metaslab groups. We can get into a state where the metaslab
	* is no longer active since we dirty metaslabs as we remove a
	* a device, thus potentially making the metaslab group eligible
	* for preloading.
	*/
	if (mg->mg_activation_count > 0) {
	metaslab_group_preload(mg);
	}
	spa_config_exit(spa, SCL_ALLOC, FTAG);
	}

	/*
	* When writing a ditto block (i.e. more than one DVA for a given BP) on
	* the same vdev as an existing DVA of this BP, then try to allocate it
	* on a different metaslab than existing DVAs (i.e. a unique metaslab).
	*/
	static boolean_t
	metaslab_is_unique(metaslab_t msp, dva_t dva)
	{
	uint64_t dva_ms_id;

	if (DVA_GET_ASIZE(dva) == 0)
	return (B_TRUE);

	if (msp->ms_group->mg_vd->vdev_id != DVA_GET_VDEV(dva))
	return (B_TRUE);

	dva_ms_id = DVA_GET_OFFSET(dva) >> msp->ms_group->mg_vd->vdev_ms_shift;

	return (msp->ms_id != dva_ms_id);
	}

	/*
	* ==========================================================================
	* Metaslab allocation tracing facility
	* ==========================================================================
	*/

	/*
	* Add an allocation trace element to the allocation tracing list.
	*/
	static void
	metaslab_trace_add(zio_alloc_list_t zal, metaslab_group_t mg,
	metaslab_t *msp, uint64_t psize, uint32_t dva_id, uint64_t offset,
	int allocator)
	{
	metaslab_alloc_trace_t *mat;

	if (!metaslab_trace_enabled)
	return;

	/*
	* When the tracing list reaches its maximum we remove
	* the second element in the list before adding a new one.
	* By removing the second element we preserve the original
	* entry as a clue to what allocations steps have already been
	* performed.
	*/
	if (zal->zal_size == metaslab_trace_max_entries) {
	metaslab_alloc_trace_t *mat_next;
	#ifdef ZFS_DEBUG
	panic("too many entries in allocation list");
	#endif
	METASLABSTAT_BUMP(metaslabstat_trace_over_limit);
	zal->zal_size--;
	mat_next = list_next(&zal->zal_list, list_head(&zal->zal_list));
	list_remove(&zal->zal_list, mat_next);
	kmem_cache_free(metaslab_alloc_trace_cache, mat_next);
	}

	mat = kmem_cache_alloc(metaslab_alloc_trace_cache, KM_SLEEP);
	list_link_init(&mat->mat_list_node);
	mat->mat_mg = mg;
	mat->mat_msp = msp;
	mat->mat_size = psize;
	mat->mat_dva_id = dva_id;
	mat->mat_offset = offset;
	mat->mat_weight = 0;
	mat->mat_allocator = allocator;

	if (msp != NULL)
	mat->mat_weight = msp->ms_weight;

	/*
	* The list is part of the zio so locking is not required. Only
	* a single thread will perform allocations for a given zio.
	*/
	list_insert_tail(&zal->zal_list, mat);
	zal->zal_size++;

	ASSERT3U(zal->zal_size, <=, metaslab_trace_max_entries);
	}

	void
	metaslab_trace_init(zio_alloc_list_t *zal)
	{
	list_create(&zal->zal_list, sizeof (metaslab_alloc_trace_t),
	offsetof(metaslab_alloc_trace_t, mat_list_node));
	zal->zal_size = 0;
	}

	void
	metaslab_trace_fini(zio_alloc_list_t *zal)
	{
	metaslab_alloc_trace_t *mat;

	while ((mat = list_remove_head(&zal->zal_list)) != NULL)
	kmem_cache_free(metaslab_alloc_trace_cache, mat);
	list_destroy(&zal->zal_list);
	zal->zal_size = 0;
	}

	/*
	* ==========================================================================
	* Metaslab block operations
	* ==========================================================================
	*/

	static void
	metaslab_group_alloc_increment(spa_t spa, uint64_t vdev, const void tag,
	int flags, int allocator)
	{
	if (!(flags & METASLAB_ASYNC_ALLOC) \|\|
	(flags & METASLAB_DONT_THROTTLE))
	return;

	metaslab_group_t *mg = vdev_lookup_top(spa, vdev)->vdev_mg;
	if (!mg->mg_class->mc_alloc_throttle_enabled)
	return;

	metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator];
	(void) zfs_refcount_add(&mga->mga_alloc_queue_depth, tag);
	}

	static void
	metaslab_group_increment_qdepth(metaslab_group_t *mg, int allocator)
	{
	metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator];
	metaslab_class_allocator_t *mca =
	&mg->mg_class->mc_allocator[allocator];
	uint64_t max = mg->mg_max_alloc_queue_depth;
	uint64_t cur = mga->mga_cur_max_alloc_queue_depth;
	while (cur < max) {
	if (atomic_cas_64(&mga->mga_cur_max_alloc_queue_depth,
	cur, cur + 1) == cur) {
	atomic_inc_64(&mca->mca_alloc_max_slots);
	return;
	}
	cur = mga->mga_cur_max_alloc_queue_depth;
	}
	}

	void
	metaslab_group_alloc_decrement(spa_t spa, uint64_t vdev, const void tag,
	int flags, int allocator, boolean_t io_complete)
	{
	if (!(flags & METASLAB_ASYNC_ALLOC) \|\|
	(flags & METASLAB_DONT_THROTTLE))
	return;

	metaslab_group_t *mg = vdev_lookup_top(spa, vdev)->vdev_mg;
	if (!mg->mg_class->mc_alloc_throttle_enabled)
	return;

	metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator];
	(void) zfs_refcount_remove(&mga->mga_alloc_queue_depth, tag);
	if (io_complete)
	metaslab_group_increment_qdepth(mg, allocator);
	}

	void
	metaslab_group_alloc_verify(spa_t spa, const blkptr_t bp, const void *tag,
	int allocator)
	{
	#ifdef ZFS_DEBUG
	const dva_t *dva = bp->blk_dva;
	int ndvas = BP_GET_NDVAS(bp);

	for (int d = 0; d < ndvas; d++) {
	uint64_t vdev = DVA_GET_VDEV(&dva[d]);
	metaslab_group_t *mg = vdev_lookup_top(spa, vdev)->vdev_mg;
	metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator];
	VERIFY(zfs_refcount_not_held(&mga->mga_alloc_queue_depth, tag));
	}
	#endif
	}

	static uint64_t
	metaslab_block_alloc(metaslab_t *msp, uint64_t size, uint64_t txg)
	{
	uint64_t start;
	range_tree_t *rt = msp->ms_allocatable;
	metaslab_class_t *mc = msp->ms_group->mg_class;

	ASSERT(MUTEX_HELD(&msp->ms_lock));
	VERIFY(!msp->ms_condensing);
	VERIFY0(msp->ms_disabled);

	start = mc->mc_ops->msop_alloc(msp, size);
	if (start != -1ULL) {
	metaslab_group_t *mg = msp->ms_group;
	vdev_t *vd = mg->mg_vd;

	VERIFY0(P2PHASE(start, 1ULL << vd->vdev_ashift));
	VERIFY0(P2PHASE(size, 1ULL << vd->vdev_ashift));
	VERIFY3U(range_tree_space(rt) - size, <=, msp->ms_size);
	range_tree_remove(rt, start, size);
	range_tree_clear(msp->ms_trim, start, size);

	if (range_tree_is_empty(msp->ms_allocating[txg & TXG_MASK]))
	vdev_dirty(mg->mg_vd, VDD_METASLAB, msp, txg);

	range_tree_add(msp->ms_allocating[txg & TXG_MASK], start, size);
	msp->ms_allocating_total += size;

	/* Track the last successful allocation */
	msp->ms_alloc_txg = txg;
	metaslab_verify_space(msp, txg);
	}

	/*
	* Now that we've attempted the allocation we need to update the
	* metaslab's maximum block size since it may have changed.
	*/
	msp->ms_max_size = metaslab_largest_allocatable(msp);
	return (start);
	}

	/*
	* Find the metaslab with the highest weight that is less than what we've
	* already tried. In the common case, this means that we will examine each
	* metaslab at most once. Note that concurrent callers could reorder metaslabs
	* by activation/passivation once we have dropped the mg_lock. If a metaslab is
	* activated by another thread, and we fail to allocate from the metaslab we
	* have selected, we may not try the newly-activated metaslab, and instead
	* activate another metaslab. This is not optimal, but generally does not cause
	* any problems (a possible exception being if every metaslab is completely full
	* except for the newly-activated metaslab which we fail to examine).
	*/
	static metaslab_t *
	find_valid_metaslab(metaslab_group_t *mg, uint64_t activation_weight,
	dva_t *dva, int d, boolean_t want_unique, uint64_t asize, int allocator,
	boolean_t try_hard, zio_alloc_list_t zal, metaslab_t search,
	boolean_t *was_active)
	{
	avl_index_t idx;
	avl_tree_t *t = &mg->mg_metaslab_tree;
	metaslab_t *msp = avl_find(t, search, &idx);
	if (msp == NULL)
	msp = avl_nearest(t, idx, AVL_AFTER);

	uint_t tries = 0;
	for (; msp != NULL; msp = AVL_NEXT(t, msp)) {
	int i;

	if (!try_hard && tries > zfs_metaslab_find_max_tries) {
	METASLABSTAT_BUMP(metaslabstat_too_many_tries);
	return (NULL);
	}
	tries++;

	if (!metaslab_should_allocate(msp, asize, try_hard)) {
	metaslab_trace_add(zal, mg, msp, asize, d,
	TRACE_TOO_SMALL, allocator);
	continue;
	}

	/*
	* If the selected metaslab is condensing or disabled,
	* skip it.
	*/
	if (msp->ms_condensing \|\| msp->ms_disabled > 0)
	continue;

	*was_active = msp->ms_allocator != -1;
	/*
	* If we're activating as primary, this is our first allocation
	* from this disk, so we don't need to check how close we are.
	* If the metaslab under consideration was already active,
	* we're getting desperate enough to steal another allocator's
	* metaslab, so we still don't care about distances.
	*/
	if (activation_weight == METASLAB_WEIGHT_PRIMARY \|\| *was_active)
	break;

	for (i = 0; i < d; i++) {
	if (want_unique &&
	!metaslab_is_unique(msp, &dva[i]))
	break; /* try another metaslab */
	}
	if (i == d)
	break;
	}

	if (msp != NULL) {
	search->ms_weight = msp->ms_weight;
	search->ms_start = msp->ms_start + 1;
	search->ms_allocator = msp->ms_allocator;
	search->ms_primary = msp->ms_primary;
	}
	return (msp);
	}

	static void
	metaslab_active_mask_verify(metaslab_t *msp)
	{
	ASSERT(MUTEX_HELD(&msp->ms_lock));

	if ((zfs_flags & ZFS_DEBUG_METASLAB_VERIFY) == 0)
	return;

	if ((msp->ms_weight & METASLAB_ACTIVE_MASK) == 0)
	return;

	if (msp->ms_weight & METASLAB_WEIGHT_PRIMARY) {
	VERIFY0(msp->ms_weight & METASLAB_WEIGHT_SECONDARY);
	VERIFY0(msp->ms_weight & METASLAB_WEIGHT_CLAIM);
	VERIFY3S(msp->ms_allocator, !=, -1);
	VERIFY(msp->ms_primary);
	return;
	}

	if (msp->ms_weight & METASLAB_WEIGHT_SECONDARY) {
	VERIFY0(msp->ms_weight & METASLAB_WEIGHT_PRIMARY);
	VERIFY0(msp->ms_weight & METASLAB_WEIGHT_CLAIM);
	VERIFY3S(msp->ms_allocator, !=, -1);
	VERIFY(!msp->ms_primary);
	return;
	}

	if (msp->ms_weight & METASLAB_WEIGHT_CLAIM) {
	VERIFY0(msp->ms_weight & METASLAB_WEIGHT_PRIMARY);
	VERIFY0(msp->ms_weight & METASLAB_WEIGHT_SECONDARY);
	VERIFY3S(msp->ms_allocator, ==, -1);
	return;
	}
	}

	static uint64_t
	metaslab_group_alloc_normal(metaslab_group_t mg, zio_alloc_list_t zal,
	uint64_t asize, uint64_t txg, boolean_t want_unique, dva_t *dva, int d,
	int allocator, boolean_t try_hard)
	{
	metaslab_t *msp = NULL;
	uint64_t offset = -1ULL;

	uint64_t activation_weight = METASLAB_WEIGHT_PRIMARY;
	for (int i = 0; i < d; i++) {
	if (activation_weight == METASLAB_WEIGHT_PRIMARY &&
	DVA_GET_VDEV(&dva[i]) == mg->mg_vd->vdev_id) {
	activation_weight = METASLAB_WEIGHT_SECONDARY;
	} else if (activation_weight == METASLAB_WEIGHT_SECONDARY &&
	DVA_GET_VDEV(&dva[i]) == mg->mg_vd->vdev_id) {
	activation_weight = METASLAB_WEIGHT_CLAIM;
	break;
	}
	}

	/*
	* If we don't have enough metaslabs active to fill the entire array, we
	* just use the 0th slot.
	*/
	if (mg->mg_ms_ready < mg->mg_allocators * 3)
	allocator = 0;
	metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator];

	ASSERT3U(mg->mg_vd->vdev_ms_count, >=, 2);

	metaslab_t search = kmem_alloc(sizeof (search), KM_SLEEP);
	search->ms_weight = UINT64_MAX;
	search->ms_start = 0;
	/*
	* At the end of the metaslab tree are the already-active metaslabs,
	* first the primaries, then the secondaries. When we resume searching
	* through the tree, we need to consider ms_allocator and ms_primary so
	* we start in the location right after where we left off, and don't
	* accidentally loop forever considering the same metaslabs.
	*/
	search->ms_allocator = -1;
	search->ms_primary = B_TRUE;
	for (;;) {
	boolean_t was_active = B_FALSE;

	mutex_enter(&mg->mg_lock);

	if (activation_weight == METASLAB_WEIGHT_PRIMARY &&
	mga->mga_primary != NULL) {
	msp = mga->mga_primary;

	/*
	* Even though we don't hold the ms_lock for the
	* primary metaslab, those fields should not
	* change while we hold the mg_lock. Thus it is
	* safe to make assertions on them.
	*/
	ASSERT(msp->ms_primary);
	ASSERT3S(msp->ms_allocator, ==, allocator);
	ASSERT(msp->ms_loaded);

	was_active = B_TRUE;
	ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK);
	} else if (activation_weight == METASLAB_WEIGHT_SECONDARY &&
	mga->mga_secondary != NULL) {
	msp = mga->mga_secondary;

	/*
	* See comment above about the similar assertions
	* for the primary metaslab.
	*/
	ASSERT(!msp->ms_primary);
	ASSERT3S(msp->ms_allocator, ==, allocator);
	ASSERT(msp->ms_loaded);

	was_active = B_TRUE;
	ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK);
	} else {
	msp = find_valid_metaslab(mg, activation_weight, dva, d,
	want_unique, asize, allocator, try_hard, zal,
	search, &was_active);
	}

	mutex_exit(&mg->mg_lock);
	if (msp == NULL) {
	kmem_free(search, sizeof (*search));
	return (-1ULL);
	}
	mutex_enter(&msp->ms_lock);

	metaslab_active_mask_verify(msp);

	/*
	* This code is disabled out because of issues with
	* tracepoints in non-gpl kernel modules.
	*/
	#if 0
	DTRACE_PROBE3(ms__activation__attempt,
	metaslab_t *, msp, uint64_t, activation_weight,
	boolean_t, was_active);
	#endif

	/*
	* Ensure that the metaslab we have selected is still
	* capable of handling our request. It's possible that
	* another thread may have changed the weight while we
	* were blocked on the metaslab lock. We check the
	* active status first to see if we need to set_selected_txg
	* a new metaslab.
	*/
	if (was_active && !(msp->ms_weight & METASLAB_ACTIVE_MASK)) {
	ASSERT3S(msp->ms_allocator, ==, -1);
	mutex_exit(&msp->ms_lock);
	continue;
	}

	/*
	* If the metaslab was activated for another allocator
	* while we were waiting in the ms_lock above, or it's
	* a primary and we're seeking a secondary (or vice versa),
	* we go back and select a new metaslab.
	*/
	if (!was_active && (msp->ms_weight & METASLAB_ACTIVE_MASK) &&
	(msp->ms_allocator != -1) &&
	(msp->ms_allocator != allocator \|\| ((activation_weight ==
	METASLAB_WEIGHT_PRIMARY) != msp->ms_primary))) {
	ASSERT(msp->ms_loaded);
	ASSERT((msp->ms_weight & METASLAB_WEIGHT_CLAIM) \|\|
	msp->ms_allocator != -1);
	mutex_exit(&msp->ms_lock);
	continue;
	}

	/*
	* This metaslab was used for claiming regions allocated
	* by the ZIL during pool import. Once these regions are
	* claimed we don't need to keep the CLAIM bit set
	* anymore. Passivate this metaslab to zero its activation
	* mask.
	*/
	if (msp->ms_weight & METASLAB_WEIGHT_CLAIM &&
	activation_weight != METASLAB_WEIGHT_CLAIM) {
	ASSERT(msp->ms_loaded);
	ASSERT3S(msp->ms_allocator, ==, -1);
	metaslab_passivate(msp, msp->ms_weight &
	~METASLAB_WEIGHT_CLAIM);
	mutex_exit(&msp->ms_lock);
	continue;
	}

	metaslab_set_selected_txg(msp, txg);

	int activation_error =
	metaslab_activate(msp, allocator, activation_weight);
	metaslab_active_mask_verify(msp);

	/*
	* If the metaslab was activated by another thread for
	* another allocator or activation_weight (EBUSY), or it
	* failed because another metaslab was assigned as primary
	* for this allocator (EEXIST) we continue using this
	* metaslab for our allocation, rather than going on to a
	* worse metaslab (we waited for that metaslab to be loaded
	* after all).
	*
	* If the activation failed due to an I/O error or ENOSPC we
	* skip to the next metaslab.
	*/
	boolean_t activated;
	if (activation_error == 0) {
	activated = B_TRUE;
	} else if (activation_error == EBUSY \|\|
	activation_error == EEXIST) {
	activated = B_FALSE;
	} else {
	mutex_exit(&msp->ms_lock);
	continue;
	}
	ASSERT(msp->ms_loaded);

	/*
	* Now that we have the lock, recheck to see if we should
	* continue to use this metaslab for this allocation. The
	* the metaslab is now loaded so metaslab_should_allocate()
	* can accurately determine if the allocation attempt should
	* proceed.
	*/
	if (!metaslab_should_allocate(msp, asize, try_hard)) {
	/* Passivate this metaslab and select a new one. */
	metaslab_trace_add(zal, mg, msp, asize, d,
	TRACE_TOO_SMALL, allocator);
	goto next;
	}

	/*
	* If this metaslab is currently condensing then pick again
	* as we can't manipulate this metaslab until it's committed
	* to disk. If this metaslab is being initialized, we shouldn't
	* allocate from it since the allocated region might be
	* overwritten after allocation.
	*/
	if (msp->ms_condensing) {
	metaslab_trace_add(zal, mg, msp, asize, d,
	TRACE_CONDENSING, allocator);
	if (activated) {
	metaslab_passivate(msp, msp->ms_weight &
	~METASLAB_ACTIVE_MASK);
	}
	mutex_exit(&msp->ms_lock);
	continue;
	} else if (msp->ms_disabled > 0) {
	metaslab_trace_add(zal, mg, msp, asize, d,
	TRACE_DISABLED, allocator);
	if (activated) {
	metaslab_passivate(msp, msp->ms_weight &
	~METASLAB_ACTIVE_MASK);
	}
	mutex_exit(&msp->ms_lock);
	continue;
	}

	offset = metaslab_block_alloc(msp, asize, txg);
	metaslab_trace_add(zal, mg, msp, asize, d, offset, allocator);

	if (offset != -1ULL) {
	/* Proactively passivate the metaslab, if needed */
	if (activated)
	metaslab_segment_may_passivate(msp);
	break;
	}
	next:
	ASSERT(msp->ms_loaded);

	/*
	* This code is disabled out because of issues with
	* tracepoints in non-gpl kernel modules.
	*/
	#if 0
	DTRACE_PROBE2(ms__alloc__failure, metaslab_t *, msp,
	uint64_t, asize);
	#endif

	/*
	* We were unable to allocate from this metaslab so determine
	* a new weight for this metaslab. Now that we have loaded
	* the metaslab we can provide a better hint to the metaslab
	* selector.
	*
	* For space-based metaslabs, we use the maximum block size.
	* This information is only available when the metaslab
	* is loaded and is more accurate than the generic free
	* space weight that was calculated by metaslab_weight().
	* This information allows us to quickly compare the maximum
	* available allocation in the metaslab to the allocation
	* size being requested.
	*
	* For segment-based metaslabs, determine the new weight
	* based on the highest bucket in the range tree. We
	* explicitly use the loaded segment weight (i.e. the range
	* tree histogram) since it contains the space that is
	* currently available for allocation and is accurate
	* even within a sync pass.
	*/
	uint64_t weight;
	if (WEIGHT_IS_SPACEBASED(msp->ms_weight)) {
	weight = metaslab_largest_allocatable(msp);
	WEIGHT_SET_SPACEBASED(weight);
	} else {
	weight = metaslab_weight_from_range_tree(msp);
	}

	if (activated) {
	metaslab_passivate(msp, weight);
	} else {
	/*
	* For the case where we use the metaslab that is
	* active for another allocator we want to make
	* sure that we retain the activation mask.
	*
	* Note that we could attempt to use something like
	* metaslab_recalculate_weight_and_sort() that
	* retains the activation mask here. That function
	* uses metaslab_weight() to set the weight though
	* which is not as accurate as the calculations
	* above.
	*/
	weight \|= msp->ms_weight & METASLAB_ACTIVE_MASK;
	metaslab_group_sort(mg, msp, weight);
	}
	metaslab_active_mask_verify(msp);

	/*
	* We have just failed an allocation attempt, check
	* that metaslab_should_allocate() agrees. Otherwise,
	* we may end up in an infinite loop retrying the same
	* metaslab.
	*/
	ASSERT(!metaslab_should_allocate(msp, asize, try_hard));

	mutex_exit(&msp->ms_lock);
	}
	mutex_exit(&msp->ms_lock);
	kmem_free(search, sizeof (*search));
	return (offset);
	}

	static uint64_t
	metaslab_group_alloc(metaslab_group_t mg, zio_alloc_list_t zal,
	uint64_t asize, uint64_t txg, boolean_t want_unique, dva_t *dva, int d,
	int allocator, boolean_t try_hard)
	{
	uint64_t offset;

	offset = metaslab_group_alloc_normal(mg, zal, asize, txg, want_unique,
	dva, d, allocator, try_hard);

	mutex_enter(&mg->mg_lock);
	if (offset == -1ULL) {
	mg->mg_failed_allocations++;
	metaslab_trace_add(zal, mg, NULL, asize, d,
	TRACE_GROUP_FAILURE, allocator);
	if (asize == SPA_GANGBLOCKSIZE) {
	/*
	* This metaslab group was unable to allocate
	* the minimum gang block size so it must be out of
	* space. We must notify the allocation throttle
	* to start skipping allocation attempts to this
	* metaslab group until more space becomes available.
	* Note: this failure cannot be caused by the
	* allocation throttle since the allocation throttle
	* is only responsible for skipping devices and
	* not failing block allocations.
	*/
	mg->mg_no_free_space = B_TRUE;
	}
	}
	mg->mg_allocations++;
	mutex_exit(&mg->mg_lock);
	return (offset);
	}

	/*
	* Allocate a block for the specified i/o.
	*/
	int
	metaslab_alloc_dva(spa_t spa, metaslab_class_t mc, uint64_t psize,
	dva_t dva, int d, dva_t hintdva, uint64_t txg, int flags,
	zio_alloc_list_t *zal, int allocator)
	{
	metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator];
	metaslab_group_t mg, rotor;
	vdev_t *vd;
	boolean_t try_hard = B_FALSE;

	ASSERT(!DVA_IS_VALID(&dva[d]));

	/*
	* For testing, make some blocks above a certain size be gang blocks.
	* This will result in more split blocks when using device removal,
	* and a large number of split blocks coupled with ztest-induced
	* damage can result in extremely long reconstruction times. This
	* will also test spilling from special to normal.
	*/
	if (psize >= metaslab_force_ganging &&
	metaslab_force_ganging_pct > 0 &&
	(random_in_range(100) < MIN(metaslab_force_ganging_pct, 100))) {
	metaslab_trace_add(zal, NULL, NULL, psize, d, TRACE_FORCE_GANG,
	allocator);
	return (SET_ERROR(ENOSPC));
	}

	/*
	* Start at the rotor and loop through all mgs until we find something.
	* Note that there's no locking on mca_rotor or mca_aliquot because
	* nothing actually breaks if we miss a few updates -- we just won't
	* allocate quite as evenly. It all balances out over time.
	*
	* If we are doing ditto or log blocks, try to spread them across
	* consecutive vdevs. If we're forced to reuse a vdev before we've
	* allocated all of our ditto blocks, then try and spread them out on
	* that vdev as much as possible. If it turns out to not be possible,
	* gradually lower our standards until anything becomes acceptable.
	* Also, allocating on consecutive vdevs (as opposed to random vdevs)
	* gives us hope of containing our fault domains to something we're
	* able to reason about. Otherwise, any two top-level vdev failures
	* will guarantee the loss of data. With consecutive allocation,
	* only two adjacent top-level vdev failures will result in data loss.
	*
	* If we are doing gang blocks (hintdva is non-NULL), try to keep
	* ourselves on the same vdev as our gang block header. That
	* way, we can hope for locality in vdev_cache, plus it makes our
	* fault domains something tractable.
	*/
	if (hintdva) {
	vd = vdev_lookup_top(spa, DVA_GET_VDEV(&hintdva[d]));

	/*
	* It's possible the vdev we're using as the hint no
	* longer exists or its mg has been closed (e.g. by
	* device removal). Consult the rotor when
	* all else fails.
	*/
	if (vd != NULL && vd->vdev_mg != NULL) {
	mg = vdev_get_mg(vd, mc);

	if (flags & METASLAB_HINTBP_AVOID)
	mg = mg->mg_next;
	} else {
	mg = mca->mca_rotor;
	}
	} else if (d != 0) {
	vd = vdev_lookup_top(spa, DVA_GET_VDEV(&dva[d - 1]));
	mg = vd->vdev_mg->mg_next;
	} else {
	ASSERT(mca->mca_rotor != NULL);
	mg = mca->mca_rotor;
	}

	/*
	* If the hint put us into the wrong metaslab class, or into a
	* metaslab group that has been passivated, just follow the rotor.
	*/
	if (mg->mg_class != mc \|\| mg->mg_activation_count <= 0)
	mg = mca->mca_rotor;

	rotor = mg;
	top:
	do {
	boolean_t allocatable;

	ASSERT(mg->mg_activation_count == 1);
	vd = mg->mg_vd;

	/*
	* Don't allocate from faulted devices.
	*/
	if (try_hard) {
	spa_config_enter(spa, SCL_ZIO, FTAG, RW_READER);
	allocatable = vdev_allocatable(vd);
	spa_config_exit(spa, SCL_ZIO, FTAG);
	} else {
	allocatable = vdev_allocatable(vd);
	}

	/*
	* Determine if the selected metaslab group is eligible
	* for allocations. If we're ganging then don't allow
	* this metaslab group to skip allocations since that would
	* inadvertently return ENOSPC and suspend the pool
	* even though space is still available.
	*/
	if (allocatable && !GANG_ALLOCATION(flags) && !try_hard) {
	allocatable = metaslab_group_allocatable(mg, rotor,
	flags, psize, allocator, d);
	}

	if (!allocatable) {
	metaslab_trace_add(zal, mg, NULL, psize, d,
	TRACE_NOT_ALLOCATABLE, allocator);
	goto next;
	}

	/*
	* Avoid writing single-copy data to an unhealthy,
	* non-redundant vdev, unless we've already tried all
	* other vdevs.
	*/
	if (vd->vdev_state < VDEV_STATE_HEALTHY &&
	d == 0 && !try_hard && vd->vdev_children == 0) {
	metaslab_trace_add(zal, mg, NULL, psize, d,
	TRACE_VDEV_ERROR, allocator);
	goto next;
	}

	ASSERT(mg->mg_class == mc);

	uint64_t asize = vdev_psize_to_asize(vd, psize);
	ASSERT(P2PHASE(asize, 1ULL << vd->vdev_ashift) == 0);

	/*
	* If we don't need to try hard, then require that the
	* block be on a different metaslab from any other DVAs
	* in this BP (unique=true). If we are trying hard, then
	* allow any metaslab to be used (unique=false).
	*/
	uint64_t offset = metaslab_group_alloc(mg, zal, asize, txg,
	!try_hard, dva, d, allocator, try_hard);

	if (offset != -1ULL) {
	/*
	* If we've just selected this metaslab group,
	* figure out whether the corresponding vdev is
	* over- or under-used relative to the pool,
	* and set an allocation bias to even it out.
	*
	* Bias is also used to compensate for unequally
	* sized vdevs so that space is allocated fairly.
	*/
	if (mca->mca_aliquot == 0 && metaslab_bias_enabled) {
	vdev_stat_t *vs = &vd->vdev_stat;
	int64_t vs_free = vs->vs_space - vs->vs_alloc;
	int64_t mc_free = mc->mc_space - mc->mc_alloc;
	int64_t ratio;

	/*
	* Calculate how much more or less we should
	* try to allocate from this device during
	* this iteration around the rotor.
	*
	* This basically introduces a zero-centered
	* bias towards the devices with the most
	* free space, while compensating for vdev
	* size differences.
	*
	* Examples:
	* vdev V1 = 16M/128M
	* vdev V2 = 16M/128M
	* ratio(V1) = 100% ratio(V2) = 100%
	*
	* vdev V1 = 16M/128M
	* vdev V2 = 64M/128M
	* ratio(V1) = 127% ratio(V2) = 72%
	*
	* vdev V1 = 16M/128M
	* vdev V2 = 64M/512M
	* ratio(V1) = 40% ratio(V2) = 160%
	*/
	ratio = (vs_free * mc->mc_alloc_groups * 100) /
	(mc_free + 1);
	mg->mg_bias = ((ratio - 100) *
	(int64_t)mg->mg_aliquot) / 100;
	} else if (!metaslab_bias_enabled) {
	mg->mg_bias = 0;
	}

	if ((flags & METASLAB_ZIL) \|\|
	atomic_add_64_nv(&mca->mca_aliquot, asize) >=
	mg->mg_aliquot + mg->mg_bias) {
	mca->mca_rotor = mg->mg_next;
	mca->mca_aliquot = 0;
	}

	DVA_SET_VDEV(&dva[d], vd->vdev_id);
	DVA_SET_OFFSET(&dva[d], offset);
	DVA_SET_GANG(&dva[d],
	((flags & METASLAB_GANG_HEADER) ? 1 : 0));
	DVA_SET_ASIZE(&dva[d], asize);

	return (0);
	}
	next:
	mca->mca_rotor = mg->mg_next;
	mca->mca_aliquot = 0;
	} while ((mg = mg->mg_next) != rotor);

	/*
	* If we haven't tried hard, perhaps do so now.
	*/
	if (!try_hard && (zfs_metaslab_try_hard_before_gang \|\|
	GANG_ALLOCATION(flags) \|\| (flags & METASLAB_ZIL) != 0 \|\|
	psize <= 1 << spa->spa_min_ashift)) {
	METASLABSTAT_BUMP(metaslabstat_try_hard);
	try_hard = B_TRUE;
	goto top;
	}

	memset(&dva[d], 0, sizeof (dva_t));

	metaslab_trace_add(zal, rotor, NULL, psize, d, TRACE_ENOSPC, allocator);
	return (SET_ERROR(ENOSPC));
	}

	void
	metaslab_free_concrete(vdev_t *vd, uint64_t offset, uint64_t asize,
	boolean_t checkpoint)
	{
	metaslab_t *msp;
	spa_t *spa = vd->vdev_spa;

	ASSERT(vdev_is_concrete(vd));
	ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0);
	ASSERT3U(offset >> vd->vdev_ms_shift, <, vd->vdev_ms_count);

	msp = vd->vdev_ms[offset >> vd->vdev_ms_shift];

	VERIFY(!msp->ms_condensing);
	VERIFY3U(offset, >=, msp->ms_start);
	VERIFY3U(offset + asize, <=, msp->ms_start + msp->ms_size);
	VERIFY0(P2PHASE(offset, 1ULL << vd->vdev_ashift));
	VERIFY0(P2PHASE(asize, 1ULL << vd->vdev_ashift));

	metaslab_check_free_impl(vd, offset, asize);

	mutex_enter(&msp->ms_lock);
	if (range_tree_is_empty(msp->ms_freeing) &&
	range_tree_is_empty(msp->ms_checkpointing)) {
	vdev_dirty(vd, VDD_METASLAB, msp, spa_syncing_txg(spa));
	}

	if (checkpoint) {
	ASSERT(spa_has_checkpoint(spa));
	range_tree_add(msp->ms_checkpointing, offset, asize);
	} else {
	range_tree_add(msp->ms_freeing, offset, asize);
	}
	mutex_exit(&msp->ms_lock);
	}

	void
	metaslab_free_impl_cb(uint64_t inner_offset, vdev_t *vd, uint64_t offset,
	uint64_t size, void *arg)
	{
	(void) inner_offset;
	boolean_t *checkpoint = arg;

	ASSERT3P(checkpoint, !=, NULL);

	if (vd->vdev_ops->vdev_op_remap != NULL)
	vdev_indirect_mark_obsolete(vd, offset, size);
	else
	metaslab_free_impl(vd, offset, size, *checkpoint);
	}

	static void
	metaslab_free_impl(vdev_t *vd, uint64_t offset, uint64_t size,
	boolean_t checkpoint)
	{
	spa_t *spa = vd->vdev_spa;

	ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0);

	if (spa_syncing_txg(spa) > spa_freeze_txg(spa))
	return;

	if (spa->spa_vdev_removal != NULL &&
	spa->spa_vdev_removal->svr_vdev_id == vd->vdev_id &&
	vdev_is_concrete(vd)) {
	/*
	* Note: we check if the vdev is concrete because when
	* we complete the removal, we first change the vdev to be
	* an indirect vdev (in open context), and then (in syncing
	* context) clear spa_vdev_removal.
	*/
	free_from_removing_vdev(vd, offset, size);
	} else if (vd->vdev_ops->vdev_op_remap != NULL) {
	vdev_indirect_mark_obsolete(vd, offset, size);
	vd->vdev_ops->vdev_op_remap(vd, offset, size,
	metaslab_free_impl_cb, &checkpoint);
	} else {
	metaslab_free_concrete(vd, offset, size, checkpoint);
	}
	}

	typedef struct remap_blkptr_cb_arg {
	blkptr_t *rbca_bp;
	spa_remap_cb_t rbca_cb;
	vdev_t *rbca_remap_vd;
	uint64_t rbca_remap_offset;
	void *rbca_cb_arg;
	} remap_blkptr_cb_arg_t;

	static void
	remap_blkptr_cb(uint64_t inner_offset, vdev_t *vd, uint64_t offset,
	uint64_t size, void *arg)
	{
	remap_blkptr_cb_arg_t *rbca = arg;
	blkptr_t *bp = rbca->rbca_bp;

	/* We can not remap split blocks. */
	if (size != DVA_GET_ASIZE(&bp->blk_dva[0]))
	return;
	ASSERT0(inner_offset);

	if (rbca->rbca_cb != NULL) {
	/*
	* At this point we know that we are not handling split
	* blocks and we invoke the callback on the previous
	* vdev which must be indirect.
	*/
	ASSERT3P(rbca->rbca_remap_vd->vdev_ops, ==, &vdev_indirect_ops);

	rbca->rbca_cb(rbca->rbca_remap_vd->vdev_id,
	rbca->rbca_remap_offset, size, rbca->rbca_cb_arg);

	/* set up remap_blkptr_cb_arg for the next call */
	rbca->rbca_remap_vd = vd;
	rbca->rbca_remap_offset = offset;
	}

	/*
	* The phys birth time is that of dva[0]. This ensures that we know
	* when each dva was written, so that resilver can determine which
	* blocks need to be scrubbed (i.e. those written during the time
	* the vdev was offline). It also ensures that the key used in
	* the ARC hash table is unique (i.e. dva[0] + phys_birth). If
	* we didn't change the phys_birth, a lookup in the ARC for a
	* remapped BP could find the data that was previously stored at
	* this vdev + offset.
	*/
	vdev_t *oldvd = vdev_lookup_top(vd->vdev_spa,
	DVA_GET_VDEV(&bp->blk_dva[0]));
	vdev_indirect_births_t *vib = oldvd->vdev_indirect_births;
	bp->blk_phys_birth = vdev_indirect_births_physbirth(vib,
	DVA_GET_OFFSET(&bp->blk_dva[0]), DVA_GET_ASIZE(&bp->blk_dva[0]));

	DVA_SET_VDEV(&bp->blk_dva[0], vd->vdev_id);
	DVA_SET_OFFSET(&bp->blk_dva[0], offset);
	}

	/*
	* If the block pointer contains any indirect DVAs, modify them to refer to
	* concrete DVAs. Note that this will sometimes not be possible, leaving
	* the indirect DVA in place. This happens if the indirect DVA spans multiple
	* segments in the mapping (i.e. it is a "split block").
	*
	* If the BP was remapped, calls the callback on the original dva (note the
	* callback can be called multiple times if the original indirect DVA refers
	* to another indirect DVA, etc).
	*
	* Returns TRUE if the BP was remapped.
	*/
	boolean_t
	spa_remap_blkptr(spa_t spa, blkptr_t bp, spa_remap_cb_t callback, void *arg)
	{
	remap_blkptr_cb_arg_t rbca;

	if (!zfs_remap_blkptr_enable)
	return (B_FALSE);

	if (!spa_feature_is_enabled(spa, SPA_FEATURE_OBSOLETE_COUNTS))
	return (B_FALSE);

	/*
	* Dedup BP's can not be remapped, because ddt_phys_select() depends
	* on DVA[0] being the same in the BP as in the DDT (dedup table).
	*/
	if (BP_GET_DEDUP(bp))
	return (B_FALSE);

	/*
	* Gang blocks can not be remapped, because
	* zio_checksum_gang_verifier() depends on the DVA[0] that's in
	* the BP used to read the gang block header (GBH) being the same
	* as the DVA[0] that we allocated for the GBH.
	*/
	if (BP_IS_GANG(bp))
	return (B_FALSE);

	/*
	* Embedded BP's have no DVA to remap.
	*/
	if (BP_GET_NDVAS(bp) < 1)
	return (B_FALSE);

	/*
	* Note: we only remap dva[0]. If we remapped other dvas, we
	* would no longer know what their phys birth txg is.
	*/
	dva_t *dva = &bp->blk_dva[0];

	uint64_t offset = DVA_GET_OFFSET(dva);
	uint64_t size = DVA_GET_ASIZE(dva);
	vdev_t *vd = vdev_lookup_top(spa, DVA_GET_VDEV(dva));

	if (vd->vdev_ops->vdev_op_remap == NULL)
	return (B_FALSE);

	rbca.rbca_bp = bp;
	rbca.rbca_cb = callback;
	rbca.rbca_remap_vd = vd;
	rbca.rbca_remap_offset = offset;
	rbca.rbca_cb_arg = arg;

	/*
	* remap_blkptr_cb() will be called in order for each level of
	* indirection, until a concrete vdev is reached or a split block is
	* encountered. old_vd and old_offset are updated within the callback
	* as we go from the one indirect vdev to the next one (either concrete
	* or indirect again) in that order.
	*/
	vd->vdev_ops->vdev_op_remap(vd, offset, size, remap_blkptr_cb, &rbca);

	/* Check if the DVA wasn't remapped because it is a split block */
	if (DVA_GET_VDEV(&rbca.rbca_bp->blk_dva[0]) == vd->vdev_id)
	return (B_FALSE);

	return (B_TRUE);
	}

	/*
	* Undo the allocation of a DVA which happened in the given transaction group.
	*/
	void
	metaslab_unalloc_dva(spa_t spa, const dva_t dva, uint64_t txg)
	{
	metaslab_t *msp;
	vdev_t *vd;
	uint64_t vdev = DVA_GET_VDEV(dva);
	uint64_t offset = DVA_GET_OFFSET(dva);
	uint64_t size = DVA_GET_ASIZE(dva);

	ASSERT(DVA_IS_VALID(dva));
	ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0);

	if (txg > spa_freeze_txg(spa))
	return;

	if ((vd = vdev_lookup_top(spa, vdev)) == NULL \|\| !DVA_IS_VALID(dva) \|\|
	(offset >> vd->vdev_ms_shift) >= vd->vdev_ms_count) {
	zfs_panic_recover("metaslab_free_dva(): bad DVA %llu:%llu:%llu",
	(u_longlong_t)vdev, (u_longlong_t)offset,
	(u_longlong_t)size);
	return;
	}

	ASSERT(!vd->vdev_removing);
	ASSERT(vdev_is_concrete(vd));
	ASSERT0(vd->vdev_indirect_config.vic_mapping_object);
	ASSERT3P(vd->vdev_indirect_mapping, ==, NULL);

	if (DVA_GET_GANG(dva))
	size = vdev_gang_header_asize(vd);

	msp = vd->vdev_ms[offset >> vd->vdev_ms_shift];

	mutex_enter(&msp->ms_lock);
	range_tree_remove(msp->ms_allocating[txg & TXG_MASK],
	offset, size);
	msp->ms_allocating_total -= size;

	VERIFY(!msp->ms_condensing);
	VERIFY3U(offset, >=, msp->ms_start);
	VERIFY3U(offset + size, <=, msp->ms_start + msp->ms_size);
	VERIFY3U(range_tree_space(msp->ms_allocatable) + size, <=,
	msp->ms_size);
	VERIFY0(P2PHASE(offset, 1ULL << vd->vdev_ashift));
	VERIFY0(P2PHASE(size, 1ULL << vd->vdev_ashift));
	range_tree_add(msp->ms_allocatable, offset, size);
	mutex_exit(&msp->ms_lock);
	}

	/*
	* Free the block represented by the given DVA.
	*/
	void
	metaslab_free_dva(spa_t spa, const dva_t dva, boolean_t checkpoint)
	{
	uint64_t vdev = DVA_GET_VDEV(dva);
	uint64_t offset = DVA_GET_OFFSET(dva);
	uint64_t size = DVA_GET_ASIZE(dva);
	vdev_t *vd = vdev_lookup_top(spa, vdev);

	ASSERT(DVA_IS_VALID(dva));
	ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0);

	if (DVA_GET_GANG(dva)) {
	size = vdev_gang_header_asize(vd);
	}

	metaslab_free_impl(vd, offset, size, checkpoint);
	}

	/*
	* Reserve some allocation slots. The reservation system must be called
	* before we call into the allocator. If there aren't any available slots
	* then the I/O will be throttled until an I/O completes and its slots are
	* freed up. The function returns true if it was successful in placing
	* the reservation.
	*/
	boolean_t
	metaslab_class_throttle_reserve(metaslab_class_t *mc, int slots, int allocator,
	zio_t *zio, int flags)
	{
	metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator];
	uint64_t max = mca->mca_alloc_max_slots;

	ASSERT(mc->mc_alloc_throttle_enabled);
	if (GANG_ALLOCATION(flags) \|\| (flags & METASLAB_MUST_RESERVE) \|\|
	zfs_refcount_count(&mca->mca_alloc_slots) + slots <= max) {
	/*
	* The potential race between _count() and _add() is covered
	* by the allocator lock in most cases, or irrelevant due to
	* GANG_ALLOCATION() or METASLAB_MUST_RESERVE set in others.
	* But even if we assume some other non-existing scenario, the
	* worst that can happen is few more I/Os get to allocation
	* earlier, that is not a problem.
	*
	* We reserve the slots individually so that we can unreserve
	* them individually when an I/O completes.
	*/
	zfs_refcount_add_few(&mca->mca_alloc_slots, slots, zio);
	zio->io_flags \|= ZIO_FLAG_IO_ALLOCATING;
	return (B_TRUE);
	}
	return (B_FALSE);
	}

	void
	metaslab_class_throttle_unreserve(metaslab_class_t *mc, int slots,
	int allocator, zio_t *zio)
	{
	metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator];

	ASSERT(mc->mc_alloc_throttle_enabled);
	zfs_refcount_remove_few(&mca->mca_alloc_slots, slots, zio);
	}

	static int
	metaslab_claim_concrete(vdev_t *vd, uint64_t offset, uint64_t size,
	uint64_t txg)
	{
	metaslab_t *msp;
	spa_t *spa = vd->vdev_spa;
	int error = 0;

	if (offset >> vd->vdev_ms_shift >= vd->vdev_ms_count)
	return (SET_ERROR(ENXIO));

	ASSERT3P(vd->vdev_ms, !=, NULL);
	msp = vd->vdev_ms[offset >> vd->vdev_ms_shift];

	mutex_enter(&msp->ms_lock);

	if ((txg != 0 && spa_writeable(spa)) \|\| !msp->ms_loaded) {
	error = metaslab_activate(msp, 0, METASLAB_WEIGHT_CLAIM);
	if (error == EBUSY) {
	ASSERT(msp->ms_loaded);
	ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK);
	error = 0;
	}
	}

	if (error == 0 &&
	!range_tree_contains(msp->ms_allocatable, offset, size))
	error = SET_ERROR(ENOENT);

	if (error \|\| txg == 0) { /* txg == 0 indicates dry run */
	mutex_exit(&msp->ms_lock);
	return (error);
	}

	VERIFY(!msp->ms_condensing);
	VERIFY0(P2PHASE(offset, 1ULL << vd->vdev_ashift));
	VERIFY0(P2PHASE(size, 1ULL << vd->vdev_ashift));
	VERIFY3U(range_tree_space(msp->ms_allocatable) - size, <=,
	msp->ms_size);
	range_tree_remove(msp->ms_allocatable, offset, size);
	range_tree_clear(msp->ms_trim, offset, size);

	if (spa_writeable(spa)) { /* don't dirty if we're zdb(8) */
	metaslab_class_t *mc = msp->ms_group->mg_class;
	multilist_sublist_t *mls =
	multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp);
	if (!multilist_link_active(&msp->ms_class_txg_node)) {
	msp->ms_selected_txg = txg;
	multilist_sublist_insert_head(mls, msp);
	}
	multilist_sublist_unlock(mls);

	if (range_tree_is_empty(msp->ms_allocating[txg & TXG_MASK]))
	vdev_dirty(vd, VDD_METASLAB, msp, txg);
	range_tree_add(msp->ms_allocating[txg & TXG_MASK],
	offset, size);
	msp->ms_allocating_total += size;
	}

	mutex_exit(&msp->ms_lock);

	return (0);
	}

	typedef struct metaslab_claim_cb_arg_t {
	uint64_t mcca_txg;
	int mcca_error;
	} metaslab_claim_cb_arg_t;

	static void
	metaslab_claim_impl_cb(uint64_t inner_offset, vdev_t *vd, uint64_t offset,
	uint64_t size, void *arg)
	{
	(void) inner_offset;
	metaslab_claim_cb_arg_t *mcca_arg = arg;

	if (mcca_arg->mcca_error == 0) {
	mcca_arg->mcca_error = metaslab_claim_concrete(vd, offset,
	size, mcca_arg->mcca_txg);
	}
	}

	int
	metaslab_claim_impl(vdev_t *vd, uint64_t offset, uint64_t size, uint64_t txg)
	{
	if (vd->vdev_ops->vdev_op_remap != NULL) {
	metaslab_claim_cb_arg_t arg;

	/*
	* Only zdb(8) can claim on indirect vdevs. This is used
	* to detect leaks of mapped space (that are not accounted
	* for in the obsolete counts, spacemap, or bpobj).
	*/
	ASSERT(!spa_writeable(vd->vdev_spa));
	arg.mcca_error = 0;
	arg.mcca_txg = txg;

	vd->vdev_ops->vdev_op_remap(vd, offset, size,
	metaslab_claim_impl_cb, &arg);

	if (arg.mcca_error == 0) {
	arg.mcca_error = metaslab_claim_concrete(vd,
	offset, size, txg);
	}
	return (arg.mcca_error);
	} else {
	return (metaslab_claim_concrete(vd, offset, size, txg));
	}
	}

	/*
	* Intent log support: upon opening the pool after a crash, notify the SPA
	* of blocks that the intent log has allocated for immediate write, but
	* which are still considered free by the SPA because the last transaction
	* group didn't commit yet.
	*/
	static int
	metaslab_claim_dva(spa_t spa, const dva_t dva, uint64_t txg)
	{
	uint64_t vdev = DVA_GET_VDEV(dva);
	uint64_t offset = DVA_GET_OFFSET(dva);
	uint64_t size = DVA_GET_ASIZE(dva);
	vdev_t *vd;

	if ((vd = vdev_lookup_top(spa, vdev)) == NULL) {
	return (SET_ERROR(ENXIO));
	}

	ASSERT(DVA_IS_VALID(dva));

	if (DVA_GET_GANG(dva))
	size = vdev_gang_header_asize(vd);

	return (metaslab_claim_impl(vd, offset, size, txg));
	}

	int
	metaslab_alloc(spa_t spa, metaslab_class_t mc, uint64_t psize, blkptr_t *bp,
	int ndvas, uint64_t txg, blkptr_t *hintbp, int flags,
	zio_alloc_list_t zal, zio_t zio, int allocator)
	{
	dva_t *dva = bp->blk_dva;
	dva_t *hintdva = (hintbp != NULL) ? hintbp->blk_dva : NULL;
	int error = 0;

	ASSERT(bp->blk_birth == 0);
	ASSERT(BP_PHYSICAL_BIRTH(bp) == 0);

	spa_config_enter(spa, SCL_ALLOC, FTAG, RW_READER);

	if (mc->mc_allocator[allocator].mca_rotor == NULL) {
	/* no vdevs in this class */
	spa_config_exit(spa, SCL_ALLOC, FTAG);
	return (SET_ERROR(ENOSPC));
	}

	ASSERT(ndvas > 0 && ndvas <= spa_max_replication(spa));
	ASSERT(BP_GET_NDVAS(bp) == 0);
	ASSERT(hintbp == NULL \|\| ndvas <= BP_GET_NDVAS(hintbp));
	ASSERT3P(zal, !=, NULL);

	for (int d = 0; d < ndvas; d++) {
	error = metaslab_alloc_dva(spa, mc, psize, dva, d, hintdva,
	txg, flags, zal, allocator);
	if (error != 0) {
	for (d--; d >= 0; d--) {
	metaslab_unalloc_dva(spa, &dva[d], txg);
	metaslab_group_alloc_decrement(spa,
	DVA_GET_VDEV(&dva[d]), zio, flags,
	allocator, B_FALSE);
	memset(&dva[d], 0, sizeof (dva_t));
	}
	spa_config_exit(spa, SCL_ALLOC, FTAG);
	return (error);
	} else {
	/*
	* Update the metaslab group's queue depth
	* based on the newly allocated dva.
	*/
	metaslab_group_alloc_increment(spa,
	DVA_GET_VDEV(&dva[d]), zio, flags, allocator);
	}
	}
	ASSERT(error == 0);
	ASSERT(BP_GET_NDVAS(bp) == ndvas);

	spa_config_exit(spa, SCL_ALLOC, FTAG);

	BP_SET_BIRTH(bp, txg, 0);

	return (0);
	}

	void
	metaslab_free(spa_t spa, const blkptr_t bp, uint64_t txg, boolean_t now)
	{
	const dva_t *dva = bp->blk_dva;
	int ndvas = BP_GET_NDVAS(bp);

	ASSERT(!BP_IS_HOLE(bp));
	ASSERT(!now \|\| bp->blk_birth >= spa_syncing_txg(spa));

	/*
	* If we have a checkpoint for the pool we need to make sure that
	* the blocks that we free that are part of the checkpoint won't be
	* reused until the checkpoint is discarded or we revert to it.
	*
	* The checkpoint flag is passed down the metaslab_free code path
	* and is set whenever we want to add a block to the checkpoint's
	* accounting. That is, we "checkpoint" blocks that existed at the
	* time the checkpoint was created and are therefore referenced by
	* the checkpointed uberblock.
	*
	* Note that, we don't checkpoint any blocks if the current
	* syncing txg <= spa_checkpoint_txg. We want these frees to sync
	* normally as they will be referenced by the checkpointed uberblock.
	*/
	boolean_t checkpoint = B_FALSE;
	if (bp->blk_birth <= spa->spa_checkpoint_txg &&
	spa_syncing_txg(spa) > spa->spa_checkpoint_txg) {
	/*
	* At this point, if the block is part of the checkpoint
	* there is no way it was created in the current txg.
	*/
	ASSERT(!now);
	ASSERT3U(spa_syncing_txg(spa), ==, txg);
	checkpoint = B_TRUE;
	}

	spa_config_enter(spa, SCL_FREE, FTAG, RW_READER);

	for (int d = 0; d < ndvas; d++) {
	if (now) {
	metaslab_unalloc_dva(spa, &dva[d], txg);
	} else {
	ASSERT3U(txg, ==, spa_syncing_txg(spa));
	metaslab_free_dva(spa, &dva[d], checkpoint);
	}
	}

	spa_config_exit(spa, SCL_FREE, FTAG);
	}

	int
	metaslab_claim(spa_t spa, const blkptr_t bp, uint64_t txg)
	{
	const dva_t *dva = bp->blk_dva;
	int ndvas = BP_GET_NDVAS(bp);
	int error = 0;

	ASSERT(!BP_IS_HOLE(bp));

	if (txg != 0) {
	/*
	* First do a dry run to make sure all DVAs are claimable,
	* so we don't have to unwind from partial failures below.
	*/
	if ((error = metaslab_claim(spa, bp, 0)) != 0)
	return (error);
	}

	spa_config_enter(spa, SCL_ALLOC, FTAG, RW_READER);

	for (int d = 0; d < ndvas; d++) {
	error = metaslab_claim_dva(spa, &dva[d], txg);
	if (error != 0)
	break;
	}

	spa_config_exit(spa, SCL_ALLOC, FTAG);

	ASSERT(error == 0 \|\| txg == 0);

	return (error);
	}

	static void
	metaslab_check_free_impl_cb(uint64_t inner, vdev_t *vd, uint64_t offset,
	uint64_t size, void *arg)
	{
	(void) inner, (void) arg;

	if (vd->vdev_ops == &vdev_indirect_ops)
	return;

	metaslab_check_free_impl(vd, offset, size);
	}

	static void
	metaslab_check_free_impl(vdev_t *vd, uint64_t offset, uint64_t size)
	{
	metaslab_t *msp;
	spa_t *spa __maybe_unused = vd->vdev_spa;

	if ((zfs_flags & ZFS_DEBUG_ZIO_FREE) == 0)
	return;

	if (vd->vdev_ops->vdev_op_remap != NULL) {
	vd->vdev_ops->vdev_op_remap(vd, offset, size,
	metaslab_check_free_impl_cb, NULL);
	return;
	}

	ASSERT(vdev_is_concrete(vd));
	ASSERT3U(offset >> vd->vdev_ms_shift, <, vd->vdev_ms_count);
	ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0);

	msp = vd->vdev_ms[offset >> vd->vdev_ms_shift];

	mutex_enter(&msp->ms_lock);
	if (msp->ms_loaded) {
	range_tree_verify_not_present(msp->ms_allocatable,
	offset, size);
	}

	/*
	* Check all segments that currently exist in the freeing pipeline.
	*
	* It would intuitively make sense to also check the current allocating
	* tree since metaslab_unalloc_dva() exists for extents that are
	* allocated and freed in the same sync pass within the same txg.
	* Unfortunately there are places (e.g. the ZIL) where we allocate a
	* segment but then we free part of it within the same txg
	* [see zil_sync()]. Thus, we don't call range_tree_verify() in the
	* current allocating tree.
	*/
	range_tree_verify_not_present(msp->ms_freeing, offset, size);
	range_tree_verify_not_present(msp->ms_checkpointing, offset, size);
	range_tree_verify_not_present(msp->ms_freed, offset, size);
	for (int j = 0; j < TXG_DEFER_SIZE; j++)
	range_tree_verify_not_present(msp->ms_defer[j], offset, size);
	range_tree_verify_not_present(msp->ms_trim, offset, size);
	mutex_exit(&msp->ms_lock);
	}

	void
	metaslab_check_free(spa_t spa, const blkptr_t bp)
	{
	if ((zfs_flags & ZFS_DEBUG_ZIO_FREE) == 0)
	return;

	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
	for (int i = 0; i < BP_GET_NDVAS(bp); i++) {
	uint64_t vdev = DVA_GET_VDEV(&bp->blk_dva[i]);
	vdev_t *vd = vdev_lookup_top(spa, vdev);
	uint64_t offset = DVA_GET_OFFSET(&bp->blk_dva[i]);
	uint64_t size = DVA_GET_ASIZE(&bp->blk_dva[i]);

	if (DVA_GET_GANG(&bp->blk_dva[i]))
	size = vdev_gang_header_asize(vd);

	ASSERT3P(vd, !=, NULL);

	metaslab_check_free_impl(vd, offset, size);
	}
	spa_config_exit(spa, SCL_VDEV, FTAG);
	}

	static void
	metaslab_group_disable_wait(metaslab_group_t *mg)
	{
	ASSERT(MUTEX_HELD(&mg->mg_ms_disabled_lock));
	while (mg->mg_disabled_updating) {
	cv_wait(&mg->mg_ms_disabled_cv, &mg->mg_ms_disabled_lock);
	}
	}

	static void
	metaslab_group_disabled_increment(metaslab_group_t *mg)
	{
	ASSERT(MUTEX_HELD(&mg->mg_ms_disabled_lock));
	ASSERT(mg->mg_disabled_updating);

	while (mg->mg_ms_disabled >= max_disabled_ms) {
	cv_wait(&mg->mg_ms_disabled_cv, &mg->mg_ms_disabled_lock);
	}
	mg->mg_ms_disabled++;
	ASSERT3U(mg->mg_ms_disabled, <=, max_disabled_ms);
	}

	/*
	* Mark the metaslab as disabled to prevent any allocations on this metaslab.
	* We must also track how many metaslabs are currently disabled within a
	* metaslab group and limit them to prevent allocation failures from
	* occurring because all metaslabs are disabled.
	*/
	void
	metaslab_disable(metaslab_t *msp)
	{
	ASSERT(!MUTEX_HELD(&msp->ms_lock));
	metaslab_group_t *mg = msp->ms_group;

	mutex_enter(&mg->mg_ms_disabled_lock);

	/*
	* To keep an accurate count of how many threads have disabled
	* a specific metaslab group, we only allow one thread to mark
	* the metaslab group at a time. This ensures that the value of
	* ms_disabled will be accurate when we decide to mark a metaslab
	* group as disabled. To do this we force all other threads
	* to wait till the metaslab's mg_disabled_updating flag is no
	* longer set.
	*/
	metaslab_group_disable_wait(mg);
	mg->mg_disabled_updating = B_TRUE;
	if (msp->ms_disabled == 0) {
	metaslab_group_disabled_increment(mg);
	}
	mutex_enter(&msp->ms_lock);
	msp->ms_disabled++;
	mutex_exit(&msp->ms_lock);

	mg->mg_disabled_updating = B_FALSE;
	cv_broadcast(&mg->mg_ms_disabled_cv);
	mutex_exit(&mg->mg_ms_disabled_lock);
	}

	void
	metaslab_enable(metaslab_t *msp, boolean_t sync, boolean_t unload)
	{
	metaslab_group_t *mg = msp->ms_group;
	spa_t *spa = mg->mg_vd->vdev_spa;

	/*
	* Wait for the outstanding IO to be synced to prevent newly
	* allocated blocks from being overwritten. This used by
	* initialize and TRIM which are modifying unallocated space.
	*/
	if (sync)
	txg_wait_synced(spa_get_dsl(spa), 0);

	mutex_enter(&mg->mg_ms_disabled_lock);
	mutex_enter(&msp->ms_lock);
	if (--msp->ms_disabled == 0) {
	mg->mg_ms_disabled--;
	cv_broadcast(&mg->mg_ms_disabled_cv);
	if (unload)
	metaslab_unload(msp);
	}
	mutex_exit(&msp->ms_lock);
	mutex_exit(&mg->mg_ms_disabled_lock);
	}

	void
	metaslab_set_unflushed_dirty(metaslab_t *ms, boolean_t dirty)
	{
	ms->ms_unflushed_dirty = dirty;
	}

	static void
	metaslab_update_ondisk_flush_data(metaslab_t ms, dmu_tx_t tx)
	{
	vdev_t *vd = ms->ms_group->mg_vd;
	spa_t *spa = vd->vdev_spa;
	objset_t *mos = spa_meta_objset(spa);

	ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP));

	metaslab_unflushed_phys_t entry = {
	.msp_unflushed_txg = metaslab_unflushed_txg(ms),
	};
	uint64_t entry_size = sizeof (entry);
	uint64_t entry_offset = ms->ms_id * entry_size;

	uint64_t object = 0;
	int err = zap_lookup(mos, vd->vdev_top_zap,
	VDEV_TOP_ZAP_MS_UNFLUSHED_PHYS_TXGS, sizeof (uint64_t), 1,
	&object);
	if (err == ENOENT) {
	object = dmu_object_alloc(mos, DMU_OTN_UINT64_METADATA,
	SPA_OLD_MAXBLOCKSIZE, DMU_OT_NONE, 0, tx);
	VERIFY0(zap_add(mos, vd->vdev_top_zap,
	VDEV_TOP_ZAP_MS_UNFLUSHED_PHYS_TXGS, sizeof (uint64_t), 1,
	&object, tx));
	} else {
	VERIFY0(err);
	}

	dmu_write(spa_meta_objset(spa), object, entry_offset, entry_size,
	&entry, tx);
	}

	void
	metaslab_set_unflushed_txg(metaslab_t ms, uint64_t txg, dmu_tx_t tx)
	{
	ms->ms_unflushed_txg = txg;
	metaslab_update_ondisk_flush_data(ms, tx);
	}

	boolean_t
	metaslab_unflushed_dirty(metaslab_t *ms)
	{
	return (ms->ms_unflushed_dirty);
	}

	uint64_t
	metaslab_unflushed_txg(metaslab_t *ms)
	{
	return (ms->ms_unflushed_txg);
	}

	ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, aliquot, U64, ZMOD_RW,
	"Allocation granularity (a.k.a. stripe size)");

	ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, debug_load, INT, ZMOD_RW,
	"Load all metaslabs when pool is first opened");

	ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, debug_unload, INT, ZMOD_RW,
	"Prevent metaslabs from being unloaded");

	ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, preload_enabled, INT, ZMOD_RW,
	"Preload potential metaslabs during reassessment");

	ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, preload_limit, UINT, ZMOD_RW,
	"Max number of metaslabs per group to preload");

	ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, unload_delay, UINT, ZMOD_RW,
	"Delay in txgs after metaslab was last used before unloading");

	ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, unload_delay_ms, UINT, ZMOD_RW,
	"Delay in milliseconds after metaslab was last used before unloading");

	/* BEGIN CSTYLED */
	ZFS_MODULE_PARAM(zfs_mg, zfs_mg_, noalloc_threshold, UINT, ZMOD_RW,
	"Percentage of metaslab group size that should be free to make it "
	"eligible for allocation");

	ZFS_MODULE_PARAM(zfs_mg, zfs_mg_, fragmentation_threshold, UINT, ZMOD_RW,
	"Percentage of metaslab group size that should be considered eligible "
	"for allocations unless all metaslab groups within the metaslab class "
	"have also crossed this threshold");

	ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, fragmentation_factor_enabled, INT,
	ZMOD_RW,
	"Use the fragmentation metric to prefer less fragmented metaslabs");
	/* END CSTYLED */

	ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, fragmentation_threshold, UINT,
	ZMOD_RW, "Fragmentation for metaslab to allow allocation");

	ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, lba_weighting_enabled, INT, ZMOD_RW,
	"Prefer metaslabs with lower LBAs");

	ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, bias_enabled, INT, ZMOD_RW,
	"Enable metaslab group biasing");

	ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, segment_weight_enabled, INT,
	ZMOD_RW, "Enable segment-based metaslab selection");

	ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, switch_threshold, INT, ZMOD_RW,
	"Segment-based metaslab selection maximum buckets before switching");

	ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, force_ganging, U64, ZMOD_RW,
	"Blocks larger than this size are sometimes forced to be gang blocks");

	ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, force_ganging_pct, UINT, ZMOD_RW,
	"Percentage of large blocks that will be forced to be gang blocks");

	ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, df_max_search, UINT, ZMOD_RW,
	"Max distance (bytes) to search forward before using size tree");

	ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, df_use_largest_segment, INT, ZMOD_RW,
	"When looking in size tree, use largest segment instead of exact fit");

	ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, max_size_cache_sec, U64,
	ZMOD_RW, "How long to trust the cached max chunk size of a metaslab");

	ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, mem_limit, UINT, ZMOD_RW,
	"Percentage of memory that can be used to store metaslab range trees");

	ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, try_hard_before_gang, INT,
	ZMOD_RW, "Try hard to allocate before ganging");

	ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, find_max_tries, UINT, ZMOD_RW,
	"Normally only consider this many of the best metaslabs in each vdev");
	diff --git a/sys/contrib/openzfs/module/zfs/mmp.c b/sys/contrib/openzfs/module/zfs/mmp.c
	index 25eea0752941..8144d8965085 100644
	--- a/sys/contrib/openzfs/module/zfs/mmp.c
	+++ b/sys/contrib/openzfs/module/zfs/mmp.c
	@@ -1,747 +1,748 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
	*/

	#include <sys/abd.h>
	#include <sys/mmp.h>
	#include <sys/spa.h>
	#include <sys/spa_impl.h>
	#include <sys/time.h>
	#include <sys/vdev.h>
	#include <sys/vdev_impl.h>
	#include <sys/zfs_context.h>
	#include <sys/callb.h>

	/*
	* Multi-Modifier Protection (MMP) attempts to prevent a user from importing
	* or opening a pool on more than one host at a time. In particular, it
	* prevents "zpool import -f" on a host from succeeding while the pool is
	* already imported on another host. There are many other ways in which a
	* device could be used by two hosts for different purposes at the same time
	* resulting in pool damage. This implementation does not attempt to detect
	* those cases.
	*
	* MMP operates by ensuring there are frequent visible changes on disk (a
	* "heartbeat") at all times. And by altering the import process to check
	* for these changes and failing the import when they are detected. This
	* functionality is enabled by setting the 'multihost' pool property to on.
	*
	* Uberblocks written by the txg_sync thread always go into the first
	* (N-MMP_BLOCKS_PER_LABEL) slots, the remaining slots are reserved for MMP.
	* They are used to hold uberblocks which are exactly the same as the last
	* synced uberblock except that the ub_timestamp and mmp_config are frequently
	* updated. Like all other uberblocks, the slot is written with an embedded
	* checksum, and slots with invalid checksums are ignored. This provides the
	* "heartbeat", with no risk of overwriting good uberblocks that must be
	* preserved, e.g. previous txgs and associated block pointers.
	*
	* Three optional fields are added to uberblock structure; ub_mmp_magic,
	* ub_mmp_config, and ub_mmp_delay. The ub_mmp_magic value allows zfs to tell
	* whether the other ub_mmp_* fields are valid. The ub_mmp_config field tells
	* the importing host the settings of zfs_multihost_interval and
	* zfs_multihost_fail_intervals on the host which last had (or currently has)
	* the pool imported. These determine how long a host must wait to detect
	* activity in the pool, before concluding the pool is not in use. The
	* mmp_delay field is a decaying average of the amount of time between
	* completion of successive MMP writes, in nanoseconds. It indicates whether
	* MMP is enabled.
	*
	* During import an activity test may now be performed to determine if
	* the pool is in use. The activity test is typically required if the
	* ZPOOL_CONFIG_HOSTID does not match the system hostid, the pool state is
	* POOL_STATE_ACTIVE, and the pool is not a root pool.
	*
	* The activity test finds the "best" uberblock (highest txg, timestamp, and, if
	* ub_mmp_magic is valid, sequence number from ub_mmp_config). It then waits
	* some time, and finds the "best" uberblock again. If any of the mentioned
	* fields have different values in the newly read uberblock, the pool is in use
	* by another host and the import fails. In order to assure the accuracy of the
	* activity test, the default values result in an activity test duration of 20x
	* the mmp write interval.
	*
	* The duration of the "zpool import" activity test depends on the information
	* available in the "best" uberblock:
	*
	* 1) If uberblock was written by zfs-0.8 or newer and fail_intervals > 0:
	* ub_mmp_config.fail_intervals * ub_mmp_config.multihost_interval * 2
	*
	* In this case, a weak guarantee is provided. Since the host which last had
	* the pool imported will suspend the pool if no mmp writes land within
	* fail_intervals * multihost_interval ms, the absence of writes during that
	* time means either the pool is not imported, or it is imported but the pool
	* is suspended and no further writes will occur.
	*
	* Note that resuming the suspended pool on the remote host would invalidate
	* this guarantee, and so it is not allowed.
	*
	* The factor of 2 provides a conservative safety factor and derives from
	* MMP_IMPORT_SAFETY_FACTOR;
	*
	* 2) If uberblock was written by zfs-0.8 or newer and fail_intervals == 0:
	* (ub_mmp_config.multihost_interval + ub_mmp_delay) *
	* zfs_multihost_import_intervals
	*
	* In this case no guarantee can provided. However, as long as some devices
	* are healthy and connected, it is likely that at least one write will land
	* within (multihost_interval + mmp_delay) because multihost_interval is
	* enough time for a write to be attempted to each leaf vdev, and mmp_delay
	* is enough for one to land, based on past delays. Multiplying by
	* zfs_multihost_import_intervals provides a conservative safety factor.
	*
	* 3) If uberblock was written by zfs-0.7:
	* (zfs_multihost_interval + ub_mmp_delay) * zfs_multihost_import_intervals
	*
	* The same logic as case #2 applies, but we do not know remote tunables.
	*
	* We use the local value for zfs_multihost_interval because the original MMP
	* did not record this value in the uberblock.
	*
	* ub_mmp_delay >= (zfs_multihost_interval / leaves), so if the other host
	* has a much larger zfs_multihost_interval set, ub_mmp_delay will reflect
	* that. We will have waited enough time for zfs_multihost_import_intervals
	* writes to be issued and all but one to land.
	*
	* single device pool example delays
	*
	* import_delay = (1 + 1) * 20 = 40s #defaults, no I/O delay
	* import_delay = (1 + 10) * 20 = 220s #defaults, 10s I/O delay
	* import_delay = (10 + 10) * 20 = 400s #10s multihost_interval,
	* no I/O delay
	* 100 device pool example delays
	*
	* import_delay = (1 + .01) * 20 = 20s #defaults, no I/O delay
	* import_delay = (1 + 10) * 20 = 220s #defaults, 10s I/O delay
	* import_delay = (10 + .1) * 20 = 202s #10s multihost_interval,
	* no I/O delay
	*
	* 4) Otherwise, this uberblock was written by a pre-MMP zfs:
	* zfs_multihost_import_intervals * zfs_multihost_interval
	*
	* In this case local tunables are used. By default this product = 10s, long
	* enough for a pool with any activity at all to write at least one
	* uberblock. No guarantee can be provided.
	*
	* Additionally, the duration is then extended by a random 25% to attempt to to
	* detect simultaneous imports. For example, if both partner hosts are rebooted
	* at the same time and automatically attempt to import the pool.
	*/

	/*
	* Used to control the frequency of mmp writes which are performed when the
	* 'multihost' pool property is on. This is one factor used to determine the
	* length of the activity check during import.
	*
	* On average an mmp write will be issued for each leaf vdev every
	* zfs_multihost_interval milliseconds. In practice, the observed period can
	* vary with the I/O load and this observed value is the ub_mmp_delay which is
	* stored in the uberblock. The minimum allowed value is 100 ms.
	*/
	uint64_t zfs_multihost_interval = MMP_DEFAULT_INTERVAL;

	/*
	* Used to control the duration of the activity test on import. Smaller values
	* of zfs_multihost_import_intervals will reduce the import time but increase
	* the risk of failing to detect an active pool. The total activity check time
	* is never allowed to drop below one second. A value of 0 is ignored and
	* treated as if it was set to 1.
	*/
	uint_t zfs_multihost_import_intervals = MMP_DEFAULT_IMPORT_INTERVALS;

	/*
	* Controls the behavior of the pool when mmp write failures or delays are
	* detected.
	*
	* When zfs_multihost_fail_intervals = 0, mmp write failures or delays are
	* ignored. The failures will still be reported to the ZED which depending on
	* its configuration may take action such as suspending the pool or taking a
	* device offline.
	*
	* When zfs_multihost_fail_intervals > 0, the pool will be suspended if
	* zfs_multihost_fail_intervals * zfs_multihost_interval milliseconds pass
	* without a successful mmp write. This guarantees the activity test will see
	* mmp writes if the pool is imported. A value of 1 is ignored and treated as
	* if it was set to 2, because a single leaf vdev pool will issue a write once
	* per multihost_interval and thus any variation in latency would cause the
	* pool to be suspended.
	*/
	uint_t zfs_multihost_fail_intervals = MMP_DEFAULT_FAIL_INTERVALS;

	static const void *const mmp_tag = "mmp_write_uberblock";
	static __attribute__((noreturn)) void mmp_thread(void *arg);

	void
	mmp_init(spa_t *spa)
	{
	mmp_thread_t *mmp = &spa->spa_mmp;

	mutex_init(&mmp->mmp_thread_lock, NULL, MUTEX_DEFAULT, NULL);
	cv_init(&mmp->mmp_thread_cv, NULL, CV_DEFAULT, NULL);
	mutex_init(&mmp->mmp_io_lock, NULL, MUTEX_DEFAULT, NULL);
	mmp->mmp_kstat_id = 1;
	}

	void
	mmp_fini(spa_t *spa)
	{
	mmp_thread_t *mmp = &spa->spa_mmp;

	mutex_destroy(&mmp->mmp_thread_lock);
	cv_destroy(&mmp->mmp_thread_cv);
	mutex_destroy(&mmp->mmp_io_lock);
	}

	static void
	mmp_thread_enter(mmp_thread_t mmp, callb_cpr_t cpr)
	{
	CALLB_CPR_INIT(cpr, &mmp->mmp_thread_lock, callb_generic_cpr, FTAG);
	mutex_enter(&mmp->mmp_thread_lock);
	}

	static void
	mmp_thread_exit(mmp_thread_t mmp, kthread_t mpp, callb_cpr_t cpr)
	{
	ASSERT(*mpp != NULL);
	*mpp = NULL;
	cv_broadcast(&mmp->mmp_thread_cv);
	CALLB_CPR_EXIT(cpr); /* drops &mmp->mmp_thread_lock */
	}

	void
	mmp_thread_start(spa_t *spa)
	{
	mmp_thread_t *mmp = &spa->spa_mmp;

	if (spa_writeable(spa)) {
	mutex_enter(&mmp->mmp_thread_lock);
	if (!mmp->mmp_thread) {
	mmp->mmp_thread = thread_create(NULL, 0, mmp_thread,
	spa, 0, &p0, TS_RUN, defclsyspri);
	zfs_dbgmsg("MMP thread started pool '%s' "
	"gethrtime %llu", spa_name(spa), gethrtime());
	}
	mutex_exit(&mmp->mmp_thread_lock);
	}
	}

	void
	mmp_thread_stop(spa_t *spa)
	{
	mmp_thread_t *mmp = &spa->spa_mmp;

	mutex_enter(&mmp->mmp_thread_lock);
	mmp->mmp_thread_exiting = 1;
	cv_broadcast(&mmp->mmp_thread_cv);

	while (mmp->mmp_thread) {
	cv_wait(&mmp->mmp_thread_cv, &mmp->mmp_thread_lock);
	}
	mutex_exit(&mmp->mmp_thread_lock);
	zfs_dbgmsg("MMP thread stopped pool '%s' gethrtime %llu",
	spa_name(spa), gethrtime());

	ASSERT(mmp->mmp_thread == NULL);
	mmp->mmp_thread_exiting = 0;
	}

	typedef enum mmp_vdev_state_flag {
	MMP_FAIL_NOT_WRITABLE = (1 << 0),
	MMP_FAIL_WRITE_PENDING = (1 << 1),
	} mmp_vdev_state_flag_t;

	/*
	* Find a leaf vdev to write an MMP block to. It must not have an outstanding
	* mmp write (if so a new write will also likely block). If there is no usable
	* leaf, a nonzero error value is returned. The error value returned is a bit
	* field.
	*
	* MMP_FAIL_WRITE_PENDING One or more leaf vdevs are writeable, but have an
	* outstanding MMP write.
	* MMP_FAIL_NOT_WRITABLE One or more leaf vdevs are not writeable.
	*/

	static int
	mmp_next_leaf(spa_t *spa)
	{
	vdev_t *leaf;
	vdev_t *starting_leaf;
	int fail_mask = 0;

	ASSERT(MUTEX_HELD(&spa->spa_mmp.mmp_io_lock));
	ASSERT(spa_config_held(spa, SCL_STATE, RW_READER));
	ASSERT(list_link_active(&spa->spa_leaf_list.list_head) == B_TRUE);
	ASSERT(!list_is_empty(&spa->spa_leaf_list));

	if (spa->spa_mmp.mmp_leaf_last_gen != spa->spa_leaf_list_gen) {
	spa->spa_mmp.mmp_last_leaf = list_head(&spa->spa_leaf_list);
	spa->spa_mmp.mmp_leaf_last_gen = spa->spa_leaf_list_gen;
	}

	leaf = spa->spa_mmp.mmp_last_leaf;
	if (leaf == NULL)
	leaf = list_head(&spa->spa_leaf_list);
	starting_leaf = leaf;

	do {
	leaf = list_next(&spa->spa_leaf_list, leaf);
	if (leaf == NULL) {
	leaf = list_head(&spa->spa_leaf_list);
	ASSERT3P(leaf, !=, NULL);
	}

	/*
	* We skip unwritable, offline, detached, and dRAID spare
	* devices as they are either not legal targets or the write
	* may fail or not be seen by other hosts. Skipped dRAID
	* spares can never be written so the fail mask is not set.
	*/
	if (!vdev_writeable(leaf) \|\| leaf->vdev_offline \|\|
	leaf->vdev_detached) {
	fail_mask \|= MMP_FAIL_NOT_WRITABLE;
	} else if (leaf->vdev_ops == &vdev_draid_spare_ops) {
	continue;
	} else if (leaf->vdev_mmp_pending != 0) {
	fail_mask \|= MMP_FAIL_WRITE_PENDING;
	} else {
	spa->spa_mmp.mmp_last_leaf = leaf;
	return (0);
	}
	} while (leaf != starting_leaf);

	ASSERT(fail_mask);

	return (fail_mask);
	}

	/*
	* MMP writes are issued on a fixed schedule, but may complete at variable,
	* much longer, intervals. The mmp_delay captures long periods between
	* successful writes for any reason, including disk latency, scheduling delays,
	* etc.
	*
	* The mmp_delay is usually calculated as a decaying average, but if the latest
	* delay is higher we do not average it, so that we do not hide sudden spikes
	* which the importing host must wait for.
	*
	* If writes are occurring frequently, such as due to a high rate of txg syncs,
	* the mmp_delay could become very small. Since those short delays depend on
	* activity we cannot count on, we never allow mmp_delay to get lower than rate
	* expected if only mmp_thread writes occur.
	*
	* If an mmp write was skipped or fails, and we have already waited longer than
	* mmp_delay, we need to update it so the next write reflects the longer delay.
	*
	* Do not set mmp_delay if the multihost property is not on, so as not to
	* trigger an activity check on import.
	*/
	static void
	mmp_delay_update(spa_t *spa, boolean_t write_completed)
	{
	mmp_thread_t *mts = &spa->spa_mmp;
	hrtime_t delay = gethrtime() - mts->mmp_last_write;

	ASSERT(MUTEX_HELD(&mts->mmp_io_lock));

	if (spa_multihost(spa) == B_FALSE) {
	mts->mmp_delay = 0;
	return;
	}

	if (delay > mts->mmp_delay)
	mts->mmp_delay = delay;

	if (write_completed == B_FALSE)
	return;

	mts->mmp_last_write = gethrtime();

	/*
	* strictly less than, in case delay was changed above.
	*/
	if (delay < mts->mmp_delay) {
	hrtime_t min_delay =
	MSEC2NSEC(MMP_INTERVAL_OK(zfs_multihost_interval)) /
	MAX(1, vdev_count_leaves(spa));
	mts->mmp_delay = MAX(((delay + mts->mmp_delay * 127) / 128),
	min_delay);
	}
	}

	static void
	mmp_write_done(zio_t *zio)
	{
	spa_t *spa = zio->io_spa;
	vdev_t *vd = zio->io_vd;
	mmp_thread_t *mts = zio->io_private;

	mutex_enter(&mts->mmp_io_lock);
	uint64_t mmp_kstat_id = vd->vdev_mmp_kstat_id;
	hrtime_t mmp_write_duration = gethrtime() - vd->vdev_mmp_pending;

	mmp_delay_update(spa, (zio->io_error == 0));

	vd->vdev_mmp_pending = 0;
	vd->vdev_mmp_kstat_id = 0;

	mutex_exit(&mts->mmp_io_lock);
	spa_config_exit(spa, SCL_STATE, mmp_tag);

	spa_mmp_history_set(spa, mmp_kstat_id, zio->io_error,
	mmp_write_duration);

	abd_free(zio->io_abd);
	}

	/*
	* When the uberblock on-disk is updated by a spa_sync,
	* creating a new "best" uberblock, update the one stored
	* in the mmp thread state, used for mmp writes.
	*/
	void
	mmp_update_uberblock(spa_t spa, uberblock_t ub)
	{
	mmp_thread_t *mmp = &spa->spa_mmp;

	mutex_enter(&mmp->mmp_io_lock);
	mmp->mmp_ub = *ub;
	mmp->mmp_seq = 1;
	mmp->mmp_ub.ub_timestamp = gethrestime_sec();
	mmp_delay_update(spa, B_TRUE);
	mutex_exit(&mmp->mmp_io_lock);
	}

	/*
	* Choose a random vdev, label, and MMP block, and write over it
	* with a copy of the last-synced uberblock, whose timestamp
	* has been updated to reflect that the pool is in use.
	*/
	static void
	mmp_write_uberblock(spa_t *spa)
	{
	int flags = ZIO_FLAG_CONFIG_WRITER \| ZIO_FLAG_CANFAIL;
	mmp_thread_t *mmp = &spa->spa_mmp;
	uberblock_t *ub;
	vdev_t *vd = NULL;
	int label, error;
	uint64_t offset;

	hrtime_t lock_acquire_time = gethrtime();
	spa_config_enter_mmp(spa, SCL_STATE, mmp_tag, RW_READER);
	lock_acquire_time = gethrtime() - lock_acquire_time;
	if (lock_acquire_time > (MSEC2NSEC(MMP_MIN_INTERVAL) / 10))
	zfs_dbgmsg("MMP SCL_STATE acquisition pool '%s' took %llu ns "
	"gethrtime %llu", spa_name(spa), lock_acquire_time,
	gethrtime());

	mutex_enter(&mmp->mmp_io_lock);

	error = mmp_next_leaf(spa);

	/*
	* spa_mmp_history has two types of entries:
	* Issued MMP write: records time issued, error status, etc.
	* Skipped MMP write: an MMP write could not be issued because no
	* suitable leaf vdev was available. See comment above struct
	* spa_mmp_history for details.
	*/

	if (error) {
	mmp_delay_update(spa, B_FALSE);
	if (mmp->mmp_skip_error == error) {
	spa_mmp_history_set_skip(spa, mmp->mmp_kstat_id - 1);
	} else {
	mmp->mmp_skip_error = error;
	spa_mmp_history_add(spa, mmp->mmp_ub.ub_txg,
	gethrestime_sec(), mmp->mmp_delay, NULL, 0,
	mmp->mmp_kstat_id++, error);
	zfs_dbgmsg("MMP error choosing leaf pool '%s' "
	"gethrtime %llu fail_mask %#x", spa_name(spa),
	gethrtime(), error);
	}
	mutex_exit(&mmp->mmp_io_lock);
	spa_config_exit(spa, SCL_STATE, mmp_tag);
	return;
	}

	vd = spa->spa_mmp.mmp_last_leaf;
	if (mmp->mmp_skip_error != 0) {
	mmp->mmp_skip_error = 0;
	zfs_dbgmsg("MMP write after skipping due to unavailable "
	"leaves, pool '%s' gethrtime %llu leaf %llu",
	spa_name(spa), (u_longlong_t)gethrtime(),
	(u_longlong_t)vd->vdev_guid);
	}

	if (mmp->mmp_zio_root == NULL)
	mmp->mmp_zio_root = zio_root(spa, NULL, NULL,
	flags \| ZIO_FLAG_GODFATHER);

	if (mmp->mmp_ub.ub_timestamp != gethrestime_sec()) {
	/*
	* Want to reset mmp_seq when timestamp advances because after
	* an mmp_seq wrap new values will not be chosen by
	* uberblock_compare() as the "best".
	*/
	mmp->mmp_ub.ub_timestamp = gethrestime_sec();
	mmp->mmp_seq = 1;
	}

	ub = &mmp->mmp_ub;
	ub->ub_mmp_magic = MMP_MAGIC;
	ub->ub_mmp_delay = mmp->mmp_delay;
	ub->ub_mmp_config = MMP_SEQ_SET(mmp->mmp_seq) \|
	MMP_INTERVAL_SET(MMP_INTERVAL_OK(zfs_multihost_interval)) \|
	MMP_FAIL_INT_SET(MMP_FAIL_INTVS_OK(
	zfs_multihost_fail_intervals));
	vd->vdev_mmp_pending = gethrtime();
	vd->vdev_mmp_kstat_id = mmp->mmp_kstat_id;

	zio_t *zio = zio_null(mmp->mmp_zio_root, spa, NULL, NULL, NULL, flags);
	abd_t *ub_abd = abd_alloc_for_io(VDEV_UBERBLOCK_SIZE(vd), B_TRUE);
	abd_zero(ub_abd, VDEV_UBERBLOCK_SIZE(vd));
	abd_copy_from_buf(ub_abd, ub, sizeof (uberblock_t));

	mmp->mmp_seq++;
	mmp->mmp_kstat_id++;
	mutex_exit(&mmp->mmp_io_lock);

	offset = VDEV_UBERBLOCK_OFFSET(vd, VDEV_UBERBLOCK_COUNT(vd) -
	MMP_BLOCKS_PER_LABEL + random_in_range(MMP_BLOCKS_PER_LABEL));

	label = random_in_range(VDEV_LABELS);
	vdev_label_write(zio, vd, label, ub_abd, offset,
	VDEV_UBERBLOCK_SIZE(vd), mmp_write_done, mmp,
	flags \| ZIO_FLAG_DONT_PROPAGATE);

	(void) spa_mmp_history_add(spa, ub->ub_txg, ub->ub_timestamp,
	ub->ub_mmp_delay, vd, label, vd->vdev_mmp_kstat_id, 0);

	zio_nowait(zio);
	}

	static __attribute__((noreturn)) void
	mmp_thread(void *arg)
	{
	spa_t spa = (spa_t )arg;
	mmp_thread_t *mmp = &spa->spa_mmp;
	boolean_t suspended = spa_suspended(spa);
	boolean_t multihost = spa_multihost(spa);
	uint64_t mmp_interval = MSEC2NSEC(MMP_INTERVAL_OK(
	zfs_multihost_interval));
	uint32_t mmp_fail_intervals = MMP_FAIL_INTVS_OK(
	zfs_multihost_fail_intervals);
	hrtime_t mmp_fail_ns = mmp_fail_intervals * mmp_interval;
	boolean_t last_spa_suspended;
	boolean_t last_spa_multihost;
	uint64_t last_mmp_interval;
	uint32_t last_mmp_fail_intervals;
	hrtime_t last_mmp_fail_ns;
	callb_cpr_t cpr;
	int skip_wait = 0;

	mmp_thread_enter(mmp, &cpr);

	/*
	* There have been no MMP writes yet. Setting mmp_last_write here gives
	* us one mmp_fail_ns period, which is consistent with the activity
	* check duration, to try to land an MMP write before MMP suspends the
	* pool (if so configured).
	*/

	mutex_enter(&mmp->mmp_io_lock);
	mmp->mmp_last_write = gethrtime();
	mmp->mmp_delay = MSEC2NSEC(MMP_INTERVAL_OK(zfs_multihost_interval));
	mutex_exit(&mmp->mmp_io_lock);

	while (!mmp->mmp_thread_exiting) {
	hrtime_t next_time = gethrtime() +
	MSEC2NSEC(MMP_DEFAULT_INTERVAL);
	int leaves = MAX(vdev_count_leaves(spa), 1);

	/* Detect changes in tunables or state */

	last_spa_suspended = suspended;
	last_spa_multihost = multihost;
	suspended = spa_suspended(spa);
	multihost = spa_multihost(spa);

	last_mmp_interval = mmp_interval;
	last_mmp_fail_intervals = mmp_fail_intervals;
	last_mmp_fail_ns = mmp_fail_ns;
	mmp_interval = MSEC2NSEC(MMP_INTERVAL_OK(
	zfs_multihost_interval));
	mmp_fail_intervals = MMP_FAIL_INTVS_OK(
	zfs_multihost_fail_intervals);

	/* Smooth so pool is not suspended when reducing tunables */
	if (mmp_fail_intervals * mmp_interval < mmp_fail_ns) {
	mmp_fail_ns = (mmp_fail_ns * 31 +
	mmp_fail_intervals * mmp_interval) / 32;
	} else {
	mmp_fail_ns = mmp_fail_intervals *
	mmp_interval;
	}

	if (mmp_interval != last_mmp_interval \|\|
	mmp_fail_intervals != last_mmp_fail_intervals) {
	/*
	* We want other hosts to see new tunables as quickly as
	* possible. Write out at higher frequency than usual.
	*/
	skip_wait += leaves;
	}

	if (multihost)
	next_time = gethrtime() + mmp_interval / leaves;

	if (mmp_fail_ns != last_mmp_fail_ns) {
	zfs_dbgmsg("MMP interval change pool '%s' "
	"gethrtime %llu last_mmp_interval %llu "
	"mmp_interval %llu last_mmp_fail_intervals %u "
	"mmp_fail_intervals %u mmp_fail_ns %llu "
	"skip_wait %d leaves %d next_time %llu",
	spa_name(spa), (u_longlong_t)gethrtime(),
	(u_longlong_t)last_mmp_interval,
	(u_longlong_t)mmp_interval, last_mmp_fail_intervals,
	mmp_fail_intervals, (u_longlong_t)mmp_fail_ns,
	skip_wait, leaves, (u_longlong_t)next_time);
	}

	/*
	* MMP off => on, or suspended => !suspended:
	* No writes occurred recently. Update mmp_last_write to give
	* us some time to try.
	*/
	if ((!last_spa_multihost && multihost) \|\|
	(last_spa_suspended && !suspended)) {
	zfs_dbgmsg("MMP state change pool '%s': gethrtime %llu "
	"last_spa_multihost %u multihost %u "
	"last_spa_suspended %u suspended %u",
	spa_name(spa), (u_longlong_t)gethrtime(),
	last_spa_multihost, multihost, last_spa_suspended,
	suspended);
	mutex_enter(&mmp->mmp_io_lock);
	mmp->mmp_last_write = gethrtime();
	mmp->mmp_delay = mmp_interval;
	mutex_exit(&mmp->mmp_io_lock);
	}

	/*
	* MMP on => off:
	* mmp_delay == 0 tells importing node to skip activity check.
	*/
	if (last_spa_multihost && !multihost) {
	mutex_enter(&mmp->mmp_io_lock);
	mmp->mmp_delay = 0;
	mutex_exit(&mmp->mmp_io_lock);
	}

	/*
	* Suspend the pool if no MMP write has succeeded in over
	* mmp_interval * mmp_fail_intervals nanoseconds.
	*/
	if (multihost && !suspended && mmp_fail_intervals &&
	(gethrtime() - mmp->mmp_last_write) > mmp_fail_ns) {
	zfs_dbgmsg("MMP suspending pool '%s': gethrtime %llu "
	"mmp_last_write %llu mmp_interval %llu "
	- "mmp_fail_intervals %llu mmp_fail_ns %llu",
	+ "mmp_fail_intervals %llu mmp_fail_ns %llu txg %llu",
	spa_name(spa), (u_longlong_t)gethrtime(),
	(u_longlong_t)mmp->mmp_last_write,
	(u_longlong_t)mmp_interval,
	(u_longlong_t)mmp_fail_intervals,
	- (u_longlong_t)mmp_fail_ns);
	+ (u_longlong_t)mmp_fail_ns,
	+ (u_longlong_t)spa->spa_uberblock.ub_txg);
	cmn_err(CE_WARN, "MMP writes to pool '%s' have not "
	"succeeded in over %llu ms; suspending pool. "
	"Hrtime %llu",
	spa_name(spa),
	NSEC2MSEC(gethrtime() - mmp->mmp_last_write),
	gethrtime());
	zio_suspend(spa, NULL, ZIO_SUSPEND_MMP);
	}

	if (multihost && !suspended)
	mmp_write_uberblock(spa);

	if (skip_wait > 0) {
	next_time = gethrtime() + MSEC2NSEC(MMP_MIN_INTERVAL) /
	leaves;
	skip_wait--;
	}

	CALLB_CPR_SAFE_BEGIN(&cpr);
	(void) cv_timedwait_idle_hires(&mmp->mmp_thread_cv,
	&mmp->mmp_thread_lock, next_time, USEC2NSEC(100),
	CALLOUT_FLAG_ABSOLUTE);
	CALLB_CPR_SAFE_END(&cpr, &mmp->mmp_thread_lock);
	}

	/* Outstanding writes are allowed to complete. */
	zio_wait(mmp->mmp_zio_root);

	mmp->mmp_zio_root = NULL;
	mmp_thread_exit(mmp, &mmp->mmp_thread, &cpr);

	thread_exit();
	}

	/*
	* Signal the MMP thread to wake it, when it is sleeping on
	* its cv. Used when some module parameter has changed and
	* we want the thread to know about it.
	* Only signal if the pool is active and mmp thread is
	* running, otherwise there is no thread to wake.
	*/
	static void
	mmp_signal_thread(spa_t *spa)
	{
	mmp_thread_t *mmp = &spa->spa_mmp;

	mutex_enter(&mmp->mmp_thread_lock);
	if (mmp->mmp_thread)
	cv_broadcast(&mmp->mmp_thread_cv);
	mutex_exit(&mmp->mmp_thread_lock);
	}

	void
	mmp_signal_all_threads(void)
	{
	spa_t *spa = NULL;

	mutex_enter(&spa_namespace_lock);
	while ((spa = spa_next(spa))) {
	if (spa->spa_state == POOL_STATE_ACTIVE)
	mmp_signal_thread(spa);
	}
	mutex_exit(&spa_namespace_lock);
	}

	/* BEGIN CSTYLED */
	ZFS_MODULE_PARAM_CALL(zfs_multihost, zfs_multihost_, interval,
	param_set_multihost_interval, spl_param_get_u64, ZMOD_RW,
	"Milliseconds between mmp writes to each leaf");
	/* END CSTYLED */

	ZFS_MODULE_PARAM(zfs_multihost, zfs_multihost_, fail_intervals, UINT, ZMOD_RW,
	"Max allowed period without a successful mmp write");

	ZFS_MODULE_PARAM(zfs_multihost, zfs_multihost_, import_intervals, UINT, ZMOD_RW,
	"Number of zfs_multihost_interval periods to wait for activity");
	diff --git a/sys/contrib/openzfs/module/zfs/multilist.c b/sys/contrib/openzfs/module/zfs/multilist.c
	index b1cdf1c5c5f4..3d3ef86e6839 100644
	--- a/sys/contrib/openzfs/module/zfs/multilist.c
	+++ b/sys/contrib/openzfs/module/zfs/multilist.c
	@@ -1,429 +1,451 @@
	/*
	* CDDL HEADER START
	*
	* This file and its contents are supplied under the terms of the
	* Common Development and Distribution License ("CDDL"), version 1.0.
	* You may only use this file in accordance with the terms of version
	* 1.0 of the CDDL.
	*
	* A full copy of the text of the CDDL should have accompanied this
	* source. A copy of the CDDL is also available via the Internet at
	* http://www.illumos.org/license/CDDL.
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2013, 2017 by Delphix. All rights reserved.
	*/

	#include <sys/zfs_context.h>
	#include <sys/multilist.h>
	#include <sys/trace_zfs.h>

	/*
	* This overrides the number of sublists in each multilist_t, which defaults
	* to the number of CPUs in the system (see multilist_create()).
	*/
	uint_t zfs_multilist_num_sublists = 0;

	/*
	* Given the object contained on the list, return a pointer to the
	* object's multilist_node_t structure it contains.
	*/
	#ifdef ZFS_DEBUG
	static multilist_node_t *
	multilist_d2l(multilist_t ml, void obj)
	{
	return ((multilist_node_t )((char )obj + ml->ml_offset));
	}
	#else
	#define multilist_d2l(ml, obj) ((void) sizeof (ml), (void) sizeof (obj), NULL)
	#endif

	/*
	* Initialize a new mutlilist using the parameters specified.
	*
	* - 'size' denotes the size of the structure containing the
	* multilist_node_t.
	* - 'offset' denotes the byte offset of the mutlilist_node_t within
	* the structure that contains it.
	* - 'num' specifies the number of internal sublists to create.
	* - 'index_func' is used to determine which sublist to insert into
	* when the multilist_insert() function is called; as well as which
	* sublist to remove from when multilist_remove() is called. The
	* requirements this function must meet, are the following:
	*
	* - It must always return the same value when called on the same
	* object (to ensure the object is removed from the list it was
	* inserted into).
	*
	* - It must return a value in the range [0, number of sublists).
	* The multilist_get_num_sublists() function may be used to
	* determine the number of sublists in the multilist.
	*
	* Also, in order to reduce internal contention between the sublists
	* during insertion and removal, this function should choose evenly
	* between all available sublists when inserting. This isn't a hard
	* requirement, but a general rule of thumb in order to garner the
	* best multi-threaded performance out of the data structure.
	*/
	static void
	multilist_create_impl(multilist_t *ml, size_t size, size_t offset,
	uint_t num, multilist_sublist_index_func_t *index_func)
	{
	ASSERT3U(size, >, 0);
	ASSERT3U(size, >=, offset + sizeof (multilist_node_t));
	ASSERT3U(num, >, 0);
	ASSERT3P(index_func, !=, NULL);

	ml->ml_offset = offset;
	ml->ml_num_sublists = num;
	ml->ml_index_func = index_func;

	ml->ml_sublists = kmem_zalloc(sizeof (multilist_sublist_t) *
	ml->ml_num_sublists, KM_SLEEP);

	ASSERT3P(ml->ml_sublists, !=, NULL);

	for (int i = 0; i < ml->ml_num_sublists; i++) {
	multilist_sublist_t *mls = &ml->ml_sublists[i];
	mutex_init(&mls->mls_lock, NULL, MUTEX_NOLOCKDEP, NULL);
	list_create(&mls->mls_list, size, offset);
	}
	}

	/*
	* Allocate a new multilist, using the default number of sublists (the number
	* of CPUs, or at least 4, or the tunable zfs_multilist_num_sublists). Note
	* that the multilists do not expand if more CPUs are hot-added. In that case,
	* we will have less fanout than boot_ncpus, but we don't want to always
	* reserve the RAM necessary to create the extra slots for additional CPUs up
	* front, and dynamically adding them is a complex task.
	*/
	void
	multilist_create(multilist_t *ml, size_t size, size_t offset,
	multilist_sublist_index_func_t *index_func)
	{
	uint_t num_sublists;

	if (zfs_multilist_num_sublists > 0) {
	num_sublists = zfs_multilist_num_sublists;
	} else {
	num_sublists = MAX(boot_ncpus, 4);
	}

	multilist_create_impl(ml, size, offset, num_sublists, index_func);
	}

	/*
	* Destroy the given multilist object, and free up any memory it holds.
	*/
	void
	multilist_destroy(multilist_t *ml)
	{
	ASSERT(multilist_is_empty(ml));

	for (int i = 0; i < ml->ml_num_sublists; i++) {
	multilist_sublist_t *mls = &ml->ml_sublists[i];

	ASSERT(list_is_empty(&mls->mls_list));

	list_destroy(&mls->mls_list);
	mutex_destroy(&mls->mls_lock);
	}

	ASSERT3P(ml->ml_sublists, !=, NULL);
	kmem_free(ml->ml_sublists,
	sizeof (multilist_sublist_t) * ml->ml_num_sublists);

	ml->ml_num_sublists = 0;
	ml->ml_offset = 0;
	ml->ml_sublists = NULL;
	}

	/*
	* Insert the given object into the multilist.
	*
	* This function will insert the object specified into the sublist
	* determined using the function given at multilist creation time.
	*
	* The sublist locks are automatically acquired if not already held, to
	* ensure consistency when inserting and removing from multiple threads.
	*/
	void
	multilist_insert(multilist_t ml, void obj)
	{
	unsigned int sublist_idx = ml->ml_index_func(ml, obj);
	multilist_sublist_t *mls;
	boolean_t need_lock;

	DTRACE_PROBE3(multilist__insert, multilist_t *, ml,
	unsigned int, sublist_idx, void *, obj);

	ASSERT3U(sublist_idx, <, ml->ml_num_sublists);

	mls = &ml->ml_sublists[sublist_idx];

	/*
	* Note: Callers may already hold the sublist lock by calling
	* multilist_sublist_lock(). Here we rely on MUTEX_HELD()
	* returning TRUE if and only if the current thread holds the
	* lock. While it's a little ugly to make the lock recursive in
	* this way, it works and allows the calling code to be much
	* simpler -- otherwise it would have to pass around a flag
	* indicating that it already has the lock.
	*/
	need_lock = !MUTEX_HELD(&mls->mls_lock);

	if (need_lock)
	mutex_enter(&mls->mls_lock);

	ASSERT(!multilist_link_active(multilist_d2l(ml, obj)));

	multilist_sublist_insert_head(mls, obj);

	if (need_lock)
	mutex_exit(&mls->mls_lock);
	}

	/*
	* Remove the given object from the multilist.
	*
	* This function will remove the object specified from the sublist
	* determined using the function given at multilist creation time.
	*
	* The necessary sublist locks are automatically acquired, to ensure
	* consistency when inserting and removing from multiple threads.
	*/
	void
	multilist_remove(multilist_t ml, void obj)
	{
	unsigned int sublist_idx = ml->ml_index_func(ml, obj);
	multilist_sublist_t *mls;
	boolean_t need_lock;

	DTRACE_PROBE3(multilist__remove, multilist_t *, ml,
	unsigned int, sublist_idx, void *, obj);

	ASSERT3U(sublist_idx, <, ml->ml_num_sublists);

	mls = &ml->ml_sublists[sublist_idx];
	/* See comment in multilist_insert(). */
	need_lock = !MUTEX_HELD(&mls->mls_lock);

	if (need_lock)
	mutex_enter(&mls->mls_lock);

	ASSERT(multilist_link_active(multilist_d2l(ml, obj)));

	multilist_sublist_remove(mls, obj);

	if (need_lock)
	mutex_exit(&mls->mls_lock);
	}

	/*
	* Check to see if this multilist object is empty.
	*
	* This will return TRUE if it finds all of the sublists of this
	* multilist to be empty, and FALSE otherwise. Each sublist lock will be
	* automatically acquired as necessary.
	*
	* If concurrent insertions and removals are occurring, the semantics
	* of this function become a little fuzzy. Instead of locking all
	* sublists for the entire call time of the function, each sublist is
	* only locked as it is individually checked for emptiness. Thus, it's
	* possible for this function to return TRUE with non-empty sublists at
	* the time the function returns. This would be due to another thread
	* inserting into a given sublist, after that specific sublist was check
	* and deemed empty, but before all sublists have been checked.
	*/
	int
	multilist_is_empty(multilist_t *ml)
	{
	for (int i = 0; i < ml->ml_num_sublists; i++) {
	multilist_sublist_t *mls = &ml->ml_sublists[i];
	/* See comment in multilist_insert(). */
	boolean_t need_lock = !MUTEX_HELD(&mls->mls_lock);

	if (need_lock)
	mutex_enter(&mls->mls_lock);

	if (!list_is_empty(&mls->mls_list)) {
	if (need_lock)
	mutex_exit(&mls->mls_lock);

	return (FALSE);
	}

	if (need_lock)
	mutex_exit(&mls->mls_lock);
	}

	return (TRUE);
	}

	/* Return the number of sublists composing this multilist */
	unsigned int
	multilist_get_num_sublists(multilist_t *ml)
	{
	return (ml->ml_num_sublists);
	}

	/* Return a randomly selected, valid sublist index for this multilist */
	unsigned int
	multilist_get_random_index(multilist_t *ml)
	{
	return (random_in_range(ml->ml_num_sublists));
	}

	+void
	+multilist_sublist_lock(multilist_sublist_t *mls)
	+{
	+ mutex_enter(&mls->mls_lock);
	+}
	+
	/* Lock and return the sublist specified at the given index */
	multilist_sublist_t *
	-multilist_sublist_lock(multilist_t *ml, unsigned int sublist_idx)
	+multilist_sublist_lock_idx(multilist_t *ml, unsigned int sublist_idx)
	{
	multilist_sublist_t *mls;

	ASSERT3U(sublist_idx, <, ml->ml_num_sublists);
	mls = &ml->ml_sublists[sublist_idx];
	mutex_enter(&mls->mls_lock);

	return (mls);
	}

	/* Lock and return the sublist that would be used to store the specified obj */
	multilist_sublist_t *
	multilist_sublist_lock_obj(multilist_t ml, void obj)
	{
	- return (multilist_sublist_lock(ml, ml->ml_index_func(ml, obj)));
	+ return (multilist_sublist_lock_idx(ml, ml->ml_index_func(ml, obj)));
	}

	void
	multilist_sublist_unlock(multilist_sublist_t *mls)
	{
	mutex_exit(&mls->mls_lock);
	}

	/*
	* We're allowing any object to be inserted into this specific sublist,
	* but this can lead to trouble if multilist_remove() is called to
	* remove this object. Specifically, if calling ml_index_func on this
	* object returns an index for sublist different than what is passed as
	* a parameter here, any call to multilist_remove() with this newly
	* inserted object is undefined! (the call to multilist_remove() will
	* remove the object from a list that it isn't contained in)
	*/
	void
	multilist_sublist_insert_head(multilist_sublist_t mls, void obj)
	{
	ASSERT(MUTEX_HELD(&mls->mls_lock));
	list_insert_head(&mls->mls_list, obj);
	}

	/* please see comment above multilist_sublist_insert_head */
	void
	multilist_sublist_insert_tail(multilist_sublist_t mls, void obj)
	{
	ASSERT(MUTEX_HELD(&mls->mls_lock));
	list_insert_tail(&mls->mls_list, obj);
	}

	+/* please see comment above multilist_sublist_insert_head */
	+void
	+multilist_sublist_insert_after(multilist_sublist_t mls, void prev, void *obj)
	+{
	+ ASSERT(MUTEX_HELD(&mls->mls_lock));
	+ list_insert_after(&mls->mls_list, prev, obj);
	+}
	+
	+/* please see comment above multilist_sublist_insert_head */
	+void
	+multilist_sublist_insert_before(multilist_sublist_t mls, void next, void *obj)
	+{
	+ ASSERT(MUTEX_HELD(&mls->mls_lock));
	+ list_insert_before(&mls->mls_list, next, obj);
	+}
	+
	/*
	* Move the object one element forward in the list.
	*
	* This function will move the given object forward in the list (towards
	* the head) by one object. So, in essence, it will swap its position in
	* the list with its "prev" pointer. If the given object is already at the
	* head of the list, it cannot be moved forward any more than it already
	* is, so no action is taken.
	*
	* NOTE: This function must not remove any object from the list other
	* than the object given as the parameter. This is relied upon in
	* arc_evict_state_impl().
	*/
	void
	multilist_sublist_move_forward(multilist_sublist_t mls, void obj)
	{
	void *prev = list_prev(&mls->mls_list, obj);

	ASSERT(MUTEX_HELD(&mls->mls_lock));
	ASSERT(!list_is_empty(&mls->mls_list));

	/* 'obj' must be at the head of the list, nothing to do */
	if (prev == NULL)
	return;

	list_remove(&mls->mls_list, obj);
	list_insert_before(&mls->mls_list, prev, obj);
	}

	void
	multilist_sublist_remove(multilist_sublist_t mls, void obj)
	{
	ASSERT(MUTEX_HELD(&mls->mls_lock));
	list_remove(&mls->mls_list, obj);
	}

	int
	multilist_sublist_is_empty(multilist_sublist_t *mls)
	{
	ASSERT(MUTEX_HELD(&mls->mls_lock));
	return (list_is_empty(&mls->mls_list));
	}

	int
	multilist_sublist_is_empty_idx(multilist_t *ml, unsigned int sublist_idx)
	{
	multilist_sublist_t *mls;
	int empty;

	ASSERT3U(sublist_idx, <, ml->ml_num_sublists);
	mls = &ml->ml_sublists[sublist_idx];
	ASSERT(!MUTEX_HELD(&mls->mls_lock));
	mutex_enter(&mls->mls_lock);
	empty = list_is_empty(&mls->mls_list);
	mutex_exit(&mls->mls_lock);
	return (empty);
	}

	void *
	multilist_sublist_head(multilist_sublist_t *mls)
	{
	ASSERT(MUTEX_HELD(&mls->mls_lock));
	return (list_head(&mls->mls_list));
	}

	void *
	multilist_sublist_tail(multilist_sublist_t *mls)
	{
	ASSERT(MUTEX_HELD(&mls->mls_lock));
	return (list_tail(&mls->mls_list));
	}

	void *
	multilist_sublist_next(multilist_sublist_t mls, void obj)
	{
	ASSERT(MUTEX_HELD(&mls->mls_lock));
	return (list_next(&mls->mls_list, obj));
	}

	void *
	multilist_sublist_prev(multilist_sublist_t mls, void obj)
	{
	ASSERT(MUTEX_HELD(&mls->mls_lock));
	return (list_prev(&mls->mls_list, obj));
	}

	void
	multilist_link_init(multilist_node_t *link)
	{
	list_link_init(link);
	}

	int
	multilist_link_active(multilist_node_t *link)
	{
	return (list_link_active(link));
	}

	ZFS_MODULE_PARAM(zfs, zfs_, multilist_num_sublists, UINT, ZMOD_RW,
	"Number of sublists used in each multilist");
	diff --git a/sys/contrib/openzfs/module/zfs/spa.c b/sys/contrib/openzfs/module/zfs/spa.c
	index d7fe96cde6a4..251dd8a4d1c7 100644
	--- a/sys/contrib/openzfs/module/zfs/spa.c
	+++ b/sys/contrib/openzfs/module/zfs/spa.c
	@@ -1,10491 +1,10604 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	- * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
	+ * Copyright (c) 2011, 2024 by Delphix. All rights reserved.
	* Copyright (c) 2018, Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
	* Copyright 2013 Saso Kiselkov. All rights reserved.
	* Copyright (c) 2014 Integros [integros.com]
	* Copyright 2016 Toomas Soome <tsoome@me.com>
	* Copyright (c) 2016 Actifio, Inc. All rights reserved.
	* Copyright 2018 Joyent, Inc.
	* Copyright (c) 2017, 2019, Datto Inc. All rights reserved.
	* Copyright 2017 Joyent, Inc.
	* Copyright (c) 2017, Intel Corporation.
	* Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
	* Copyright (c) 2023 Hewlett Packard Enterprise Development LP.
	*/

	/*
	* SPA: Storage Pool Allocator
	*
	* This file contains all the routines used when modifying on-disk SPA state.
	* This includes opening, importing, destroying, exporting a pool, and syncing a
	* pool.
	*/

	#include <sys/zfs_context.h>
	#include <sys/fm/fs/zfs.h>
	#include <sys/spa_impl.h>
	#include <sys/zio.h>
	#include <sys/zio_checksum.h>
	#include <sys/dmu.h>
	#include <sys/dmu_tx.h>
	#include <sys/zap.h>
	#include <sys/zil.h>
	#include <sys/brt.h>
	#include <sys/ddt.h>
	#include <sys/vdev_impl.h>
	#include <sys/vdev_removal.h>
	#include <sys/vdev_indirect_mapping.h>
	#include <sys/vdev_indirect_births.h>
	#include <sys/vdev_initialize.h>
	#include <sys/vdev_rebuild.h>
	#include <sys/vdev_trim.h>
	#include <sys/vdev_disk.h>
	#include <sys/vdev_draid.h>
	#include <sys/metaslab.h>
	#include <sys/metaslab_impl.h>
	#include <sys/mmp.h>
	#include <sys/uberblock_impl.h>
	#include <sys/txg.h>
	#include <sys/avl.h>
	#include <sys/bpobj.h>
	#include <sys/dmu_traverse.h>
	#include <sys/dmu_objset.h>
	#include <sys/unique.h>
	#include <sys/dsl_pool.h>
	#include <sys/dsl_dataset.h>
	#include <sys/dsl_dir.h>
	#include <sys/dsl_prop.h>
	#include <sys/dsl_synctask.h>
	#include <sys/fs/zfs.h>
	#include <sys/arc.h>
	#include <sys/callb.h>
	#include <sys/systeminfo.h>
	#include <sys/zfs_ioctl.h>
	#include <sys/dsl_scan.h>
	#include <sys/zfeature.h>
	#include <sys/dsl_destroy.h>
	#include <sys/zvol.h>

	#ifdef _KERNEL
	#include <sys/fm/protocol.h>
	#include <sys/fm/util.h>
	#include <sys/callb.h>
	#include <sys/zone.h>
	#include <sys/vmsystm.h>
	#endif /* _KERNEL */

	#include "zfs_prop.h"
	#include "zfs_comutil.h"

	/*
	* The interval, in seconds, at which failed configuration cache file writes
	* should be retried.
	*/
	int zfs_ccw_retry_interval = 300;

	typedef enum zti_modes {
	ZTI_MODE_FIXED, /* value is # of threads (min 1) */
	ZTI_MODE_BATCH, /* cpu-intensive; value is ignored */
	ZTI_MODE_SCALE, /* Taskqs scale with CPUs. */
	ZTI_MODE_NULL, /* don't create a taskq */
	ZTI_NMODES
	} zti_modes_t;

	#define ZTI_P(n, q) { ZTI_MODE_FIXED, (n), (q) }
	#define ZTI_PCT(n) { ZTI_MODE_ONLINE_PERCENT, (n), 1 }
	#define ZTI_BATCH { ZTI_MODE_BATCH, 0, 1 }
	#define ZTI_SCALE { ZTI_MODE_SCALE, 0, 1 }
	#define ZTI_NULL { ZTI_MODE_NULL, 0, 0 }

	#define ZTI_N(n) ZTI_P(n, 1)
	#define ZTI_ONE ZTI_N(1)

	typedef struct zio_taskq_info {
	zti_modes_t zti_mode;
	uint_t zti_value;
	uint_t zti_count;
	} zio_taskq_info_t;

	static const char *const zio_taskq_types[ZIO_TASKQ_TYPES] = {
	"iss", "iss_h", "int", "int_h"
	};

	/*
	* This table defines the taskq settings for each ZFS I/O type. When
	* initializing a pool, we use this table to create an appropriately sized
	* taskq. Some operations are low volume and therefore have a small, static
	* number of threads assigned to their taskqs using the ZTI_N(#) or ZTI_ONE
	* macros. Other operations process a large amount of data; the ZTI_BATCH
	* macro causes us to create a taskq oriented for throughput. Some operations
	* are so high frequency and short-lived that the taskq itself can become a
	* point of lock contention. The ZTI_P(#, #) macro indicates that we need an
	* additional degree of parallelism specified by the number of threads per-
	* taskq and the number of taskqs; when dispatching an event in this case, the
	* particular taskq is chosen at random. ZTI_SCALE is similar to ZTI_BATCH,
	* but with number of taskqs also scaling with number of CPUs.
	*
	* The different taskq priorities are to handle the different contexts (issue
	* and interrupt) and then to reserve threads for ZIO_PRIORITY_NOW I/Os that
	* need to be handled with minimum delay.
	*/
	static zio_taskq_info_t zio_taskqs[ZIO_TYPES][ZIO_TASKQ_TYPES] = {
	/* ISSUE ISSUE_HIGH INTR INTR_HIGH */
	{ ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* NULL */
	{ ZTI_N(8), ZTI_NULL, ZTI_SCALE, ZTI_NULL }, /* READ */
	{ ZTI_BATCH, ZTI_N(5), ZTI_SCALE, ZTI_N(5) }, /* WRITE */
	{ ZTI_SCALE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* FREE */
	{ ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* CLAIM */
	{ ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* IOCTL */
	{ ZTI_N(4), ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* TRIM */
	};

	static void spa_sync_version(void arg, dmu_tx_t tx);
	static void spa_sync_props(void arg, dmu_tx_t tx);
	static boolean_t spa_has_active_shared_spare(spa_t *spa);
	static int spa_load_impl(spa_t *spa, spa_import_type_t type,
	const char **ereport);
	static void spa_vdev_resilver_done(spa_t *spa);

	/*
	* Percentage of all CPUs that can be used by the metaslab preload taskq.
	*/
	static uint_t metaslab_preload_pct = 50;

	static uint_t zio_taskq_batch_pct = 80; /* 1 thread per cpu in pset */
	static uint_t zio_taskq_batch_tpq; /* threads per taskq */
	static const boolean_t zio_taskq_sysdc = B_TRUE; /* use SDC scheduling class */
	static const uint_t zio_taskq_basedc = 80; /* base duty cycle */

	static const boolean_t spa_create_process = B_TRUE; /* no process => no sysdc */

	/*
	* Report any spa_load_verify errors found, but do not fail spa_load.
	* This is used by zdb to analyze non-idle pools.
	*/
	boolean_t spa_load_verify_dryrun = B_FALSE;

	/*
	* Allow read spacemaps in case of readonly import (spa_mode == SPA_MODE_READ).
	* This is used by zdb for spacemaps verification.
	*/
	boolean_t spa_mode_readable_spacemaps = B_FALSE;

	/*
	* This (illegal) pool name is used when temporarily importing a spa_t in order
	* to get the vdev stats associated with the imported devices.
	*/
	#define TRYIMPORT_NAME "$import"

	/*
	* For debugging purposes: print out vdev tree during pool import.
	*/
	static int spa_load_print_vdev_tree = B_FALSE;

	/*
	* A non-zero value for zfs_max_missing_tvds means that we allow importing
	* pools with missing top-level vdevs. This is strictly intended for advanced
	* pool recovery cases since missing data is almost inevitable. Pools with
	* missing devices can only be imported read-only for safety reasons, and their
	* fail-mode will be automatically set to "continue".
	*
	* With 1 missing vdev we should be able to import the pool and mount all
	* datasets. User data that was not modified after the missing device has been
	* added should be recoverable. This means that snapshots created prior to the
	* addition of that device should be completely intact.
	*
	* With 2 missing vdevs, some datasets may fail to mount since there are
	* dataset statistics that are stored as regular metadata. Some data might be
	* recoverable if those vdevs were added recently.
	*
	* With 3 or more missing vdevs, the pool is severely damaged and MOS entries
	* may be missing entirely. Chances of data recovery are very low. Note that
	* there are also risks of performing an inadvertent rewind as we might be
	* missing all the vdevs with the latest uberblocks.
	*/
	uint64_t zfs_max_missing_tvds = 0;

	/*
	* The parameters below are similar to zfs_max_missing_tvds but are only
	* intended for a preliminary open of the pool with an untrusted config which
	* might be incomplete or out-dated.
	*
	* We are more tolerant for pools opened from a cachefile since we could have
	* an out-dated cachefile where a device removal was not registered.
	* We could have set the limit arbitrarily high but in the case where devices
	* are really missing we would want to return the proper error codes; we chose
	* SPA_DVAS_PER_BP - 1 so that some copies of the MOS would still be available
	* and we get a chance to retrieve the trusted config.
	*/
	uint64_t zfs_max_missing_tvds_cachefile = SPA_DVAS_PER_BP - 1;

	/*
	* In the case where config was assembled by scanning device paths (/dev/dsks
	* by default) we are less tolerant since all the existing devices should have
	* been detected and we want spa_load to return the right error codes.
	*/
	uint64_t zfs_max_missing_tvds_scan = 0;

	/*
	* Debugging aid that pauses spa_sync() towards the end.
	*/
	static const boolean_t zfs_pause_spa_sync = B_FALSE;

	/*
	* Variables to indicate the livelist condense zthr func should wait at certain
	* points for the livelist to be removed - used to test condense/destroy races
	*/
	static int zfs_livelist_condense_zthr_pause = 0;
	static int zfs_livelist_condense_sync_pause = 0;

	/*
	* Variables to track whether or not condense cancellation has been
	* triggered in testing.
	*/
	static int zfs_livelist_condense_sync_cancel = 0;
	static int zfs_livelist_condense_zthr_cancel = 0;

	/*
	* Variable to track whether or not extra ALLOC blkptrs were added to a
	* livelist entry while it was being condensed (caused by the way we track
	* remapped blkptrs in dbuf_remap_impl)
	*/
	static int zfs_livelist_condense_new_alloc = 0;

	/*
	* ==========================================================================
	* SPA properties routines
	* ==========================================================================
	*/

	/*
	* Add a (source=src, propname=propval) list to an nvlist.
	*/
	static void
	spa_prop_add_list(nvlist_t nvl, zpool_prop_t prop, const char strval,
	uint64_t intval, zprop_source_t src)
	{
	const char *propname = zpool_prop_to_name(prop);
	nvlist_t *propval;

	propval = fnvlist_alloc();
	fnvlist_add_uint64(propval, ZPROP_SOURCE, src);

	if (strval != NULL)
	fnvlist_add_string(propval, ZPROP_VALUE, strval);
	else
	fnvlist_add_uint64(propval, ZPROP_VALUE, intval);

	fnvlist_add_nvlist(nvl, propname, propval);
	nvlist_free(propval);
	}

	/*
	* Add a user property (source=src, propname=propval) to an nvlist.
	*/
	static void
	spa_prop_add_user(nvlist_t nvl, const char propname, char *strval,
	zprop_source_t src)
	{
	nvlist_t *propval;

	VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0);
	VERIFY(nvlist_add_uint64(propval, ZPROP_SOURCE, src) == 0);
	VERIFY(nvlist_add_string(propval, ZPROP_VALUE, strval) == 0);
	VERIFY(nvlist_add_nvlist(nvl, propname, propval) == 0);
	nvlist_free(propval);
	}

	/*
	* Get property values from the spa configuration.
	*/
	static void
	spa_prop_get_config(spa_t spa, nvlist_t *nvp)
	{
	vdev_t *rvd = spa->spa_root_vdev;
	dsl_pool_t *pool = spa->spa_dsl_pool;
	uint64_t size, alloc, cap, version;
	const zprop_source_t src = ZPROP_SRC_NONE;
	spa_config_dirent_t *dp;
	metaslab_class_t *mc = spa_normal_class(spa);

	ASSERT(MUTEX_HELD(&spa->spa_props_lock));

	if (rvd != NULL) {
	alloc = metaslab_class_get_alloc(mc);
	alloc += metaslab_class_get_alloc(spa_special_class(spa));
	alloc += metaslab_class_get_alloc(spa_dedup_class(spa));
	alloc += metaslab_class_get_alloc(spa_embedded_log_class(spa));

	size = metaslab_class_get_space(mc);
	size += metaslab_class_get_space(spa_special_class(spa));
	size += metaslab_class_get_space(spa_dedup_class(spa));
	size += metaslab_class_get_space(spa_embedded_log_class(spa));

	spa_prop_add_list(*nvp, ZPOOL_PROP_NAME, spa_name(spa), 0, src);
	spa_prop_add_list(*nvp, ZPOOL_PROP_SIZE, NULL, size, src);
	spa_prop_add_list(*nvp, ZPOOL_PROP_ALLOCATED, NULL, alloc, src);
	spa_prop_add_list(*nvp, ZPOOL_PROP_FREE, NULL,
	size - alloc, src);
	spa_prop_add_list(*nvp, ZPOOL_PROP_CHECKPOINT, NULL,
	spa->spa_checkpoint_info.sci_dspace, src);

	spa_prop_add_list(*nvp, ZPOOL_PROP_FRAGMENTATION, NULL,
	metaslab_class_fragmentation(mc), src);
	spa_prop_add_list(*nvp, ZPOOL_PROP_EXPANDSZ, NULL,
	metaslab_class_expandable_space(mc), src);
	spa_prop_add_list(*nvp, ZPOOL_PROP_READONLY, NULL,
	(spa_mode(spa) == SPA_MODE_READ), src);

	cap = (size == 0) ? 0 : (alloc * 100 / size);
	spa_prop_add_list(*nvp, ZPOOL_PROP_CAPACITY, NULL, cap, src);

	spa_prop_add_list(*nvp, ZPOOL_PROP_DEDUPRATIO, NULL,
	ddt_get_pool_dedup_ratio(spa), src);
	spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONEUSED, NULL,
	brt_get_used(spa), src);
	spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONESAVED, NULL,
	brt_get_saved(spa), src);
	spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONERATIO, NULL,
	brt_get_ratio(spa), src);

	spa_prop_add_list(*nvp, ZPOOL_PROP_HEALTH, NULL,
	rvd->vdev_state, src);

	version = spa_version(spa);
	if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION)) {
	spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL,
	version, ZPROP_SRC_DEFAULT);
	} else {
	spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL,
	version, ZPROP_SRC_LOCAL);
	}
	spa_prop_add_list(*nvp, ZPOOL_PROP_LOAD_GUID,
	NULL, spa_load_guid(spa), src);
	}

	if (pool != NULL) {
	/*
	* The $FREE directory was introduced in SPA_VERSION_DEADLISTS,
	* when opening pools before this version freedir will be NULL.
	*/
	if (pool->dp_free_dir != NULL) {
	spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, NULL,
	dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes,
	src);
	} else {
	spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING,
	NULL, 0, src);
	}

	if (pool->dp_leak_dir != NULL) {
	spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, NULL,
	dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes,
	src);
	} else {
	spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED,
	NULL, 0, src);
	}
	}

	spa_prop_add_list(*nvp, ZPOOL_PROP_GUID, NULL, spa_guid(spa), src);

	if (spa->spa_comment != NULL) {
	spa_prop_add_list(*nvp, ZPOOL_PROP_COMMENT, spa->spa_comment,
	0, ZPROP_SRC_LOCAL);
	}

	if (spa->spa_compatibility != NULL) {
	spa_prop_add_list(*nvp, ZPOOL_PROP_COMPATIBILITY,
	spa->spa_compatibility, 0, ZPROP_SRC_LOCAL);
	}

	if (spa->spa_root != NULL)
	spa_prop_add_list(*nvp, ZPOOL_PROP_ALTROOT, spa->spa_root,
	0, ZPROP_SRC_LOCAL);

	if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) {
	spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL,
	MIN(zfs_max_recordsize, SPA_MAXBLOCKSIZE), ZPROP_SRC_NONE);
	} else {
	spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL,
	SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE);
	}

	if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE)) {
	spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL,
	DNODE_MAX_SIZE, ZPROP_SRC_NONE);
	} else {
	spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL,
	DNODE_MIN_SIZE, ZPROP_SRC_NONE);
	}

	if ((dp = list_head(&spa->spa_config_list)) != NULL) {
	if (dp->scd_path == NULL) {
	spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE,
	"none", 0, ZPROP_SRC_LOCAL);
	} else if (strcmp(dp->scd_path, spa_config_path) != 0) {
	spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE,
	dp->scd_path, 0, ZPROP_SRC_LOCAL);
	}
	}
	}

	/*
	* Get zpool property values.
	*/
	int
	spa_prop_get(spa_t spa, nvlist_t *nvp)
	{
	objset_t *mos = spa->spa_meta_objset;
	zap_cursor_t zc;
	zap_attribute_t za;
	dsl_pool_t *dp;
	int err;

	err = nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP);
	if (err)
	return (err);

	dp = spa_get_dsl(spa);
	dsl_pool_config_enter(dp, FTAG);
	mutex_enter(&spa->spa_props_lock);

	/*
	* Get properties from the spa config.
	*/
	spa_prop_get_config(spa, nvp);

	/* If no pool property object, no more prop to get. */
	if (mos == NULL \|\| spa->spa_pool_props_object == 0)
	goto out;

	/*
	* Get properties from the MOS pool property object.
	*/
	for (zap_cursor_init(&zc, mos, spa->spa_pool_props_object);
	(err = zap_cursor_retrieve(&zc, &za)) == 0;
	zap_cursor_advance(&zc)) {
	uint64_t intval = 0;
	char *strval = NULL;
	zprop_source_t src = ZPROP_SRC_DEFAULT;
	zpool_prop_t prop;

	if ((prop = zpool_name_to_prop(za.za_name)) ==
	ZPOOL_PROP_INVAL && !zfs_prop_user(za.za_name))
	continue;

	switch (za.za_integer_length) {
	case 8:
	/* integer property */
	if (za.za_first_integer !=
	zpool_prop_default_numeric(prop))
	src = ZPROP_SRC_LOCAL;

	if (prop == ZPOOL_PROP_BOOTFS) {
	dsl_dataset_t *ds = NULL;

	err = dsl_dataset_hold_obj(dp,
	za.za_first_integer, FTAG, &ds);
	if (err != 0)
	break;

	strval = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN,
	KM_SLEEP);
	dsl_dataset_name(ds, strval);
	dsl_dataset_rele(ds, FTAG);
	} else {
	strval = NULL;
	intval = za.za_first_integer;
	}

	spa_prop_add_list(*nvp, prop, strval, intval, src);

	if (strval != NULL)
	kmem_free(strval, ZFS_MAX_DATASET_NAME_LEN);

	break;

	case 1:
	/* string property */
	strval = kmem_alloc(za.za_num_integers, KM_SLEEP);
	err = zap_lookup(mos, spa->spa_pool_props_object,
	za.za_name, 1, za.za_num_integers, strval);
	if (err) {
	kmem_free(strval, za.za_num_integers);
	break;
	}
	if (prop != ZPOOL_PROP_INVAL) {
	spa_prop_add_list(*nvp, prop, strval, 0, src);
	} else {
	src = ZPROP_SRC_LOCAL;
	spa_prop_add_user(*nvp, za.za_name, strval,
	src);
	}
	kmem_free(strval, za.za_num_integers);
	break;

	default:
	break;
	}
	}
	zap_cursor_fini(&zc);
	out:
	mutex_exit(&spa->spa_props_lock);
	dsl_pool_config_exit(dp, FTAG);
	if (err && err != ENOENT) {
	nvlist_free(*nvp);
	*nvp = NULL;
	return (err);
	}

	return (0);
	}

	/*
	* Validate the given pool properties nvlist and modify the list
	* for the property values to be set.
	*/
	static int
	spa_prop_validate(spa_t spa, nvlist_t props)
	{
	nvpair_t *elem;
	int error = 0, reset_bootfs = 0;
	uint64_t objnum = 0;
	boolean_t has_feature = B_FALSE;

	elem = NULL;
	while ((elem = nvlist_next_nvpair(props, elem)) != NULL) {
	uint64_t intval;
	const char strval, slash, check, fname;
	const char *propname = nvpair_name(elem);
	zpool_prop_t prop = zpool_name_to_prop(propname);

	switch (prop) {
	case ZPOOL_PROP_INVAL:
	/*
	* Sanitize the input.
	*/
	if (zfs_prop_user(propname)) {
	if (strlen(propname) >= ZAP_MAXNAMELEN) {
	error = SET_ERROR(ENAMETOOLONG);
	break;
	}

	if (strlen(fnvpair_value_string(elem)) >=
	ZAP_MAXVALUELEN) {
	error = SET_ERROR(E2BIG);
	break;
	}
	} else if (zpool_prop_feature(propname)) {
	if (nvpair_type(elem) != DATA_TYPE_UINT64) {
	error = SET_ERROR(EINVAL);
	break;
	}

	if (nvpair_value_uint64(elem, &intval) != 0) {
	error = SET_ERROR(EINVAL);
	break;
	}

	if (intval != 0) {
	error = SET_ERROR(EINVAL);
	break;
	}

	fname = strchr(propname, '@') + 1;
	if (zfeature_lookup_name(fname, NULL) != 0) {
	error = SET_ERROR(EINVAL);
	break;
	}

	has_feature = B_TRUE;
	} else {
	error = SET_ERROR(EINVAL);
	break;
	}
	break;

	case ZPOOL_PROP_VERSION:
	error = nvpair_value_uint64(elem, &intval);
	if (!error &&
	(intval < spa_version(spa) \|\|
	intval > SPA_VERSION_BEFORE_FEATURES \|\|
	has_feature))
	error = SET_ERROR(EINVAL);
	break;

	case ZPOOL_PROP_DELEGATION:
	case ZPOOL_PROP_AUTOREPLACE:
	case ZPOOL_PROP_LISTSNAPS:
	case ZPOOL_PROP_AUTOEXPAND:
	case ZPOOL_PROP_AUTOTRIM:
	error = nvpair_value_uint64(elem, &intval);
	if (!error && intval > 1)
	error = SET_ERROR(EINVAL);
	break;

	case ZPOOL_PROP_MULTIHOST:
	error = nvpair_value_uint64(elem, &intval);
	if (!error && intval > 1)
	error = SET_ERROR(EINVAL);

	if (!error) {
	uint32_t hostid = zone_get_hostid(NULL);
	if (hostid)
	spa->spa_hostid = hostid;
	else
	error = SET_ERROR(ENOTSUP);
	}

	break;

	case ZPOOL_PROP_BOOTFS:
	/*
	* If the pool version is less than SPA_VERSION_BOOTFS,
	* or the pool is still being created (version == 0),
	* the bootfs property cannot be set.
	*/
	if (spa_version(spa) < SPA_VERSION_BOOTFS) {
	error = SET_ERROR(ENOTSUP);
	break;
	}

	/*
	* Make sure the vdev config is bootable
	*/
	if (!vdev_is_bootable(spa->spa_root_vdev)) {
	error = SET_ERROR(ENOTSUP);
	break;
	}

	reset_bootfs = 1;

	error = nvpair_value_string(elem, &strval);

	if (!error) {
	objset_t *os;

	if (strval == NULL \|\| strval[0] == '\0') {
	objnum = zpool_prop_default_numeric(
	ZPOOL_PROP_BOOTFS);
	break;
	}

	error = dmu_objset_hold(strval, FTAG, &os);
	if (error != 0)
	break;

	/* Must be ZPL. */
	if (dmu_objset_type(os) != DMU_OST_ZFS) {
	error = SET_ERROR(ENOTSUP);
	} else {
	objnum = dmu_objset_id(os);
	}
	dmu_objset_rele(os, FTAG);
	}
	break;

	case ZPOOL_PROP_FAILUREMODE:
	error = nvpair_value_uint64(elem, &intval);
	if (!error && intval > ZIO_FAILURE_MODE_PANIC)
	error = SET_ERROR(EINVAL);

	/*
	* This is a special case which only occurs when
	* the pool has completely failed. This allows
	* the user to change the in-core failmode property
	* without syncing it out to disk (I/Os might
	* currently be blocked). We do this by returning
	* EIO to the caller (spa_prop_set) to trick it
	* into thinking we encountered a property validation
	* error.
	*/
	if (!error && spa_suspended(spa)) {
	spa->spa_failmode = intval;
	error = SET_ERROR(EIO);
	}
	break;

	case ZPOOL_PROP_CACHEFILE:
	if ((error = nvpair_value_string(elem, &strval)) != 0)
	break;

	if (strval[0] == '\0')
	break;

	if (strcmp(strval, "none") == 0)
	break;

	if (strval[0] != '/') {
	error = SET_ERROR(EINVAL);
	break;
	}

	slash = strrchr(strval, '/');
	ASSERT(slash != NULL);

	if (slash[1] == '\0' \|\| strcmp(slash, "/.") == 0 \|\|
	strcmp(slash, "/..") == 0)
	error = SET_ERROR(EINVAL);
	break;

	case ZPOOL_PROP_COMMENT:
	if ((error = nvpair_value_string(elem, &strval)) != 0)
	break;
	for (check = strval; *check != '\0'; check++) {
	if (!isprint(*check)) {
	error = SET_ERROR(EINVAL);
	break;
	}
	}
	if (strlen(strval) > ZPROP_MAX_COMMENT)
	error = SET_ERROR(E2BIG);
	break;

	default:
	break;
	}

	if (error)
	break;
	}

	(void) nvlist_remove_all(props,
	zpool_prop_to_name(ZPOOL_PROP_DEDUPDITTO));

	if (!error && reset_bootfs) {
	error = nvlist_remove(props,
	zpool_prop_to_name(ZPOOL_PROP_BOOTFS), DATA_TYPE_STRING);

	if (!error) {
	error = nvlist_add_uint64(props,
	zpool_prop_to_name(ZPOOL_PROP_BOOTFS), objnum);
	}
	}

	return (error);
	}

	void
	spa_configfile_set(spa_t spa, nvlist_t nvp, boolean_t need_sync)
	{
	const char *cachefile;
	spa_config_dirent_t *dp;

	if (nvlist_lookup_string(nvp, zpool_prop_to_name(ZPOOL_PROP_CACHEFILE),
	&cachefile) != 0)
	return;

	dp = kmem_alloc(sizeof (spa_config_dirent_t),
	KM_SLEEP);

	if (cachefile[0] == '\0')
	dp->scd_path = spa_strdup(spa_config_path);
	else if (strcmp(cachefile, "none") == 0)
	dp->scd_path = NULL;
	else
	dp->scd_path = spa_strdup(cachefile);

	list_insert_head(&spa->spa_config_list, dp);
	if (need_sync)
	spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
	}

	int
	spa_prop_set(spa_t spa, nvlist_t nvp)
	{
	int error;
	nvpair_t *elem = NULL;
	boolean_t need_sync = B_FALSE;

	if ((error = spa_prop_validate(spa, nvp)) != 0)
	return (error);

	while ((elem = nvlist_next_nvpair(nvp, elem)) != NULL) {
	zpool_prop_t prop = zpool_name_to_prop(nvpair_name(elem));

	if (prop == ZPOOL_PROP_CACHEFILE \|\|
	prop == ZPOOL_PROP_ALTROOT \|\|
	prop == ZPOOL_PROP_READONLY)
	continue;

	if (prop == ZPOOL_PROP_INVAL &&
	zfs_prop_user(nvpair_name(elem))) {
	need_sync = B_TRUE;
	break;
	}

	if (prop == ZPOOL_PROP_VERSION \|\| prop == ZPOOL_PROP_INVAL) {
	uint64_t ver = 0;

	if (prop == ZPOOL_PROP_VERSION) {
	VERIFY(nvpair_value_uint64(elem, &ver) == 0);
	} else {
	ASSERT(zpool_prop_feature(nvpair_name(elem)));
	ver = SPA_VERSION_FEATURES;
	need_sync = B_TRUE;
	}

	/* Save time if the version is already set. */
	if (ver == spa_version(spa))
	continue;

	/*
	* In addition to the pool directory object, we might
	* create the pool properties object, the features for
	* read object, the features for write object, or the
	* feature descriptions object.
	*/
	error = dsl_sync_task(spa->spa_name, NULL,
	spa_sync_version, &ver,
	6, ZFS_SPACE_CHECK_RESERVED);
	if (error)
	return (error);
	continue;
	}

	need_sync = B_TRUE;
	break;
	}

	if (need_sync) {
	return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props,
	nvp, 6, ZFS_SPACE_CHECK_RESERVED));
	}

	return (0);
	}

	/*
	* If the bootfs property value is dsobj, clear it.
	*/
	void
	spa_prop_clear_bootfs(spa_t spa, uint64_t dsobj, dmu_tx_t tx)
	{
	if (spa->spa_bootfs == dsobj && spa->spa_pool_props_object != 0) {
	VERIFY(zap_remove(spa->spa_meta_objset,
	spa->spa_pool_props_object,
	zpool_prop_to_name(ZPOOL_PROP_BOOTFS), tx) == 0);
	spa->spa_bootfs = 0;
	}
	}

	static int
	spa_change_guid_check(void arg, dmu_tx_t tx)
	{
	uint64_t *newguid __maybe_unused = arg;
	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
	vdev_t *rvd = spa->spa_root_vdev;
	uint64_t vdev_state;

	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
	int error = (spa_has_checkpoint(spa)) ?
	ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
	return (SET_ERROR(error));
	}

	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
	vdev_state = rvd->vdev_state;
	spa_config_exit(spa, SCL_STATE, FTAG);

	if (vdev_state != VDEV_STATE_HEALTHY)
	return (SET_ERROR(ENXIO));

	ASSERT3U(spa_guid(spa), !=, *newguid);

	return (0);
	}

	static void
	spa_change_guid_sync(void arg, dmu_tx_t tx)
	{
	uint64_t *newguid = arg;
	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
	uint64_t oldguid;
	vdev_t *rvd = spa->spa_root_vdev;

	oldguid = spa_guid(spa);

	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
	rvd->vdev_guid = *newguid;
	rvd->vdev_guid_sum += (*newguid - oldguid);
	vdev_config_dirty(rvd);
	spa_config_exit(spa, SCL_STATE, FTAG);

	spa_history_log_internal(spa, "guid change", tx, "old=%llu new=%llu",
	(u_longlong_t)oldguid, (u_longlong_t)*newguid);
	}

	/*
	* Change the GUID for the pool. This is done so that we can later
	* re-import a pool built from a clone of our own vdevs. We will modify
	* the root vdev's guid, our own pool guid, and then mark all of our
	* vdevs dirty. Note that we must make sure that all our vdevs are
	* online when we do this, or else any vdevs that weren't present
	* would be orphaned from our pool. We are also going to issue a
	* sysevent to update any watchers.
	*/
	int
	spa_change_guid(spa_t *spa)
	{
	int error;
	uint64_t guid;

	mutex_enter(&spa->spa_vdev_top_lock);
	mutex_enter(&spa_namespace_lock);
	guid = spa_generate_guid(NULL);

	error = dsl_sync_task(spa->spa_name, spa_change_guid_check,
	spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED);

	if (error == 0) {
	/*
	* Clear the kobj flag from all the vdevs to allow
	* vdev_cache_process_kobj_evt() to post events to all the
	* vdevs since GUID is updated.
	*/
	vdev_clear_kobj_evt(spa->spa_root_vdev);
	for (int i = 0; i < spa->spa_l2cache.sav_count; i++)
	vdev_clear_kobj_evt(spa->spa_l2cache.sav_vdevs[i]);

	spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE);
	spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_REGUID);
	}

	mutex_exit(&spa_namespace_lock);
	mutex_exit(&spa->spa_vdev_top_lock);

	return (error);
	}

	/*
	* ==========================================================================
	* SPA state manipulation (open/create/destroy/import/export)
	* ==========================================================================
	*/

	static int
	spa_error_entry_compare(const void a, const void b)
	{
	const spa_error_entry_t sa = (const spa_error_entry_t )a;
	const spa_error_entry_t sb = (const spa_error_entry_t )b;
	int ret;

	ret = memcmp(&sa->se_bookmark, &sb->se_bookmark,
	sizeof (zbookmark_phys_t));

	return (TREE_ISIGN(ret));
	}

	/*
	* Utility function which retrieves copies of the current logs and
	* re-initializes them in the process.
	*/
	void
	spa_get_errlists(spa_t spa, avl_tree_t last, avl_tree_t *scrub)
	{
	ASSERT(MUTEX_HELD(&spa->spa_errlist_lock));

	memcpy(last, &spa->spa_errlist_last, sizeof (avl_tree_t));
	memcpy(scrub, &spa->spa_errlist_scrub, sizeof (avl_tree_t));

	avl_create(&spa->spa_errlist_scrub,
	spa_error_entry_compare, sizeof (spa_error_entry_t),
	offsetof(spa_error_entry_t, se_avl));
	avl_create(&spa->spa_errlist_last,
	spa_error_entry_compare, sizeof (spa_error_entry_t),
	offsetof(spa_error_entry_t, se_avl));
	}

	static void
	spa_taskqs_init(spa_t *spa, zio_type_t t, zio_taskq_type_t q)
	{
	const zio_taskq_info_t *ztip = &zio_taskqs[t][q];
	enum zti_modes mode = ztip->zti_mode;
	uint_t value = ztip->zti_value;
	uint_t count = ztip->zti_count;
	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
	uint_t cpus, flags = TASKQ_DYNAMIC;
	boolean_t batch = B_FALSE;

	switch (mode) {
	case ZTI_MODE_FIXED:
	ASSERT3U(value, >, 0);
	break;

	case ZTI_MODE_BATCH:
	batch = B_TRUE;
	flags \|= TASKQ_THREADS_CPU_PCT;
	value = MIN(zio_taskq_batch_pct, 100);
	break;

	case ZTI_MODE_SCALE:
	flags \|= TASKQ_THREADS_CPU_PCT;
	/*
	* We want more taskqs to reduce lock contention, but we want
	* less for better request ordering and CPU utilization.
	*/
	cpus = MAX(1, boot_ncpus * zio_taskq_batch_pct / 100);
	if (zio_taskq_batch_tpq > 0) {
	count = MAX(1, (cpus + zio_taskq_batch_tpq / 2) /
	zio_taskq_batch_tpq);
	} else {
	/*
	* Prefer 6 threads per taskq, but no more taskqs
	* than threads in them on large systems. For 80%:
	*
	* taskq taskq total
	* cpus taskqs percent threads threads
	* ------- ------- ------- ------- -------
	* 1 1 80% 1 1
	* 2 1 80% 1 1
	* 4 1 80% 3 3
	* 8 2 40% 3 6
	* 16 3 27% 4 12
	* 32 5 16% 5 25
	* 64 7 11% 7 49
	* 128 10 8% 10 100
	* 256 14 6% 15 210
	*/
	count = 1 + cpus / 6;
	while (count * count > cpus)
	count--;
	}
	/* Limit each taskq within 100% to not trigger assertion. */
	count = MAX(count, (zio_taskq_batch_pct + 99) / 100);
	value = (zio_taskq_batch_pct + count / 2) / count;
	break;

	case ZTI_MODE_NULL:
	tqs->stqs_count = 0;
	tqs->stqs_taskq = NULL;
	return;

	default:
	panic("unrecognized mode for %s_%s taskq (%u:%u) in "
	"spa_activate()",
	zio_type_name[t], zio_taskq_types[q], mode, value);
	break;
	}

	ASSERT3U(count, >, 0);
	tqs->stqs_count = count;
	tqs->stqs_taskq = kmem_alloc(count * sizeof (taskq_t *), KM_SLEEP);

	for (uint_t i = 0; i < count; i++) {
	taskq_t *tq;
	char name[32];

	if (count > 1)
	(void) snprintf(name, sizeof (name), "%s_%s_%u",
	zio_type_name[t], zio_taskq_types[q], i);
	else
	(void) snprintf(name, sizeof (name), "%s_%s",
	zio_type_name[t], zio_taskq_types[q]);

	if (zio_taskq_sysdc && spa->spa_proc != &p0) {
	if (batch)
	flags \|= TASKQ_DC_BATCH;

	(void) zio_taskq_basedc;
	tq = taskq_create_sysdc(name, value, 50, INT_MAX,
	spa->spa_proc, zio_taskq_basedc, flags);
	} else {
	pri_t pri = maxclsyspri;
	/*
	* The write issue taskq can be extremely CPU
	* intensive. Run it at slightly less important
	* priority than the other taskqs.
	*
	* Under Linux and FreeBSD this means incrementing
	* the priority value as opposed to platforms like
	* illumos where it should be decremented.
	*
	* On FreeBSD, if priorities divided by four (RQ_PPQ)
	* are equal then a difference between them is
	* insignificant.
	*/
	if (t == ZIO_TYPE_WRITE && q == ZIO_TASKQ_ISSUE) {
	#if defined(__linux__)
	pri++;
	#elif defined(__FreeBSD__)
	pri += 4;
	#else
	#error "unknown OS"
	#endif
	}
	tq = taskq_create_proc(name, value, pri, 50,
	INT_MAX, spa->spa_proc, flags);
	}

	tqs->stqs_taskq[i] = tq;
	}
	}

	static void
	spa_taskqs_fini(spa_t *spa, zio_type_t t, zio_taskq_type_t q)
	{
	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];

	if (tqs->stqs_taskq == NULL) {
	ASSERT3U(tqs->stqs_count, ==, 0);
	return;
	}

	for (uint_t i = 0; i < tqs->stqs_count; i++) {
	ASSERT3P(tqs->stqs_taskq[i], !=, NULL);
	taskq_destroy(tqs->stqs_taskq[i]);
	}

	kmem_free(tqs->stqs_taskq, tqs->stqs_count * sizeof (taskq_t *));
	tqs->stqs_taskq = NULL;
	}

	#ifdef _KERNEL
	/*
	* The READ and WRITE rows of zio_taskqs are configurable at module load time
	* by setting zio_taskq_read or zio_taskq_write.
	*
	* Example (the defaults for READ and WRITE)
	* zio_taskq_read='fixed,1,8 null scale null'
	* zio_taskq_write='batch fixed,1,5 scale fixed,1,5'
	*
	* Each sets the entire row at a time.
	*
	* 'fixed' is parameterised: fixed,Q,T where Q is number of taskqs, T is number
	* of threads per taskq.
	*
	* 'null' can only be set on the high-priority queues (queue selection for
	* high-priority queues will fall back to the regular queue if the high-pri
	* is NULL.
	*/
	static const char *const modes[ZTI_NMODES] = {
	"fixed", "batch", "scale", "null"
	};

	/* Parse the incoming config string. Modifies cfg */
	static int
	spa_taskq_param_set(zio_type_t t, char *cfg)
	{
	int err = 0;

	zio_taskq_info_t row[ZIO_TASKQ_TYPES] = {{0}};

	char next = cfg, tok, *c;

	/*
	* Parse out each element from the string and fill `row`. The entire
	* row has to be set at once, so any errors are flagged by just
	* breaking out of this loop early.
	*/
	uint_t q;
	for (q = 0; q < ZIO_TASKQ_TYPES; q++) {
	/* `next` is the start of the config */
	if (next == NULL)
	break;

	/* Eat up leading space */
	while (isspace(*next))
	next++;
	if (*next == '\0')
	break;

	/* Mode ends at space or end of string */
	tok = next;
	next = strchr(tok, ' ');
	if (next != NULL) *next++ = '\0';

	/* Parameters start after a comma */
	c = strchr(tok, ',');
	if (c != NULL) *c++ = '\0';

	/* Match mode string */
	uint_t mode;
	for (mode = 0; mode < ZTI_NMODES; mode++)
	if (strcmp(tok, modes[mode]) == 0)
	break;
	if (mode == ZTI_NMODES)
	break;

	/* Invalid canary */
	row[q].zti_mode = ZTI_NMODES;

	/* Per-mode setup */
	switch (mode) {

	/*
	* FIXED is parameterised: number of queues, and number of
	* threads per queue.
	*/
	case ZTI_MODE_FIXED: {
	/* No parameters? */
	if (c == NULL \|\| *c == '\0')
	break;

	/* Find next parameter */
	tok = c;
	c = strchr(tok, ',');
	if (c == NULL)
	break;

	/* Take digits and convert */
	unsigned long long nq;
	if (!(isdigit(*tok)))
	break;
	err = ddi_strtoull(tok, &tok, 10, &nq);
	/* Must succeed and also end at the next param sep */
	if (err != 0 \|\| tok != c)
	break;

	/* Move past the comma */
	tok++;
	/* Need another number */
	if (!(isdigit(*tok)))
	break;
	/* Remember start to make sure we moved */
	c = tok;

	/* Take digits */
	unsigned long long ntpq;
	err = ddi_strtoull(tok, &tok, 10, &ntpq);
	/* Must succeed, and moved forward */
	if (err != 0 \|\| tok == c \|\| *tok != '\0')
	break;

	/*
	* sanity; zero queues/threads make no sense, and
	* 16K is almost certainly more than anyone will ever
	* need and avoids silly numbers like UINT32_MAX
	*/
	if (nq == 0 \|\| nq >= 16384 \|\|
	ntpq == 0 \|\| ntpq >= 16384)
	break;

	const zio_taskq_info_t zti = ZTI_P(ntpq, nq);
	row[q] = zti;
	break;
	}

	case ZTI_MODE_BATCH: {
	const zio_taskq_info_t zti = ZTI_BATCH;
	row[q] = zti;
	break;
	}

	case ZTI_MODE_SCALE: {
	const zio_taskq_info_t zti = ZTI_SCALE;
	row[q] = zti;
	break;
	}

	case ZTI_MODE_NULL: {
	/*
	* Can only null the high-priority queues; the general-
	* purpose ones have to exist.
	*/
	if (q != ZIO_TASKQ_ISSUE_HIGH &&
	q != ZIO_TASKQ_INTERRUPT_HIGH)
	break;

	const zio_taskq_info_t zti = ZTI_NULL;
	row[q] = zti;
	break;
	}

	default:
	break;
	}

	/* Ensure we set a mode */
	if (row[q].zti_mode == ZTI_NMODES)
	break;
	}

	/* Didn't get a full row, fail */
	if (q < ZIO_TASKQ_TYPES)
	return (SET_ERROR(EINVAL));

	/* Eat trailing space */
	if (next != NULL)
	while (isspace(*next))
	next++;

	/* If there's anything left over then fail */
	if (next != NULL && *next != '\0')
	return (SET_ERROR(EINVAL));

	/* Success! Copy it into the real config */
	for (q = 0; q < ZIO_TASKQ_TYPES; q++)
	zio_taskqs[t][q] = row[q];

	return (0);
	}

	static int
	spa_taskq_param_get(zio_type_t t, char *buf, boolean_t add_newline)
	{
	int pos = 0;

	/* Build paramater string from live config */
	const char *sep = "";
	for (uint_t q = 0; q < ZIO_TASKQ_TYPES; q++) {
	const zio_taskq_info_t *zti = &zio_taskqs[t][q];
	if (zti->zti_mode == ZTI_MODE_FIXED)
	pos += sprintf(&buf[pos], "%s%s,%u,%u", sep,
	modes[zti->zti_mode], zti->zti_count,
	zti->zti_value);
	else
	pos += sprintf(&buf[pos], "%s%s", sep,
	modes[zti->zti_mode]);
	sep = " ";
	}

	if (add_newline)
	buf[pos++] = '\n';
	buf[pos] = '\0';

	return (pos);
	}

	#ifdef __linux__
	static int
	spa_taskq_read_param_set(const char val, zfs_kernel_param_t kp)
	{
	char *cfg = kmem_strdup(val);
	int err = spa_taskq_param_set(ZIO_TYPE_READ, cfg);
	kmem_free(cfg, strlen(val)+1);
	return (-err);
	}
	static int
	spa_taskq_read_param_get(char buf, zfs_kernel_param_t kp)
	{
	return (spa_taskq_param_get(ZIO_TYPE_READ, buf, TRUE));
	}

	static int
	spa_taskq_write_param_set(const char val, zfs_kernel_param_t kp)
	{
	char *cfg = kmem_strdup(val);
	int err = spa_taskq_param_set(ZIO_TYPE_WRITE, cfg);
	kmem_free(cfg, strlen(val)+1);
	return (-err);
	}
	static int
	spa_taskq_write_param_get(char buf, zfs_kernel_param_t kp)
	{
	return (spa_taskq_param_get(ZIO_TYPE_WRITE, buf, TRUE));
	}
	#else
	/*
	* On FreeBSD load-time parameters can be set up before malloc() is available,
	* so we have to do all the parsing work on the stack.
	*/
	#define SPA_TASKQ_PARAM_MAX (128)

	static int
	spa_taskq_read_param(ZFS_MODULE_PARAM_ARGS)
	{
	char buf[SPA_TASKQ_PARAM_MAX];
	int err;

	(void) spa_taskq_param_get(ZIO_TYPE_READ, buf, FALSE);
	err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
	if (err \|\| req->newptr == NULL)
	return (err);
	return (spa_taskq_param_set(ZIO_TYPE_READ, buf));
	}

	static int
	spa_taskq_write_param(ZFS_MODULE_PARAM_ARGS)
	{
	char buf[SPA_TASKQ_PARAM_MAX];
	int err;

	(void) spa_taskq_param_get(ZIO_TYPE_WRITE, buf, FALSE);
	err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
	if (err \|\| req->newptr == NULL)
	return (err);
	return (spa_taskq_param_set(ZIO_TYPE_WRITE, buf));
	}
	#endif
	#endif /* _KERNEL */

	/*
	* Dispatch a task to the appropriate taskq for the ZFS I/O type and priority.
	* Note that a type may have multiple discrete taskqs to avoid lock contention
	* on the taskq itself. In that case we choose which taskq at random by using
	* the low bits of gethrtime().
	*/
	void
	spa_taskq_dispatch_ent(spa_t *spa, zio_type_t t, zio_taskq_type_t q,
	task_func_t func, void arg, uint_t flags, taskq_ent_t *ent)
	{
	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
	taskq_t *tq;

	ASSERT3P(tqs->stqs_taskq, !=, NULL);
	ASSERT3U(tqs->stqs_count, !=, 0);

	if (tqs->stqs_count == 1) {
	tq = tqs->stqs_taskq[0];
	} else {
	tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count];
	}

	taskq_dispatch_ent(tq, func, arg, flags, ent);
	}

	/*
	* Same as spa_taskq_dispatch_ent() but block on the task until completion.
	*/
	void
	spa_taskq_dispatch_sync(spa_t *spa, zio_type_t t, zio_taskq_type_t q,
	task_func_t func, void arg, uint_t flags)
	{
	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
	taskq_t *tq;
	taskqid_t id;

	ASSERT3P(tqs->stqs_taskq, !=, NULL);
	ASSERT3U(tqs->stqs_count, !=, 0);

	if (tqs->stqs_count == 1) {
	tq = tqs->stqs_taskq[0];
	} else {
	tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count];
	}

	id = taskq_dispatch(tq, func, arg, flags);
	if (id)
	taskq_wait_id(tq, id);
	}

	static void
	spa_create_zio_taskqs(spa_t *spa)
	{
	for (int t = 0; t < ZIO_TYPES; t++) {
	for (int q = 0; q < ZIO_TASKQ_TYPES; q++) {
	spa_taskqs_init(spa, t, q);
	}
	}
	}

	/*
	* Disabled until spa_thread() can be adapted for Linux.
	*/
	#undef HAVE_SPA_THREAD

	#if defined(_KERNEL) && defined(HAVE_SPA_THREAD)
	static void
	spa_thread(void *arg)
	{
	psetid_t zio_taskq_psrset_bind = PS_NONE;
	callb_cpr_t cprinfo;

	spa_t *spa = arg;
	user_t *pu = PTOU(curproc);

	CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr,
	spa->spa_name);

	ASSERT(curproc != &p0);
	(void) snprintf(pu->u_psargs, sizeof (pu->u_psargs),
	"zpool-%s", spa->spa_name);
	(void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm));

	/* bind this thread to the requested psrset */
	if (zio_taskq_psrset_bind != PS_NONE) {
	pool_lock();
	mutex_enter(&cpu_lock);
	mutex_enter(&pidlock);
	mutex_enter(&curproc->p_lock);

	if (cpupart_bind_thread(curthread, zio_taskq_psrset_bind,
	0, NULL, NULL) == 0) {
	curthread->t_bind_pset = zio_taskq_psrset_bind;
	} else {
	cmn_err(CE_WARN,
	"Couldn't bind process for zfs pool \"%s\" to "
	"pset %d\n", spa->spa_name, zio_taskq_psrset_bind);
	}

	mutex_exit(&curproc->p_lock);
	mutex_exit(&pidlock);
	mutex_exit(&cpu_lock);
	pool_unlock();
	}

	if (zio_taskq_sysdc) {
	sysdc_thread_enter(curthread, 100, 0);
	}

	spa->spa_proc = curproc;
	spa->spa_did = curthread->t_did;

	spa_create_zio_taskqs(spa);

	mutex_enter(&spa->spa_proc_lock);
	ASSERT(spa->spa_proc_state == SPA_PROC_CREATED);

	spa->spa_proc_state = SPA_PROC_ACTIVE;
	cv_broadcast(&spa->spa_proc_cv);

	CALLB_CPR_SAFE_BEGIN(&cprinfo);
	while (spa->spa_proc_state == SPA_PROC_ACTIVE)
	cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock);
	CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_proc_lock);

	ASSERT(spa->spa_proc_state == SPA_PROC_DEACTIVATE);
	spa->spa_proc_state = SPA_PROC_GONE;
	spa->spa_proc = &p0;
	cv_broadcast(&spa->spa_proc_cv);
	CALLB_CPR_EXIT(&cprinfo); /* drops spa_proc_lock */

	mutex_enter(&curproc->p_lock);
	lwp_exit();
	}
	#endif

	/*
	* Activate an uninitialized pool.
	*/
	static void
	spa_activate(spa_t *spa, spa_mode_t mode)
	{
	ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED);

	spa->spa_state = POOL_STATE_ACTIVE;
	spa->spa_mode = mode;
	spa->spa_read_spacemaps = spa_mode_readable_spacemaps;

	spa->spa_normal_class = metaslab_class_create(spa, &zfs_metaslab_ops);
	spa->spa_log_class = metaslab_class_create(spa, &zfs_metaslab_ops);
	spa->spa_embedded_log_class =
	metaslab_class_create(spa, &zfs_metaslab_ops);
	spa->spa_special_class = metaslab_class_create(spa, &zfs_metaslab_ops);
	spa->spa_dedup_class = metaslab_class_create(spa, &zfs_metaslab_ops);

	/* Try to create a covering process */
	mutex_enter(&spa->spa_proc_lock);
	ASSERT(spa->spa_proc_state == SPA_PROC_NONE);
	ASSERT(spa->spa_proc == &p0);
	spa->spa_did = 0;

	(void) spa_create_process;
	#ifdef HAVE_SPA_THREAD
	/* Only create a process if we're going to be around a while. */
	if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) {
	if (newproc(spa_thread, (caddr_t)spa, syscid, maxclsyspri,
	NULL, 0) == 0) {
	spa->spa_proc_state = SPA_PROC_CREATED;
	while (spa->spa_proc_state == SPA_PROC_CREATED) {
	cv_wait(&spa->spa_proc_cv,
	&spa->spa_proc_lock);
	}
	ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE);
	ASSERT(spa->spa_proc != &p0);
	ASSERT(spa->spa_did != 0);
	} else {
	#ifdef _KERNEL
	cmn_err(CE_WARN,
	"Couldn't create process for zfs pool \"%s\"\n",
	spa->spa_name);
	#endif
	}
	}
	#endif /* HAVE_SPA_THREAD */
	mutex_exit(&spa->spa_proc_lock);

	/* If we didn't create a process, we need to create our taskqs. */
	if (spa->spa_proc == &p0) {
	spa_create_zio_taskqs(spa);
	}

	for (size_t i = 0; i < TXG_SIZE; i++) {
	spa->spa_txg_zio[i] = zio_root(spa, NULL, NULL,
	ZIO_FLAG_CANFAIL);
	}

	list_create(&spa->spa_config_dirty_list, sizeof (vdev_t),
	offsetof(vdev_t, vdev_config_dirty_node));
	list_create(&spa->spa_evicting_os_list, sizeof (objset_t),
	offsetof(objset_t, os_evicting_node));
	list_create(&spa->spa_state_dirty_list, sizeof (vdev_t),
	offsetof(vdev_t, vdev_state_dirty_node));

	txg_list_create(&spa->spa_vdev_txg_list, spa,
	offsetof(struct vdev, vdev_txg_node));

	avl_create(&spa->spa_errlist_scrub,
	spa_error_entry_compare, sizeof (spa_error_entry_t),
	offsetof(spa_error_entry_t, se_avl));
	avl_create(&spa->spa_errlist_last,
	spa_error_entry_compare, sizeof (spa_error_entry_t),
	offsetof(spa_error_entry_t, se_avl));
	avl_create(&spa->spa_errlist_healed,
	spa_error_entry_compare, sizeof (spa_error_entry_t),
	offsetof(spa_error_entry_t, se_avl));

	spa_activate_os(spa);

	spa_keystore_init(&spa->spa_keystore);

	/*
	* This taskq is used to perform zvol-minor-related tasks
	* asynchronously. This has several advantages, including easy
	* resolution of various deadlocks.
	*
	* The taskq must be single threaded to ensure tasks are always
	* processed in the order in which they were dispatched.
	*
	* A taskq per pool allows one to keep the pools independent.
	* This way if one pool is suspended, it will not impact another.
	*
	* The preferred location to dispatch a zvol minor task is a sync
	* task. In this context, there is easy access to the spa_t and minimal
	* error handling is required because the sync task must succeed.
	*/
	spa->spa_zvol_taskq = taskq_create("z_zvol", 1, defclsyspri,
	1, INT_MAX, 0);

	/*
	* The taskq to preload metaslabs.
	*/
	spa->spa_metaslab_taskq = taskq_create("z_metaslab",
	metaslab_preload_pct, maxclsyspri, 1, INT_MAX,
	TASKQ_DYNAMIC \| TASKQ_THREADS_CPU_PCT);

	/*
	* Taskq dedicated to prefetcher threads: this is used to prevent the
	* pool traverse code from monopolizing the global (and limited)
	* system_taskq by inappropriately scheduling long running tasks on it.
	*/
	spa->spa_prefetch_taskq = taskq_create("z_prefetch", 100,
	defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC \| TASKQ_THREADS_CPU_PCT);

	/*
	* The taskq to upgrade datasets in this pool. Currently used by
	* feature SPA_FEATURE_USEROBJ_ACCOUNTING/SPA_FEATURE_PROJECT_QUOTA.
	*/
	spa->spa_upgrade_taskq = taskq_create("z_upgrade", 100,
	defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC \| TASKQ_THREADS_CPU_PCT);
	}

	/*
	* Opposite of spa_activate().
	*/
	static void
	spa_deactivate(spa_t *spa)
	{
	ASSERT(spa->spa_sync_on == B_FALSE);
	ASSERT(spa->spa_dsl_pool == NULL);
	ASSERT(spa->spa_root_vdev == NULL);
	ASSERT(spa->spa_async_zio_root == NULL);
	ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED);

	spa_evicting_os_wait(spa);

	if (spa->spa_zvol_taskq) {
	taskq_destroy(spa->spa_zvol_taskq);
	spa->spa_zvol_taskq = NULL;
	}

	if (spa->spa_metaslab_taskq) {
	taskq_destroy(spa->spa_metaslab_taskq);
	spa->spa_metaslab_taskq = NULL;
	}

	if (spa->spa_prefetch_taskq) {
	taskq_destroy(spa->spa_prefetch_taskq);
	spa->spa_prefetch_taskq = NULL;
	}

	if (spa->spa_upgrade_taskq) {
	taskq_destroy(spa->spa_upgrade_taskq);
	spa->spa_upgrade_taskq = NULL;
	}

	txg_list_destroy(&spa->spa_vdev_txg_list);

	list_destroy(&spa->spa_config_dirty_list);
	list_destroy(&spa->spa_evicting_os_list);
	list_destroy(&spa->spa_state_dirty_list);

	taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid);

	for (int t = 0; t < ZIO_TYPES; t++) {
	for (int q = 0; q < ZIO_TASKQ_TYPES; q++) {
	spa_taskqs_fini(spa, t, q);
	}
	}

	for (size_t i = 0; i < TXG_SIZE; i++) {
	ASSERT3P(spa->spa_txg_zio[i], !=, NULL);
	VERIFY0(zio_wait(spa->spa_txg_zio[i]));
	spa->spa_txg_zio[i] = NULL;
	}

	metaslab_class_destroy(spa->spa_normal_class);
	spa->spa_normal_class = NULL;

	metaslab_class_destroy(spa->spa_log_class);
	spa->spa_log_class = NULL;

	metaslab_class_destroy(spa->spa_embedded_log_class);
	spa->spa_embedded_log_class = NULL;

	metaslab_class_destroy(spa->spa_special_class);
	spa->spa_special_class = NULL;

	metaslab_class_destroy(spa->spa_dedup_class);
	spa->spa_dedup_class = NULL;

	/*
	* If this was part of an import or the open otherwise failed, we may
	* still have errors left in the queues. Empty them just in case.
	*/
	spa_errlog_drain(spa);
	avl_destroy(&spa->spa_errlist_scrub);
	avl_destroy(&spa->spa_errlist_last);
	avl_destroy(&spa->spa_errlist_healed);

	spa_keystore_fini(&spa->spa_keystore);

	spa->spa_state = POOL_STATE_UNINITIALIZED;

	mutex_enter(&spa->spa_proc_lock);
	if (spa->spa_proc_state != SPA_PROC_NONE) {
	ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE);
	spa->spa_proc_state = SPA_PROC_DEACTIVATE;
	cv_broadcast(&spa->spa_proc_cv);
	while (spa->spa_proc_state == SPA_PROC_DEACTIVATE) {
	ASSERT(spa->spa_proc != &p0);
	cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock);
	}
	ASSERT(spa->spa_proc_state == SPA_PROC_GONE);
	spa->spa_proc_state = SPA_PROC_NONE;
	}
	ASSERT(spa->spa_proc == &p0);
	mutex_exit(&spa->spa_proc_lock);

	/*
	* We want to make sure spa_thread() has actually exited the ZFS
	* module, so that the module can't be unloaded out from underneath
	* it.
	*/
	if (spa->spa_did != 0) {
	thread_join(spa->spa_did);
	spa->spa_did = 0;
	}

	spa_deactivate_os(spa);

	}

	/*
	* Verify a pool configuration, and construct the vdev tree appropriately. This
	* will create all the necessary vdevs in the appropriate layout, with each vdev
	* in the CLOSED state. This will prep the pool before open/creation/import.
	* All vdev validation is done by the vdev_alloc() routine.
	*/
	int
	spa_config_parse(spa_t spa, vdev_t vdp, nvlist_t nv, vdev_t *parent,
	uint_t id, int atype)
	{
	nvlist_t **child;
	uint_t children;
	int error;

	if ((error = vdev_alloc(spa, vdp, nv, parent, id, atype)) != 0)
	return (error);

	if ((*vdp)->vdev_ops->vdev_op_leaf)
	return (0);

	error = nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
	&child, &children);

	if (error == ENOENT)
	return (0);

	if (error) {
	vdev_free(*vdp);
	*vdp = NULL;
	return (SET_ERROR(EINVAL));
	}

	for (int c = 0; c < children; c++) {
	vdev_t *vd;
	if ((error = spa_config_parse(spa, &vd, child[c], *vdp, c,
	atype)) != 0) {
	vdev_free(*vdp);
	*vdp = NULL;
	return (error);
	}
	}

	ASSERT(*vdp != NULL);

	return (0);
	}

	static boolean_t
	spa_should_flush_logs_on_unload(spa_t *spa)
	{
	if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP))
	return (B_FALSE);

	if (!spa_writeable(spa))
	return (B_FALSE);

	if (!spa->spa_sync_on)
	return (B_FALSE);

	if (spa_state(spa) != POOL_STATE_EXPORTED)
	return (B_FALSE);

	if (zfs_keep_log_spacemaps_at_export)
	return (B_FALSE);

	return (B_TRUE);
	}

	/*
	* Opens a transaction that will set the flag that will instruct
	* spa_sync to attempt to flush all the metaslabs for that txg.
	*/
	static void
	spa_unload_log_sm_flush_all(spa_t *spa)
	{
	dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
	VERIFY0(dmu_tx_assign(tx, TXG_WAIT));

	ASSERT3U(spa->spa_log_flushall_txg, ==, 0);
	spa->spa_log_flushall_txg = dmu_tx_get_txg(tx);

	dmu_tx_commit(tx);
	txg_wait_synced(spa_get_dsl(spa), spa->spa_log_flushall_txg);
	}

	static void
	spa_unload_log_sm_metadata(spa_t *spa)
	{
	void *cookie = NULL;
	spa_log_sm_t *sls;
	log_summary_entry_t *e;

	while ((sls = avl_destroy_nodes(&spa->spa_sm_logs_by_txg,
	&cookie)) != NULL) {
	VERIFY0(sls->sls_mscount);
	kmem_free(sls, sizeof (spa_log_sm_t));
	}

	while ((e = list_remove_head(&spa->spa_log_summary)) != NULL) {
	VERIFY0(e->lse_mscount);
	kmem_free(e, sizeof (log_summary_entry_t));
	}

	spa->spa_unflushed_stats.sus_nblocks = 0;
	spa->spa_unflushed_stats.sus_memused = 0;
	spa->spa_unflushed_stats.sus_blocklimit = 0;
	}

	static void
	spa_destroy_aux_threads(spa_t *spa)
	{
	if (spa->spa_condense_zthr != NULL) {
	zthr_destroy(spa->spa_condense_zthr);
	spa->spa_condense_zthr = NULL;
	}
	if (spa->spa_checkpoint_discard_zthr != NULL) {
	zthr_destroy(spa->spa_checkpoint_discard_zthr);
	spa->spa_checkpoint_discard_zthr = NULL;
	}
	if (spa->spa_livelist_delete_zthr != NULL) {
	zthr_destroy(spa->spa_livelist_delete_zthr);
	spa->spa_livelist_delete_zthr = NULL;
	}
	if (spa->spa_livelist_condense_zthr != NULL) {
	zthr_destroy(spa->spa_livelist_condense_zthr);
	spa->spa_livelist_condense_zthr = NULL;
	}
	}

	/*
	* Opposite of spa_load().
	*/
	static void
	spa_unload(spa_t *spa)
	{
	ASSERT(MUTEX_HELD(&spa_namespace_lock));
	ASSERT(spa_state(spa) != POOL_STATE_UNINITIALIZED);

	spa_import_progress_remove(spa_guid(spa));
	spa_load_note(spa, "UNLOADING");

	spa_wake_waiters(spa);

	/*
	* If we have set the spa_final_txg, we have already performed the
	* tasks below in spa_export_common(). We should not redo it here since
	* we delay the final TXGs beyond what spa_final_txg is set at.
	*/
	if (spa->spa_final_txg == UINT64_MAX) {
	/*
	* If the log space map feature is enabled and the pool is
	* getting exported (but not destroyed), we want to spend some
	* time flushing as many metaslabs as we can in an attempt to
	* destroy log space maps and save import time.
	*/
	if (spa_should_flush_logs_on_unload(spa))
	spa_unload_log_sm_flush_all(spa);

	/*
	* Stop async tasks.
	*/
	spa_async_suspend(spa);

	if (spa->spa_root_vdev) {
	vdev_t *root_vdev = spa->spa_root_vdev;
	vdev_initialize_stop_all(root_vdev,
	VDEV_INITIALIZE_ACTIVE);
	vdev_trim_stop_all(root_vdev, VDEV_TRIM_ACTIVE);
	vdev_autotrim_stop_all(spa);
	vdev_rebuild_stop_all(spa);
	}
	}

	/*
	* Stop syncing.
	*/
	if (spa->spa_sync_on) {
	txg_sync_stop(spa->spa_dsl_pool);
	spa->spa_sync_on = B_FALSE;
	}

	/*
	* This ensures that there is no async metaslab prefetching
	* while we attempt to unload the spa.
	*/
	taskq_wait(spa->spa_metaslab_taskq);

	if (spa->spa_mmp.mmp_thread)
	mmp_thread_stop(spa);

	/*
	* Wait for any outstanding async I/O to complete.
	*/
	if (spa->spa_async_zio_root != NULL) {
	for (int i = 0; i < max_ncpus; i++)
	(void) zio_wait(spa->spa_async_zio_root[i]);
	kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *));
	spa->spa_async_zio_root = NULL;
	}

	if (spa->spa_vdev_removal != NULL) {
	spa_vdev_removal_destroy(spa->spa_vdev_removal);
	spa->spa_vdev_removal = NULL;
	}

	spa_destroy_aux_threads(spa);

	spa_condense_fini(spa);

	bpobj_close(&spa->spa_deferred_bpobj);

	spa_config_enter(spa, SCL_ALL, spa, RW_WRITER);

	/*
	* Close all vdevs.
	*/
	if (spa->spa_root_vdev)
	vdev_free(spa->spa_root_vdev);
	ASSERT(spa->spa_root_vdev == NULL);

	/*
	* Close the dsl pool.
	*/
	if (spa->spa_dsl_pool) {
	dsl_pool_close(spa->spa_dsl_pool);
	spa->spa_dsl_pool = NULL;
	spa->spa_meta_objset = NULL;
	}

	ddt_unload(spa);
	brt_unload(spa);
	spa_unload_log_sm_metadata(spa);

	/*
	* Drop and purge level 2 cache
	*/
	spa_l2cache_drop(spa);

	if (spa->spa_spares.sav_vdevs) {
	for (int i = 0; i < spa->spa_spares.sav_count; i++)
	vdev_free(spa->spa_spares.sav_vdevs[i]);
	kmem_free(spa->spa_spares.sav_vdevs,
	spa->spa_spares.sav_count * sizeof (void *));
	spa->spa_spares.sav_vdevs = NULL;
	}
	if (spa->spa_spares.sav_config) {
	nvlist_free(spa->spa_spares.sav_config);
	spa->spa_spares.sav_config = NULL;
	}
	spa->spa_spares.sav_count = 0;

	if (spa->spa_l2cache.sav_vdevs) {
	for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
	vdev_clear_stats(spa->spa_l2cache.sav_vdevs[i]);
	vdev_free(spa->spa_l2cache.sav_vdevs[i]);
	}
	kmem_free(spa->spa_l2cache.sav_vdevs,
	spa->spa_l2cache.sav_count * sizeof (void *));
	spa->spa_l2cache.sav_vdevs = NULL;
	}
	if (spa->spa_l2cache.sav_config) {
	nvlist_free(spa->spa_l2cache.sav_config);
	spa->spa_l2cache.sav_config = NULL;
	}
	spa->spa_l2cache.sav_count = 0;

	spa->spa_async_suspended = 0;

	spa->spa_indirect_vdevs_loaded = B_FALSE;

	if (spa->spa_comment != NULL) {
	spa_strfree(spa->spa_comment);
	spa->spa_comment = NULL;
	}
	if (spa->spa_compatibility != NULL) {
	spa_strfree(spa->spa_compatibility);
	spa->spa_compatibility = NULL;
	}

	spa_config_exit(spa, SCL_ALL, spa);
	}

	/*
	* Load (or re-load) the current list of vdevs describing the active spares for
	* this pool. When this is called, we have some form of basic information in
	* 'spa_spares.sav_config'. We parse this into vdevs, try to open them, and
	* then re-generate a more complete list including status information.
	*/
	void
	spa_load_spares(spa_t *spa)
	{
	nvlist_t **spares;
	uint_t nspares;
	int i;
	vdev_t vd, tvd;

	#ifndef _KERNEL
	/*
	* zdb opens both the current state of the pool and the
	* checkpointed state (if present), with a different spa_t.
	*
	* As spare vdevs are shared among open pools, we skip loading
	* them when we load the checkpointed state of the pool.
	*/
	if (!spa_writeable(spa))
	return;
	#endif

	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);

	/*
	* First, close and free any existing spare vdevs.
	*/
	if (spa->spa_spares.sav_vdevs) {
	for (i = 0; i < spa->spa_spares.sav_count; i++) {
	vd = spa->spa_spares.sav_vdevs[i];

	/* Undo the call to spa_activate() below */
	if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid,
	B_FALSE)) != NULL && tvd->vdev_isspare)
	spa_spare_remove(tvd);
	vdev_close(vd);
	vdev_free(vd);
	}

	kmem_free(spa->spa_spares.sav_vdevs,
	spa->spa_spares.sav_count * sizeof (void *));
	}

	if (spa->spa_spares.sav_config == NULL)
	nspares = 0;
	else
	VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config,
	ZPOOL_CONFIG_SPARES, &spares, &nspares));

	spa->spa_spares.sav_count = (int)nspares;
	spa->spa_spares.sav_vdevs = NULL;

	if (nspares == 0)
	return;

	/*
	* Construct the array of vdevs, opening them to get status in the
	* process. For each spare, there is potentially two different vdev_t
	* structures associated with it: one in the list of spares (used only
	* for basic validation purposes) and one in the active vdev
	* configuration (if it's spared in). During this phase we open and
	* validate each vdev on the spare list. If the vdev also exists in the
	* active configuration, then we also mark this vdev as an active spare.
	*/
	spa->spa_spares.sav_vdevs = kmem_zalloc(nspares * sizeof (void *),
	KM_SLEEP);
	for (i = 0; i < spa->spa_spares.sav_count; i++) {
	VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0,
	VDEV_ALLOC_SPARE) == 0);
	ASSERT(vd != NULL);

	spa->spa_spares.sav_vdevs[i] = vd;

	if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid,
	B_FALSE)) != NULL) {
	if (!tvd->vdev_isspare)
	spa_spare_add(tvd);

	/*
	* We only mark the spare active if we were successfully
	* able to load the vdev. Otherwise, importing a pool
	* with a bad active spare would result in strange
	* behavior, because multiple pool would think the spare
	* is actively in use.
	*
	* There is a vulnerability here to an equally bizarre
	* circumstance, where a dead active spare is later
	* brought back to life (onlined or otherwise). Given
	* the rarity of this scenario, and the extra complexity
	* it adds, we ignore the possibility.
	*/
	if (!vdev_is_dead(tvd))
	spa_spare_activate(tvd);
	}

	vd->vdev_top = vd;
	vd->vdev_aux = &spa->spa_spares;

	if (vdev_open(vd) != 0)
	continue;

	if (vdev_validate_aux(vd) == 0)
	spa_spare_add(vd);
	}

	/*
	* Recompute the stashed list of spares, with status information
	* this time.
	*/
	fnvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES);

	spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *),
	KM_SLEEP);
	for (i = 0; i < spa->spa_spares.sav_count; i++)
	spares[i] = vdev_config_generate(spa,
	spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE);
	fnvlist_add_nvlist_array(spa->spa_spares.sav_config,
	ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares,
	spa->spa_spares.sav_count);
	for (i = 0; i < spa->spa_spares.sav_count; i++)
	nvlist_free(spares[i]);
	kmem_free(spares, spa->spa_spares.sav_count * sizeof (void *));
	}

	/*
	* Load (or re-load) the current list of vdevs describing the active l2cache for
	* this pool. When this is called, we have some form of basic information in
	* 'spa_l2cache.sav_config'. We parse this into vdevs, try to open them, and
	* then re-generate a more complete list including status information.
	* Devices which are already active have their details maintained, and are
	* not re-opened.
	*/
	void
	spa_load_l2cache(spa_t *spa)
	{
	nvlist_t **l2cache = NULL;
	uint_t nl2cache;
	int i, j, oldnvdevs;
	uint64_t guid;
	vdev_t vd, oldvdevs, *newvdevs;
	spa_aux_vdev_t *sav = &spa->spa_l2cache;

	#ifndef _KERNEL
	/*
	* zdb opens both the current state of the pool and the
	* checkpointed state (if present), with a different spa_t.
	*
	* As L2 caches are part of the ARC which is shared among open
	* pools, we skip loading them when we load the checkpointed
	* state of the pool.
	*/
	if (!spa_writeable(spa))
	return;
	#endif

	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);

	oldvdevs = sav->sav_vdevs;
	oldnvdevs = sav->sav_count;
	sav->sav_vdevs = NULL;
	sav->sav_count = 0;

	if (sav->sav_config == NULL) {
	nl2cache = 0;
	newvdevs = NULL;
	goto out;
	}

	VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config,
	ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache));
	newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP);

	/*
	* Process new nvlist of vdevs.
	*/
	for (i = 0; i < nl2cache; i++) {
	guid = fnvlist_lookup_uint64(l2cache[i], ZPOOL_CONFIG_GUID);

	newvdevs[i] = NULL;
	for (j = 0; j < oldnvdevs; j++) {
	vd = oldvdevs[j];
	if (vd != NULL && guid == vd->vdev_guid) {
	/*
	* Retain previous vdev for add/remove ops.
	*/
	newvdevs[i] = vd;
	oldvdevs[j] = NULL;
	break;
	}
	}

	if (newvdevs[i] == NULL) {
	/*
	* Create new vdev
	*/
	VERIFY(spa_config_parse(spa, &vd, l2cache[i], NULL, 0,
	VDEV_ALLOC_L2CACHE) == 0);
	ASSERT(vd != NULL);
	newvdevs[i] = vd;

	/*
	* Commit this vdev as an l2cache device,
	* even if it fails to open.
	*/
	spa_l2cache_add(vd);

	vd->vdev_top = vd;
	vd->vdev_aux = sav;

	spa_l2cache_activate(vd);

	if (vdev_open(vd) != 0)
	continue;

	(void) vdev_validate_aux(vd);

	if (!vdev_is_dead(vd))
	l2arc_add_vdev(spa, vd);

	/*
	* Upon cache device addition to a pool or pool
	* creation with a cache device or if the header
	* of the device is invalid we issue an async
	* TRIM command for the whole device which will
	* execute if l2arc_trim_ahead > 0.
	*/
	spa_async_request(spa, SPA_ASYNC_L2CACHE_TRIM);
	}
	}

	sav->sav_vdevs = newvdevs;
	sav->sav_count = (int)nl2cache;

	/*
	* Recompute the stashed list of l2cache devices, with status
	* information this time.
	*/
	fnvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE);

	if (sav->sav_count > 0)
	l2cache = kmem_alloc(sav->sav_count * sizeof (void *),
	KM_SLEEP);
	for (i = 0; i < sav->sav_count; i++)
	l2cache[i] = vdev_config_generate(spa,
	sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE);
	fnvlist_add_nvlist_array(sav->sav_config, ZPOOL_CONFIG_L2CACHE,
	(const nvlist_t * const *)l2cache, sav->sav_count);

	out:
	/*
	* Purge vdevs that were dropped
	*/
	if (oldvdevs) {
	for (i = 0; i < oldnvdevs; i++) {
	uint64_t pool;

	vd = oldvdevs[i];
	if (vd != NULL) {
	ASSERT(vd->vdev_isl2cache);

	if (spa_l2cache_exists(vd->vdev_guid, &pool) &&
	pool != 0ULL && l2arc_vdev_present(vd))
	l2arc_remove_vdev(vd);
	vdev_clear_stats(vd);
	vdev_free(vd);
	}
	}

	kmem_free(oldvdevs, oldnvdevs * sizeof (void *));
	}

	for (i = 0; i < sav->sav_count; i++)
	nvlist_free(l2cache[i]);
	if (sav->sav_count)
	kmem_free(l2cache, sav->sav_count * sizeof (void *));
	}

	static int
	load_nvlist(spa_t spa, uint64_t obj, nvlist_t *value)
	{
	dmu_buf_t *db;
	char *packed = NULL;
	size_t nvsize = 0;
	int error;
	*value = NULL;

	error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db);
	if (error)
	return (error);

	nvsize = (uint64_t )db->db_data;
	dmu_buf_rele(db, FTAG);

	packed = vmem_alloc(nvsize, KM_SLEEP);
	error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed,
	DMU_READ_PREFETCH);
	if (error == 0)
	error = nvlist_unpack(packed, nvsize, value, 0);
	vmem_free(packed, nvsize);

	return (error);
	}

	/*
	* Concrete top-level vdevs that are not missing and are not logs. At every
	* spa_sync we write new uberblocks to at least SPA_SYNC_MIN_VDEVS core tvds.
	*/
	static uint64_t
	spa_healthy_core_tvds(spa_t *spa)
	{
	vdev_t *rvd = spa->spa_root_vdev;
	uint64_t tvds = 0;

	for (uint64_t i = 0; i < rvd->vdev_children; i++) {
	vdev_t *vd = rvd->vdev_child[i];
	if (vd->vdev_islog)
	continue;
	if (vdev_is_concrete(vd) && !vdev_is_dead(vd))
	tvds++;
	}

	return (tvds);
	}

	/*
	* Checks to see if the given vdev could not be opened, in which case we post a
	* sysevent to notify the autoreplace code that the device has been removed.
	*/
	static void
	spa_check_removed(vdev_t *vd)
	{
	for (uint64_t c = 0; c < vd->vdev_children; c++)
	spa_check_removed(vd->vdev_child[c]);

	if (vd->vdev_ops->vdev_op_leaf && vdev_is_dead(vd) &&
	vdev_is_concrete(vd)) {
	zfs_post_autoreplace(vd->vdev_spa, vd);
	spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_CHECK);
	}
	}

	static int
	spa_check_for_missing_logs(spa_t *spa)
	{
	vdev_t *rvd = spa->spa_root_vdev;

	/*
	* If we're doing a normal import, then build up any additional
	* diagnostic information about missing log devices.
	* We'll pass this up to the user for further processing.
	*/
	if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) {
	nvlist_t *child, nv;
	uint64_t idx = 0;

	child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t *),
	KM_SLEEP);
	nv = fnvlist_alloc();

	for (uint64_t c = 0; c < rvd->vdev_children; c++) {
	vdev_t *tvd = rvd->vdev_child[c];

	/*
	* We consider a device as missing only if it failed
	* to open (i.e. offline or faulted is not considered
	* as missing).
	*/
	if (tvd->vdev_islog &&
	tvd->vdev_state == VDEV_STATE_CANT_OPEN) {
	child[idx++] = vdev_config_generate(spa, tvd,
	B_FALSE, VDEV_CONFIG_MISSING);
	}
	}

	if (idx > 0) {
	fnvlist_add_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
	(const nvlist_t * const *)child, idx);
	fnvlist_add_nvlist(spa->spa_load_info,
	ZPOOL_CONFIG_MISSING_DEVICES, nv);

	for (uint64_t i = 0; i < idx; i++)
	nvlist_free(child[i]);
	}
	nvlist_free(nv);
	kmem_free(child, rvd->vdev_children * sizeof (char **));

	if (idx > 0) {
	spa_load_failed(spa, "some log devices are missing");
	vdev_dbgmsg_print_tree(rvd, 2);
	return (SET_ERROR(ENXIO));
	}
	} else {
	for (uint64_t c = 0; c < rvd->vdev_children; c++) {
	vdev_t *tvd = rvd->vdev_child[c];

	if (tvd->vdev_islog &&
	tvd->vdev_state == VDEV_STATE_CANT_OPEN) {
	spa_set_log_state(spa, SPA_LOG_CLEAR);
	spa_load_note(spa, "some log devices are "
	"missing, ZIL is dropped.");
	vdev_dbgmsg_print_tree(rvd, 2);
	break;
	}
	}
	}

	return (0);
	}

	/*
	* Check for missing log devices
	*/
	static boolean_t
	spa_check_logs(spa_t *spa)
	{
	boolean_t rv = B_FALSE;
	dsl_pool_t *dp = spa_get_dsl(spa);

	switch (spa->spa_log_state) {
	default:
	break;
	case SPA_LOG_MISSING:
	/* need to recheck in case slog has been restored */
	case SPA_LOG_UNKNOWN:
	rv = (dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
	zil_check_log_chain, NULL, DS_FIND_CHILDREN) != 0);
	if (rv)
	spa_set_log_state(spa, SPA_LOG_MISSING);
	break;
	}
	return (rv);
	}

	/*
	* Passivate any log vdevs (note, does not apply to embedded log metaslabs).
	*/
	static boolean_t
	spa_passivate_log(spa_t *spa)
	{
	vdev_t *rvd = spa->spa_root_vdev;
	boolean_t slog_found = B_FALSE;

	ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER));

	for (int c = 0; c < rvd->vdev_children; c++) {
	vdev_t *tvd = rvd->vdev_child[c];

	if (tvd->vdev_islog) {
	ASSERT3P(tvd->vdev_log_mg, ==, NULL);
	metaslab_group_passivate(tvd->vdev_mg);
	slog_found = B_TRUE;
	}
	}

	return (slog_found);
	}

	/*
	* Activate any log vdevs (note, does not apply to embedded log metaslabs).
	*/
	static void
	spa_activate_log(spa_t *spa)
	{
	vdev_t *rvd = spa->spa_root_vdev;

	ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER));

	for (int c = 0; c < rvd->vdev_children; c++) {
	vdev_t *tvd = rvd->vdev_child[c];

	if (tvd->vdev_islog) {
	ASSERT3P(tvd->vdev_log_mg, ==, NULL);
	metaslab_group_activate(tvd->vdev_mg);
	}
	}
	}

	int
	spa_reset_logs(spa_t *spa)
	{
	int error;

	error = dmu_objset_find(spa_name(spa), zil_reset,
	NULL, DS_FIND_CHILDREN);
	if (error == 0) {
	/*
	* We successfully offlined the log device, sync out the
	* current txg so that the "stubby" block can be removed
	* by zil_sync().
	*/
	txg_wait_synced(spa->spa_dsl_pool, 0);
	}
	return (error);
	}

	static void
	spa_aux_check_removed(spa_aux_vdev_t *sav)
	{
	for (int i = 0; i < sav->sav_count; i++)
	spa_check_removed(sav->sav_vdevs[i]);
	}

	void
	spa_claim_notify(zio_t *zio)
	{
	spa_t *spa = zio->io_spa;

	if (zio->io_error)
	return;

	mutex_enter(&spa->spa_props_lock); /* any mutex will do */
	if (spa->spa_claim_max_txg < zio->io_bp->blk_birth)
	spa->spa_claim_max_txg = zio->io_bp->blk_birth;
	mutex_exit(&spa->spa_props_lock);
	}

	typedef struct spa_load_error {
	boolean_t sle_verify_data;
	uint64_t sle_meta_count;
	uint64_t sle_data_count;
	} spa_load_error_t;

	static void
	spa_load_verify_done(zio_t *zio)
	{
	blkptr_t *bp = zio->io_bp;
	spa_load_error_t *sle = zio->io_private;
	dmu_object_type_t type = BP_GET_TYPE(bp);
	int error = zio->io_error;
	spa_t *spa = zio->io_spa;

	abd_free(zio->io_abd);
	if (error) {
	if ((BP_GET_LEVEL(bp) != 0 \|\| DMU_OT_IS_METADATA(type)) &&
	type != DMU_OT_INTENT_LOG)
	atomic_inc_64(&sle->sle_meta_count);
	else
	atomic_inc_64(&sle->sle_data_count);
	}

	mutex_enter(&spa->spa_scrub_lock);
	spa->spa_load_verify_bytes -= BP_GET_PSIZE(bp);
	cv_broadcast(&spa->spa_scrub_io_cv);
	mutex_exit(&spa->spa_scrub_lock);
	}

	/*
	* Maximum number of inflight bytes is the log2 fraction of the arc size.
	* By default, we set it to 1/16th of the arc.
	*/
	static uint_t spa_load_verify_shift = 4;
	static int spa_load_verify_metadata = B_TRUE;
	static int spa_load_verify_data = B_TRUE;

	static int
	spa_load_verify_cb(spa_t spa, zilog_t zilog, const blkptr_t *bp,
	const zbookmark_phys_t zb, const dnode_phys_t dnp, void *arg)
	{
	zio_t *rio = arg;
	spa_load_error_t *sle = rio->io_private;

	(void) zilog, (void) dnp;

	/*
	* Note: normally this routine will not be called if
	* spa_load_verify_metadata is not set. However, it may be useful
	* to manually set the flag after the traversal has begun.
	*/
	if (!spa_load_verify_metadata)
	return (0);

	/*
	* Sanity check the block pointer in order to detect obvious damage
	* before using the contents in subsequent checks or in zio_read().
	* When damaged consider it to be a metadata error since we cannot
	* trust the BP_GET_TYPE and BP_GET_LEVEL values.
	*/
	if (!zfs_blkptr_verify(spa, bp, BLK_CONFIG_NEEDED, BLK_VERIFY_LOG)) {
	atomic_inc_64(&sle->sle_meta_count);
	return (0);
	}

	if (zb->zb_level == ZB_DNODE_LEVEL \|\| BP_IS_HOLE(bp) \|\|
	BP_IS_EMBEDDED(bp) \|\| BP_IS_REDACTED(bp))
	return (0);

	if (!BP_IS_METADATA(bp) &&
	(!spa_load_verify_data \|\| !sle->sle_verify_data))
	return (0);

	uint64_t maxinflight_bytes =
	arc_target_bytes() >> spa_load_verify_shift;
	size_t size = BP_GET_PSIZE(bp);

	mutex_enter(&spa->spa_scrub_lock);
	while (spa->spa_load_verify_bytes >= maxinflight_bytes)
	cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock);
	spa->spa_load_verify_bytes += size;
	mutex_exit(&spa->spa_scrub_lock);

	zio_nowait(zio_read(rio, spa, bp, abd_alloc_for_io(size, B_FALSE), size,
	spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB,
	ZIO_FLAG_SPECULATIVE \| ZIO_FLAG_CANFAIL \|
	ZIO_FLAG_SCRUB \| ZIO_FLAG_RAW, zb));
	return (0);
	}

	static int
	verify_dataset_name_len(dsl_pool_t dp, dsl_dataset_t ds, void *arg)
	{
	(void) dp, (void) arg;

	if (dsl_dataset_namelen(ds) >= ZFS_MAX_DATASET_NAME_LEN)
	return (SET_ERROR(ENAMETOOLONG));

	return (0);
	}

	static int
	spa_load_verify(spa_t *spa)
	{
	zio_t *rio;
	spa_load_error_t sle = { 0 };
	zpool_load_policy_t policy;
	boolean_t verify_ok = B_FALSE;
	int error = 0;

	zpool_get_load_policy(spa->spa_config, &policy);

	if (policy.zlp_rewind & ZPOOL_NEVER_REWIND \|\|
	policy.zlp_maxmeta == UINT64_MAX)
	return (0);

	dsl_pool_config_enter(spa->spa_dsl_pool, FTAG);
	error = dmu_objset_find_dp(spa->spa_dsl_pool,
	spa->spa_dsl_pool->dp_root_dir_obj, verify_dataset_name_len, NULL,
	DS_FIND_CHILDREN);
	dsl_pool_config_exit(spa->spa_dsl_pool, FTAG);
	if (error != 0)
	return (error);

	/*
	* Verify data only if we are rewinding or error limit was set.
	* Otherwise nothing except dbgmsg care about it to waste time.
	*/
	sle.sle_verify_data = (policy.zlp_rewind & ZPOOL_REWIND_MASK) \|\|
	(policy.zlp_maxdata < UINT64_MAX);

	rio = zio_root(spa, NULL, &sle,
	ZIO_FLAG_CANFAIL \| ZIO_FLAG_SPECULATIVE);

	if (spa_load_verify_metadata) {
	if (spa->spa_extreme_rewind) {
	spa_load_note(spa, "performing a complete scan of the "
	"pool since extreme rewind is on. This may take "
	"a very long time.\n (spa_load_verify_data=%u, "
	"spa_load_verify_metadata=%u)",
	spa_load_verify_data, spa_load_verify_metadata);
	}

	error = traverse_pool(spa, spa->spa_verify_min_txg,
	TRAVERSE_PRE \| TRAVERSE_PREFETCH_METADATA \|
	TRAVERSE_NO_DECRYPT, spa_load_verify_cb, rio);
	}

	(void) zio_wait(rio);
	ASSERT0(spa->spa_load_verify_bytes);

	spa->spa_load_meta_errors = sle.sle_meta_count;
	spa->spa_load_data_errors = sle.sle_data_count;

	if (sle.sle_meta_count != 0 \|\| sle.sle_data_count != 0) {
	spa_load_note(spa, "spa_load_verify found %llu metadata errors "
	"and %llu data errors", (u_longlong_t)sle.sle_meta_count,
	(u_longlong_t)sle.sle_data_count);
	}

	if (spa_load_verify_dryrun \|\|
	(!error && sle.sle_meta_count <= policy.zlp_maxmeta &&
	sle.sle_data_count <= policy.zlp_maxdata)) {
	int64_t loss = 0;

	verify_ok = B_TRUE;
	spa->spa_load_txg = spa->spa_uberblock.ub_txg;
	spa->spa_load_txg_ts = spa->spa_uberblock.ub_timestamp;

	loss = spa->spa_last_ubsync_txg_ts - spa->spa_load_txg_ts;
	fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_LOAD_TIME,
	spa->spa_load_txg_ts);
	fnvlist_add_int64(spa->spa_load_info, ZPOOL_CONFIG_REWIND_TIME,
	loss);
	fnvlist_add_uint64(spa->spa_load_info,
	ZPOOL_CONFIG_LOAD_META_ERRORS, sle.sle_meta_count);
	fnvlist_add_uint64(spa->spa_load_info,
	ZPOOL_CONFIG_LOAD_DATA_ERRORS, sle.sle_data_count);
	} else {
	spa->spa_load_max_txg = spa->spa_uberblock.ub_txg;
	}

	if (spa_load_verify_dryrun)
	return (0);

	if (error) {
	if (error != ENXIO && error != EIO)
	error = SET_ERROR(EIO);
	return (error);
	}

	return (verify_ok ? 0 : EIO);
	}

	/*
	* Find a value in the pool props object.
	*/
	static void
	spa_prop_find(spa_t spa, zpool_prop_t prop, uint64_t val)
	{
	(void) zap_lookup(spa->spa_meta_objset, spa->spa_pool_props_object,
	zpool_prop_to_name(prop), sizeof (uint64_t), 1, val);
	}

	/*
	* Find a value in the pool directory object.
	*/
	static int
	spa_dir_prop(spa_t spa, const char name, uint64_t *val, boolean_t log_enoent)
	{
	int error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
	name, sizeof (uint64_t), 1, val);

	if (error != 0 && (error != ENOENT \|\| log_enoent)) {
	spa_load_failed(spa, "couldn't get '%s' value in MOS directory "
	"[error=%d]", name, error);
	}

	return (error);
	}

	static int
	spa_vdev_err(vdev_t *vdev, vdev_aux_t aux, int err)
	{
	vdev_set_state(vdev, B_TRUE, VDEV_STATE_CANT_OPEN, aux);
	return (SET_ERROR(err));
	}

	boolean_t
	spa_livelist_delete_check(spa_t *spa)
	{
	return (spa->spa_livelists_to_delete != 0);
	}

	static boolean_t
	spa_livelist_delete_cb_check(void arg, zthr_t z)
	{
	(void) z;
	spa_t *spa = arg;
	return (spa_livelist_delete_check(spa));
	}

	static int
	delete_blkptr_cb(void arg, const blkptr_t bp, dmu_tx_t *tx)
	{
	spa_t *spa = arg;
	zio_free(spa, tx->tx_txg, bp);
	dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD,
	-bp_get_dsize_sync(spa, bp),
	-BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx);
	return (0);
	}

	static int
	dsl_get_next_livelist_obj(objset_t os, uint64_t zap_obj, uint64_t llp)
	{
	int err;
	zap_cursor_t zc;
	zap_attribute_t za;
	zap_cursor_init(&zc, os, zap_obj);
	err = zap_cursor_retrieve(&zc, &za);
	zap_cursor_fini(&zc);
	if (err == 0)
	*llp = za.za_first_integer;
	return (err);
	}

	/*
	* Components of livelist deletion that must be performed in syncing
	* context: freeing block pointers and updating the pool-wide data
	* structures to indicate how much work is left to do
	*/
	typedef struct sublist_delete_arg {
	spa_t *spa;
	dsl_deadlist_t *ll;
	uint64_t key;
	bplist_t *to_free;
	} sublist_delete_arg_t;

	static void
	sublist_delete_sync(void arg, dmu_tx_t tx)
	{
	sublist_delete_arg_t *sda = arg;
	spa_t *spa = sda->spa;
	dsl_deadlist_t *ll = sda->ll;
	uint64_t key = sda->key;
	bplist_t *to_free = sda->to_free;

	bplist_iterate(to_free, delete_blkptr_cb, spa, tx);
	dsl_deadlist_remove_entry(ll, key, tx);
	}

	typedef struct livelist_delete_arg {
	spa_t *spa;
	uint64_t ll_obj;
	uint64_t zap_obj;
	} livelist_delete_arg_t;

	static void
	livelist_delete_sync(void arg, dmu_tx_t tx)
	{
	livelist_delete_arg_t *lda = arg;
	spa_t *spa = lda->spa;
	uint64_t ll_obj = lda->ll_obj;
	uint64_t zap_obj = lda->zap_obj;
	objset_t *mos = spa->spa_meta_objset;
	uint64_t count;

	/* free the livelist and decrement the feature count */
	VERIFY0(zap_remove_int(mos, zap_obj, ll_obj, tx));
	dsl_deadlist_free(mos, ll_obj, tx);
	spa_feature_decr(spa, SPA_FEATURE_LIVELIST, tx);
	VERIFY0(zap_count(mos, zap_obj, &count));
	if (count == 0) {
	/* no more livelists to delete */
	VERIFY0(zap_remove(mos, DMU_POOL_DIRECTORY_OBJECT,
	DMU_POOL_DELETED_CLONES, tx));
	VERIFY0(zap_destroy(mos, zap_obj, tx));
	spa->spa_livelists_to_delete = 0;
	spa_notify_waiters(spa);
	}
	}

	/*
	* Load in the value for the livelist to be removed and open it. Then,
	* load its first sublist and determine which block pointers should actually
	* be freed. Then, call a synctask which performs the actual frees and updates
	* the pool-wide livelist data.
	*/
	static void
	spa_livelist_delete_cb(void arg, zthr_t z)
	{
	spa_t *spa = arg;
	uint64_t ll_obj = 0, count;
	objset_t *mos = spa->spa_meta_objset;
	uint64_t zap_obj = spa->spa_livelists_to_delete;
	/*
	* Determine the next livelist to delete. This function should only
	* be called if there is at least one deleted clone.
	*/
	VERIFY0(dsl_get_next_livelist_obj(mos, zap_obj, &ll_obj));
	VERIFY0(zap_count(mos, ll_obj, &count));
	if (count > 0) {
	dsl_deadlist_t *ll;
	dsl_deadlist_entry_t *dle;
	bplist_t to_free;
	ll = kmem_zalloc(sizeof (dsl_deadlist_t), KM_SLEEP);
	dsl_deadlist_open(ll, mos, ll_obj);
	dle = dsl_deadlist_first(ll);
	ASSERT3P(dle, !=, NULL);
	bplist_create(&to_free);
	int err = dsl_process_sub_livelist(&dle->dle_bpobj, &to_free,
	z, NULL);
	if (err == 0) {
	sublist_delete_arg_t sync_arg = {
	.spa = spa,
	.ll = ll,
	.key = dle->dle_mintxg,
	.to_free = &to_free
	};
	zfs_dbgmsg("deleting sublist (id %llu) from"
	" livelist %llu, %lld remaining",
	(u_longlong_t)dle->dle_bpobj.bpo_object,
	(u_longlong_t)ll_obj, (longlong_t)count - 1);
	VERIFY0(dsl_sync_task(spa_name(spa), NULL,
	sublist_delete_sync, &sync_arg, 0,
	ZFS_SPACE_CHECK_DESTROY));
	} else {
	VERIFY3U(err, ==, EINTR);
	}
	bplist_clear(&to_free);
	bplist_destroy(&to_free);
	dsl_deadlist_close(ll);
	kmem_free(ll, sizeof (dsl_deadlist_t));
	} else {
	livelist_delete_arg_t sync_arg = {
	.spa = spa,
	.ll_obj = ll_obj,
	.zap_obj = zap_obj
	};
	zfs_dbgmsg("deletion of livelist %llu completed",
	(u_longlong_t)ll_obj);
	VERIFY0(dsl_sync_task(spa_name(spa), NULL, livelist_delete_sync,
	&sync_arg, 0, ZFS_SPACE_CHECK_DESTROY));
	}
	}

	static void
	spa_start_livelist_destroy_thread(spa_t *spa)
	{
	ASSERT3P(spa->spa_livelist_delete_zthr, ==, NULL);
	spa->spa_livelist_delete_zthr =
	zthr_create("z_livelist_destroy",
	spa_livelist_delete_cb_check, spa_livelist_delete_cb, spa,
	minclsyspri);
	}

	typedef struct livelist_new_arg {
	bplist_t *allocs;
	bplist_t *frees;
	} livelist_new_arg_t;

	static int
	livelist_track_new_cb(void arg, const blkptr_t bp, boolean_t bp_freed,
	dmu_tx_t *tx)
	{
	ASSERT(tx == NULL);
	livelist_new_arg_t *lna = arg;
	if (bp_freed) {
	bplist_append(lna->frees, bp);
	} else {
	bplist_append(lna->allocs, bp);
	zfs_livelist_condense_new_alloc++;
	}
	return (0);
	}

	typedef struct livelist_condense_arg {
	spa_t *spa;
	bplist_t to_keep;
	uint64_t first_size;
	uint64_t next_size;
	} livelist_condense_arg_t;

	static void
	spa_livelist_condense_sync(void arg, dmu_tx_t tx)
	{
	livelist_condense_arg_t *lca = arg;
	spa_t *spa = lca->spa;
	bplist_t new_frees;
	dsl_dataset_t *ds = spa->spa_to_condense.ds;

	/* Have we been cancelled? */
	if (spa->spa_to_condense.cancelled) {
	zfs_livelist_condense_sync_cancel++;
	goto out;
	}

	dsl_deadlist_entry_t *first = spa->spa_to_condense.first;
	dsl_deadlist_entry_t *next = spa->spa_to_condense.next;
	dsl_deadlist_t *ll = &ds->ds_dir->dd_livelist;

	/*
	* It's possible that the livelist was changed while the zthr was
	* running. Therefore, we need to check for new blkptrs in the two
	* entries being condensed and continue to track them in the livelist.
	* Because of the way we handle remapped blkptrs (see dbuf_remap_impl),
	* it's possible that the newly added blkptrs are FREEs or ALLOCs so
	* we need to sort them into two different bplists.
	*/
	uint64_t first_obj = first->dle_bpobj.bpo_object;
	uint64_t next_obj = next->dle_bpobj.bpo_object;
	uint64_t cur_first_size = first->dle_bpobj.bpo_phys->bpo_num_blkptrs;
	uint64_t cur_next_size = next->dle_bpobj.bpo_phys->bpo_num_blkptrs;

	bplist_create(&new_frees);
	livelist_new_arg_t new_bps = {
	.allocs = &lca->to_keep,
	.frees = &new_frees,
	};

	if (cur_first_size > lca->first_size) {
	VERIFY0(livelist_bpobj_iterate_from_nofree(&first->dle_bpobj,
	livelist_track_new_cb, &new_bps, lca->first_size));
	}
	if (cur_next_size > lca->next_size) {
	VERIFY0(livelist_bpobj_iterate_from_nofree(&next->dle_bpobj,
	livelist_track_new_cb, &new_bps, lca->next_size));
	}

	dsl_deadlist_clear_entry(first, ll, tx);
	ASSERT(bpobj_is_empty(&first->dle_bpobj));
	dsl_deadlist_remove_entry(ll, next->dle_mintxg, tx);

	bplist_iterate(&lca->to_keep, dsl_deadlist_insert_alloc_cb, ll, tx);
	bplist_iterate(&new_frees, dsl_deadlist_insert_free_cb, ll, tx);
	bplist_destroy(&new_frees);

	char dsname[ZFS_MAX_DATASET_NAME_LEN];
	dsl_dataset_name(ds, dsname);
	zfs_dbgmsg("txg %llu condensing livelist of %s (id %llu), bpobj %llu "
	"(%llu blkptrs) and bpobj %llu (%llu blkptrs) -> bpobj %llu "
	"(%llu blkptrs)", (u_longlong_t)tx->tx_txg, dsname,
	(u_longlong_t)ds->ds_object, (u_longlong_t)first_obj,
	(u_longlong_t)cur_first_size, (u_longlong_t)next_obj,
	(u_longlong_t)cur_next_size,
	(u_longlong_t)first->dle_bpobj.bpo_object,
	(u_longlong_t)first->dle_bpobj.bpo_phys->bpo_num_blkptrs);
	out:
	dmu_buf_rele(ds->ds_dbuf, spa);
	spa->spa_to_condense.ds = NULL;
	bplist_clear(&lca->to_keep);
	bplist_destroy(&lca->to_keep);
	kmem_free(lca, sizeof (livelist_condense_arg_t));
	spa->spa_to_condense.syncing = B_FALSE;
	}

	static void
	spa_livelist_condense_cb(void arg, zthr_t t)
	{
	while (zfs_livelist_condense_zthr_pause &&
	!(zthr_has_waiters(t) \|\| zthr_iscancelled(t)))
	delay(1);

	spa_t *spa = arg;
	dsl_deadlist_entry_t *first = spa->spa_to_condense.first;
	dsl_deadlist_entry_t *next = spa->spa_to_condense.next;
	uint64_t first_size, next_size;

	livelist_condense_arg_t *lca =
	kmem_alloc(sizeof (livelist_condense_arg_t), KM_SLEEP);
	bplist_create(&lca->to_keep);

	/*
	* Process the livelists (matching FREEs and ALLOCs) in open context
	* so we have minimal work in syncing context to condense.
	*
	* We save bpobj sizes (first_size and next_size) to use later in
	* syncing context to determine if entries were added to these sublists
	* while in open context. This is possible because the clone is still
	* active and open for normal writes and we want to make sure the new,
	* unprocessed blockpointers are inserted into the livelist normally.
	*
	* Note that dsl_process_sub_livelist() both stores the size number of
	* blockpointers and iterates over them while the bpobj's lock held, so
	* the sizes returned to us are consistent which what was actually
	* processed.
	*/
	int err = dsl_process_sub_livelist(&first->dle_bpobj, &lca->to_keep, t,
	&first_size);
	if (err == 0)
	err = dsl_process_sub_livelist(&next->dle_bpobj, &lca->to_keep,
	t, &next_size);

	if (err == 0) {
	while (zfs_livelist_condense_sync_pause &&
	!(zthr_has_waiters(t) \|\| zthr_iscancelled(t)))
	delay(1);

	dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
	dmu_tx_mark_netfree(tx);
	dmu_tx_hold_space(tx, 1);
	err = dmu_tx_assign(tx, TXG_NOWAIT \| TXG_NOTHROTTLE);
	if (err == 0) {
	/*
	* Prevent the condense zthr restarting before
	* the synctask completes.
	*/
	spa->spa_to_condense.syncing = B_TRUE;
	lca->spa = spa;
	lca->first_size = first_size;
	lca->next_size = next_size;
	dsl_sync_task_nowait(spa_get_dsl(spa),
	spa_livelist_condense_sync, lca, tx);
	dmu_tx_commit(tx);
	return;
	}
	}
	/*
	* Condensing can not continue: either it was externally stopped or
	* we were unable to assign to a tx because the pool has run out of
	* space. In the second case, we'll just end up trying to condense
	* again in a later txg.
	*/
	ASSERT(err != 0);
	bplist_clear(&lca->to_keep);
	bplist_destroy(&lca->to_keep);
	kmem_free(lca, sizeof (livelist_condense_arg_t));
	dmu_buf_rele(spa->spa_to_condense.ds->ds_dbuf, spa);
	spa->spa_to_condense.ds = NULL;
	if (err == EINTR)
	zfs_livelist_condense_zthr_cancel++;
	}

	/*
	* Check that there is something to condense but that a condense is not
	* already in progress and that condensing has not been cancelled.
	*/
	static boolean_t
	spa_livelist_condense_cb_check(void arg, zthr_t z)
	{
	(void) z;
	spa_t *spa = arg;
	if ((spa->spa_to_condense.ds != NULL) &&
	(spa->spa_to_condense.syncing == B_FALSE) &&
	(spa->spa_to_condense.cancelled == B_FALSE)) {
	return (B_TRUE);
	}
	return (B_FALSE);
	}

	static void
	spa_start_livelist_condensing_thread(spa_t *spa)
	{
	spa->spa_to_condense.ds = NULL;
	spa->spa_to_condense.first = NULL;
	spa->spa_to_condense.next = NULL;
	spa->spa_to_condense.syncing = B_FALSE;
	spa->spa_to_condense.cancelled = B_FALSE;

	ASSERT3P(spa->spa_livelist_condense_zthr, ==, NULL);
	spa->spa_livelist_condense_zthr =
	zthr_create("z_livelist_condense",
	spa_livelist_condense_cb_check,
	spa_livelist_condense_cb, spa, minclsyspri);
	}

	static void
	spa_spawn_aux_threads(spa_t *spa)
	{
	ASSERT(spa_writeable(spa));

	ASSERT(MUTEX_HELD(&spa_namespace_lock));

	spa_start_indirect_condensing_thread(spa);
	spa_start_livelist_destroy_thread(spa);
	spa_start_livelist_condensing_thread(spa);

	ASSERT3P(spa->spa_checkpoint_discard_zthr, ==, NULL);
	spa->spa_checkpoint_discard_zthr =
	zthr_create("z_checkpoint_discard",
	spa_checkpoint_discard_thread_check,
	spa_checkpoint_discard_thread, spa, minclsyspri);
	}

	/*
	* Fix up config after a partly-completed split. This is done with the
	* ZPOOL_CONFIG_SPLIT nvlist. Both the splitting pool and the split-off
	* pool have that entry in their config, but only the splitting one contains
	* a list of all the guids of the vdevs that are being split off.
	*
	* This function determines what to do with that list: either rejoin
	* all the disks to the pool, or complete the splitting process. To attempt
	* the rejoin, each disk that is offlined is marked online again, and
	* we do a reopen() call. If the vdev label for every disk that was
	* marked online indicates it was successfully split off (VDEV_AUX_SPLIT_POOL)
	* then we call vdev_split() on each disk, and complete the split.
	*
	* Otherwise we leave the config alone, with all the vdevs in place in
	* the original pool.
	*/
	static void
	spa_try_repair(spa_t spa, nvlist_t config)
	{
	uint_t extracted;
	uint64_t *glist;
	uint_t i, gcount;
	nvlist_t *nvl;
	vdev_t **vd;
	boolean_t attempt_reopen;

	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) != 0)
	return;

	/* check that the config is complete */
	if (nvlist_lookup_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST,
	&glist, &gcount) != 0)
	return;

	vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP);

	/* attempt to online all the vdevs & validate */
	attempt_reopen = B_TRUE;
	for (i = 0; i < gcount; i++) {
	if (glist[i] == 0) /* vdev is hole */
	continue;

	vd[i] = spa_lookup_by_guid(spa, glist[i], B_FALSE);
	if (vd[i] == NULL) {
	/*
	* Don't bother attempting to reopen the disks;
	* just do the split.
	*/
	attempt_reopen = B_FALSE;
	} else {
	/* attempt to re-online it */
	vd[i]->vdev_offline = B_FALSE;
	}
	}

	if (attempt_reopen) {
	vdev_reopen(spa->spa_root_vdev);

	/* check each device to see what state it's in */
	for (extracted = 0, i = 0; i < gcount; i++) {
	if (vd[i] != NULL &&
	vd[i]->vdev_stat.vs_aux != VDEV_AUX_SPLIT_POOL)
	break;
	++extracted;
	}
	}

	/*
	* If every disk has been moved to the new pool, or if we never
	* even attempted to look at them, then we split them off for
	* good.
	*/
	if (!attempt_reopen \|\| gcount == extracted) {
	for (i = 0; i < gcount; i++)
	if (vd[i] != NULL)
	vdev_split(vd[i]);
	vdev_reopen(spa->spa_root_vdev);
	}

	kmem_free(vd, gcount * sizeof (vdev_t *));
	}

	static int
	spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type)
	{
	const char *ereport = FM_EREPORT_ZFS_POOL;
	int error;

	spa->spa_load_state = state;
	(void) spa_import_progress_set_state(spa_guid(spa),
	spa_load_state(spa));
	+ spa_import_progress_set_notes(spa, "spa_load()");

	gethrestime(&spa->spa_loaded_ts);
	error = spa_load_impl(spa, type, &ereport);

	/*
	* Don't count references from objsets that are already closed
	* and are making their way through the eviction process.
	*/
	spa_evicting_os_wait(spa);
	spa->spa_minref = zfs_refcount_count(&spa->spa_refcount);
	if (error) {
	if (error != EEXIST) {
	spa->spa_loaded_ts.tv_sec = 0;
	spa->spa_loaded_ts.tv_nsec = 0;
	}
	if (error != EBADF) {
	(void) zfs_ereport_post(ereport, spa,
	NULL, NULL, NULL, 0);
	}
	}
	spa->spa_load_state = error ? SPA_LOAD_ERROR : SPA_LOAD_NONE;
	spa->spa_ena = 0;

	(void) spa_import_progress_set_state(spa_guid(spa),
	spa_load_state(spa));

	return (error);
	}

	#ifdef ZFS_DEBUG
	/*
	* Count the number of per-vdev ZAPs associated with all of the vdevs in the
	* vdev tree rooted in the given vd, and ensure that each ZAP is present in the
	* spa's per-vdev ZAP list.
	*/
	static uint64_t
	vdev_count_verify_zaps(vdev_t *vd)
	{
	spa_t *spa = vd->vdev_spa;
	uint64_t total = 0;

	if (spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_AVZ_V2) &&
	vd->vdev_root_zap != 0) {
	total++;
	ASSERT0(zap_lookup_int(spa->spa_meta_objset,
	spa->spa_all_vdev_zaps, vd->vdev_root_zap));
	}
	if (vd->vdev_top_zap != 0) {
	total++;
	ASSERT0(zap_lookup_int(spa->spa_meta_objset,
	spa->spa_all_vdev_zaps, vd->vdev_top_zap));
	}
	if (vd->vdev_leaf_zap != 0) {
	total++;
	ASSERT0(zap_lookup_int(spa->spa_meta_objset,
	spa->spa_all_vdev_zaps, vd->vdev_leaf_zap));
	}

	for (uint64_t i = 0; i < vd->vdev_children; i++) {
	total += vdev_count_verify_zaps(vd->vdev_child[i]);
	}

	return (total);
	}
	#else
	#define vdev_count_verify_zaps(vd) ((void) sizeof (vd), 0)
	#endif

	/*
	* Determine whether the activity check is required.
	*/
	static boolean_t
	spa_activity_check_required(spa_t spa, uberblock_t ub, nvlist_t *label,
	nvlist_t *config)
	{
	uint64_t state = 0;
	uint64_t hostid = 0;
	uint64_t tryconfig_txg = 0;
	uint64_t tryconfig_timestamp = 0;
	uint16_t tryconfig_mmp_seq = 0;
	nvlist_t *nvinfo;

	if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) {
	nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
	(void) nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG,
	&tryconfig_txg);
	(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_TIMESTAMP,
	&tryconfig_timestamp);
	(void) nvlist_lookup_uint16(nvinfo, ZPOOL_CONFIG_MMP_SEQ,
	&tryconfig_mmp_seq);
	}

	(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE, &state);

	/*
	* Disable the MMP activity check - This is used by zdb which
	* is intended to be used on potentially active pools.
	*/
	if (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP)
	return (B_FALSE);

	/*
	* Skip the activity check when the MMP feature is disabled.
	*/
	if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay == 0)
	return (B_FALSE);

	/*
	* If the tryconfig_ values are nonzero, they are the results of an
	* earlier tryimport. If they all match the uberblock we just found,
	* then the pool has not changed and we return false so we do not test
	* a second time.
	*/
	if (tryconfig_txg && tryconfig_txg == ub->ub_txg &&
	tryconfig_timestamp && tryconfig_timestamp == ub->ub_timestamp &&
	tryconfig_mmp_seq && tryconfig_mmp_seq ==
	(MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0))
	return (B_FALSE);

	/*
	* Allow the activity check to be skipped when importing the pool
	* on the same host which last imported it. Since the hostid from
	* configuration may be stale use the one read from the label.
	*/
	if (nvlist_exists(label, ZPOOL_CONFIG_HOSTID))
	hostid = fnvlist_lookup_uint64(label, ZPOOL_CONFIG_HOSTID);

	if (hostid == spa_get_hostid(spa))
	return (B_FALSE);

	/*
	* Skip the activity test when the pool was cleanly exported.
	*/
	if (state != POOL_STATE_ACTIVE)
	return (B_FALSE);

	return (B_TRUE);
	}

	/*
	* Nanoseconds the activity check must watch for changes on-disk.
	*/
	static uint64_t
	spa_activity_check_duration(spa_t spa, uberblock_t ub)
	{
	uint64_t import_intervals = MAX(zfs_multihost_import_intervals, 1);
	uint64_t multihost_interval = MSEC2NSEC(
	MMP_INTERVAL_OK(zfs_multihost_interval));
	uint64_t import_delay = MAX(NANOSEC, import_intervals *
	multihost_interval);

	/*
	* Local tunables determine a minimum duration except for the case
	* where we know when the remote host will suspend the pool if MMP
	* writes do not land.
	*
	* See Big Theory comment at the top of mmp.c for the reasoning behind
	* these cases and times.
	*/

	ASSERT(MMP_IMPORT_SAFETY_FACTOR >= 100);

	if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) &&
	MMP_FAIL_INT(ub) > 0) {

	/* MMP on remote host will suspend pool after failed writes */
	import_delay = MMP_FAIL_INT(ub) * MSEC2NSEC(MMP_INTERVAL(ub)) *
	MMP_IMPORT_SAFETY_FACTOR / 100;

	zfs_dbgmsg("fail_intvals>0 import_delay=%llu ub_mmp "
	"mmp_fails=%llu ub_mmp mmp_interval=%llu "
	"import_intervals=%llu", (u_longlong_t)import_delay,
	(u_longlong_t)MMP_FAIL_INT(ub),
	(u_longlong_t)MMP_INTERVAL(ub),
	(u_longlong_t)import_intervals);

	} else if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) &&
	MMP_FAIL_INT(ub) == 0) {

	/* MMP on remote host will never suspend pool */
	import_delay = MAX(import_delay, (MSEC2NSEC(MMP_INTERVAL(ub)) +
	ub->ub_mmp_delay) * import_intervals);

	zfs_dbgmsg("fail_intvals=0 import_delay=%llu ub_mmp "
	"mmp_interval=%llu ub_mmp_delay=%llu "
	"import_intervals=%llu", (u_longlong_t)import_delay,
	(u_longlong_t)MMP_INTERVAL(ub),
	(u_longlong_t)ub->ub_mmp_delay,
	(u_longlong_t)import_intervals);

	} else if (MMP_VALID(ub)) {
	/*
	* zfs-0.7 compatibility case
	*/

	import_delay = MAX(import_delay, (multihost_interval +
	ub->ub_mmp_delay) * import_intervals);

	zfs_dbgmsg("import_delay=%llu ub_mmp_delay=%llu "
	"import_intervals=%llu leaves=%u",
	(u_longlong_t)import_delay,
	(u_longlong_t)ub->ub_mmp_delay,
	(u_longlong_t)import_intervals,
	vdev_count_leaves(spa));
	} else {
	/* Using local tunings is the only reasonable option */
	zfs_dbgmsg("pool last imported on non-MMP aware "
	"host using import_delay=%llu multihost_interval=%llu "
	"import_intervals=%llu", (u_longlong_t)import_delay,
	(u_longlong_t)multihost_interval,
	(u_longlong_t)import_intervals);
	}

	return (import_delay);
	}

	/*
	- * Perform the import activity check. If the user canceled the import or
	- * we detected activity then fail.
	+ * Remote host activity check.
	+ *
	+ * error results:
	+ * 0 - no activity detected
	+ * EREMOTEIO - remote activity detected
	+ * EINTR - user canceled the operation
	*/
	static int
	-spa_activity_check(spa_t spa, uberblock_t ub, nvlist_t *config)
	+spa_activity_check(spa_t spa, uberblock_t ub, nvlist_t *config,
	+ boolean_t importing)
	{
	uint64_t txg = ub->ub_txg;
	uint64_t timestamp = ub->ub_timestamp;
	uint64_t mmp_config = ub->ub_mmp_config;
	uint16_t mmp_seq = MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0;
	uint64_t import_delay;
	- hrtime_t import_expire;
	+ hrtime_t import_expire, now;
	nvlist_t *mmp_label = NULL;
	vdev_t *rvd = spa->spa_root_vdev;
	kcondvar_t cv;
	kmutex_t mtx;
	int error = 0;

	cv_init(&cv, NULL, CV_DEFAULT, NULL);
	mutex_init(&mtx, NULL, MUTEX_DEFAULT, NULL);
	mutex_enter(&mtx);

	/*
	* If ZPOOL_CONFIG_MMP_TXG is present an activity check was performed
	* during the earlier tryimport. If the txg recorded there is 0 then
	* the pool is known to be active on another host.
	*
	* Otherwise, the pool might be in use on another host. Check for
	* changes in the uberblocks on disk if necessary.
	*/
	if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) {
	nvlist_t *nvinfo = fnvlist_lookup_nvlist(config,
	ZPOOL_CONFIG_LOAD_INFO);

	if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_TXG) &&
	fnvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG) == 0) {
	vdev_uberblock_load(rvd, ub, &mmp_label);
	error = SET_ERROR(EREMOTEIO);
	goto out;
	}
	}

	import_delay = spa_activity_check_duration(spa, ub);

	/* Add a small random factor in case of simultaneous imports (0-25%) */
	import_delay += import_delay * random_in_range(250) / 1000;

	import_expire = gethrtime() + import_delay;

	- while (gethrtime() < import_expire) {
	- (void) spa_import_progress_set_mmp_check(spa_guid(spa),
	- NSEC2SEC(import_expire - gethrtime()));
	+ if (importing) {
	+ spa_import_progress_set_notes(spa, "Checking MMP activity, "
	+ "waiting %llu ms", (u_longlong_t)NSEC2MSEC(import_delay));
	+ }
	+
	+ int iterations = 0;
	+ while ((now = gethrtime()) < import_expire) {
	+ if (importing && iterations++ % 30 == 0) {
	+ spa_import_progress_set_notes(spa, "Checking MMP "
	+ "activity, %llu ms remaining",
	+ (u_longlong_t)NSEC2MSEC(import_expire - now));
	+ }
	+
	+ if (importing) {
	+ (void) spa_import_progress_set_mmp_check(spa_guid(spa),
	+ NSEC2SEC(import_expire - gethrtime()));
	+ }

	vdev_uberblock_load(rvd, ub, &mmp_label);

	if (txg != ub->ub_txg \|\| timestamp != ub->ub_timestamp \|\|
	mmp_seq != (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) {
	zfs_dbgmsg("multihost activity detected "
	"txg %llu ub_txg %llu "
	"timestamp %llu ub_timestamp %llu "
	"mmp_config %#llx ub_mmp_config %#llx",
	(u_longlong_t)txg, (u_longlong_t)ub->ub_txg,
	(u_longlong_t)timestamp,
	(u_longlong_t)ub->ub_timestamp,
	(u_longlong_t)mmp_config,
	(u_longlong_t)ub->ub_mmp_config);

	error = SET_ERROR(EREMOTEIO);
	break;
	}

	if (mmp_label) {
	nvlist_free(mmp_label);
	mmp_label = NULL;
	}

	error = cv_timedwait_sig(&cv, &mtx, ddi_get_lbolt() + hz);
	if (error != -1) {
	error = SET_ERROR(EINTR);
	break;
	}
	error = 0;
	}

	out:
	mutex_exit(&mtx);
	mutex_destroy(&mtx);
	cv_destroy(&cv);

	/*
	* If the pool is determined to be active store the status in the
	* spa->spa_load_info nvlist. If the remote hostname or hostid are
	* available from configuration read from disk store them as well.
	* This allows 'zpool import' to generate a more useful message.
	*
	* ZPOOL_CONFIG_MMP_STATE - observed pool status (mandatory)
	* ZPOOL_CONFIG_MMP_HOSTNAME - hostname from the active pool
	* ZPOOL_CONFIG_MMP_HOSTID - hostid from the active pool
	*/
	if (error == EREMOTEIO) {
	const char *hostname = "<unknown>";
	uint64_t hostid = 0;

	if (mmp_label) {
	if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTNAME)) {
	hostname = fnvlist_lookup_string(mmp_label,
	ZPOOL_CONFIG_HOSTNAME);
	fnvlist_add_string(spa->spa_load_info,
	ZPOOL_CONFIG_MMP_HOSTNAME, hostname);
	}

	if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTID)) {
	hostid = fnvlist_lookup_uint64(mmp_label,
	ZPOOL_CONFIG_HOSTID);
	fnvlist_add_uint64(spa->spa_load_info,
	ZPOOL_CONFIG_MMP_HOSTID, hostid);
	}
	}

	fnvlist_add_uint64(spa->spa_load_info,
	ZPOOL_CONFIG_MMP_STATE, MMP_STATE_ACTIVE);
	fnvlist_add_uint64(spa->spa_load_info,
	ZPOOL_CONFIG_MMP_TXG, 0);

	error = spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO);
	}

	if (mmp_label)
	nvlist_free(mmp_label);

	return (error);
	}

	+/*
	+ * Called from zfs_ioc_clear for a pool that was suspended
	+ * after failing mmp write checks.
	+ */
	+boolean_t
	+spa_mmp_remote_host_activity(spa_t *spa)
	+{
	+ ASSERT(spa_multihost(spa) && spa_suspended(spa));
	+
	+ nvlist_t *best_label;
	+ uberblock_t best_ub;
	+
	+ /*
	+ * Locate the best uberblock on disk
	+ */
	+ vdev_uberblock_load(spa->spa_root_vdev, &best_ub, &best_label);
	+ if (best_label) {
	+ /*
	+ * confirm that the best hostid matches our hostid
	+ */
	+ if (nvlist_exists(best_label, ZPOOL_CONFIG_HOSTID) &&
	+ spa_get_hostid(spa) !=
	+ fnvlist_lookup_uint64(best_label, ZPOOL_CONFIG_HOSTID)) {
	+ nvlist_free(best_label);
	+ return (B_TRUE);
	+ }
	+ nvlist_free(best_label);
	+ } else {
	+ return (B_TRUE);
	+ }
	+
	+ if (!MMP_VALID(&best_ub) \|\|
	+ !MMP_FAIL_INT_VALID(&best_ub) \|\|
	+ MMP_FAIL_INT(&best_ub) == 0) {
	+ return (B_TRUE);
	+ }
	+
	+ if (best_ub.ub_txg != spa->spa_uberblock.ub_txg \|\|
	+ best_ub.ub_timestamp != spa->spa_uberblock.ub_timestamp) {
	+ zfs_dbgmsg("txg mismatch detected during pool clear "
	+ "txg %llu ub_txg %llu timestamp %llu ub_timestamp %llu",
	+ (u_longlong_t)spa->spa_uberblock.ub_txg,
	+ (u_longlong_t)best_ub.ub_txg,
	+ (u_longlong_t)spa->spa_uberblock.ub_timestamp,
	+ (u_longlong_t)best_ub.ub_timestamp);
	+ return (B_TRUE);
	+ }
	+
	+ /*
	+ * Perform an activity check looking for any remote writer
	+ */
	+ return (spa_activity_check(spa, &spa->spa_uberblock, spa->spa_config,
	+ B_FALSE) != 0);
	+}
	+
	static int
	spa_verify_host(spa_t spa, nvlist_t mos_config)
	{
	uint64_t hostid;
	const char *hostname;
	uint64_t myhostid = 0;

	if (!spa_is_root(spa) && nvlist_lookup_uint64(mos_config,
	ZPOOL_CONFIG_HOSTID, &hostid) == 0) {
	hostname = fnvlist_lookup_string(mos_config,
	ZPOOL_CONFIG_HOSTNAME);

	myhostid = zone_get_hostid(NULL);

	if (hostid != 0 && myhostid != 0 && hostid != myhostid) {
	cmn_err(CE_WARN, "pool '%s' could not be "
	"loaded as it was last accessed by "
	"another system (host: %s hostid: 0x%llx). "
	"See: https://openzfs.github.io/openzfs-docs/msg/"
	"ZFS-8000-EY",
	spa_name(spa), hostname, (u_longlong_t)hostid);
	spa_load_failed(spa, "hostid verification failed: pool "
	"last accessed by host: %s (hostid: 0x%llx)",
	hostname, (u_longlong_t)hostid);
	return (SET_ERROR(EBADF));
	}
	}

	return (0);
	}

	static int
	spa_ld_parse_config(spa_t *spa, spa_import_type_t type)
	{
	int error = 0;
	nvlist_t nvtree, nvl, *config = spa->spa_config;
	int parse;
	vdev_t *rvd;
	uint64_t pool_guid;
	const char *comment;
	const char *compatibility;

	/*
	* Versioning wasn't explicitly added to the label until later, so if
	* it's not present treat it as the initial version.
	*/
	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
	&spa->spa_ubsync.ub_version) != 0)
	spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL;

	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid)) {
	spa_load_failed(spa, "invalid config provided: '%s' missing",
	ZPOOL_CONFIG_POOL_GUID);
	return (SET_ERROR(EINVAL));
	}

	/*
	* If we are doing an import, ensure that the pool is not already
	* imported by checking if its pool guid already exists in the
	* spa namespace.
	*
	* The only case that we allow an already imported pool to be
	* imported again, is when the pool is checkpointed and we want to
	* look at its checkpointed state from userland tools like zdb.
	*/
	#ifdef _KERNEL
	if ((spa->spa_load_state == SPA_LOAD_IMPORT \|\|
	spa->spa_load_state == SPA_LOAD_TRYIMPORT) &&
	spa_guid_exists(pool_guid, 0)) {
	#else
	if ((spa->spa_load_state == SPA_LOAD_IMPORT \|\|
	spa->spa_load_state == SPA_LOAD_TRYIMPORT) &&
	spa_guid_exists(pool_guid, 0) &&
	!spa_importing_readonly_checkpoint(spa)) {
	#endif
	spa_load_failed(spa, "a pool with guid %llu is already open",
	(u_longlong_t)pool_guid);
	return (SET_ERROR(EEXIST));
	}

	spa->spa_config_guid = pool_guid;

	nvlist_free(spa->spa_load_info);
	spa->spa_load_info = fnvlist_alloc();

	ASSERT(spa->spa_comment == NULL);
	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
	spa->spa_comment = spa_strdup(comment);

	ASSERT(spa->spa_compatibility == NULL);
	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMPATIBILITY,
	&compatibility) == 0)
	spa->spa_compatibility = spa_strdup(compatibility);

	(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG,
	&spa->spa_config_txg);

	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) == 0)
	spa->spa_config_splitting = fnvlist_dup(nvl);

	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvtree)) {
	spa_load_failed(spa, "invalid config provided: '%s' missing",
	ZPOOL_CONFIG_VDEV_TREE);
	return (SET_ERROR(EINVAL));
	}

	/*
	* Create "The Godfather" zio to hold all async IOs
	*/
	spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
	KM_SLEEP);
	for (int i = 0; i < max_ncpus; i++) {
	spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
	ZIO_FLAG_CANFAIL \| ZIO_FLAG_SPECULATIVE \|
	ZIO_FLAG_GODFATHER);
	}

	/*
	* Parse the configuration into a vdev tree. We explicitly set the
	* value that will be returned by spa_version() since parsing the
	* configuration requires knowing the version number.
	*/
	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	parse = (type == SPA_IMPORT_EXISTING ?
	VDEV_ALLOC_LOAD : VDEV_ALLOC_SPLIT);
	error = spa_config_parse(spa, &rvd, nvtree, NULL, 0, parse);
	spa_config_exit(spa, SCL_ALL, FTAG);

	if (error != 0) {
	spa_load_failed(spa, "unable to parse config [error=%d]",
	error);
	return (error);
	}

	ASSERT(spa->spa_root_vdev == rvd);
	ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT);
	ASSERT3U(spa->spa_max_ashift, <=, SPA_MAXBLOCKSHIFT);

	if (type != SPA_IMPORT_ASSEMBLE) {
	ASSERT(spa_guid(spa) == pool_guid);
	}

	return (0);
	}

	/*
	* Recursively open all vdevs in the vdev tree. This function is called twice:
	* first with the untrusted config, then with the trusted config.
	*/
	static int
	spa_ld_open_vdevs(spa_t *spa)
	{
	int error = 0;

	/*
	* spa_missing_tvds_allowed defines how many top-level vdevs can be
	* missing/unopenable for the root vdev to be still considered openable.
	*/
	if (spa->spa_trust_config) {
	spa->spa_missing_tvds_allowed = zfs_max_missing_tvds;
	} else if (spa->spa_config_source == SPA_CONFIG_SRC_CACHEFILE) {
	spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_cachefile;
	} else if (spa->spa_config_source == SPA_CONFIG_SRC_SCAN) {
	spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_scan;
	} else {
	spa->spa_missing_tvds_allowed = 0;
	}

	spa->spa_missing_tvds_allowed =
	MAX(zfs_max_missing_tvds, spa->spa_missing_tvds_allowed);

	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	error = vdev_open(spa->spa_root_vdev);
	spa_config_exit(spa, SCL_ALL, FTAG);

	if (spa->spa_missing_tvds != 0) {
	spa_load_note(spa, "vdev tree has %lld missing top-level "
	"vdevs.", (u_longlong_t)spa->spa_missing_tvds);
	if (spa->spa_trust_config && (spa->spa_mode & SPA_MODE_WRITE)) {
	/*
	* Although theoretically we could allow users to open
	* incomplete pools in RW mode, we'd need to add a lot
	* of extra logic (e.g. adjust pool space to account
	* for missing vdevs).
	* This limitation also prevents users from accidentally
	* opening the pool in RW mode during data recovery and
	* damaging it further.
	*/
	spa_load_note(spa, "pools with missing top-level "
	"vdevs can only be opened in read-only mode.");
	error = SET_ERROR(ENXIO);
	} else {
	spa_load_note(spa, "current settings allow for maximum "
	"%lld missing top-level vdevs at this stage.",
	(u_longlong_t)spa->spa_missing_tvds_allowed);
	}
	}
	if (error != 0) {
	spa_load_failed(spa, "unable to open vdev tree [error=%d]",
	error);
	}
	if (spa->spa_missing_tvds != 0 \|\| error != 0)
	vdev_dbgmsg_print_tree(spa->spa_root_vdev, 2);

	return (error);
	}

	/*
	* We need to validate the vdev labels against the configuration that
	* we have in hand. This function is called twice: first with an untrusted
	* config, then with a trusted config. The validation is more strict when the
	* config is trusted.
	*/
	static int
	spa_ld_validate_vdevs(spa_t *spa)
	{
	int error = 0;
	vdev_t *rvd = spa->spa_root_vdev;

	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	error = vdev_validate(rvd);
	spa_config_exit(spa, SCL_ALL, FTAG);

	if (error != 0) {
	spa_load_failed(spa, "vdev_validate failed [error=%d]", error);
	return (error);
	}

	if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) {
	spa_load_failed(spa, "cannot open vdev tree after invalidating "
	"some vdevs");
	vdev_dbgmsg_print_tree(rvd, 2);
	return (SET_ERROR(ENXIO));
	}

	return (0);
	}

	static void
	spa_ld_select_uberblock_done(spa_t spa, uberblock_t ub)
	{
	spa->spa_state = POOL_STATE_ACTIVE;
	spa->spa_ubsync = spa->spa_uberblock;
	spa->spa_verify_min_txg = spa->spa_extreme_rewind ?
	TXG_INITIAL - 1 : spa_last_synced_txg(spa) - TXG_DEFER_SIZE - 1;
	spa->spa_first_txg = spa->spa_last_ubsync_txg ?
	spa->spa_last_ubsync_txg : spa_last_synced_txg(spa) + 1;
	spa->spa_claim_max_txg = spa->spa_first_txg;
	spa->spa_prev_software_version = ub->ub_software_version;
	}

	static int
	spa_ld_select_uberblock(spa_t *spa, spa_import_type_t type)
	{
	vdev_t *rvd = spa->spa_root_vdev;
	nvlist_t *label;
	uberblock_t *ub = &spa->spa_uberblock;
	boolean_t activity_check = B_FALSE;

	/*
	* If we are opening the checkpointed state of the pool by
	* rewinding to it, at this point we will have written the
	* checkpointed uberblock to the vdev labels, so searching
	* the labels will find the right uberblock. However, if
	* we are opening the checkpointed state read-only, we have
	* not modified the labels. Therefore, we must ignore the
	* labels and continue using the spa_uberblock that was set
	* by spa_ld_checkpoint_rewind.
	*
	* Note that it would be fine to ignore the labels when
	* rewinding (opening writeable) as well. However, if we
	* crash just after writing the labels, we will end up
	* searching the labels. Doing so in the common case means
	* that this code path gets exercised normally, rather than
	* just in the edge case.
	*/
	if (ub->ub_checkpoint_txg != 0 &&
	spa_importing_readonly_checkpoint(spa)) {
	spa_ld_select_uberblock_done(spa, ub);
	return (0);
	}

	/*
	* Find the best uberblock.
	*/
	vdev_uberblock_load(rvd, ub, &label);

	/*
	* If we weren't able to find a single valid uberblock, return failure.
	*/
	if (ub->ub_txg == 0) {
	nvlist_free(label);
	spa_load_failed(spa, "no valid uberblock found");
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, ENXIO));
	}

	if (spa->spa_load_max_txg != UINT64_MAX) {
	(void) spa_import_progress_set_max_txg(spa_guid(spa),
	(u_longlong_t)spa->spa_load_max_txg);
	}
	spa_load_note(spa, "using uberblock with txg=%llu",
	(u_longlong_t)ub->ub_txg);


	/*
	* For pools which have the multihost property on determine if the
	* pool is truly inactive and can be safely imported. Prevent
	* hosts which don't have a hostid set from importing the pool.
	*/
	activity_check = spa_activity_check_required(spa, ub, label,
	spa->spa_config);
	if (activity_check) {
	if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay &&
	spa_get_hostid(spa) == 0) {
	nvlist_free(label);
	fnvlist_add_uint64(spa->spa_load_info,
	ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID);
	return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO));
	}

	- int error = spa_activity_check(spa, ub, spa->spa_config);
	+ int error =
	+ spa_activity_check(spa, ub, spa->spa_config, B_TRUE);
	if (error) {
	nvlist_free(label);
	return (error);
	}

	fnvlist_add_uint64(spa->spa_load_info,
	ZPOOL_CONFIG_MMP_STATE, MMP_STATE_INACTIVE);
	fnvlist_add_uint64(spa->spa_load_info,
	ZPOOL_CONFIG_MMP_TXG, ub->ub_txg);
	fnvlist_add_uint16(spa->spa_load_info,
	ZPOOL_CONFIG_MMP_SEQ,
	(MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0));
	}

	/*
	* If the pool has an unsupported version we can't open it.
	*/
	if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) {
	nvlist_free(label);
	spa_load_failed(spa, "version %llu is not supported",
	(u_longlong_t)ub->ub_version);
	return (spa_vdev_err(rvd, VDEV_AUX_VERSION_NEWER, ENOTSUP));
	}

	if (ub->ub_version >= SPA_VERSION_FEATURES) {
	nvlist_t *features;

	/*
	* If we weren't able to find what's necessary for reading the
	* MOS in the label, return failure.
	*/
	if (label == NULL) {
	spa_load_failed(spa, "label config unavailable");
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA,
	ENXIO));
	}

	if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_FEATURES_FOR_READ,
	&features) != 0) {
	nvlist_free(label);
	spa_load_failed(spa, "invalid label: '%s' missing",
	ZPOOL_CONFIG_FEATURES_FOR_READ);
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA,
	ENXIO));
	}

	/*
	* Update our in-core representation with the definitive values
	* from the label.
	*/
	nvlist_free(spa->spa_label_features);
	spa->spa_label_features = fnvlist_dup(features);
	}

	nvlist_free(label);

	/*
	* Look through entries in the label nvlist's features_for_read. If
	* there is a feature listed there which we don't understand then we
	* cannot open a pool.
	*/
	if (ub->ub_version >= SPA_VERSION_FEATURES) {
	nvlist_t *unsup_feat;

	unsup_feat = fnvlist_alloc();

	for (nvpair_t *nvp = nvlist_next_nvpair(spa->spa_label_features,
	NULL); nvp != NULL;
	nvp = nvlist_next_nvpair(spa->spa_label_features, nvp)) {
	if (!zfeature_is_supported(nvpair_name(nvp))) {
	fnvlist_add_string(unsup_feat,
	nvpair_name(nvp), "");
	}
	}

	if (!nvlist_empty(unsup_feat)) {
	fnvlist_add_nvlist(spa->spa_load_info,
	ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat);
	nvlist_free(unsup_feat);
	spa_load_failed(spa, "some features are unsupported");
	return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT,
	ENOTSUP));
	}

	nvlist_free(unsup_feat);
	}

	if (type != SPA_IMPORT_ASSEMBLE && spa->spa_config_splitting) {
	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	spa_try_repair(spa, spa->spa_config);
	spa_config_exit(spa, SCL_ALL, FTAG);
	nvlist_free(spa->spa_config_splitting);
	spa->spa_config_splitting = NULL;
	}

	/*
	* Initialize internal SPA structures.
	*/
	spa_ld_select_uberblock_done(spa, ub);

	return (0);
	}

	static int
	spa_ld_open_rootbp(spa_t *spa)
	{
	int error = 0;
	vdev_t *rvd = spa->spa_root_vdev;

	error = dsl_pool_init(spa, spa->spa_first_txg, &spa->spa_dsl_pool);
	if (error != 0) {
	spa_load_failed(spa, "unable to open rootbp in dsl_pool_init "
	"[error=%d]", error);
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	}
	spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset;

	return (0);
	}

	static int
	spa_ld_trusted_config(spa_t *spa, spa_import_type_t type,
	boolean_t reloading)
	{
	vdev_t mrvd, rvd = spa->spa_root_vdev;
	nvlist_t nv, mos_config, *policy;
	int error = 0, copy_error;
	uint64_t healthy_tvds, healthy_tvds_mos;
	uint64_t mos_config_txg;

	if (spa_dir_prop(spa, DMU_POOL_CONFIG, &spa->spa_config_object, B_TRUE)
	!= 0)
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));

	/*
	* If we're assembling a pool from a split, the config provided is
	* already trusted so there is nothing to do.
	*/
	if (type == SPA_IMPORT_ASSEMBLE)
	return (0);

	healthy_tvds = spa_healthy_core_tvds(spa);

	if (load_nvlist(spa, spa->spa_config_object, &mos_config)
	!= 0) {
	spa_load_failed(spa, "unable to retrieve MOS config");
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	}

	/*
	* If we are doing an open, pool owner wasn't verified yet, thus do
	* the verification here.
	*/
	if (spa->spa_load_state == SPA_LOAD_OPEN) {
	error = spa_verify_host(spa, mos_config);
	if (error != 0) {
	nvlist_free(mos_config);
	return (error);
	}
	}

	nv = fnvlist_lookup_nvlist(mos_config, ZPOOL_CONFIG_VDEV_TREE);

	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);

	/*
	* Build a new vdev tree from the trusted config
	*/
	error = spa_config_parse(spa, &mrvd, nv, NULL, 0, VDEV_ALLOC_LOAD);
	if (error != 0) {
	nvlist_free(mos_config);
	spa_config_exit(spa, SCL_ALL, FTAG);
	spa_load_failed(spa, "spa_config_parse failed [error=%d]",
	error);
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error));
	}

	/*
	* Vdev paths in the MOS may be obsolete. If the untrusted config was
	* obtained by scanning /dev/dsk, then it will have the right vdev
	* paths. We update the trusted MOS config with this information.
	* We first try to copy the paths with vdev_copy_path_strict, which
	* succeeds only when both configs have exactly the same vdev tree.
	* If that fails, we fall back to a more flexible method that has a
	* best effort policy.
	*/
	copy_error = vdev_copy_path_strict(rvd, mrvd);
	if (copy_error != 0 \|\| spa_load_print_vdev_tree) {
	spa_load_note(spa, "provided vdev tree:");
	vdev_dbgmsg_print_tree(rvd, 2);
	spa_load_note(spa, "MOS vdev tree:");
	vdev_dbgmsg_print_tree(mrvd, 2);
	}
	if (copy_error != 0) {
	spa_load_note(spa, "vdev_copy_path_strict failed, falling "
	"back to vdev_copy_path_relaxed");
	vdev_copy_path_relaxed(rvd, mrvd);
	}

	vdev_close(rvd);
	vdev_free(rvd);
	spa->spa_root_vdev = mrvd;
	rvd = mrvd;
	spa_config_exit(spa, SCL_ALL, FTAG);

	/*
	* If 'zpool import' used a cached config, then the on-disk hostid and
	* hostname may be different to the cached config in ways that should
	* prevent import. Userspace can't discover this without a scan, but
	* we know, so we add these values to LOAD_INFO so the caller can know
	* the difference.
	*
	* Note that we have to do this before the config is regenerated,
	* because the new config will have the hostid and hostname for this
	* host, in readiness for import.
	*/
	if (nvlist_exists(mos_config, ZPOOL_CONFIG_HOSTID))
	fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_HOSTID,
	fnvlist_lookup_uint64(mos_config, ZPOOL_CONFIG_HOSTID));
	if (nvlist_exists(mos_config, ZPOOL_CONFIG_HOSTNAME))
	fnvlist_add_string(spa->spa_load_info, ZPOOL_CONFIG_HOSTNAME,
	fnvlist_lookup_string(mos_config, ZPOOL_CONFIG_HOSTNAME));

	/*
	* We will use spa_config if we decide to reload the spa or if spa_load
	* fails and we rewind. We must thus regenerate the config using the
	* MOS information with the updated paths. ZPOOL_LOAD_POLICY is used to
	* pass settings on how to load the pool and is not stored in the MOS.
	* We copy it over to our new, trusted config.
	*/
	mos_config_txg = fnvlist_lookup_uint64(mos_config,
	ZPOOL_CONFIG_POOL_TXG);
	nvlist_free(mos_config);
	mos_config = spa_config_generate(spa, NULL, mos_config_txg, B_FALSE);
	if (nvlist_lookup_nvlist(spa->spa_config, ZPOOL_LOAD_POLICY,
	&policy) == 0)
	fnvlist_add_nvlist(mos_config, ZPOOL_LOAD_POLICY, policy);
	spa_config_set(spa, mos_config);
	spa->spa_config_source = SPA_CONFIG_SRC_MOS;

	/*
	* Now that we got the config from the MOS, we should be more strict
	* in checking blkptrs and can make assumptions about the consistency
	* of the vdev tree. spa_trust_config must be set to true before opening
	* vdevs in order for them to be writeable.
	*/
	spa->spa_trust_config = B_TRUE;

	/*
	* Open and validate the new vdev tree
	*/
	error = spa_ld_open_vdevs(spa);
	if (error != 0)
	return (error);

	error = spa_ld_validate_vdevs(spa);
	if (error != 0)
	return (error);

	if (copy_error != 0 \|\| spa_load_print_vdev_tree) {
	spa_load_note(spa, "final vdev tree:");
	vdev_dbgmsg_print_tree(rvd, 2);
	}

	if (spa->spa_load_state != SPA_LOAD_TRYIMPORT &&
	!spa->spa_extreme_rewind && zfs_max_missing_tvds == 0) {
	/*
	* Sanity check to make sure that we are indeed loading the
	* latest uberblock. If we missed SPA_SYNC_MIN_VDEVS tvds
	* in the config provided and they happened to be the only ones
	* to have the latest uberblock, we could involuntarily perform
	* an extreme rewind.
	*/
	healthy_tvds_mos = spa_healthy_core_tvds(spa);
	if (healthy_tvds_mos - healthy_tvds >=
	SPA_SYNC_MIN_VDEVS) {
	spa_load_note(spa, "config provided misses too many "
	"top-level vdevs compared to MOS (%lld vs %lld). ",
	(u_longlong_t)healthy_tvds,
	(u_longlong_t)healthy_tvds_mos);
	spa_load_note(spa, "vdev tree:");
	vdev_dbgmsg_print_tree(rvd, 2);
	if (reloading) {
	spa_load_failed(spa, "config was already "
	"provided from MOS. Aborting.");
	return (spa_vdev_err(rvd,
	VDEV_AUX_CORRUPT_DATA, EIO));
	}
	spa_load_note(spa, "spa must be reloaded using MOS "
	"config");
	return (SET_ERROR(EAGAIN));
	}
	}

	error = spa_check_for_missing_logs(spa);
	if (error != 0)
	return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, ENXIO));

	if (rvd->vdev_guid_sum != spa->spa_uberblock.ub_guid_sum) {
	spa_load_failed(spa, "uberblock guid sum doesn't match MOS "
	"guid sum (%llu != %llu)",
	(u_longlong_t)spa->spa_uberblock.ub_guid_sum,
	(u_longlong_t)rvd->vdev_guid_sum);
	return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM,
	ENXIO));
	}

	return (0);
	}

	static int
	spa_ld_open_indirect_vdev_metadata(spa_t *spa)
	{
	int error = 0;
	vdev_t *rvd = spa->spa_root_vdev;

	/*
	* Everything that we read before spa_remove_init() must be stored
	* on concreted vdevs. Therefore we do this as early as possible.
	*/
	error = spa_remove_init(spa);
	if (error != 0) {
	spa_load_failed(spa, "spa_remove_init failed [error=%d]",
	error);
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	}

	/*
	* Retrieve information needed to condense indirect vdev mappings.
	*/
	error = spa_condense_init(spa);
	if (error != 0) {
	spa_load_failed(spa, "spa_condense_init failed [error=%d]",
	error);
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error));
	}

	return (0);
	}

	static int
	spa_ld_check_features(spa_t spa, boolean_t missing_feat_writep)
	{
	int error = 0;
	vdev_t *rvd = spa->spa_root_vdev;

	if (spa_version(spa) >= SPA_VERSION_FEATURES) {
	boolean_t missing_feat_read = B_FALSE;
	nvlist_t unsup_feat, enabled_feat;

	if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ,
	&spa->spa_feat_for_read_obj, B_TRUE) != 0) {
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	}

	if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_WRITE,
	&spa->spa_feat_for_write_obj, B_TRUE) != 0) {
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	}

	if (spa_dir_prop(spa, DMU_POOL_FEATURE_DESCRIPTIONS,
	&spa->spa_feat_desc_obj, B_TRUE) != 0) {
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	}

	enabled_feat = fnvlist_alloc();
	unsup_feat = fnvlist_alloc();

	if (!spa_features_check(spa, B_FALSE,
	unsup_feat, enabled_feat))
	missing_feat_read = B_TRUE;

	if (spa_writeable(spa) \|\|
	spa->spa_load_state == SPA_LOAD_TRYIMPORT) {
	if (!spa_features_check(spa, B_TRUE,
	unsup_feat, enabled_feat)) {
	*missing_feat_writep = B_TRUE;
	}
	}

	fnvlist_add_nvlist(spa->spa_load_info,
	ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat);

	if (!nvlist_empty(unsup_feat)) {
	fnvlist_add_nvlist(spa->spa_load_info,
	ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat);
	}

	fnvlist_free(enabled_feat);
	fnvlist_free(unsup_feat);

	if (!missing_feat_read) {
	fnvlist_add_boolean(spa->spa_load_info,
	ZPOOL_CONFIG_CAN_RDONLY);
	}

	/*
	* If the state is SPA_LOAD_TRYIMPORT, our objective is
	* twofold: to determine whether the pool is available for
	* import in read-write mode and (if it is not) whether the
	* pool is available for import in read-only mode. If the pool
	* is available for import in read-write mode, it is displayed
	* as available in userland; if it is not available for import
	* in read-only mode, it is displayed as unavailable in
	* userland. If the pool is available for import in read-only
	* mode but not read-write mode, it is displayed as unavailable
	* in userland with a special note that the pool is actually
	* available for open in read-only mode.
	*
	* As a result, if the state is SPA_LOAD_TRYIMPORT and we are
	* missing a feature for write, we must first determine whether
	* the pool can be opened read-only before returning to
	* userland in order to know whether to display the
	* abovementioned note.
	*/
	if (missing_feat_read \|\| (*missing_feat_writep &&
	spa_writeable(spa))) {
	spa_load_failed(spa, "pool uses unsupported features");
	return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT,
	ENOTSUP));
	}

	/*
	* Load refcounts for ZFS features from disk into an in-memory
	* cache during SPA initialization.
	*/
	for (spa_feature_t i = 0; i < SPA_FEATURES; i++) {
	uint64_t refcount;

	error = feature_get_refcount_from_disk(spa,
	&spa_feature_table[i], &refcount);
	if (error == 0) {
	spa->spa_feat_refcount_cache[i] = refcount;
	} else if (error == ENOTSUP) {
	spa->spa_feat_refcount_cache[i] =
	SPA_FEATURE_DISABLED;
	} else {
	spa_load_failed(spa, "error getting refcount "
	"for feature %s [error=%d]",
	spa_feature_table[i].fi_guid, error);
	return (spa_vdev_err(rvd,
	VDEV_AUX_CORRUPT_DATA, EIO));
	}
	}
	}

	if (spa_feature_is_active(spa, SPA_FEATURE_ENABLED_TXG)) {
	if (spa_dir_prop(spa, DMU_POOL_FEATURE_ENABLED_TXG,
	&spa->spa_feat_enabled_txg_obj, B_TRUE) != 0)
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	}

	/*
	* Encryption was added before bookmark_v2, even though bookmark_v2
	* is now a dependency. If this pool has encryption enabled without
	* bookmark_v2, trigger an errata message.
	*/
	if (spa_feature_is_enabled(spa, SPA_FEATURE_ENCRYPTION) &&
	!spa_feature_is_enabled(spa, SPA_FEATURE_BOOKMARK_V2)) {
	spa->spa_errata = ZPOOL_ERRATA_ZOL_8308_ENCRYPTION;
	}

	return (0);
	}

	static int
	spa_ld_load_special_directories(spa_t *spa)
	{
	int error = 0;
	vdev_t *rvd = spa->spa_root_vdev;

	spa->spa_is_initializing = B_TRUE;
	error = dsl_pool_open(spa->spa_dsl_pool);
	spa->spa_is_initializing = B_FALSE;
	if (error != 0) {
	spa_load_failed(spa, "dsl_pool_open failed [error=%d]", error);
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	}

	return (0);
	}

	static int
	spa_ld_get_props(spa_t *spa)
	{
	int error = 0;
	uint64_t obj;
	vdev_t *rvd = spa->spa_root_vdev;

	/* Grab the checksum salt from the MOS. */
	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
	DMU_POOL_CHECKSUM_SALT, 1,
	sizeof (spa->spa_cksum_salt.zcs_bytes),
	spa->spa_cksum_salt.zcs_bytes);
	if (error == ENOENT) {
	/* Generate a new salt for subsequent use */
	(void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes,
	sizeof (spa->spa_cksum_salt.zcs_bytes));
	} else if (error != 0) {
	spa_load_failed(spa, "unable to retrieve checksum salt from "
	"MOS [error=%d]", error);
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	}

	if (spa_dir_prop(spa, DMU_POOL_SYNC_BPOBJ, &obj, B_TRUE) != 0)
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj);
	if (error != 0) {
	spa_load_failed(spa, "error opening deferred-frees bpobj "
	"[error=%d]", error);
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	}

	/*
	* Load the bit that tells us to use the new accounting function
	* (raid-z deflation). If we have an older pool, this will not
	* be present.
	*/
	error = spa_dir_prop(spa, DMU_POOL_DEFLATE, &spa->spa_deflate, B_FALSE);
	if (error != 0 && error != ENOENT)
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));

	error = spa_dir_prop(spa, DMU_POOL_CREATION_VERSION,
	&spa->spa_creation_version, B_FALSE);
	if (error != 0 && error != ENOENT)
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));

	/*
	* Load the persistent error log. If we have an older pool, this will
	* not be present.
	*/
	error = spa_dir_prop(spa, DMU_POOL_ERRLOG_LAST, &spa->spa_errlog_last,
	B_FALSE);
	if (error != 0 && error != ENOENT)
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));

	error = spa_dir_prop(spa, DMU_POOL_ERRLOG_SCRUB,
	&spa->spa_errlog_scrub, B_FALSE);
	if (error != 0 && error != ENOENT)
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));

	/*
	* Load the livelist deletion field. If a livelist is queued for
	* deletion, indicate that in the spa
	*/
	error = spa_dir_prop(spa, DMU_POOL_DELETED_CLONES,
	&spa->spa_livelists_to_delete, B_FALSE);
	if (error != 0 && error != ENOENT)
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));

	/*
	* Load the history object. If we have an older pool, this
	* will not be present.
	*/
	error = spa_dir_prop(spa, DMU_POOL_HISTORY, &spa->spa_history, B_FALSE);
	if (error != 0 && error != ENOENT)
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));

	/*
	* Load the per-vdev ZAP map. If we have an older pool, this will not
	* be present; in this case, defer its creation to a later time to
	* avoid dirtying the MOS this early / out of sync context. See
	* spa_sync_config_object.
	*/

	/* The sentinel is only available in the MOS config. */
	nvlist_t *mos_config;
	if (load_nvlist(spa, spa->spa_config_object, &mos_config) != 0) {
	spa_load_failed(spa, "unable to retrieve MOS config");
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	}

	error = spa_dir_prop(spa, DMU_POOL_VDEV_ZAP_MAP,
	&spa->spa_all_vdev_zaps, B_FALSE);

	if (error == ENOENT) {
	VERIFY(!nvlist_exists(mos_config,
	ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS));
	spa->spa_avz_action = AVZ_ACTION_INITIALIZE;
	ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev));
	} else if (error != 0) {
	nvlist_free(mos_config);
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	} else if (!nvlist_exists(mos_config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)) {
	/*
	* An older version of ZFS overwrote the sentinel value, so
	* we have orphaned per-vdev ZAPs in the MOS. Defer their
	* destruction to later; see spa_sync_config_object.
	*/
	spa->spa_avz_action = AVZ_ACTION_DESTROY;
	/*
	* We're assuming that no vdevs have had their ZAPs created
	* before this. Better be sure of it.
	*/
	ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev));
	}
	nvlist_free(mos_config);

	spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION);

	error = spa_dir_prop(spa, DMU_POOL_PROPS, &spa->spa_pool_props_object,
	B_FALSE);
	if (error && error != ENOENT)
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));

	if (error == 0) {
	uint64_t autoreplace = 0;

	spa_prop_find(spa, ZPOOL_PROP_BOOTFS, &spa->spa_bootfs);
	spa_prop_find(spa, ZPOOL_PROP_AUTOREPLACE, &autoreplace);
	spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation);
	spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode);
	spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand);
	spa_prop_find(spa, ZPOOL_PROP_MULTIHOST, &spa->spa_multihost);
	spa_prop_find(spa, ZPOOL_PROP_AUTOTRIM, &spa->spa_autotrim);
	spa->spa_autoreplace = (autoreplace != 0);
	}

	/*
	* If we are importing a pool with missing top-level vdevs,
	* we enforce that the pool doesn't panic or get suspended on
	* error since the likelihood of missing data is extremely high.
	*/
	if (spa->spa_missing_tvds > 0 &&
	spa->spa_failmode != ZIO_FAILURE_MODE_CONTINUE &&
	spa->spa_load_state != SPA_LOAD_TRYIMPORT) {
	spa_load_note(spa, "forcing failmode to 'continue' "
	"as some top level vdevs are missing");
	spa->spa_failmode = ZIO_FAILURE_MODE_CONTINUE;
	}

	return (0);
	}

	static int
	spa_ld_open_aux_vdevs(spa_t *spa, spa_import_type_t type)
	{
	int error = 0;
	vdev_t *rvd = spa->spa_root_vdev;

	/*
	* If we're assembling the pool from the split-off vdevs of
	* an existing pool, we don't want to attach the spares & cache
	* devices.
	*/

	/*
	* Load any hot spares for this pool.
	*/
	error = spa_dir_prop(spa, DMU_POOL_SPARES, &spa->spa_spares.sav_object,
	B_FALSE);
	if (error != 0 && error != ENOENT)
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	if (error == 0 && type != SPA_IMPORT_ASSEMBLE) {
	ASSERT(spa_version(spa) >= SPA_VERSION_SPARES);
	if (load_nvlist(spa, spa->spa_spares.sav_object,
	&spa->spa_spares.sav_config) != 0) {
	spa_load_failed(spa, "error loading spares nvlist");
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	}

	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	spa_load_spares(spa);
	spa_config_exit(spa, SCL_ALL, FTAG);
	} else if (error == 0) {
	spa->spa_spares.sav_sync = B_TRUE;
	}

	/*
	* Load any level 2 ARC devices for this pool.
	*/
	error = spa_dir_prop(spa, DMU_POOL_L2CACHE,
	&spa->spa_l2cache.sav_object, B_FALSE);
	if (error != 0 && error != ENOENT)
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	if (error == 0 && type != SPA_IMPORT_ASSEMBLE) {
	ASSERT(spa_version(spa) >= SPA_VERSION_L2CACHE);
	if (load_nvlist(spa, spa->spa_l2cache.sav_object,
	&spa->spa_l2cache.sav_config) != 0) {
	spa_load_failed(spa, "error loading l2cache nvlist");
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	}

	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	spa_load_l2cache(spa);
	spa_config_exit(spa, SCL_ALL, FTAG);
	} else if (error == 0) {
	spa->spa_l2cache.sav_sync = B_TRUE;
	}

	return (0);
	}

	static int
	spa_ld_load_vdev_metadata(spa_t *spa)
	{
	int error = 0;
	vdev_t *rvd = spa->spa_root_vdev;

	/*
	* If the 'multihost' property is set, then never allow a pool to
	* be imported when the system hostid is zero. The exception to
	* this rule is zdb which is always allowed to access pools.
	*/
	if (spa_multihost(spa) && spa_get_hostid(spa) == 0 &&
	(spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) == 0) {
	fnvlist_add_uint64(spa->spa_load_info,
	ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID);
	return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO));
	}

	/*
	* If the 'autoreplace' property is set, then post a resource notifying
	* the ZFS DE that it should not issue any faults for unopenable
	* devices. We also iterate over the vdevs, and post a sysevent for any
	* unopenable vdevs so that the normal autoreplace handler can take
	* over.
	*/
	if (spa->spa_autoreplace && spa->spa_load_state != SPA_LOAD_TRYIMPORT) {
	spa_check_removed(spa->spa_root_vdev);
	/*
	* For the import case, this is done in spa_import(), because
	* at this point we're using the spare definitions from
	* the MOS config, not necessarily from the userland config.
	*/
	if (spa->spa_load_state != SPA_LOAD_IMPORT) {
	spa_aux_check_removed(&spa->spa_spares);
	spa_aux_check_removed(&spa->spa_l2cache);
	}
	}

	/*
	* Load the vdev metadata such as metaslabs, DTLs, spacemap object, etc.
	*/
	error = vdev_load(rvd);
	if (error != 0) {
	spa_load_failed(spa, "vdev_load failed [error=%d]", error);
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error));
	}

	error = spa_ld_log_spacemaps(spa);
	if (error != 0) {
	spa_load_failed(spa, "spa_ld_log_spacemaps failed [error=%d]",
	error);
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error));
	}

	/*
	* Propagate the leaf DTLs we just loaded all the way up the vdev tree.
	*/
	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	vdev_dtl_reassess(rvd, 0, 0, B_FALSE, B_FALSE);
	spa_config_exit(spa, SCL_ALL, FTAG);

	return (0);
	}

	static int
	spa_ld_load_dedup_tables(spa_t *spa)
	{
	int error = 0;
	vdev_t *rvd = spa->spa_root_vdev;

	error = ddt_load(spa);
	if (error != 0) {
	spa_load_failed(spa, "ddt_load failed [error=%d]", error);
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	}

	return (0);
	}

	static int
	spa_ld_load_brt(spa_t *spa)
	{
	int error = 0;
	vdev_t *rvd = spa->spa_root_vdev;

	error = brt_load(spa);
	if (error != 0) {
	spa_load_failed(spa, "brt_load failed [error=%d]", error);
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
	}

	return (0);
	}

	static int
	spa_ld_verify_logs(spa_t spa, spa_import_type_t type, const char *ereport)
	{
	vdev_t *rvd = spa->spa_root_vdev;

	if (type != SPA_IMPORT_ASSEMBLE && spa_writeable(spa)) {
	boolean_t missing = spa_check_logs(spa);
	if (missing) {
	if (spa->spa_missing_tvds != 0) {
	spa_load_note(spa, "spa_check_logs failed "
	"so dropping the logs");
	} else {
	*ereport = FM_EREPORT_ZFS_LOG_REPLAY;
	spa_load_failed(spa, "spa_check_logs failed");
	return (spa_vdev_err(rvd, VDEV_AUX_BAD_LOG,
	ENXIO));
	}
	}
	}

	return (0);
	}

	static int
	spa_ld_verify_pool_data(spa_t *spa)
	{
	int error = 0;
	vdev_t *rvd = spa->spa_root_vdev;

	/*
	* We've successfully opened the pool, verify that we're ready
	* to start pushing transactions.
	*/
	if (spa->spa_load_state != SPA_LOAD_TRYIMPORT) {
	error = spa_load_verify(spa);
	if (error != 0) {
	spa_load_failed(spa, "spa_load_verify failed "
	"[error=%d]", error);
	return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA,
	error));
	}
	}

	return (0);
	}

	static void
	spa_ld_claim_log_blocks(spa_t *spa)
	{
	dmu_tx_t *tx;
	dsl_pool_t *dp = spa_get_dsl(spa);

	/*
	* Claim log blocks that haven't been committed yet.
	* This must all happen in a single txg.
	* Note: spa_claim_max_txg is updated by spa_claim_notify(),
	* invoked from zil_claim_log_block()'s i/o done callback.
	* Price of rollback is that we abandon the log.
	*/
	spa->spa_claiming = B_TRUE;

	tx = dmu_tx_create_assigned(dp, spa_first_txg(spa));
	(void) dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
	zil_claim, tx, DS_FIND_CHILDREN);
	dmu_tx_commit(tx);

	spa->spa_claiming = B_FALSE;

	spa_set_log_state(spa, SPA_LOG_GOOD);
	}

	static void
	spa_ld_check_for_config_update(spa_t *spa, uint64_t config_cache_txg,
	boolean_t update_config_cache)
	{
	vdev_t *rvd = spa->spa_root_vdev;
	int need_update = B_FALSE;

	/*
	* If the config cache is stale, or we have uninitialized
	* metaslabs (see spa_vdev_add()), then update the config.
	*
	* If this is a verbatim import, trust the current
	* in-core spa_config and update the disk labels.
	*/
	if (update_config_cache \|\| config_cache_txg != spa->spa_config_txg \|\|
	spa->spa_load_state == SPA_LOAD_IMPORT \|\|
	spa->spa_load_state == SPA_LOAD_RECOVER \|\|
	(spa->spa_import_flags & ZFS_IMPORT_VERBATIM))
	need_update = B_TRUE;

	for (int c = 0; c < rvd->vdev_children; c++)
	if (rvd->vdev_child[c]->vdev_ms_array == 0)
	need_update = B_TRUE;

	/*
	* Update the config cache asynchronously in case we're the
	* root pool, in which case the config cache isn't writable yet.
	*/
	if (need_update)
	spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
	}

	static void
	spa_ld_prepare_for_reload(spa_t *spa)
	{
	spa_mode_t mode = spa->spa_mode;
	int async_suspended = spa->spa_async_suspended;

	spa_unload(spa);
	spa_deactivate(spa);
	spa_activate(spa, mode);

	/*
	* We save the value of spa_async_suspended as it gets reset to 0 by
	* spa_unload(). We want to restore it back to the original value before
	* returning as we might be calling spa_async_resume() later.
	*/
	spa->spa_async_suspended = async_suspended;
	}

	static int
	spa_ld_read_checkpoint_txg(spa_t *spa)
	{
	uberblock_t checkpoint;
	int error = 0;

	ASSERT0(spa->spa_checkpoint_txg);
	ASSERT(MUTEX_HELD(&spa_namespace_lock));

	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
	DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t),
	sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint);

	if (error == ENOENT)
	return (0);

	if (error != 0)
	return (error);

	ASSERT3U(checkpoint.ub_txg, !=, 0);
	ASSERT3U(checkpoint.ub_checkpoint_txg, !=, 0);
	ASSERT3U(checkpoint.ub_timestamp, !=, 0);
	spa->spa_checkpoint_txg = checkpoint.ub_txg;
	spa->spa_checkpoint_info.sci_timestamp = checkpoint.ub_timestamp;

	return (0);
	}

	static int
	spa_ld_mos_init(spa_t *spa, spa_import_type_t type)
	{
	int error = 0;

	ASSERT(MUTEX_HELD(&spa_namespace_lock));
	ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE);

	/*
	* Never trust the config that is provided unless we are assembling
	* a pool following a split.
	* This means don't trust blkptrs and the vdev tree in general. This
	* also effectively puts the spa in read-only mode since
	* spa_writeable() checks for spa_trust_config to be true.
	* We will later load a trusted config from the MOS.
	*/
	if (type != SPA_IMPORT_ASSEMBLE)
	spa->spa_trust_config = B_FALSE;

	/*
	* Parse the config provided to create a vdev tree.
	*/
	error = spa_ld_parse_config(spa, type);
	if (error != 0)
	return (error);

	spa_import_progress_add(spa);

	/*
	* Now that we have the vdev tree, try to open each vdev. This involves
	* opening the underlying physical device, retrieving its geometry and
	* probing the vdev with a dummy I/O. The state of each vdev will be set
	* based on the success of those operations. After this we'll be ready
	* to read from the vdevs.
	*/
	error = spa_ld_open_vdevs(spa);
	if (error != 0)
	return (error);

	/*
	* Read the label of each vdev and make sure that the GUIDs stored
	* there match the GUIDs in the config provided.
	* If we're assembling a new pool that's been split off from an
	* existing pool, the labels haven't yet been updated so we skip
	* validation for now.
	*/
	if (type != SPA_IMPORT_ASSEMBLE) {
	error = spa_ld_validate_vdevs(spa);
	if (error != 0)
	return (error);
	}

	/*
	* Read all vdev labels to find the best uberblock (i.e. latest,
	* unless spa_load_max_txg is set) and store it in spa_uberblock. We
	* get the list of features required to read blkptrs in the MOS from
	* the vdev label with the best uberblock and verify that our version
	* of zfs supports them all.
	*/
	error = spa_ld_select_uberblock(spa, type);
	if (error != 0)
	return (error);

	/*
	* Pass that uberblock to the dsl_pool layer which will open the root
	* blkptr. This blkptr points to the latest version of the MOS and will
	* allow us to read its contents.
	*/
	error = spa_ld_open_rootbp(spa);
	if (error != 0)
	return (error);

	return (0);
	}

	static int
	spa_ld_checkpoint_rewind(spa_t *spa)
	{
	uberblock_t checkpoint;
	int error = 0;

	ASSERT(MUTEX_HELD(&spa_namespace_lock));
	ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT);

	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
	DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t),
	sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint);

	if (error != 0) {
	spa_load_failed(spa, "unable to retrieve checkpointed "
	"uberblock from the MOS config [error=%d]", error);

	if (error == ENOENT)
	error = ZFS_ERR_NO_CHECKPOINT;

	return (error);
	}

	ASSERT3U(checkpoint.ub_txg, <, spa->spa_uberblock.ub_txg);
	ASSERT3U(checkpoint.ub_txg, ==, checkpoint.ub_checkpoint_txg);

	/*
	* We need to update the txg and timestamp of the checkpointed
	* uberblock to be higher than the latest one. This ensures that
	* the checkpointed uberblock is selected if we were to close and
	* reopen the pool right after we've written it in the vdev labels.
	* (also see block comment in vdev_uberblock_compare)
	*/
	checkpoint.ub_txg = spa->spa_uberblock.ub_txg + 1;
	checkpoint.ub_timestamp = gethrestime_sec();

	/*
	* Set current uberblock to be the checkpointed uberblock.
	*/
	spa->spa_uberblock = checkpoint;

	/*
	* If we are doing a normal rewind, then the pool is open for
	* writing and we sync the "updated" checkpointed uberblock to
	* disk. Once this is done, we've basically rewound the whole
	* pool and there is no way back.
	*
	* There are cases when we don't want to attempt and sync the
	* checkpointed uberblock to disk because we are opening a
	* pool as read-only. Specifically, verifying the checkpointed
	* state with zdb, and importing the checkpointed state to get
	* a "preview" of its content.
	*/
	if (spa_writeable(spa)) {
	vdev_t *rvd = spa->spa_root_vdev;

	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL };
	int svdcount = 0;
	int children = rvd->vdev_children;
	int c0 = random_in_range(children);

	for (int c = 0; c < children; c++) {
	vdev_t *vd = rvd->vdev_child[(c0 + c) % children];

	/* Stop when revisiting the first vdev */
	if (c > 0 && svd[0] == vd)
	break;

	if (vd->vdev_ms_array == 0 \|\| vd->vdev_islog \|\|
	!vdev_is_concrete(vd))
	continue;

	svd[svdcount++] = vd;
	if (svdcount == SPA_SYNC_MIN_VDEVS)
	break;
	}
	error = vdev_config_sync(svd, svdcount, spa->spa_first_txg);
	if (error == 0)
	spa->spa_last_synced_guid = rvd->vdev_guid;
	spa_config_exit(spa, SCL_ALL, FTAG);

	if (error != 0) {
	spa_load_failed(spa, "failed to write checkpointed "
	"uberblock to the vdev labels [error=%d]", error);
	return (error);
	}
	}

	return (0);
	}

	static int
	spa_ld_mos_with_trusted_config(spa_t *spa, spa_import_type_t type,
	boolean_t *update_config_cache)
	{
	int error;

	/*
	* Parse the config for pool, open and validate vdevs,
	* select an uberblock, and use that uberblock to open
	* the MOS.
	*/
	error = spa_ld_mos_init(spa, type);
	if (error != 0)
	return (error);

	/*
	* Retrieve the trusted config stored in the MOS and use it to create
	* a new, exact version of the vdev tree, then reopen all vdevs.
	*/
	error = spa_ld_trusted_config(spa, type, B_FALSE);
	if (error == EAGAIN) {
	if (update_config_cache != NULL)
	*update_config_cache = B_TRUE;

	/*
	* Redo the loading process with the trusted config if it is
	* too different from the untrusted config.
	*/
	spa_ld_prepare_for_reload(spa);
	spa_load_note(spa, "RELOADING");
	error = spa_ld_mos_init(spa, type);
	if (error != 0)
	return (error);

	error = spa_ld_trusted_config(spa, type, B_TRUE);
	if (error != 0)
	return (error);

	} else if (error != 0) {
	return (error);
	}

	return (0);
	}

	/*
	* Load an existing storage pool, using the config provided. This config
	* describes which vdevs are part of the pool and is later validated against
	* partial configs present in each vdev's label and an entire copy of the
	* config stored in the MOS.
	*/
	static int
	spa_load_impl(spa_t spa, spa_import_type_t type, const char *ereport)
	{
	int error = 0;
	boolean_t missing_feat_write = B_FALSE;
	boolean_t checkpoint_rewind =
	(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT);
	boolean_t update_config_cache = B_FALSE;

	ASSERT(MUTEX_HELD(&spa_namespace_lock));
	ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE);

	spa_load_note(spa, "LOADING");

	error = spa_ld_mos_with_trusted_config(spa, type, &update_config_cache);
	if (error != 0)
	return (error);

	/*
	* If we are rewinding to the checkpoint then we need to repeat
	* everything we've done so far in this function but this time
	* selecting the checkpointed uberblock and using that to open
	* the MOS.
	*/
	if (checkpoint_rewind) {
	/*
	* If we are rewinding to the checkpoint update config cache
	* anyway.
	*/
	update_config_cache = B_TRUE;

	/*
	* Extract the checkpointed uberblock from the current MOS
	* and use this as the pool's uberblock from now on. If the
	* pool is imported as writeable we also write the checkpoint
	* uberblock to the labels, making the rewind permanent.
	*/
	error = spa_ld_checkpoint_rewind(spa);
	if (error != 0)
	return (error);

	/*
	* Redo the loading process again with the
	* checkpointed uberblock.
	*/
	spa_ld_prepare_for_reload(spa);
	spa_load_note(spa, "LOADING checkpointed uberblock");
	error = spa_ld_mos_with_trusted_config(spa, type, NULL);
	if (error != 0)
	return (error);
	}

	/*
	* Retrieve the checkpoint txg if the pool has a checkpoint.
	*/
	+ spa_import_progress_set_notes(spa, "Loading checkpoint txg");
	error = spa_ld_read_checkpoint_txg(spa);
	if (error != 0)
	return (error);

	/*
	* Retrieve the mapping of indirect vdevs. Those vdevs were removed
	* from the pool and their contents were re-mapped to other vdevs. Note
	* that everything that we read before this step must have been
	* rewritten on concrete vdevs after the last device removal was
	* initiated. Otherwise we could be reading from indirect vdevs before
	* we have loaded their mappings.
	*/
	+ spa_import_progress_set_notes(spa, "Loading indirect vdev metadata");
	error = spa_ld_open_indirect_vdev_metadata(spa);
	if (error != 0)
	return (error);

	/*
	* Retrieve the full list of active features from the MOS and check if
	* they are all supported.
	*/
	+ spa_import_progress_set_notes(spa, "Checking feature flags");
	error = spa_ld_check_features(spa, &missing_feat_write);
	if (error != 0)
	return (error);

	/*
	* Load several special directories from the MOS needed by the dsl_pool
	* layer.
	*/
	+ spa_import_progress_set_notes(spa, "Loading special MOS directories");
	error = spa_ld_load_special_directories(spa);
	if (error != 0)
	return (error);

	/*
	* Retrieve pool properties from the MOS.
	*/
	+ spa_import_progress_set_notes(spa, "Loading properties");
	error = spa_ld_get_props(spa);
	if (error != 0)
	return (error);

	/*
	* Retrieve the list of auxiliary devices - cache devices and spares -
	* and open them.
	*/
	+ spa_import_progress_set_notes(spa, "Loading AUX vdevs");
	error = spa_ld_open_aux_vdevs(spa, type);
	if (error != 0)
	return (error);

	/*
	* Load the metadata for all vdevs. Also check if unopenable devices
	* should be autoreplaced.
	*/
	+ spa_import_progress_set_notes(spa, "Loading vdev metadata");
	error = spa_ld_load_vdev_metadata(spa);
	if (error != 0)
	return (error);

	+ spa_import_progress_set_notes(spa, "Loading dedup tables");
	error = spa_ld_load_dedup_tables(spa);
	if (error != 0)
	return (error);

	+ spa_import_progress_set_notes(spa, "Loading BRT");
	error = spa_ld_load_brt(spa);
	if (error != 0)
	return (error);

	/*
	* Verify the logs now to make sure we don't have any unexpected errors
	* when we claim log blocks later.
	*/
	+ spa_import_progress_set_notes(spa, "Verifying Log Devices");
	error = spa_ld_verify_logs(spa, type, ereport);
	if (error != 0)
	return (error);

	if (missing_feat_write) {
	ASSERT(spa->spa_load_state == SPA_LOAD_TRYIMPORT);

	/*
	* At this point, we know that we can open the pool in
	* read-only mode but not read-write mode. We now have enough
	* information and can return to userland.
	*/
	return (spa_vdev_err(spa->spa_root_vdev, VDEV_AUX_UNSUP_FEAT,
	ENOTSUP));
	}

	/*
	* Traverse the last txgs to make sure the pool was left off in a safe
	* state. When performing an extreme rewind, we verify the whole pool,
	* which can take a very long time.
	*/
	+ spa_import_progress_set_notes(spa, "Verifying pool data");
	error = spa_ld_verify_pool_data(spa);
	if (error != 0)
	return (error);

	/*
	* Calculate the deflated space for the pool. This must be done before
	* we write anything to the pool because we'd need to update the space
	* accounting using the deflated sizes.
	*/
	+ spa_import_progress_set_notes(spa, "Calculating deflated space");
	spa_update_dspace(spa);

	/*
	* We have now retrieved all the information we needed to open the
	* pool. If we are importing the pool in read-write mode, a few
	* additional steps must be performed to finish the import.
	*/
	+ spa_import_progress_set_notes(spa, "Starting import");
	if (spa_writeable(spa) && (spa->spa_load_state == SPA_LOAD_RECOVER \|\|
	spa->spa_load_max_txg == UINT64_MAX)) {
	uint64_t config_cache_txg = spa->spa_config_txg;

	ASSERT(spa->spa_load_state != SPA_LOAD_TRYIMPORT);

	/*
	* In case of a checkpoint rewind, log the original txg
	* of the checkpointed uberblock.
	*/
	if (checkpoint_rewind) {
	spa_history_log_internal(spa, "checkpoint rewind",
	NULL, "rewound state to txg=%llu",
	(u_longlong_t)spa->spa_uberblock.ub_checkpoint_txg);
	}

	+ spa_import_progress_set_notes(spa, "Claiming ZIL blocks");
	/*
	* Traverse the ZIL and claim all blocks.
	*/
	spa_ld_claim_log_blocks(spa);

	/*
	* Kick-off the syncing thread.
	*/
	spa->spa_sync_on = B_TRUE;
	txg_sync_start(spa->spa_dsl_pool);
	mmp_thread_start(spa);

	/*
	* Wait for all claims to sync. We sync up to the highest
	* claimed log block birth time so that claimed log blocks
	* don't appear to be from the future. spa_claim_max_txg
	* will have been set for us by ZIL traversal operations
	* performed above.
	*/
	+ spa_import_progress_set_notes(spa, "Syncing ZIL claims");
	txg_wait_synced(spa->spa_dsl_pool, spa->spa_claim_max_txg);

	/*
	* Check if we need to request an update of the config. On the
	* next sync, we would update the config stored in vdev labels
	* and the cachefile (by default /etc/zfs/zpool.cache).
	*/
	+ spa_import_progress_set_notes(spa, "Updating configs");
	spa_ld_check_for_config_update(spa, config_cache_txg,
	update_config_cache);

	/*
	* Check if a rebuild was in progress and if so resume it.
	* Then check all DTLs to see if anything needs resilvering.
	* The resilver will be deferred if a rebuild was started.
	*/
	+ spa_import_progress_set_notes(spa, "Starting resilvers");
	if (vdev_rebuild_active(spa->spa_root_vdev)) {
	vdev_rebuild_restart(spa);
	} else if (!dsl_scan_resilvering(spa->spa_dsl_pool) &&
	vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) {
	spa_async_request(spa, SPA_ASYNC_RESILVER);
	}

	/*
	* Log the fact that we booted up (so that we can detect if
	* we rebooted in the middle of an operation).
	*/
	spa_history_log_version(spa, "open", NULL);

	+ spa_import_progress_set_notes(spa,
	+ "Restarting device removals");
	spa_restart_removal(spa);
	spa_spawn_aux_threads(spa);

	/*
	* Delete any inconsistent datasets.
	*
	* Note:
	* Since we may be issuing deletes for clones here,
	* we make sure to do so after we've spawned all the
	* auxiliary threads above (from which the livelist
	* deletion zthr is part of).
	*/
	+ spa_import_progress_set_notes(spa,
	+ "Cleaning up inconsistent objsets");
	(void) dmu_objset_find(spa_name(spa),
	dsl_destroy_inconsistent, NULL, DS_FIND_CHILDREN);

	/*
	* Clean up any stale temporary dataset userrefs.
	*/
	+ spa_import_progress_set_notes(spa,
	+ "Cleaning up temporary userrefs");
	dsl_pool_clean_tmp_userrefs(spa->spa_dsl_pool);

	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
	+ spa_import_progress_set_notes(spa, "Restarting initialize");
	vdev_initialize_restart(spa->spa_root_vdev);
	+ spa_import_progress_set_notes(spa, "Restarting TRIM");
	vdev_trim_restart(spa->spa_root_vdev);
	vdev_autotrim_restart(spa);
	spa_config_exit(spa, SCL_CONFIG, FTAG);
	+ spa_import_progress_set_notes(spa, "Finished importing");
	}

	spa_import_progress_remove(spa_guid(spa));
	spa_async_request(spa, SPA_ASYNC_L2CACHE_REBUILD);

	spa_load_note(spa, "LOADED");

	return (0);
	}

	static int
	spa_load_retry(spa_t *spa, spa_load_state_t state)
	{
	spa_mode_t mode = spa->spa_mode;

	spa_unload(spa);
	spa_deactivate(spa);

	spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1;

	spa_activate(spa, mode);
	spa_async_suspend(spa);

	spa_load_note(spa, "spa_load_retry: rewind, max txg: %llu",
	(u_longlong_t)spa->spa_load_max_txg);

	return (spa_load(spa, state, SPA_IMPORT_EXISTING));
	}

	/*
	* If spa_load() fails this function will try loading prior txg's. If
	* 'state' is SPA_LOAD_RECOVER and one of these loads succeeds the pool
	* will be rewound to that txg. If 'state' is not SPA_LOAD_RECOVER this
	* function will not rewind the pool and will return the same error as
	* spa_load().
	*/
	static int
	spa_load_best(spa_t *spa, spa_load_state_t state, uint64_t max_request,
	int rewind_flags)
	{
	nvlist_t *loadinfo = NULL;
	nvlist_t *config = NULL;
	int load_error, rewind_error;
	uint64_t safe_rewind_txg;
	uint64_t min_txg;

	if (spa->spa_load_txg && state == SPA_LOAD_RECOVER) {
	spa->spa_load_max_txg = spa->spa_load_txg;
	spa_set_log_state(spa, SPA_LOG_CLEAR);
	} else {
	spa->spa_load_max_txg = max_request;
	if (max_request != UINT64_MAX)
	spa->spa_extreme_rewind = B_TRUE;
	}

	load_error = rewind_error = spa_load(spa, state, SPA_IMPORT_EXISTING);
	if (load_error == 0)
	return (0);
	if (load_error == ZFS_ERR_NO_CHECKPOINT) {
	/*
	* When attempting checkpoint-rewind on a pool with no
	* checkpoint, we should not attempt to load uberblocks
	* from previous txgs when spa_load fails.
	*/
	ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT);
	spa_import_progress_remove(spa_guid(spa));
	return (load_error);
	}

	if (spa->spa_root_vdev != NULL)
	config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);

	spa->spa_last_ubsync_txg = spa->spa_uberblock.ub_txg;
	spa->spa_last_ubsync_txg_ts = spa->spa_uberblock.ub_timestamp;

	if (rewind_flags & ZPOOL_NEVER_REWIND) {
	nvlist_free(config);
	spa_import_progress_remove(spa_guid(spa));
	return (load_error);
	}

	if (state == SPA_LOAD_RECOVER) {
	/* Price of rolling back is discarding txgs, including log */
	spa_set_log_state(spa, SPA_LOG_CLEAR);
	} else {
	/*
	* If we aren't rolling back save the load info from our first
	* import attempt so that we can restore it after attempting
	* to rewind.
	*/
	loadinfo = spa->spa_load_info;
	spa->spa_load_info = fnvlist_alloc();
	}

	spa->spa_load_max_txg = spa->spa_last_ubsync_txg;
	safe_rewind_txg = spa->spa_last_ubsync_txg - TXG_DEFER_SIZE;
	min_txg = (rewind_flags & ZPOOL_EXTREME_REWIND) ?
	TXG_INITIAL : safe_rewind_txg;

	/*
	* Continue as long as we're finding errors, we're still within
	* the acceptable rewind range, and we're still finding uberblocks
	*/
	while (rewind_error && spa->spa_uberblock.ub_txg >= min_txg &&
	spa->spa_uberblock.ub_txg <= spa->spa_load_max_txg) {
	if (spa->spa_load_max_txg < safe_rewind_txg)
	spa->spa_extreme_rewind = B_TRUE;
	rewind_error = spa_load_retry(spa, state);
	}

	spa->spa_extreme_rewind = B_FALSE;
	spa->spa_load_max_txg = UINT64_MAX;

	if (config && (rewind_error \|\| state != SPA_LOAD_RECOVER))
	spa_config_set(spa, config);
	else
	nvlist_free(config);

	if (state == SPA_LOAD_RECOVER) {
	ASSERT3P(loadinfo, ==, NULL);
	spa_import_progress_remove(spa_guid(spa));
	return (rewind_error);
	} else {
	/* Store the rewind info as part of the initial load info */
	fnvlist_add_nvlist(loadinfo, ZPOOL_CONFIG_REWIND_INFO,
	spa->spa_load_info);

	/* Restore the initial load info */
	fnvlist_free(spa->spa_load_info);
	spa->spa_load_info = loadinfo;

	spa_import_progress_remove(spa_guid(spa));
	return (load_error);
	}
	}

	/*
	* Pool Open/Import
	*
	* The import case is identical to an open except that the configuration is sent
	* down from userland, instead of grabbed from the configuration cache. For the
	* case of an open, the pool configuration will exist in the
	* POOL_STATE_UNINITIALIZED state.
	*
	* The stats information (gen/count/ustats) is used to gather vdev statistics at
	* the same time open the pool, without having to keep around the spa_t in some
	* ambiguous state.
	*/
	static int
	spa_open_common(const char pool, spa_t spapp, const void tag,
	nvlist_t nvpolicy, nvlist_t *config)
	{
	spa_t *spa;
	spa_load_state_t state = SPA_LOAD_OPEN;
	int error;
	int locked = B_FALSE;
	int firstopen = B_FALSE;

	*spapp = NULL;

	/*
	* As disgusting as this is, we need to support recursive calls to this
	* function because dsl_dir_open() is called during spa_load(), and ends
	* up calling spa_open() again. The real fix is to figure out how to
	* avoid dsl_dir_open() calling this in the first place.
	*/
	if (MUTEX_NOT_HELD(&spa_namespace_lock)) {
	mutex_enter(&spa_namespace_lock);
	locked = B_TRUE;
	}

	if ((spa = spa_lookup(pool)) == NULL) {
	if (locked)
	mutex_exit(&spa_namespace_lock);
	return (SET_ERROR(ENOENT));
	}

	if (spa->spa_state == POOL_STATE_UNINITIALIZED) {
	zpool_load_policy_t policy;

	firstopen = B_TRUE;

	zpool_get_load_policy(nvpolicy ? nvpolicy : spa->spa_config,
	&policy);
	if (policy.zlp_rewind & ZPOOL_DO_REWIND)
	state = SPA_LOAD_RECOVER;

	spa_activate(spa, spa_mode_global);

	if (state != SPA_LOAD_RECOVER)
	spa->spa_last_ubsync_txg = spa->spa_load_txg = 0;
	spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE;

	zfs_dbgmsg("spa_open_common: opening %s", pool);
	error = spa_load_best(spa, state, policy.zlp_txg,
	policy.zlp_rewind);

	if (error == EBADF) {
	/*
	* If vdev_validate() returns failure (indicated by
	* EBADF), it indicates that one of the vdevs indicates
	* that the pool has been exported or destroyed. If
	* this is the case, the config cache is out of sync and
	* we should remove the pool from the namespace.
	*/
	spa_unload(spa);
	spa_deactivate(spa);
	spa_write_cachefile(spa, B_TRUE, B_TRUE, B_FALSE);
	spa_remove(spa);
	if (locked)
	mutex_exit(&spa_namespace_lock);
	return (SET_ERROR(ENOENT));
	}

	if (error) {
	/*
	* We can't open the pool, but we still have useful
	* information: the state of each vdev after the
	* attempted vdev_open(). Return this to the user.
	*/
	if (config != NULL && spa->spa_config) {
	*config = fnvlist_dup(spa->spa_config);
	fnvlist_add_nvlist(*config,
	ZPOOL_CONFIG_LOAD_INFO,
	spa->spa_load_info);
	}
	spa_unload(spa);
	spa_deactivate(spa);
	spa->spa_last_open_failed = error;
	if (locked)
	mutex_exit(&spa_namespace_lock);
	*spapp = NULL;
	return (error);
	}
	}

	spa_open_ref(spa, tag);

	if (config != NULL)
	*config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);

	/*
	* If we've recovered the pool, pass back any information we
	* gathered while doing the load.
	*/
	if (state == SPA_LOAD_RECOVER && config != NULL) {
	fnvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO,
	spa->spa_load_info);
	}

	if (locked) {
	spa->spa_last_open_failed = 0;
	spa->spa_last_ubsync_txg = 0;
	spa->spa_load_txg = 0;
	mutex_exit(&spa_namespace_lock);
	}

	if (firstopen)
	zvol_create_minors_recursive(spa_name(spa));

	*spapp = spa;

	return (0);
	}

	int
	spa_open_rewind(const char name, spa_t spapp, const void tag,
	nvlist_t policy, nvlist_t *config)
	{
	return (spa_open_common(name, spapp, tag, policy, config));
	}

	int
	spa_open(const char name, spa_t spapp, const void tag)
	{
	return (spa_open_common(name, spapp, tag, NULL, NULL));
	}

	/*
	* Lookup the given spa_t, incrementing the inject count in the process,
	* preventing it from being exported or destroyed.
	*/
	spa_t *
	spa_inject_addref(char *name)
	{
	spa_t *spa;

	mutex_enter(&spa_namespace_lock);
	if ((spa = spa_lookup(name)) == NULL) {
	mutex_exit(&spa_namespace_lock);
	return (NULL);
	}
	spa->spa_inject_ref++;
	mutex_exit(&spa_namespace_lock);

	return (spa);
	}

	void
	spa_inject_delref(spa_t *spa)
	{
	mutex_enter(&spa_namespace_lock);
	spa->spa_inject_ref--;
	mutex_exit(&spa_namespace_lock);
	}

	/*
	* Add spares device information to the nvlist.
	*/
	static void
	spa_add_spares(spa_t spa, nvlist_t config)
	{
	nvlist_t **spares;
	uint_t i, nspares;
	nvlist_t *nvroot;
	uint64_t guid;
	vdev_stat_t *vs;
	uint_t vsc;
	uint64_t pool;

	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));

	if (spa->spa_spares.sav_count == 0)
	return;

	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
	VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config,
	ZPOOL_CONFIG_SPARES, &spares, &nspares));
	if (nspares != 0) {
	fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
	(const nvlist_t * const *)spares, nspares);
	VERIFY0(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
	&spares, &nspares));

	/*
	* Go through and find any spares which have since been
	* repurposed as an active spare. If this is the case, update
	* their status appropriately.
	*/
	for (i = 0; i < nspares; i++) {
	guid = fnvlist_lookup_uint64(spares[i],
	ZPOOL_CONFIG_GUID);
	VERIFY0(nvlist_lookup_uint64_array(spares[i],
	ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc));
	if (spa_spare_exists(guid, &pool, NULL) &&
	pool != 0ULL) {
	vs->vs_state = VDEV_STATE_CANT_OPEN;
	vs->vs_aux = VDEV_AUX_SPARED;
	} else {
	vs->vs_state =
	spa->spa_spares.sav_vdevs[i]->vdev_state;
	}
	}
	}
	}

	/*
	* Add l2cache device information to the nvlist, including vdev stats.
	*/
	static void
	spa_add_l2cache(spa_t spa, nvlist_t config)
	{
	nvlist_t **l2cache;
	uint_t i, j, nl2cache;
	nvlist_t *nvroot;
	uint64_t guid;
	vdev_t *vd;
	vdev_stat_t *vs;
	uint_t vsc;

	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));

	if (spa->spa_l2cache.sav_count == 0)
	return;

	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
	VERIFY0(nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config,
	ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache));
	if (nl2cache != 0) {
	fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
	(const nvlist_t * const *)l2cache, nl2cache);
	VERIFY0(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
	&l2cache, &nl2cache));

	/*
	* Update level 2 cache device stats.
	*/

	for (i = 0; i < nl2cache; i++) {
	guid = fnvlist_lookup_uint64(l2cache[i],
	ZPOOL_CONFIG_GUID);

	vd = NULL;
	for (j = 0; j < spa->spa_l2cache.sav_count; j++) {
	if (guid ==
	spa->spa_l2cache.sav_vdevs[j]->vdev_guid) {
	vd = spa->spa_l2cache.sav_vdevs[j];
	break;
	}
	}
	ASSERT(vd != NULL);

	VERIFY0(nvlist_lookup_uint64_array(l2cache[i],
	ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc));
	vdev_get_stats(vd, vs);
	vdev_config_generate_stats(vd, l2cache[i]);

	}
	}
	}

	static void
	spa_feature_stats_from_disk(spa_t spa, nvlist_t features)
	{
	zap_cursor_t zc;
	zap_attribute_t za;

	if (spa->spa_feat_for_read_obj != 0) {
	for (zap_cursor_init(&zc, spa->spa_meta_objset,
	spa->spa_feat_for_read_obj);
	zap_cursor_retrieve(&zc, &za) == 0;
	zap_cursor_advance(&zc)) {
	ASSERT(za.za_integer_length == sizeof (uint64_t) &&
	za.za_num_integers == 1);
	VERIFY0(nvlist_add_uint64(features, za.za_name,
	za.za_first_integer));
	}
	zap_cursor_fini(&zc);
	}

	if (spa->spa_feat_for_write_obj != 0) {
	for (zap_cursor_init(&zc, spa->spa_meta_objset,
	spa->spa_feat_for_write_obj);
	zap_cursor_retrieve(&zc, &za) == 0;
	zap_cursor_advance(&zc)) {
	ASSERT(za.za_integer_length == sizeof (uint64_t) &&
	za.za_num_integers == 1);
	VERIFY0(nvlist_add_uint64(features, za.za_name,
	za.za_first_integer));
	}
	zap_cursor_fini(&zc);
	}
	}

	static void
	spa_feature_stats_from_cache(spa_t spa, nvlist_t features)
	{
	int i;

	for (i = 0; i < SPA_FEATURES; i++) {
	zfeature_info_t feature = spa_feature_table[i];
	uint64_t refcount;

	if (feature_get_refcount(spa, &feature, &refcount) != 0)
	continue;

	VERIFY0(nvlist_add_uint64(features, feature.fi_guid, refcount));
	}
	}

	/*
	* Store a list of pool features and their reference counts in the
	* config.
	*
	* The first time this is called on a spa, allocate a new nvlist, fetch
	* the pool features and reference counts from disk, then save the list
	* in the spa. In subsequent calls on the same spa use the saved nvlist
	* and refresh its values from the cached reference counts. This
	* ensures we don't block here on I/O on a suspended pool so 'zpool
	* clear' can resume the pool.
	*/
	static void
	spa_add_feature_stats(spa_t spa, nvlist_t config)
	{
	nvlist_t *features;

	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));

	mutex_enter(&spa->spa_feat_stats_lock);
	features = spa->spa_feat_stats;

	if (features != NULL) {
	spa_feature_stats_from_cache(spa, features);
	} else {
	VERIFY0(nvlist_alloc(&features, NV_UNIQUE_NAME, KM_SLEEP));
	spa->spa_feat_stats = features;
	spa_feature_stats_from_disk(spa, features);
	}

	VERIFY0(nvlist_add_nvlist(config, ZPOOL_CONFIG_FEATURE_STATS,
	features));

	mutex_exit(&spa->spa_feat_stats_lock);
	}

	int
	spa_get_stats(const char name, nvlist_t *config,
	char *altroot, size_t buflen)
	{
	int error;
	spa_t *spa;

	*config = NULL;
	error = spa_open_common(name, &spa, FTAG, NULL, config);

	if (spa != NULL) {
	/*
	* This still leaves a window of inconsistency where the spares
	* or l2cache devices could change and the config would be
	* self-inconsistent.
	*/
	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);

	if (*config != NULL) {
	uint64_t loadtimes[2];

	loadtimes[0] = spa->spa_loaded_ts.tv_sec;
	loadtimes[1] = spa->spa_loaded_ts.tv_nsec;
	fnvlist_add_uint64_array(*config,
	ZPOOL_CONFIG_LOADED_TIME, loadtimes, 2);

	fnvlist_add_uint64(*config,
	ZPOOL_CONFIG_ERRCOUNT,
	spa_approx_errlog_size(spa));

	if (spa_suspended(spa)) {
	fnvlist_add_uint64(*config,
	ZPOOL_CONFIG_SUSPENDED,
	spa->spa_failmode);
	fnvlist_add_uint64(*config,
	ZPOOL_CONFIG_SUSPENDED_REASON,
	spa->spa_suspended);
	}

	spa_add_spares(spa, *config);
	spa_add_l2cache(spa, *config);
	spa_add_feature_stats(spa, *config);
	}
	}

	/*
	* We want to get the alternate root even for faulted pools, so we cheat
	* and call spa_lookup() directly.
	*/
	if (altroot) {
	if (spa == NULL) {
	mutex_enter(&spa_namespace_lock);
	spa = spa_lookup(name);
	if (spa)
	spa_altroot(spa, altroot, buflen);
	else
	altroot[0] = '\0';
	spa = NULL;
	mutex_exit(&spa_namespace_lock);
	} else {
	spa_altroot(spa, altroot, buflen);
	}
	}

	if (spa != NULL) {
	spa_config_exit(spa, SCL_CONFIG, FTAG);
	spa_close(spa, FTAG);
	}

	return (error);
	}

	/*
	* Validate that the auxiliary device array is well formed. We must have an
	* array of nvlists, each which describes a valid leaf vdev. If this is an
	* import (mode is VDEV_ALLOC_SPARE), then we allow corrupted spares to be
	* specified, as long as they are well-formed.
	*/
	static int
	spa_validate_aux_devs(spa_t spa, nvlist_t nvroot, uint64_t crtxg, int mode,
	spa_aux_vdev_t sav, const char config, uint64_t version,
	vdev_labeltype_t label)
	{
	nvlist_t **dev;
	uint_t i, ndev;
	vdev_t *vd;
	int error;

	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);

	/*
	* It's acceptable to have no devs specified.
	*/
	if (nvlist_lookup_nvlist_array(nvroot, config, &dev, &ndev) != 0)
	return (0);

	if (ndev == 0)
	return (SET_ERROR(EINVAL));

	/*
	* Make sure the pool is formatted with a version that supports this
	* device type.
	*/
	if (spa_version(spa) < version)
	return (SET_ERROR(ENOTSUP));

	/*
	* Set the pending device list so we correctly handle device in-use
	* checking.
	*/
	sav->sav_pending = dev;
	sav->sav_npending = ndev;

	for (i = 0; i < ndev; i++) {
	if ((error = spa_config_parse(spa, &vd, dev[i], NULL, 0,
	mode)) != 0)
	goto out;

	if (!vd->vdev_ops->vdev_op_leaf) {
	vdev_free(vd);
	error = SET_ERROR(EINVAL);
	goto out;
	}

	vd->vdev_top = vd;

	if ((error = vdev_open(vd)) == 0 &&
	(error = vdev_label_init(vd, crtxg, label)) == 0) {
	fnvlist_add_uint64(dev[i], ZPOOL_CONFIG_GUID,
	vd->vdev_guid);
	}

	vdev_free(vd);

	if (error &&
	(mode != VDEV_ALLOC_SPARE && mode != VDEV_ALLOC_L2CACHE))
	goto out;
	else
	error = 0;
	}

	out:
	sav->sav_pending = NULL;
	sav->sav_npending = 0;
	return (error);
	}

	static int
	spa_validate_aux(spa_t spa, nvlist_t nvroot, uint64_t crtxg, int mode)
	{
	int error;

	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);

	if ((error = spa_validate_aux_devs(spa, nvroot, crtxg, mode,
	&spa->spa_spares, ZPOOL_CONFIG_SPARES, SPA_VERSION_SPARES,
	VDEV_LABEL_SPARE)) != 0) {
	return (error);
	}

	return (spa_validate_aux_devs(spa, nvroot, crtxg, mode,
	&spa->spa_l2cache, ZPOOL_CONFIG_L2CACHE, SPA_VERSION_L2CACHE,
	VDEV_LABEL_L2CACHE));
	}

	static void
	spa_set_aux_vdevs(spa_aux_vdev_t sav, nvlist_t *devs, int ndevs,
	const char *config)
	{
	int i;

	if (sav->sav_config != NULL) {
	nvlist_t **olddevs;
	uint_t oldndevs;
	nvlist_t **newdevs;

	/*
	* Generate new dev list by concatenating with the
	* current dev list.
	*/
	VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config, config,
	&olddevs, &oldndevs));

	newdevs = kmem_alloc(sizeof (void )
	(ndevs + oldndevs), KM_SLEEP);
	for (i = 0; i < oldndevs; i++)
	newdevs[i] = fnvlist_dup(olddevs[i]);
	for (i = 0; i < ndevs; i++)
	newdevs[i + oldndevs] = fnvlist_dup(devs[i]);

	fnvlist_remove(sav->sav_config, config);

	fnvlist_add_nvlist_array(sav->sav_config, config,
	(const nvlist_t * const *)newdevs, ndevs + oldndevs);
	for (i = 0; i < oldndevs + ndevs; i++)
	nvlist_free(newdevs[i]);
	kmem_free(newdevs, (oldndevs + ndevs) * sizeof (void *));
	} else {
	/*
	* Generate a new dev list.
	*/
	sav->sav_config = fnvlist_alloc();
	fnvlist_add_nvlist_array(sav->sav_config, config,
	(const nvlist_t * const *)devs, ndevs);
	}
	}

	/*
	* Stop and drop level 2 ARC devices
	*/
	void
	spa_l2cache_drop(spa_t *spa)
	{
	vdev_t *vd;
	int i;
	spa_aux_vdev_t *sav = &spa->spa_l2cache;

	for (i = 0; i < sav->sav_count; i++) {
	uint64_t pool;

	vd = sav->sav_vdevs[i];
	ASSERT(vd != NULL);

	if (spa_l2cache_exists(vd->vdev_guid, &pool) &&
	pool != 0ULL && l2arc_vdev_present(vd))
	l2arc_remove_vdev(vd);
	}
	}

	/*
	* Verify encryption parameters for spa creation. If we are encrypting, we must
	* have the encryption feature flag enabled.
	*/
	static int
	spa_create_check_encryption_params(dsl_crypto_params_t *dcp,
	boolean_t has_encryption)
	{
	if (dcp->cp_crypt != ZIO_CRYPT_OFF &&
	dcp->cp_crypt != ZIO_CRYPT_INHERIT &&
	!has_encryption)
	return (SET_ERROR(ENOTSUP));

	return (dmu_objset_create_crypt_check(NULL, dcp, NULL));
	}

	/*
	* Pool Creation
	*/
	int
	spa_create(const char pool, nvlist_t nvroot, nvlist_t *props,
	nvlist_t zplprops, dsl_crypto_params_t dcp)
	{
	spa_t *spa;
	const char *altroot = NULL;
	vdev_t *rvd;
	dsl_pool_t *dp;
	dmu_tx_t *tx;
	int error = 0;
	uint64_t txg = TXG_INITIAL;
	nvlist_t spares, l2cache;
	uint_t nspares, nl2cache;
	uint64_t version, obj, ndraid = 0;
	boolean_t has_features;
	boolean_t has_encryption;
	boolean_t has_allocclass;
	spa_feature_t feat;
	const char *feat_name;
	const char *poolname;
	nvlist_t *nvl;

	if (props == NULL \|\|
	nvlist_lookup_string(props, "tname", &poolname) != 0)
	poolname = (char *)pool;

	/*
	* If this pool already exists, return failure.
	*/
	mutex_enter(&spa_namespace_lock);
	if (spa_lookup(poolname) != NULL) {
	mutex_exit(&spa_namespace_lock);
	return (SET_ERROR(EEXIST));
	}

	/*
	* Allocate a new spa_t structure.
	*/
	nvl = fnvlist_alloc();
	fnvlist_add_string(nvl, ZPOOL_CONFIG_POOL_NAME, pool);
	(void) nvlist_lookup_string(props,
	zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
	spa = spa_add(poolname, nvl, altroot);
	fnvlist_free(nvl);
	spa_activate(spa, spa_mode_global);

	if (props && (error = spa_prop_validate(spa, props))) {
	spa_deactivate(spa);
	spa_remove(spa);
	mutex_exit(&spa_namespace_lock);
	return (error);
	}

	/*
	* Temporary pool names should never be written to disk.
	*/
	if (poolname != pool)
	spa->spa_import_flags \|= ZFS_IMPORT_TEMP_NAME;

	has_features = B_FALSE;
	has_encryption = B_FALSE;
	has_allocclass = B_FALSE;
	for (nvpair_t *elem = nvlist_next_nvpair(props, NULL);
	elem != NULL; elem = nvlist_next_nvpair(props, elem)) {
	if (zpool_prop_feature(nvpair_name(elem))) {
	has_features = B_TRUE;

	feat_name = strchr(nvpair_name(elem), '@') + 1;
	VERIFY0(zfeature_lookup_name(feat_name, &feat));
	if (feat == SPA_FEATURE_ENCRYPTION)
	has_encryption = B_TRUE;
	if (feat == SPA_FEATURE_ALLOCATION_CLASSES)
	has_allocclass = B_TRUE;
	}
	}

	/* verify encryption params, if they were provided */
	if (dcp != NULL) {
	error = spa_create_check_encryption_params(dcp, has_encryption);
	if (error != 0) {
	spa_deactivate(spa);
	spa_remove(spa);
	mutex_exit(&spa_namespace_lock);
	return (error);
	}
	}
	if (!has_allocclass && zfs_special_devs(nvroot, NULL)) {
	spa_deactivate(spa);
	spa_remove(spa);
	mutex_exit(&spa_namespace_lock);
	return (ENOTSUP);
	}

	if (has_features \|\| nvlist_lookup_uint64(props,
	zpool_prop_to_name(ZPOOL_PROP_VERSION), &version) != 0) {
	version = SPA_VERSION;
	}
	ASSERT(SPA_VERSION_IS_SUPPORTED(version));

	spa->spa_first_txg = txg;
	spa->spa_uberblock.ub_txg = txg - 1;
	spa->spa_uberblock.ub_version = version;
	spa->spa_ubsync = spa->spa_uberblock;
	spa->spa_load_state = SPA_LOAD_CREATE;
	spa->spa_removing_phys.sr_state = DSS_NONE;
	spa->spa_removing_phys.sr_removing_vdev = -1;
	spa->spa_removing_phys.sr_prev_indirect_vdev = -1;
	spa->spa_indirect_vdevs_loaded = B_TRUE;

	/*
	* Create "The Godfather" zio to hold all async IOs
	*/
	spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
	KM_SLEEP);
	for (int i = 0; i < max_ncpus; i++) {
	spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
	ZIO_FLAG_CANFAIL \| ZIO_FLAG_SPECULATIVE \|
	ZIO_FLAG_GODFATHER);
	}

	/*
	* Create the root vdev.
	*/
	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);

	error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_ADD);

	ASSERT(error != 0 \|\| rvd != NULL);
	ASSERT(error != 0 \|\| spa->spa_root_vdev == rvd);

	if (error == 0 && !zfs_allocatable_devs(nvroot))
	error = SET_ERROR(EINVAL);

	if (error == 0 &&
	(error = vdev_create(rvd, txg, B_FALSE)) == 0 &&
	(error = vdev_draid_spare_create(nvroot, rvd, &ndraid, 0)) == 0 &&
	(error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) == 0) {
	/*
	* instantiate the metaslab groups (this will dirty the vdevs)
	* we can no longer error exit past this point
	*/
	for (int c = 0; error == 0 && c < rvd->vdev_children; c++) {
	vdev_t *vd = rvd->vdev_child[c];

	vdev_metaslab_set_size(vd);
	vdev_expand(vd, txg);
	}
	}

	spa_config_exit(spa, SCL_ALL, FTAG);

	if (error != 0) {
	spa_unload(spa);
	spa_deactivate(spa);
	spa_remove(spa);
	mutex_exit(&spa_namespace_lock);
	return (error);
	}

	/*
	* Get the list of spares, if specified.
	*/
	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
	&spares, &nspares) == 0) {
	spa->spa_spares.sav_config = fnvlist_alloc();
	fnvlist_add_nvlist_array(spa->spa_spares.sav_config,
	ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares,
	nspares);
	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	spa_load_spares(spa);
	spa_config_exit(spa, SCL_ALL, FTAG);
	spa->spa_spares.sav_sync = B_TRUE;
	}

	/*
	* Get the list of level 2 cache devices, if specified.
	*/
	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
	&l2cache, &nl2cache) == 0) {
	VERIFY0(nvlist_alloc(&spa->spa_l2cache.sav_config,
	NV_UNIQUE_NAME, KM_SLEEP));
	fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config,
	ZPOOL_CONFIG_L2CACHE, (const nvlist_t * const *)l2cache,
	nl2cache);
	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	spa_load_l2cache(spa);
	spa_config_exit(spa, SCL_ALL, FTAG);
	spa->spa_l2cache.sav_sync = B_TRUE;
	}

	spa->spa_is_initializing = B_TRUE;
	spa->spa_dsl_pool = dp = dsl_pool_create(spa, zplprops, dcp, txg);
	spa->spa_is_initializing = B_FALSE;

	/*
	* Create DDTs (dedup tables).
	*/
	ddt_create(spa);
	/*
	* Create BRT table and BRT table object.
	*/
	brt_create(spa);

	spa_update_dspace(spa);

	tx = dmu_tx_create_assigned(dp, txg);

	/*
	* Create the pool's history object.
	*/
	if (version >= SPA_VERSION_ZPOOL_HISTORY && !spa->spa_history)
	spa_history_create_obj(spa, tx);

	spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_CREATE);
	spa_history_log_version(spa, "create", tx);

	/*
	* Create the pool config object.
	*/
	spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset,
	DMU_OT_PACKED_NVLIST, SPA_CONFIG_BLOCKSIZE,
	DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx);

	if (zap_add(spa->spa_meta_objset,
	DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG,
	sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) {
	cmn_err(CE_PANIC, "failed to add pool config");
	}

	if (zap_add(spa->spa_meta_objset,
	DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CREATION_VERSION,
	sizeof (uint64_t), 1, &version, tx) != 0) {
	cmn_err(CE_PANIC, "failed to add pool version");
	}

	/* Newly created pools with the right version are always deflated. */
	if (version >= SPA_VERSION_RAIDZ_DEFLATE) {
	spa->spa_deflate = TRUE;
	if (zap_add(spa->spa_meta_objset,
	DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE,
	sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) {
	cmn_err(CE_PANIC, "failed to add deflate");
	}
	}

	/*
	* Create the deferred-free bpobj. Turn off compression
	* because sync-to-convergence takes longer if the blocksize
	* keeps changing.
	*/
	obj = bpobj_alloc(spa->spa_meta_objset, 1 << 14, tx);
	dmu_object_set_compress(spa->spa_meta_objset, obj,
	ZIO_COMPRESS_OFF, tx);
	if (zap_add(spa->spa_meta_objset,
	DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPOBJ,
	sizeof (uint64_t), 1, &obj, tx) != 0) {
	cmn_err(CE_PANIC, "failed to add bpobj");
	}
	VERIFY3U(0, ==, bpobj_open(&spa->spa_deferred_bpobj,
	spa->spa_meta_objset, obj));

	/*
	* Generate some random noise for salted checksums to operate on.
	*/
	(void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes,
	sizeof (spa->spa_cksum_salt.zcs_bytes));

	/*
	* Set pool properties.
	*/
	spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS);
	spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION);
	spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE);
	spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND);
	spa->spa_multihost = zpool_prop_default_numeric(ZPOOL_PROP_MULTIHOST);
	spa->spa_autotrim = zpool_prop_default_numeric(ZPOOL_PROP_AUTOTRIM);

	if (props != NULL) {
	spa_configfile_set(spa, props, B_FALSE);
	spa_sync_props(props, tx);
	}

	for (int i = 0; i < ndraid; i++)
	spa_feature_incr(spa, SPA_FEATURE_DRAID, tx);

	dmu_tx_commit(tx);

	spa->spa_sync_on = B_TRUE;
	txg_sync_start(dp);
	mmp_thread_start(spa);
	txg_wait_synced(dp, txg);

	spa_spawn_aux_threads(spa);

	spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE);

	/*
	* Don't count references from objsets that are already closed
	* and are making their way through the eviction process.
	*/
	spa_evicting_os_wait(spa);
	spa->spa_minref = zfs_refcount_count(&spa->spa_refcount);
	spa->spa_load_state = SPA_LOAD_NONE;

	spa_import_os(spa);

	mutex_exit(&spa_namespace_lock);

	return (0);
	}

	/*
	* Import a non-root pool into the system.
	*/
	int
	spa_import(char pool, nvlist_t config, nvlist_t *props, uint64_t flags)
	{
	spa_t *spa;
	const char *altroot = NULL;
	spa_load_state_t state = SPA_LOAD_IMPORT;
	zpool_load_policy_t policy;
	spa_mode_t mode = spa_mode_global;
	uint64_t readonly = B_FALSE;
	int error;
	nvlist_t *nvroot;
	nvlist_t spares, l2cache;
	uint_t nspares, nl2cache;

	/*
	* If a pool with this name exists, return failure.
	*/
	mutex_enter(&spa_namespace_lock);
	if (spa_lookup(pool) != NULL) {
	mutex_exit(&spa_namespace_lock);
	return (SET_ERROR(EEXIST));
	}

	/*
	* Create and initialize the spa structure.
	*/
	(void) nvlist_lookup_string(props,
	zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
	(void) nvlist_lookup_uint64(props,
	zpool_prop_to_name(ZPOOL_PROP_READONLY), &readonly);
	if (readonly)
	mode = SPA_MODE_READ;
	spa = spa_add(pool, config, altroot);
	spa->spa_import_flags = flags;

	/*
	* Verbatim import - Take a pool and insert it into the namespace
	* as if it had been loaded at boot.
	*/
	if (spa->spa_import_flags & ZFS_IMPORT_VERBATIM) {
	if (props != NULL)
	spa_configfile_set(spa, props, B_FALSE);

	spa_write_cachefile(spa, B_FALSE, B_TRUE, B_FALSE);
	spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT);
	zfs_dbgmsg("spa_import: verbatim import of %s", pool);
	mutex_exit(&spa_namespace_lock);
	return (0);
	}

	spa_activate(spa, mode);

	/*
	* Don't start async tasks until we know everything is healthy.
	*/
	spa_async_suspend(spa);

	zpool_get_load_policy(config, &policy);
	if (policy.zlp_rewind & ZPOOL_DO_REWIND)
	state = SPA_LOAD_RECOVER;

	spa->spa_config_source = SPA_CONFIG_SRC_TRYIMPORT;

	if (state != SPA_LOAD_RECOVER) {
	spa->spa_last_ubsync_txg = spa->spa_load_txg = 0;
	zfs_dbgmsg("spa_import: importing %s", pool);
	} else {
	zfs_dbgmsg("spa_import: importing %s, max_txg=%lld "
	"(RECOVERY MODE)", pool, (longlong_t)policy.zlp_txg);
	}
	error = spa_load_best(spa, state, policy.zlp_txg, policy.zlp_rewind);

	/*
	* Propagate anything learned while loading the pool and pass it
	* back to caller (i.e. rewind info, missing devices, etc).
	*/
	fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, spa->spa_load_info);

	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	/*
	* Toss any existing sparelist, as it doesn't have any validity
	* anymore, and conflicts with spa_has_spare().
	*/
	if (spa->spa_spares.sav_config) {
	nvlist_free(spa->spa_spares.sav_config);
	spa->spa_spares.sav_config = NULL;
	spa_load_spares(spa);
	}
	if (spa->spa_l2cache.sav_config) {
	nvlist_free(spa->spa_l2cache.sav_config);
	spa->spa_l2cache.sav_config = NULL;
	spa_load_l2cache(spa);
	}

	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
	spa_config_exit(spa, SCL_ALL, FTAG);

	if (props != NULL)
	spa_configfile_set(spa, props, B_FALSE);

	if (error != 0 \|\| (props && spa_writeable(spa) &&
	(error = spa_prop_set(spa, props)))) {
	spa_unload(spa);
	spa_deactivate(spa);
	spa_remove(spa);
	mutex_exit(&spa_namespace_lock);
	return (error);
	}

	spa_async_resume(spa);

	/*
	* Override any spares and level 2 cache devices as specified by
	* the user, as these may have correct device names/devids, etc.
	*/
	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
	&spares, &nspares) == 0) {
	if (spa->spa_spares.sav_config)
	fnvlist_remove(spa->spa_spares.sav_config,
	ZPOOL_CONFIG_SPARES);
	else
	spa->spa_spares.sav_config = fnvlist_alloc();
	fnvlist_add_nvlist_array(spa->spa_spares.sav_config,
	ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares,
	nspares);
	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	spa_load_spares(spa);
	spa_config_exit(spa, SCL_ALL, FTAG);
	spa->spa_spares.sav_sync = B_TRUE;
	}
	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
	&l2cache, &nl2cache) == 0) {
	if (spa->spa_l2cache.sav_config)
	fnvlist_remove(spa->spa_l2cache.sav_config,
	ZPOOL_CONFIG_L2CACHE);
	else
	spa->spa_l2cache.sav_config = fnvlist_alloc();
	fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config,
	ZPOOL_CONFIG_L2CACHE, (const nvlist_t * const *)l2cache,
	nl2cache);
	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	spa_load_l2cache(spa);
	spa_config_exit(spa, SCL_ALL, FTAG);
	spa->spa_l2cache.sav_sync = B_TRUE;
	}

	/*
	* Check for any removed devices.
	*/
	if (spa->spa_autoreplace) {
	spa_aux_check_removed(&spa->spa_spares);
	spa_aux_check_removed(&spa->spa_l2cache);
	}

	if (spa_writeable(spa)) {
	/*
	* Update the config cache to include the newly-imported pool.
	*/
	spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
	}

	/*
	* It's possible that the pool was expanded while it was exported.
	* We kick off an async task to handle this for us.
	*/
	spa_async_request(spa, SPA_ASYNC_AUTOEXPAND);

	spa_history_log_version(spa, "import", NULL);

	spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT);

	mutex_exit(&spa_namespace_lock);

	zvol_create_minors_recursive(pool);

	spa_import_os(spa);

	return (0);
	}

	nvlist_t *
	spa_tryimport(nvlist_t *tryconfig)
	{
	nvlist_t *config = NULL;
	const char poolname, cachefile;
	spa_t *spa;
	uint64_t state;
	int error;
	zpool_load_policy_t policy;

	if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_POOL_NAME, &poolname))
	return (NULL);

	if (nvlist_lookup_uint64(tryconfig, ZPOOL_CONFIG_POOL_STATE, &state))
	return (NULL);

	/*
	* Create and initialize the spa structure.
	*/
	mutex_enter(&spa_namespace_lock);
	spa = spa_add(TRYIMPORT_NAME, tryconfig, NULL);
	spa_activate(spa, SPA_MODE_READ);

	/*
	* Rewind pool if a max txg was provided.
	*/
	zpool_get_load_policy(spa->spa_config, &policy);
	if (policy.zlp_txg != UINT64_MAX) {
	spa->spa_load_max_txg = policy.zlp_txg;
	spa->spa_extreme_rewind = B_TRUE;
	zfs_dbgmsg("spa_tryimport: importing %s, max_txg=%lld",
	poolname, (longlong_t)policy.zlp_txg);
	} else {
	zfs_dbgmsg("spa_tryimport: importing %s", poolname);
	}

	if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_CACHEFILE, &cachefile)
	== 0) {
	zfs_dbgmsg("spa_tryimport: using cachefile '%s'", cachefile);
	spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE;
	} else {
	spa->spa_config_source = SPA_CONFIG_SRC_SCAN;
	}

	/*
	* spa_import() relies on a pool config fetched by spa_try_import()
	* for spare/cache devices. Import flags are not passed to
	* spa_tryimport(), which makes it return early due to a missing log
	* device and missing retrieving the cache device and spare eventually.
	* Passing ZFS_IMPORT_MISSING_LOG to spa_tryimport() makes it fetch
	* the correct configuration regardless of the missing log device.
	*/
	spa->spa_import_flags \|= ZFS_IMPORT_MISSING_LOG;

	error = spa_load(spa, SPA_LOAD_TRYIMPORT, SPA_IMPORT_EXISTING);

	/*
	* If 'tryconfig' was at least parsable, return the current config.
	*/
	if (spa->spa_root_vdev != NULL) {
	config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
	fnvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, poolname);
	fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, state);
	fnvlist_add_uint64(config, ZPOOL_CONFIG_TIMESTAMP,
	spa->spa_uberblock.ub_timestamp);
	fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO,
	spa->spa_load_info);
	fnvlist_add_uint64(config, ZPOOL_CONFIG_ERRATA,
	spa->spa_errata);

	/*
	* If the bootfs property exists on this pool then we
	* copy it out so that external consumers can tell which
	* pools are bootable.
	*/
	if ((!error \|\| error == EEXIST) && spa->spa_bootfs) {
	char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP);

	/*
	* We have to play games with the name since the
	* pool was opened as TRYIMPORT_NAME.
	*/
	if (dsl_dsobj_to_dsname(spa_name(spa),
	spa->spa_bootfs, tmpname) == 0) {
	char *cp;
	char *dsname;

	dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP);

	cp = strchr(tmpname, '/');
	if (cp == NULL) {
	(void) strlcpy(dsname, tmpname,
	MAXPATHLEN);
	} else {
	(void) snprintf(dsname, MAXPATHLEN,
	"%s/%s", poolname, ++cp);
	}
	fnvlist_add_string(config, ZPOOL_CONFIG_BOOTFS,
	dsname);
	kmem_free(dsname, MAXPATHLEN);
	}
	kmem_free(tmpname, MAXPATHLEN);
	}

	/*
	* Add the list of hot spares and level 2 cache devices.
	*/
	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
	spa_add_spares(spa, config);
	spa_add_l2cache(spa, config);
	spa_config_exit(spa, SCL_CONFIG, FTAG);
	}

	spa_unload(spa);
	spa_deactivate(spa);
	spa_remove(spa);
	mutex_exit(&spa_namespace_lock);

	return (config);
	}

	/*
	* Pool export/destroy
	*
	* The act of destroying or exporting a pool is very simple. We make sure there
	* is no more pending I/O and any references to the pool are gone. Then, we
	* update the pool state and sync all the labels to disk, removing the
	* configuration from the cache afterwards. If the 'hardforce' flag is set, then
	* we don't sync the labels or remove the configuration cache.
	*/
	static int
	spa_export_common(const char pool, int new_state, nvlist_t *oldconfig,
	boolean_t force, boolean_t hardforce)
	{
	int error;
	spa_t *spa;

	if (oldconfig)
	*oldconfig = NULL;

	if (!(spa_mode_global & SPA_MODE_WRITE))
	return (SET_ERROR(EROFS));

	mutex_enter(&spa_namespace_lock);
	if ((spa = spa_lookup(pool)) == NULL) {
	mutex_exit(&spa_namespace_lock);
	return (SET_ERROR(ENOENT));
	}

	if (spa->spa_is_exporting) {
	/* the pool is being exported by another thread */
	mutex_exit(&spa_namespace_lock);
	return (SET_ERROR(ZFS_ERR_EXPORT_IN_PROGRESS));
	}
	spa->spa_is_exporting = B_TRUE;

	/*
	* Put a hold on the pool, drop the namespace lock, stop async tasks,
	* reacquire the namespace lock, and see if we can export.
	*/
	spa_open_ref(spa, FTAG);
	mutex_exit(&spa_namespace_lock);
	spa_async_suspend(spa);
	if (spa->spa_zvol_taskq) {
	zvol_remove_minors(spa, spa_name(spa), B_TRUE);
	taskq_wait(spa->spa_zvol_taskq);
	}
	mutex_enter(&spa_namespace_lock);
	spa_close(spa, FTAG);

	if (spa->spa_state == POOL_STATE_UNINITIALIZED)
	goto export_spa;
	/*
	* The pool will be in core if it's openable, in which case we can
	* modify its state. Objsets may be open only because they're dirty,
	* so we have to force it to sync before checking spa_refcnt.
	*/
	if (spa->spa_sync_on) {
	txg_wait_synced(spa->spa_dsl_pool, 0);
	spa_evicting_os_wait(spa);
	}

	/*
	* A pool cannot be exported or destroyed if there are active
	* references. If we are resetting a pool, allow references by
	* fault injection handlers.
	*/
	if (!spa_refcount_zero(spa) \|\| (spa->spa_inject_ref != 0)) {
	error = SET_ERROR(EBUSY);
	goto fail;
	}

	if (spa->spa_sync_on) {
	vdev_t *rvd = spa->spa_root_vdev;
	/*
	* A pool cannot be exported if it has an active shared spare.
	* This is to prevent other pools stealing the active spare
	* from an exported pool. At user's own will, such pool can
	* be forcedly exported.
	*/
	if (!force && new_state == POOL_STATE_EXPORTED &&
	spa_has_active_shared_spare(spa)) {
	error = SET_ERROR(EXDEV);
	goto fail;
	}

	/*
	* We're about to export or destroy this pool. Make sure
	* we stop all initialization and trim activity here before
	* we set the spa_final_txg. This will ensure that all
	* dirty data resulting from the initialization is
	* committed to disk before we unload the pool.
	*/
	vdev_initialize_stop_all(rvd, VDEV_INITIALIZE_ACTIVE);
	vdev_trim_stop_all(rvd, VDEV_TRIM_ACTIVE);
	vdev_autotrim_stop_all(spa);
	vdev_rebuild_stop_all(spa);

	/*
	* We want this to be reflected on every label,
	* so mark them all dirty. spa_unload() will do the
	* final sync that pushes these changes out.
	*/
	if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) {
	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	spa->spa_state = new_state;
	vdev_config_dirty(rvd);
	spa_config_exit(spa, SCL_ALL, FTAG);
	}

	/*
	* If the log space map feature is enabled and the pool is
	* getting exported (but not destroyed), we want to spend some
	* time flushing as many metaslabs as we can in an attempt to
	* destroy log space maps and save import time. This has to be
	* done before we set the spa_final_txg, otherwise
	* spa_sync() -> spa_flush_metaslabs() may dirty the final TXGs.
	* spa_should_flush_logs_on_unload() should be called after
	* spa_state has been set to the new_state.
	*/
	if (spa_should_flush_logs_on_unload(spa))
	spa_unload_log_sm_flush_all(spa);

	if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) {
	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	spa->spa_final_txg = spa_last_synced_txg(spa) +
	TXG_DEFER_SIZE + 1;
	spa_config_exit(spa, SCL_ALL, FTAG);
	}
	}

	export_spa:
	spa_export_os(spa);

	if (new_state == POOL_STATE_DESTROYED)
	spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_DESTROY);
	else if (new_state == POOL_STATE_EXPORTED)
	spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_EXPORT);

	if (spa->spa_state != POOL_STATE_UNINITIALIZED) {
	spa_unload(spa);
	spa_deactivate(spa);
	}

	if (oldconfig && spa->spa_config)
	*oldconfig = fnvlist_dup(spa->spa_config);

	if (new_state != POOL_STATE_UNINITIALIZED) {
	if (!hardforce)
	spa_write_cachefile(spa, B_TRUE, B_TRUE, B_FALSE);
	spa_remove(spa);
	} else {
	/*
	* If spa_remove() is not called for this spa_t and
	* there is any possibility that it can be reused,
	* we make sure to reset the exporting flag.
	*/
	spa->spa_is_exporting = B_FALSE;
	}

	mutex_exit(&spa_namespace_lock);
	return (0);

	fail:
	spa->spa_is_exporting = B_FALSE;
	spa_async_resume(spa);
	mutex_exit(&spa_namespace_lock);
	return (error);
	}

	/*
	* Destroy a storage pool.
	*/
	int
	spa_destroy(const char *pool)
	{
	return (spa_export_common(pool, POOL_STATE_DESTROYED, NULL,
	B_FALSE, B_FALSE));
	}

	/*
	* Export a storage pool.
	*/
	int
	spa_export(const char pool, nvlist_t *oldconfig, boolean_t force,
	boolean_t hardforce)
	{
	return (spa_export_common(pool, POOL_STATE_EXPORTED, oldconfig,
	force, hardforce));
	}

	/*
	* Similar to spa_export(), this unloads the spa_t without actually removing it
	* from the namespace in any way.
	*/
	int
	spa_reset(const char *pool)
	{
	return (spa_export_common(pool, POOL_STATE_UNINITIALIZED, NULL,
	B_FALSE, B_FALSE));
	}

	/*
	* ==========================================================================
	* Device manipulation
	* ==========================================================================
	*/

	/*
	* This is called as a synctask to increment the draid feature flag
	*/
	static void
	spa_draid_feature_incr(void arg, dmu_tx_t tx)
	{
	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
	int draid = (int)(uintptr_t)arg;

	for (int c = 0; c < draid; c++)
	spa_feature_incr(spa, SPA_FEATURE_DRAID, tx);
	}

	/*
	* Add a device to a storage pool.
	*/
	int
	-spa_vdev_add(spa_t spa, nvlist_t nvroot)
	+spa_vdev_add(spa_t spa, nvlist_t nvroot, boolean_t check_ashift)
	{
	uint64_t txg, ndraid = 0;
	int error;
	vdev_t *rvd = spa->spa_root_vdev;
	vdev_t vd, tvd;
	nvlist_t spares, l2cache;
	uint_t nspares, nl2cache;

	ASSERT(spa_writeable(spa));

	txg = spa_vdev_enter(spa);

	if ((error = spa_config_parse(spa, &vd, nvroot, NULL, 0,
	VDEV_ALLOC_ADD)) != 0)
	return (spa_vdev_exit(spa, NULL, txg, error));

	spa->spa_pending_vdev = vd; /* spa_vdev_exit() will clear this */

	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares,
	&nspares) != 0)
	nspares = 0;

	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache,
	&nl2cache) != 0)
	nl2cache = 0;

	if (vd->vdev_children == 0 && nspares == 0 && nl2cache == 0)
	return (spa_vdev_exit(spa, vd, txg, EINVAL));

	if (vd->vdev_children != 0 &&
	(error = vdev_create(vd, txg, B_FALSE)) != 0) {
	return (spa_vdev_exit(spa, vd, txg, error));
	}

	/*
	* The virtual dRAID spares must be added after vdev tree is created
	* and the vdev guids are generated. The guid of their associated
	* dRAID is stored in the config and used when opening the spare.
	*/
	if ((error = vdev_draid_spare_create(nvroot, vd, &ndraid,
	rvd->vdev_children)) == 0) {
	if (ndraid > 0 && nvlist_lookup_nvlist_array(nvroot,
	ZPOOL_CONFIG_SPARES, &spares, &nspares) != 0)
	nspares = 0;
	} else {
	return (spa_vdev_exit(spa, vd, txg, error));
	}

	/*
	* We must validate the spares and l2cache devices after checking the
	* children. Otherwise, vdev_inuse() will blindly overwrite the spare.
	*/
	if ((error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) != 0)
	return (spa_vdev_exit(spa, vd, txg, error));

	/*
	* If we are in the middle of a device removal, we can only add
	* devices which match the existing devices in the pool.
	* If we are in the middle of a removal, or have some indirect
	* vdevs, we can not add raidz or dRAID top levels.
	*/
	if (spa->spa_vdev_removal != NULL \|\|
	spa->spa_removing_phys.sr_prev_indirect_vdev != -1) {
	for (int c = 0; c < vd->vdev_children; c++) {
	tvd = vd->vdev_child[c];
	if (spa->spa_vdev_removal != NULL &&
	tvd->vdev_ashift != spa->spa_max_ashift) {
	return (spa_vdev_exit(spa, vd, txg, EINVAL));
	}
	/* Fail if top level vdev is raidz or a dRAID */
	if (vdev_get_nparity(tvd) != 0)
	return (spa_vdev_exit(spa, vd, txg, EINVAL));

	/*
	* Need the top level mirror to be
	* a mirror of leaf vdevs only
	*/
	if (tvd->vdev_ops == &vdev_mirror_ops) {
	for (uint64_t cid = 0;
	cid < tvd->vdev_children; cid++) {
	vdev_t *cvd = tvd->vdev_child[cid];
	if (!cvd->vdev_ops->vdev_op_leaf) {
	return (spa_vdev_exit(spa, vd,
	txg, EINVAL));
	}
	}
	}
	}
	}

	+ if (check_ashift && spa->spa_max_ashift == spa->spa_min_ashift) {
	+ for (int c = 0; c < vd->vdev_children; c++) {
	+ tvd = vd->vdev_child[c];
	+ if (tvd->vdev_ashift != spa->spa_max_ashift) {
	+ return (spa_vdev_exit(spa, vd, txg,
	+ ZFS_ERR_ASHIFT_MISMATCH));
	+ }
	+ }
	+ }
	+
	for (int c = 0; c < vd->vdev_children; c++) {
	tvd = vd->vdev_child[c];
	vdev_remove_child(vd, tvd);
	tvd->vdev_id = rvd->vdev_children;
	vdev_add_child(rvd, tvd);
	vdev_config_dirty(tvd);
	}

	if (nspares != 0) {
	spa_set_aux_vdevs(&spa->spa_spares, spares, nspares,
	ZPOOL_CONFIG_SPARES);
	spa_load_spares(spa);
	spa->spa_spares.sav_sync = B_TRUE;
	}

	if (nl2cache != 0) {
	spa_set_aux_vdevs(&spa->spa_l2cache, l2cache, nl2cache,
	ZPOOL_CONFIG_L2CACHE);
	spa_load_l2cache(spa);
	spa->spa_l2cache.sav_sync = B_TRUE;
	}

	/*
	* We can't increment a feature while holding spa_vdev so we
	* have to do it in a synctask.
	*/
	if (ndraid != 0) {
	dmu_tx_t *tx;

	tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
	dsl_sync_task_nowait(spa->spa_dsl_pool, spa_draid_feature_incr,
	(void *)(uintptr_t)ndraid, tx);
	dmu_tx_commit(tx);
	}

	/*
	* We have to be careful when adding new vdevs to an existing pool.
	* If other threads start allocating from these vdevs before we
	* sync the config cache, and we lose power, then upon reboot we may
	* fail to open the pool because there are DVAs that the config cache
	* can't translate. Therefore, we first add the vdevs without
	* initializing metaslabs; sync the config cache (via spa_vdev_exit());
	* and then let spa_config_update() initialize the new metaslabs.
	*
	* spa_load() checks for added-but-not-initialized vdevs, so that
	* if we lose power at any point in this sequence, the remaining
	* steps will be completed the next time we load the pool.
	*/
	(void) spa_vdev_exit(spa, vd, txg, 0);

	mutex_enter(&spa_namespace_lock);
	spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
	spa_event_notify(spa, NULL, NULL, ESC_ZFS_VDEV_ADD);
	mutex_exit(&spa_namespace_lock);

	return (0);
	}

	/*
	* Attach a device to a mirror. The arguments are the path to any device
	* in the mirror, and the nvroot for the new device. If the path specifies
	* a device that is not mirrored, we automatically insert the mirror vdev.
	*
	* If 'replacing' is specified, the new device is intended to replace the
	* existing device; in this case the two devices are made into their own
	* mirror using the 'replacing' vdev, which is functionally identical to
	* the mirror vdev (it actually reuses all the same ops) but has a few
	* extra rules: you can't attach to it after it's been created, and upon
	* completion of resilvering, the first disk (the one being replaced)
	* is automatically detached.
	*
	* If 'rebuild' is specified, then sequential reconstruction (a.ka. rebuild)
	* should be performed instead of traditional healing reconstruction. From
	* an administrators perspective these are both resilver operations.
	*/
	int
	spa_vdev_attach(spa_t spa, uint64_t guid, nvlist_t nvroot, int replacing,
	int rebuild)
	{
	uint64_t txg, dtl_max_txg;
	vdev_t *rvd = spa->spa_root_vdev;
	vdev_t oldvd, newvd, newrootvd, pvd, *tvd;
	vdev_ops_t *pvops;
	char oldvdpath, newvdpath;
	int newvd_isspare;
	int error;

	ASSERT(spa_writeable(spa));

	txg = spa_vdev_enter(spa);

	oldvd = spa_lookup_by_guid(spa, guid, B_FALSE);

	ASSERT(MUTEX_HELD(&spa_namespace_lock));
	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
	error = (spa_has_checkpoint(spa)) ?
	ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
	return (spa_vdev_exit(spa, NULL, txg, error));
	}

	if (rebuild) {
	if (!spa_feature_is_enabled(spa, SPA_FEATURE_DEVICE_REBUILD))
	return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));

	if (dsl_scan_resilvering(spa_get_dsl(spa)) \|\|
	dsl_scan_resilver_scheduled(spa_get_dsl(spa))) {
	return (spa_vdev_exit(spa, NULL, txg,
	ZFS_ERR_RESILVER_IN_PROGRESS));
	}
	} else {
	if (vdev_rebuild_active(rvd))
	return (spa_vdev_exit(spa, NULL, txg,
	ZFS_ERR_REBUILD_IN_PROGRESS));
	}

	if (spa->spa_vdev_removal != NULL)
	return (spa_vdev_exit(spa, NULL, txg, EBUSY));

	if (oldvd == NULL)
	return (spa_vdev_exit(spa, NULL, txg, ENODEV));

	if (!oldvd->vdev_ops->vdev_op_leaf)
	return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));

	pvd = oldvd->vdev_parent;

	if (spa_config_parse(spa, &newrootvd, nvroot, NULL, 0,
	VDEV_ALLOC_ATTACH) != 0)
	return (spa_vdev_exit(spa, NULL, txg, EINVAL));

	if (newrootvd->vdev_children != 1)
	return (spa_vdev_exit(spa, newrootvd, txg, EINVAL));

	newvd = newrootvd->vdev_child[0];

	if (!newvd->vdev_ops->vdev_op_leaf)
	return (spa_vdev_exit(spa, newrootvd, txg, EINVAL));

	if ((error = vdev_create(newrootvd, txg, replacing)) != 0)
	return (spa_vdev_exit(spa, newrootvd, txg, error));

	/*
	* log, dedup and special vdevs should not be replaced by spares.
	*/
	if ((oldvd->vdev_top->vdev_alloc_bias != VDEV_BIAS_NONE \|\|
	oldvd->vdev_top->vdev_islog) && newvd->vdev_isspare) {
	return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
	}

	/*
	* A dRAID spare can only replace a child of its parent dRAID vdev.
	*/
	if (newvd->vdev_ops == &vdev_draid_spare_ops &&
	oldvd->vdev_top != vdev_draid_spare_get_parent(newvd)) {
	return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
	}

	if (rebuild) {
	/*
	* For rebuilds, the top vdev must support reconstruction
	* using only space maps. This means the only allowable
	* vdevs types are the root vdev, a mirror, or dRAID.
	*/
	tvd = pvd;
	if (pvd->vdev_top != NULL)
	tvd = pvd->vdev_top;

	if (tvd->vdev_ops != &vdev_mirror_ops &&
	tvd->vdev_ops != &vdev_root_ops &&
	tvd->vdev_ops != &vdev_draid_ops) {
	return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
	}
	}

	if (!replacing) {
	/*
	* For attach, the only allowable parent is a mirror or the root
	* vdev.
	*/
	if (pvd->vdev_ops != &vdev_mirror_ops &&
	pvd->vdev_ops != &vdev_root_ops)
	return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));

	pvops = &vdev_mirror_ops;
	} else {
	/*
	* Active hot spares can only be replaced by inactive hot
	* spares.
	*/
	if (pvd->vdev_ops == &vdev_spare_ops &&
	oldvd->vdev_isspare &&
	!spa_has_spare(spa, newvd->vdev_guid))
	return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));

	/*
	* If the source is a hot spare, and the parent isn't already a
	* spare, then we want to create a new hot spare. Otherwise, we
	* want to create a replacing vdev. The user is not allowed to
	* attach to a spared vdev child unless the 'isspare' state is
	* the same (spare replaces spare, non-spare replaces
	* non-spare).
	*/
	if (pvd->vdev_ops == &vdev_replacing_ops &&
	spa_version(spa) < SPA_VERSION_MULTI_REPLACE) {
	return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
	} else if (pvd->vdev_ops == &vdev_spare_ops &&
	newvd->vdev_isspare != oldvd->vdev_isspare) {
	return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
	}

	if (newvd->vdev_isspare)
	pvops = &vdev_spare_ops;
	else
	pvops = &vdev_replacing_ops;
	}

	/*
	* Make sure the new device is big enough.
	*/
	if (newvd->vdev_asize < vdev_get_min_asize(oldvd))
	return (spa_vdev_exit(spa, newrootvd, txg, EOVERFLOW));

	/*
	* The new device cannot have a higher alignment requirement
	* than the top-level vdev.
	*/
	if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift)
	return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));

	/*
	* If this is an in-place replacement, update oldvd's path and devid
	* to make it distinguishable from newvd, and unopenable from now on.
	*/
	if (strcmp(oldvd->vdev_path, newvd->vdev_path) == 0) {
	spa_strfree(oldvd->vdev_path);
	oldvd->vdev_path = kmem_alloc(strlen(newvd->vdev_path) + 5,
	KM_SLEEP);
	(void) snprintf(oldvd->vdev_path, strlen(newvd->vdev_path) + 5,
	"%s/%s", newvd->vdev_path, "old");
	if (oldvd->vdev_devid != NULL) {
	spa_strfree(oldvd->vdev_devid);
	oldvd->vdev_devid = NULL;
	}
	}

	/*
	* If the parent is not a mirror, or if we're replacing, insert the new
	* mirror/replacing/spare vdev above oldvd.
	*/
	if (pvd->vdev_ops != pvops)
	pvd = vdev_add_parent(oldvd, pvops);

	ASSERT(pvd->vdev_top->vdev_parent == rvd);
	ASSERT(pvd->vdev_ops == pvops);
	ASSERT(oldvd->vdev_parent == pvd);

	/*
	* Extract the new device from its root and add it to pvd.
	*/
	vdev_remove_child(newrootvd, newvd);
	newvd->vdev_id = pvd->vdev_children;
	newvd->vdev_crtxg = oldvd->vdev_crtxg;
	vdev_add_child(pvd, newvd);

	/*
	* Reevaluate the parent vdev state.
	*/
	vdev_propagate_state(pvd);

	tvd = newvd->vdev_top;
	ASSERT(pvd->vdev_top == tvd);
	ASSERT(tvd->vdev_parent == rvd);

	vdev_config_dirty(tvd);

	/*
	* Set newvd's DTL to [TXG_INITIAL, dtl_max_txg) so that we account
	* for any dmu_sync-ed blocks. It will propagate upward when
	* spa_vdev_exit() calls vdev_dtl_reassess().
	*/
	dtl_max_txg = txg + TXG_CONCURRENT_STATES;

	vdev_dtl_dirty(newvd, DTL_MISSING,
	TXG_INITIAL, dtl_max_txg - TXG_INITIAL);

	if (newvd->vdev_isspare) {
	spa_spare_activate(newvd);
	spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_SPARE);
	}

	oldvdpath = spa_strdup(oldvd->vdev_path);
	newvdpath = spa_strdup(newvd->vdev_path);
	newvd_isspare = newvd->vdev_isspare;

	/*
	* Mark newvd's DTL dirty in this txg.
	*/
	vdev_dirty(tvd, VDD_DTL, newvd, txg);

	/*
	* Schedule the resilver or rebuild to restart in the future. We do
	* this to ensure that dmu_sync-ed blocks have been stitched into the
	* respective datasets.
	*/
	if (rebuild) {
	newvd->vdev_rebuild_txg = txg;

	vdev_rebuild(tvd);
	} else {
	newvd->vdev_resilver_txg = txg;

	if (dsl_scan_resilvering(spa_get_dsl(spa)) &&
	spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) {
	vdev_defer_resilver(newvd);
	} else {
	dsl_scan_restart_resilver(spa->spa_dsl_pool,
	dtl_max_txg);
	}
	}

	if (spa->spa_bootfs)
	spa_event_notify(spa, newvd, NULL, ESC_ZFS_BOOTFS_VDEV_ATTACH);

	spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_ATTACH);

	/*
	* Commit the config
	*/
	(void) spa_vdev_exit(spa, newrootvd, dtl_max_txg, 0);

	spa_history_log_internal(spa, "vdev attach", NULL,
	"%s vdev=%s %s vdev=%s",
	replacing && newvd_isspare ? "spare in" :
	replacing ? "replace" : "attach", newvdpath,
	replacing ? "for" : "to", oldvdpath);

	spa_strfree(oldvdpath);
	spa_strfree(newvdpath);

	return (0);
	}

	/*
	* Detach a device from a mirror or replacing vdev.
	*
	* If 'replace_done' is specified, only detach if the parent
	* is a replacing or a spare vdev.
	*/
	int
	spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done)
	{
	uint64_t txg;
	int error;
	vdev_t *rvd __maybe_unused = spa->spa_root_vdev;
	vdev_t vd, pvd, cvd, tvd;
	boolean_t unspare = B_FALSE;
	uint64_t unspare_guid = 0;
	char *vdpath;

	ASSERT(spa_writeable(spa));

	txg = spa_vdev_detach_enter(spa, guid);

	vd = spa_lookup_by_guid(spa, guid, B_FALSE);

	/*
	* Besides being called directly from the userland through the
	* ioctl interface, spa_vdev_detach() can be potentially called
	* at the end of spa_vdev_resilver_done().
	*
	* In the regular case, when we have a checkpoint this shouldn't
	* happen as we never empty the DTLs of a vdev during the scrub
	* [see comment in dsl_scan_done()]. Thus spa_vdev_resilvering_done()
	* should never get here when we have a checkpoint.
	*
	* That said, even in a case when we checkpoint the pool exactly
	* as spa_vdev_resilver_done() calls this function everything
	* should be fine as the resilver will return right away.
	*/
	ASSERT(MUTEX_HELD(&spa_namespace_lock));
	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
	error = (spa_has_checkpoint(spa)) ?
	ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
	return (spa_vdev_exit(spa, NULL, txg, error));
	}

	if (vd == NULL)
	return (spa_vdev_exit(spa, NULL, txg, ENODEV));

	if (!vd->vdev_ops->vdev_op_leaf)
	return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));

	pvd = vd->vdev_parent;

	/*
	* If the parent/child relationship is not as expected, don't do it.
	* Consider M(A,R(B,C)) -- that is, a mirror of A with a replacing
	* vdev that's replacing B with C. The user's intent in replacing
	* is to go from M(A,B) to M(A,C). If the user decides to cancel
	* the replace by detaching C, the expected behavior is to end up
	* M(A,B). But suppose that right after deciding to detach C,
	* the replacement of B completes. We would have M(A,C), and then
	* ask to detach C, which would leave us with just A -- not what
	* the user wanted. To prevent this, we make sure that the
	* parent/child relationship hasn't changed -- in this example,
	* that C's parent is still the replacing vdev R.
	*/
	if (pvd->vdev_guid != pguid && pguid != 0)
	return (spa_vdev_exit(spa, NULL, txg, EBUSY));

	/*
	* Only 'replacing' or 'spare' vdevs can be replaced.
	*/
	if (replace_done && pvd->vdev_ops != &vdev_replacing_ops &&
	pvd->vdev_ops != &vdev_spare_ops)
	return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));

	ASSERT(pvd->vdev_ops != &vdev_spare_ops \|\|
	spa_version(spa) >= SPA_VERSION_SPARES);

	/*
	* Only mirror, replacing, and spare vdevs support detach.
	*/
	if (pvd->vdev_ops != &vdev_replacing_ops &&
	pvd->vdev_ops != &vdev_mirror_ops &&
	pvd->vdev_ops != &vdev_spare_ops)
	return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));

	/*
	* If this device has the only valid copy of some data,
	* we cannot safely detach it.
	*/
	if (vdev_dtl_required(vd))
	return (spa_vdev_exit(spa, NULL, txg, EBUSY));

	ASSERT(pvd->vdev_children >= 2);

	/*
	* If we are detaching the second disk from a replacing vdev, then
	* check to see if we changed the original vdev's path to have "/old"
	* at the end in spa_vdev_attach(). If so, undo that change now.
	*/
	if (pvd->vdev_ops == &vdev_replacing_ops && vd->vdev_id > 0 &&
	vd->vdev_path != NULL) {
	size_t len = strlen(vd->vdev_path);

	for (int c = 0; c < pvd->vdev_children; c++) {
	cvd = pvd->vdev_child[c];

	if (cvd == vd \|\| cvd->vdev_path == NULL)
	continue;

	if (strncmp(cvd->vdev_path, vd->vdev_path, len) == 0 &&
	strcmp(cvd->vdev_path + len, "/old") == 0) {
	spa_strfree(cvd->vdev_path);
	cvd->vdev_path = spa_strdup(vd->vdev_path);
	break;
	}
	}
	}

	/*
	* If we are detaching the original disk from a normal spare, then it
	* implies that the spare should become a real disk, and be removed
	* from the active spare list for the pool. dRAID spares on the
	* other hand are coupled to the pool and thus should never be removed
	* from the spares list.
	*/
	if (pvd->vdev_ops == &vdev_spare_ops && vd->vdev_id == 0) {
	vdev_t *last_cvd = pvd->vdev_child[pvd->vdev_children - 1];

	if (last_cvd->vdev_isspare &&
	last_cvd->vdev_ops != &vdev_draid_spare_ops) {
	unspare = B_TRUE;
	}
	}

	/*
	* Erase the disk labels so the disk can be used for other things.
	* This must be done after all other error cases are handled,
	* but before we disembowel vd (so we can still do I/O to it).
	* But if we can't do it, don't treat the error as fatal --
	* it may be that the unwritability of the disk is the reason
	* it's being detached!
	*/
	(void) vdev_label_init(vd, 0, VDEV_LABEL_REMOVE);

	/*
	* Remove vd from its parent and compact the parent's children.
	*/
	vdev_remove_child(pvd, vd);
	vdev_compact_children(pvd);

	/*
	* Remember one of the remaining children so we can get tvd below.
	*/
	cvd = pvd->vdev_child[pvd->vdev_children - 1];

	/*
	* If we need to remove the remaining child from the list of hot spares,
	* do it now, marking the vdev as no longer a spare in the process.
	* We must do this before vdev_remove_parent(), because that can
	* change the GUID if it creates a new toplevel GUID. For a similar
	* reason, we must remove the spare now, in the same txg as the detach;
	* otherwise someone could attach a new sibling, change the GUID, and
	* the subsequent attempt to spa_vdev_remove(unspare_guid) would fail.
	*/
	if (unspare) {
	ASSERT(cvd->vdev_isspare);
	spa_spare_remove(cvd);
	unspare_guid = cvd->vdev_guid;
	(void) spa_vdev_remove(spa, unspare_guid, B_TRUE);
	cvd->vdev_unspare = B_TRUE;
	}

	/*
	* If the parent mirror/replacing vdev only has one child,
	* the parent is no longer needed. Remove it from the tree.
	*/
	if (pvd->vdev_children == 1) {
	if (pvd->vdev_ops == &vdev_spare_ops)
	cvd->vdev_unspare = B_FALSE;
	vdev_remove_parent(cvd);
	}

	/*
	* We don't set tvd until now because the parent we just removed
	* may have been the previous top-level vdev.
	*/
	tvd = cvd->vdev_top;
	ASSERT(tvd->vdev_parent == rvd);

	/*
	* Reevaluate the parent vdev state.
	*/
	vdev_propagate_state(cvd);

	/*
	* If the 'autoexpand' property is set on the pool then automatically
	* try to expand the size of the pool. For example if the device we
	* just detached was smaller than the others, it may be possible to
	* add metaslabs (i.e. grow the pool). We need to reopen the vdev
	* first so that we can obtain the updated sizes of the leaf vdevs.
	*/
	if (spa->spa_autoexpand) {
	vdev_reopen(tvd);
	vdev_expand(tvd, txg);
	}

	vdev_config_dirty(tvd);

	/*
	* Mark vd's DTL as dirty in this txg. vdev_dtl_sync() will see that
	* vd->vdev_detached is set and free vd's DTL object in syncing context.
	* But first make sure we're not on any other txg's DTL list, to
	* prevent vd from being accessed after it's freed.
	*/
	vdpath = spa_strdup(vd->vdev_path ? vd->vdev_path : "none");
	for (int t = 0; t < TXG_SIZE; t++)
	(void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t);
	vd->vdev_detached = B_TRUE;
	vdev_dirty(tvd, VDD_DTL, vd, txg);

	spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_REMOVE);
	spa_notify_waiters(spa);

	/* hang on to the spa before we release the lock */
	spa_open_ref(spa, FTAG);

	error = spa_vdev_exit(spa, vd, txg, 0);

	spa_history_log_internal(spa, "detach", NULL,
	"vdev=%s", vdpath);
	spa_strfree(vdpath);

	/*
	* If this was the removal of the original device in a hot spare vdev,
	* then we want to go through and remove the device from the hot spare
	* list of every other pool.
	*/
	if (unspare) {
	spa_t *altspa = NULL;

	mutex_enter(&spa_namespace_lock);
	while ((altspa = spa_next(altspa)) != NULL) {
	if (altspa->spa_state != POOL_STATE_ACTIVE \|\|
	altspa == spa)
	continue;

	spa_open_ref(altspa, FTAG);
	mutex_exit(&spa_namespace_lock);
	(void) spa_vdev_remove(altspa, unspare_guid, B_TRUE);
	mutex_enter(&spa_namespace_lock);
	spa_close(altspa, FTAG);
	}
	mutex_exit(&spa_namespace_lock);

	/* search the rest of the vdevs for spares to remove */
	spa_vdev_resilver_done(spa);
	}

	/* all done with the spa; OK to release */
	mutex_enter(&spa_namespace_lock);
	spa_close(spa, FTAG);
	mutex_exit(&spa_namespace_lock);

	return (error);
	}

	static int
	spa_vdev_initialize_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type,
	list_t *vd_list)
	{
	ASSERT(MUTEX_HELD(&spa_namespace_lock));

	spa_config_enter(spa, SCL_CONFIG \| SCL_STATE, FTAG, RW_READER);

	/* Look up vdev and ensure it's a leaf. */
	vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE);
	if (vd == NULL \|\| vd->vdev_detached) {
	spa_config_exit(spa, SCL_CONFIG \| SCL_STATE, FTAG);
	return (SET_ERROR(ENODEV));
	} else if (!vd->vdev_ops->vdev_op_leaf \|\| !vdev_is_concrete(vd)) {
	spa_config_exit(spa, SCL_CONFIG \| SCL_STATE, FTAG);
	return (SET_ERROR(EINVAL));
	} else if (!vdev_writeable(vd)) {
	spa_config_exit(spa, SCL_CONFIG \| SCL_STATE, FTAG);
	return (SET_ERROR(EROFS));
	}
	mutex_enter(&vd->vdev_initialize_lock);
	spa_config_exit(spa, SCL_CONFIG \| SCL_STATE, FTAG);

	/*
	* When we activate an initialize action we check to see
	* if the vdev_initialize_thread is NULL. We do this instead
	* of using the vdev_initialize_state since there might be
	* a previous initialization process which has completed but
	* the thread is not exited.
	*/
	if (cmd_type == POOL_INITIALIZE_START &&
	(vd->vdev_initialize_thread != NULL \|\|
	vd->vdev_top->vdev_removing)) {
	mutex_exit(&vd->vdev_initialize_lock);
	return (SET_ERROR(EBUSY));
	} else if (cmd_type == POOL_INITIALIZE_CANCEL &&
	(vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE &&
	vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED)) {
	mutex_exit(&vd->vdev_initialize_lock);
	return (SET_ERROR(ESRCH));
	} else if (cmd_type == POOL_INITIALIZE_SUSPEND &&
	vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE) {
	mutex_exit(&vd->vdev_initialize_lock);
	return (SET_ERROR(ESRCH));
	} else if (cmd_type == POOL_INITIALIZE_UNINIT &&
	vd->vdev_initialize_thread != NULL) {
	mutex_exit(&vd->vdev_initialize_lock);
	return (SET_ERROR(EBUSY));
	}

	switch (cmd_type) {
	case POOL_INITIALIZE_START:
	vdev_initialize(vd);
	break;
	case POOL_INITIALIZE_CANCEL:
	vdev_initialize_stop(vd, VDEV_INITIALIZE_CANCELED, vd_list);
	break;
	case POOL_INITIALIZE_SUSPEND:
	vdev_initialize_stop(vd, VDEV_INITIALIZE_SUSPENDED, vd_list);
	break;
	case POOL_INITIALIZE_UNINIT:
	vdev_uninitialize(vd);
	break;
	default:
	panic("invalid cmd_type %llu", (unsigned long long)cmd_type);
	}
	mutex_exit(&vd->vdev_initialize_lock);

	return (0);
	}

	int
	spa_vdev_initialize(spa_t spa, nvlist_t nv, uint64_t cmd_type,
	nvlist_t *vdev_errlist)
	{
	int total_errors = 0;
	list_t vd_list;

	list_create(&vd_list, sizeof (vdev_t),
	offsetof(vdev_t, vdev_initialize_node));

	/*
	* We hold the namespace lock through the whole function
	* to prevent any changes to the pool while we're starting or
	* stopping initialization. The config and state locks are held so that
	* we can properly assess the vdev state before we commit to
	* the initializing operation.
	*/
	mutex_enter(&spa_namespace_lock);

	for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL);
	pair != NULL; pair = nvlist_next_nvpair(nv, pair)) {
	uint64_t vdev_guid = fnvpair_value_uint64(pair);

	int error = spa_vdev_initialize_impl(spa, vdev_guid, cmd_type,
	&vd_list);
	if (error != 0) {
	char guid_as_str[MAXNAMELEN];

	(void) snprintf(guid_as_str, sizeof (guid_as_str),
	"%llu", (unsigned long long)vdev_guid);
	fnvlist_add_int64(vdev_errlist, guid_as_str, error);
	total_errors++;
	}
	}

	/* Wait for all initialize threads to stop. */
	vdev_initialize_stop_wait(spa, &vd_list);

	/* Sync out the initializing state */
	txg_wait_synced(spa->spa_dsl_pool, 0);
	mutex_exit(&spa_namespace_lock);

	list_destroy(&vd_list);

	return (total_errors);
	}

	static int
	spa_vdev_trim_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type,
	uint64_t rate, boolean_t partial, boolean_t secure, list_t *vd_list)
	{
	ASSERT(MUTEX_HELD(&spa_namespace_lock));

	spa_config_enter(spa, SCL_CONFIG \| SCL_STATE, FTAG, RW_READER);

	/* Look up vdev and ensure it's a leaf. */
	vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE);
	if (vd == NULL \|\| vd->vdev_detached) {
	spa_config_exit(spa, SCL_CONFIG \| SCL_STATE, FTAG);
	return (SET_ERROR(ENODEV));
	} else if (!vd->vdev_ops->vdev_op_leaf \|\| !vdev_is_concrete(vd)) {
	spa_config_exit(spa, SCL_CONFIG \| SCL_STATE, FTAG);
	return (SET_ERROR(EINVAL));
	} else if (!vdev_writeable(vd)) {
	spa_config_exit(spa, SCL_CONFIG \| SCL_STATE, FTAG);
	return (SET_ERROR(EROFS));
	} else if (!vd->vdev_has_trim) {
	spa_config_exit(spa, SCL_CONFIG \| SCL_STATE, FTAG);
	return (SET_ERROR(EOPNOTSUPP));
	} else if (secure && !vd->vdev_has_securetrim) {
	spa_config_exit(spa, SCL_CONFIG \| SCL_STATE, FTAG);
	return (SET_ERROR(EOPNOTSUPP));
	}
	mutex_enter(&vd->vdev_trim_lock);
	spa_config_exit(spa, SCL_CONFIG \| SCL_STATE, FTAG);

	/*
	* When we activate a TRIM action we check to see if the
	* vdev_trim_thread is NULL. We do this instead of using the
	* vdev_trim_state since there might be a previous TRIM process
	* which has completed but the thread is not exited.
	*/
	if (cmd_type == POOL_TRIM_START &&
	(vd->vdev_trim_thread != NULL \|\| vd->vdev_top->vdev_removing)) {
	mutex_exit(&vd->vdev_trim_lock);
	return (SET_ERROR(EBUSY));
	} else if (cmd_type == POOL_TRIM_CANCEL &&
	(vd->vdev_trim_state != VDEV_TRIM_ACTIVE &&
	vd->vdev_trim_state != VDEV_TRIM_SUSPENDED)) {
	mutex_exit(&vd->vdev_trim_lock);
	return (SET_ERROR(ESRCH));
	} else if (cmd_type == POOL_TRIM_SUSPEND &&
	vd->vdev_trim_state != VDEV_TRIM_ACTIVE) {
	mutex_exit(&vd->vdev_trim_lock);
	return (SET_ERROR(ESRCH));
	}

	switch (cmd_type) {
	case POOL_TRIM_START:
	vdev_trim(vd, rate, partial, secure);
	break;
	case POOL_TRIM_CANCEL:
	vdev_trim_stop(vd, VDEV_TRIM_CANCELED, vd_list);
	break;
	case POOL_TRIM_SUSPEND:
	vdev_trim_stop(vd, VDEV_TRIM_SUSPENDED, vd_list);
	break;
	default:
	panic("invalid cmd_type %llu", (unsigned long long)cmd_type);
	}
	mutex_exit(&vd->vdev_trim_lock);

	return (0);
	}

	/*
	* Initiates a manual TRIM for the requested vdevs. This kicks off individual
	* TRIM threads for each child vdev. These threads pass over all of the free
	* space in the vdev's metaslabs and issues TRIM commands for that space.
	*/
	int
	spa_vdev_trim(spa_t spa, nvlist_t nv, uint64_t cmd_type, uint64_t rate,
	boolean_t partial, boolean_t secure, nvlist_t *vdev_errlist)
	{
	int total_errors = 0;
	list_t vd_list;

	list_create(&vd_list, sizeof (vdev_t),
	offsetof(vdev_t, vdev_trim_node));

	/*
	* We hold the namespace lock through the whole function
	* to prevent any changes to the pool while we're starting or
	* stopping TRIM. The config and state locks are held so that
	* we can properly assess the vdev state before we commit to
	* the TRIM operation.
	*/
	mutex_enter(&spa_namespace_lock);

	for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL);
	pair != NULL; pair = nvlist_next_nvpair(nv, pair)) {
	uint64_t vdev_guid = fnvpair_value_uint64(pair);

	int error = spa_vdev_trim_impl(spa, vdev_guid, cmd_type,
	rate, partial, secure, &vd_list);
	if (error != 0) {
	char guid_as_str[MAXNAMELEN];

	(void) snprintf(guid_as_str, sizeof (guid_as_str),
	"%llu", (unsigned long long)vdev_guid);
	fnvlist_add_int64(vdev_errlist, guid_as_str, error);
	total_errors++;
	}
	}

	/* Wait for all TRIM threads to stop. */
	vdev_trim_stop_wait(spa, &vd_list);

	/* Sync out the TRIM state */
	txg_wait_synced(spa->spa_dsl_pool, 0);
	mutex_exit(&spa_namespace_lock);

	list_destroy(&vd_list);

	return (total_errors);
	}

	/*
	* Split a set of devices from their mirrors, and create a new pool from them.
	*/
	int
	spa_vdev_split_mirror(spa_t spa, const char newname, nvlist_t *config,
	nvlist_t *props, boolean_t exp)
	{
	int error = 0;
	uint64_t txg, *glist;
	spa_t *newspa;
	uint_t c, children, lastlog;
	nvlist_t *child, nvl, *tmp;
	dmu_tx_t *tx;
	const char *altroot = NULL;
	vdev_t rvd, vml = NULL; / vdev modify list */
	boolean_t activate_slog;

	ASSERT(spa_writeable(spa));

	txg = spa_vdev_enter(spa);

	ASSERT(MUTEX_HELD(&spa_namespace_lock));
	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
	error = (spa_has_checkpoint(spa)) ?
	ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
	return (spa_vdev_exit(spa, NULL, txg, error));
	}

	/* clear the log and flush everything up to now */
	activate_slog = spa_passivate_log(spa);
	(void) spa_vdev_config_exit(spa, NULL, txg, 0, FTAG);
	error = spa_reset_logs(spa);
	txg = spa_vdev_config_enter(spa);

	if (activate_slog)
	spa_activate_log(spa);

	if (error != 0)
	return (spa_vdev_exit(spa, NULL, txg, error));

	/* check new spa name before going any further */
	if (spa_lookup(newname) != NULL)
	return (spa_vdev_exit(spa, NULL, txg, EEXIST));

	/*
	* scan through all the children to ensure they're all mirrors
	*/
	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvl) != 0 \|\|
	nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, &child,
	&children) != 0)
	return (spa_vdev_exit(spa, NULL, txg, EINVAL));

	/* first, check to ensure we've got the right child count */
	rvd = spa->spa_root_vdev;
	lastlog = 0;
	for (c = 0; c < rvd->vdev_children; c++) {
	vdev_t *vd = rvd->vdev_child[c];

	/* don't count the holes & logs as children */
	if (vd->vdev_islog \|\| (vd->vdev_ops != &vdev_indirect_ops &&
	!vdev_is_concrete(vd))) {
	if (lastlog == 0)
	lastlog = c;
	continue;
	}

	lastlog = 0;
	}
	if (children != (lastlog != 0 ? lastlog : rvd->vdev_children))
	return (spa_vdev_exit(spa, NULL, txg, EINVAL));

	/* next, ensure no spare or cache devices are part of the split */
	if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_SPARES, &tmp) == 0 \|\|
	nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0)
	return (spa_vdev_exit(spa, NULL, txg, EINVAL));

	vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP);
	glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP);

	/* then, loop over each vdev and validate it */
	for (c = 0; c < children; c++) {
	uint64_t is_hole = 0;

	(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
	&is_hole);

	if (is_hole != 0) {
	if (spa->spa_root_vdev->vdev_child[c]->vdev_ishole \|\|
	spa->spa_root_vdev->vdev_child[c]->vdev_islog) {
	continue;
	} else {
	error = SET_ERROR(EINVAL);
	break;
	}
	}

	/* deal with indirect vdevs */
	if (spa->spa_root_vdev->vdev_child[c]->vdev_ops ==
	&vdev_indirect_ops)
	continue;

	/* which disk is going to be split? */
	if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_GUID,
	&glist[c]) != 0) {
	error = SET_ERROR(EINVAL);
	break;
	}

	/* look it up in the spa */
	vml[c] = spa_lookup_by_guid(spa, glist[c], B_FALSE);
	if (vml[c] == NULL) {
	error = SET_ERROR(ENODEV);
	break;
	}

	/* make sure there's nothing stopping the split */
	if (vml[c]->vdev_parent->vdev_ops != &vdev_mirror_ops \|\|
	vml[c]->vdev_islog \|\|
	!vdev_is_concrete(vml[c]) \|\|
	vml[c]->vdev_isspare \|\|
	vml[c]->vdev_isl2cache \|\|
	!vdev_writeable(vml[c]) \|\|
	vml[c]->vdev_children != 0 \|\|
	vml[c]->vdev_state != VDEV_STATE_HEALTHY \|\|
	c != spa->spa_root_vdev->vdev_child[c]->vdev_id) {
	error = SET_ERROR(EINVAL);
	break;
	}

	if (vdev_dtl_required(vml[c]) \|\|
	vdev_resilver_needed(vml[c], NULL, NULL)) {
	error = SET_ERROR(EBUSY);
	break;
	}

	/* we need certain info from the top level */
	fnvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_ARRAY,
	vml[c]->vdev_top->vdev_ms_array);
	fnvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_SHIFT,
	vml[c]->vdev_top->vdev_ms_shift);
	fnvlist_add_uint64(child[c], ZPOOL_CONFIG_ASIZE,
	vml[c]->vdev_top->vdev_asize);
	fnvlist_add_uint64(child[c], ZPOOL_CONFIG_ASHIFT,
	vml[c]->vdev_top->vdev_ashift);

	/* transfer per-vdev ZAPs */
	ASSERT3U(vml[c]->vdev_leaf_zap, !=, 0);
	VERIFY0(nvlist_add_uint64(child[c],
	ZPOOL_CONFIG_VDEV_LEAF_ZAP, vml[c]->vdev_leaf_zap));

	ASSERT3U(vml[c]->vdev_top->vdev_top_zap, !=, 0);
	VERIFY0(nvlist_add_uint64(child[c],
	ZPOOL_CONFIG_VDEV_TOP_ZAP,
	vml[c]->vdev_parent->vdev_top_zap));
	}

	if (error != 0) {
	kmem_free(vml, children * sizeof (vdev_t *));
	kmem_free(glist, children * sizeof (uint64_t));
	return (spa_vdev_exit(spa, NULL, txg, error));
	}

	/* stop writers from using the disks */
	for (c = 0; c < children; c++) {
	if (vml[c] != NULL)
	vml[c]->vdev_offline = B_TRUE;
	}
	vdev_reopen(spa->spa_root_vdev);

	/*
	* Temporarily record the splitting vdevs in the spa config. This
	* will disappear once the config is regenerated.
	*/
	nvl = fnvlist_alloc();
	fnvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, glist, children);
	kmem_free(glist, children * sizeof (uint64_t));

	mutex_enter(&spa->spa_props_lock);
	fnvlist_add_nvlist(spa->spa_config, ZPOOL_CONFIG_SPLIT, nvl);
	mutex_exit(&spa->spa_props_lock);
	spa->spa_config_splitting = nvl;
	vdev_config_dirty(spa->spa_root_vdev);

	/* configure and create the new pool */
	fnvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, newname);
	fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE,
	exp ? POOL_STATE_EXPORTED : POOL_STATE_ACTIVE);
	fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, spa_version(spa));
	fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG, spa->spa_config_txg);
	fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID,
	spa_generate_guid(NULL));
	VERIFY0(nvlist_add_boolean(config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS));
	(void) nvlist_lookup_string(props,
	zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);

	/* add the new pool to the namespace */
	newspa = spa_add(newname, config, altroot);
	newspa->spa_avz_action = AVZ_ACTION_REBUILD;
	newspa->spa_config_txg = spa->spa_config_txg;
	spa_set_log_state(newspa, SPA_LOG_CLEAR);

	/* release the spa config lock, retaining the namespace lock */
	spa_vdev_config_exit(spa, NULL, txg, 0, FTAG);

	if (zio_injection_enabled)
	zio_handle_panic_injection(spa, FTAG, 1);

	spa_activate(newspa, spa_mode_global);
	spa_async_suspend(newspa);

	/*
	* Temporarily stop the initializing and TRIM activity. We set the
	* state to ACTIVE so that we know to resume initializing or TRIM
	* once the split has completed.
	*/
	list_t vd_initialize_list;
	list_create(&vd_initialize_list, sizeof (vdev_t),
	offsetof(vdev_t, vdev_initialize_node));

	list_t vd_trim_list;
	list_create(&vd_trim_list, sizeof (vdev_t),
	offsetof(vdev_t, vdev_trim_node));

	for (c = 0; c < children; c++) {
	if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) {
	mutex_enter(&vml[c]->vdev_initialize_lock);
	vdev_initialize_stop(vml[c],
	VDEV_INITIALIZE_ACTIVE, &vd_initialize_list);
	mutex_exit(&vml[c]->vdev_initialize_lock);

	mutex_enter(&vml[c]->vdev_trim_lock);
	vdev_trim_stop(vml[c], VDEV_TRIM_ACTIVE, &vd_trim_list);
	mutex_exit(&vml[c]->vdev_trim_lock);
	}
	}

	vdev_initialize_stop_wait(spa, &vd_initialize_list);
	vdev_trim_stop_wait(spa, &vd_trim_list);

	list_destroy(&vd_initialize_list);
	list_destroy(&vd_trim_list);

	newspa->spa_config_source = SPA_CONFIG_SRC_SPLIT;
	newspa->spa_is_splitting = B_TRUE;

	/* create the new pool from the disks of the original pool */
	error = spa_load(newspa, SPA_LOAD_IMPORT, SPA_IMPORT_ASSEMBLE);
	if (error)
	goto out;

	/* if that worked, generate a real config for the new pool */
	if (newspa->spa_root_vdev != NULL) {
	newspa->spa_config_splitting = fnvlist_alloc();
	fnvlist_add_uint64(newspa->spa_config_splitting,
	ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa));
	spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL,
	B_TRUE));
	}

	/* set the props */
	if (props != NULL) {
	spa_configfile_set(newspa, props, B_FALSE);
	error = spa_prop_set(newspa, props);
	if (error)
	goto out;
	}

	/* flush everything */
	txg = spa_vdev_config_enter(newspa);
	vdev_config_dirty(newspa->spa_root_vdev);
	(void) spa_vdev_config_exit(newspa, NULL, txg, 0, FTAG);

	if (zio_injection_enabled)
	zio_handle_panic_injection(spa, FTAG, 2);

	spa_async_resume(newspa);

	/* finally, update the original pool's config */
	txg = spa_vdev_config_enter(spa);
	tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error != 0)
	dmu_tx_abort(tx);
	for (c = 0; c < children; c++) {
	if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) {
	vdev_t *tvd = vml[c]->vdev_top;

	/*
	* Need to be sure the detachable VDEV is not
	* on any other txg's DTL list to prevent it
	* from being accessed after it's freed.
	*/
	for (int t = 0; t < TXG_SIZE; t++) {
	(void) txg_list_remove_this(
	&tvd->vdev_dtl_list, vml[c], t);
	}

	vdev_split(vml[c]);
	if (error == 0)
	spa_history_log_internal(spa, "detach", tx,
	"vdev=%s", vml[c]->vdev_path);

	vdev_free(vml[c]);
	}
	}
	spa->spa_avz_action = AVZ_ACTION_REBUILD;
	vdev_config_dirty(spa->spa_root_vdev);
	spa->spa_config_splitting = NULL;
	nvlist_free(nvl);
	if (error == 0)
	dmu_tx_commit(tx);
	(void) spa_vdev_exit(spa, NULL, txg, 0);

	if (zio_injection_enabled)
	zio_handle_panic_injection(spa, FTAG, 3);

	/* split is complete; log a history record */
	spa_history_log_internal(newspa, "split", NULL,
	"from pool %s", spa_name(spa));

	newspa->spa_is_splitting = B_FALSE;
	kmem_free(vml, children * sizeof (vdev_t *));

	/* if we're not going to mount the filesystems in userland, export */
	if (exp)
	error = spa_export_common(newname, POOL_STATE_EXPORTED, NULL,
	B_FALSE, B_FALSE);

	return (error);

	out:
	spa_unload(newspa);
	spa_deactivate(newspa);
	spa_remove(newspa);

	txg = spa_vdev_config_enter(spa);

	/* re-online all offlined disks */
	for (c = 0; c < children; c++) {
	if (vml[c] != NULL)
	vml[c]->vdev_offline = B_FALSE;
	}

	/* restart initializing or trimming disks as necessary */
	spa_async_request(spa, SPA_ASYNC_INITIALIZE_RESTART);
	spa_async_request(spa, SPA_ASYNC_TRIM_RESTART);
	spa_async_request(spa, SPA_ASYNC_AUTOTRIM_RESTART);

	vdev_reopen(spa->spa_root_vdev);

	nvlist_free(spa->spa_config_splitting);
	spa->spa_config_splitting = NULL;
	(void) spa_vdev_exit(spa, NULL, txg, error);

	kmem_free(vml, children * sizeof (vdev_t *));
	return (error);
	}

	/*
	* Find any device that's done replacing, or a vdev marked 'unspare' that's
	* currently spared, so we can detach it.
	*/
	static vdev_t *
	spa_vdev_resilver_done_hunt(vdev_t *vd)
	{
	vdev_t newvd, oldvd;

	for (int c = 0; c < vd->vdev_children; c++) {
	oldvd = spa_vdev_resilver_done_hunt(vd->vdev_child[c]);
	if (oldvd != NULL)
	return (oldvd);
	}

	/*
	* Check for a completed replacement. We always consider the first
	* vdev in the list to be the oldest vdev, and the last one to be
	* the newest (see spa_vdev_attach() for how that works). In
	* the case where the newest vdev is faulted, we will not automatically
	* remove it after a resilver completes. This is OK as it will require
	* user intervention to determine which disk the admin wishes to keep.
	*/
	if (vd->vdev_ops == &vdev_replacing_ops) {
	ASSERT(vd->vdev_children > 1);

	newvd = vd->vdev_child[vd->vdev_children - 1];
	oldvd = vd->vdev_child[0];

	if (vdev_dtl_empty(newvd, DTL_MISSING) &&
	vdev_dtl_empty(newvd, DTL_OUTAGE) &&
	!vdev_dtl_required(oldvd))
	return (oldvd);
	}

	/*
	* Check for a completed resilver with the 'unspare' flag set.
	* Also potentially update faulted state.
	*/
	if (vd->vdev_ops == &vdev_spare_ops) {
	vdev_t *first = vd->vdev_child[0];
	vdev_t *last = vd->vdev_child[vd->vdev_children - 1];

	if (last->vdev_unspare) {
	oldvd = first;
	newvd = last;
	} else if (first->vdev_unspare) {
	oldvd = last;
	newvd = first;
	} else {
	oldvd = NULL;
	}

	if (oldvd != NULL &&
	vdev_dtl_empty(newvd, DTL_MISSING) &&
	vdev_dtl_empty(newvd, DTL_OUTAGE) &&
	!vdev_dtl_required(oldvd))
	return (oldvd);

	vdev_propagate_state(vd);

	/*
	* If there are more than two spares attached to a disk,
	* and those spares are not required, then we want to
	* attempt to free them up now so that they can be used
	* by other pools. Once we're back down to a single
	* disk+spare, we stop removing them.
	*/
	if (vd->vdev_children > 2) {
	newvd = vd->vdev_child[1];

	if (newvd->vdev_isspare && last->vdev_isspare &&
	vdev_dtl_empty(last, DTL_MISSING) &&
	vdev_dtl_empty(last, DTL_OUTAGE) &&
	!vdev_dtl_required(newvd))
	return (newvd);
	}
	}

	return (NULL);
	}

	static void
	spa_vdev_resilver_done(spa_t *spa)
	{
	vdev_t vd, pvd, *ppvd;
	uint64_t guid, sguid, pguid, ppguid;

	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);

	while ((vd = spa_vdev_resilver_done_hunt(spa->spa_root_vdev)) != NULL) {
	pvd = vd->vdev_parent;
	ppvd = pvd->vdev_parent;
	guid = vd->vdev_guid;
	pguid = pvd->vdev_guid;
	ppguid = ppvd->vdev_guid;
	sguid = 0;
	/*
	* If we have just finished replacing a hot spared device, then
	* we need to detach the parent's first child (the original hot
	* spare) as well.
	*/
	if (ppvd->vdev_ops == &vdev_spare_ops && pvd->vdev_id == 0 &&
	ppvd->vdev_children == 2) {
	ASSERT(pvd->vdev_ops == &vdev_replacing_ops);
	sguid = ppvd->vdev_child[1]->vdev_guid;
	}
	ASSERT(vd->vdev_resilver_txg == 0 \|\| !vdev_dtl_required(vd));

	spa_config_exit(spa, SCL_ALL, FTAG);
	if (spa_vdev_detach(spa, guid, pguid, B_TRUE) != 0)
	return;
	if (sguid && spa_vdev_detach(spa, sguid, ppguid, B_TRUE) != 0)
	return;
	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	}

	spa_config_exit(spa, SCL_ALL, FTAG);

	/*
	* If a detach was not performed above replace waiters will not have
	* been notified. In which case we must do so now.
	*/
	spa_notify_waiters(spa);
	}

	/*
	* Update the stored path or FRU for this vdev.
	*/
	static int
	spa_vdev_set_common(spa_t spa, uint64_t guid, const char value,
	boolean_t ispath)
	{
	vdev_t *vd;
	boolean_t sync = B_FALSE;

	ASSERT(spa_writeable(spa));

	spa_vdev_state_enter(spa, SCL_ALL);

	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
	return (spa_vdev_state_exit(spa, NULL, ENOENT));

	if (!vd->vdev_ops->vdev_op_leaf)
	return (spa_vdev_state_exit(spa, NULL, ENOTSUP));

	if (ispath) {
	if (strcmp(value, vd->vdev_path) != 0) {
	spa_strfree(vd->vdev_path);
	vd->vdev_path = spa_strdup(value);
	sync = B_TRUE;
	}
	} else {
	if (vd->vdev_fru == NULL) {
	vd->vdev_fru = spa_strdup(value);
	sync = B_TRUE;
	} else if (strcmp(value, vd->vdev_fru) != 0) {
	spa_strfree(vd->vdev_fru);
	vd->vdev_fru = spa_strdup(value);
	sync = B_TRUE;
	}
	}

	return (spa_vdev_state_exit(spa, sync ? vd : NULL, 0));
	}

	int
	spa_vdev_setpath(spa_t spa, uint64_t guid, const char newpath)
	{
	return (spa_vdev_set_common(spa, guid, newpath, B_TRUE));
	}

	int
	spa_vdev_setfru(spa_t spa, uint64_t guid, const char newfru)
	{
	return (spa_vdev_set_common(spa, guid, newfru, B_FALSE));
	}

	/*
	* ==========================================================================
	* SPA Scanning
	* ==========================================================================
	*/
	int
	spa_scrub_pause_resume(spa_t *spa, pool_scrub_cmd_t cmd)
	{
	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);

	if (dsl_scan_resilvering(spa->spa_dsl_pool))
	return (SET_ERROR(EBUSY));

	return (dsl_scrub_set_pause_resume(spa->spa_dsl_pool, cmd));
	}

	int
	spa_scan_stop(spa_t *spa)
	{
	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
	if (dsl_scan_resilvering(spa->spa_dsl_pool))
	return (SET_ERROR(EBUSY));

	return (dsl_scan_cancel(spa->spa_dsl_pool));
	}

	int
	spa_scan(spa_t *spa, pool_scan_func_t func)
	{
	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);

	if (func >= POOL_SCAN_FUNCS \|\| func == POOL_SCAN_NONE)
	return (SET_ERROR(ENOTSUP));

	if (func == POOL_SCAN_RESILVER &&
	!spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER))
	return (SET_ERROR(ENOTSUP));

	/*
	* If a resilver was requested, but there is no DTL on a
	* writeable leaf device, we have nothing to do.
	*/
	if (func == POOL_SCAN_RESILVER &&
	!vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) {
	spa_async_request(spa, SPA_ASYNC_RESILVER_DONE);
	return (0);
	}

	if (func == POOL_SCAN_ERRORSCRUB &&
	!spa_feature_is_enabled(spa, SPA_FEATURE_HEAD_ERRLOG))
	return (SET_ERROR(ENOTSUP));

	return (dsl_scan(spa->spa_dsl_pool, func));
	}

	/*
	* ==========================================================================
	* SPA async task processing
	* ==========================================================================
	*/

	static void
	spa_async_remove(spa_t spa, vdev_t vd)
	{
	if (vd->vdev_remove_wanted) {
	vd->vdev_remove_wanted = B_FALSE;
	vd->vdev_delayed_close = B_FALSE;
	vdev_set_state(vd, B_FALSE, VDEV_STATE_REMOVED, VDEV_AUX_NONE);

	/*
	* We want to clear the stats, but we don't want to do a full
	* vdev_clear() as that will cause us to throw away
	* degraded/faulted state as well as attempt to reopen the
	* device, all of which is a waste.
	*/
	vd->vdev_stat.vs_read_errors = 0;
	vd->vdev_stat.vs_write_errors = 0;
	vd->vdev_stat.vs_checksum_errors = 0;

	vdev_state_dirty(vd->vdev_top);

	/* Tell userspace that the vdev is gone. */
	zfs_post_remove(spa, vd);
	}

	for (int c = 0; c < vd->vdev_children; c++)
	spa_async_remove(spa, vd->vdev_child[c]);
	}

	static void
	-spa_async_probe(spa_t spa, vdev_t vd)
	+spa_async_fault_vdev(spa_t spa, vdev_t vd)
	{
	- if (vd->vdev_probe_wanted) {
	- vd->vdev_probe_wanted = B_FALSE;
	- vdev_reopen(vd); /* vdev_open() does the actual probe */
	+ if (vd->vdev_fault_wanted) {
	+ vd->vdev_fault_wanted = B_FALSE;
	+ vdev_set_state(vd, B_TRUE, VDEV_STATE_FAULTED,
	+ VDEV_AUX_ERR_EXCEEDED);
	}

	for (int c = 0; c < vd->vdev_children; c++)
	- spa_async_probe(spa, vd->vdev_child[c]);
	+ spa_async_fault_vdev(spa, vd->vdev_child[c]);
	}

	static void
	spa_async_autoexpand(spa_t spa, vdev_t vd)
	{
	if (!spa->spa_autoexpand)
	return;

	for (int c = 0; c < vd->vdev_children; c++) {
	vdev_t *cvd = vd->vdev_child[c];
	spa_async_autoexpand(spa, cvd);
	}

	if (!vd->vdev_ops->vdev_op_leaf \|\| vd->vdev_physpath == NULL)
	return;

	spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_AUTOEXPAND);
	}

	static __attribute__((noreturn)) void
	spa_async_thread(void *arg)
	{
	spa_t spa = (spa_t )arg;
	dsl_pool_t *dp = spa->spa_dsl_pool;
	int tasks;

	ASSERT(spa->spa_sync_on);

	mutex_enter(&spa->spa_async_lock);
	tasks = spa->spa_async_tasks;
	spa->spa_async_tasks = 0;
	mutex_exit(&spa->spa_async_lock);

	/*
	* See if the config needs to be updated.
	*/
	if (tasks & SPA_ASYNC_CONFIG_UPDATE) {
	uint64_t old_space, new_space;

	mutex_enter(&spa_namespace_lock);
	old_space = metaslab_class_get_space(spa_normal_class(spa));
	old_space += metaslab_class_get_space(spa_special_class(spa));
	old_space += metaslab_class_get_space(spa_dedup_class(spa));
	old_space += metaslab_class_get_space(
	spa_embedded_log_class(spa));

	spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);

	new_space = metaslab_class_get_space(spa_normal_class(spa));
	new_space += metaslab_class_get_space(spa_special_class(spa));
	new_space += metaslab_class_get_space(spa_dedup_class(spa));
	new_space += metaslab_class_get_space(
	spa_embedded_log_class(spa));
	mutex_exit(&spa_namespace_lock);

	/*
	* If the pool grew as a result of the config update,
	* then log an internal history event.
	*/
	if (new_space != old_space) {
	spa_history_log_internal(spa, "vdev online", NULL,
	"pool '%s' size: %llu(+%llu)",
	spa_name(spa), (u_longlong_t)new_space,
	(u_longlong_t)(new_space - old_space));
	}
	}

	/*
	* See if any devices need to be marked REMOVED.
	*/
	if (tasks & SPA_ASYNC_REMOVE) {
	spa_vdev_state_enter(spa, SCL_NONE);
	spa_async_remove(spa, spa->spa_root_vdev);
	for (int i = 0; i < spa->spa_l2cache.sav_count; i++)
	spa_async_remove(spa, spa->spa_l2cache.sav_vdevs[i]);
	for (int i = 0; i < spa->spa_spares.sav_count; i++)
	spa_async_remove(spa, spa->spa_spares.sav_vdevs[i]);
	(void) spa_vdev_state_exit(spa, NULL, 0);
	}

	if ((tasks & SPA_ASYNC_AUTOEXPAND) && !spa_suspended(spa)) {
	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
	spa_async_autoexpand(spa, spa->spa_root_vdev);
	spa_config_exit(spa, SCL_CONFIG, FTAG);
	}

	/*
	- * See if any devices need to be probed.
	+ * See if any devices need to be marked faulted.
	*/
	- if (tasks & SPA_ASYNC_PROBE) {
	+ if (tasks & SPA_ASYNC_FAULT_VDEV) {
	spa_vdev_state_enter(spa, SCL_NONE);
	- spa_async_probe(spa, spa->spa_root_vdev);
	+ spa_async_fault_vdev(spa, spa->spa_root_vdev);
	(void) spa_vdev_state_exit(spa, NULL, 0);
	}

	/*
	* If any devices are done replacing, detach them.
	*/
	if (tasks & SPA_ASYNC_RESILVER_DONE \|\|
	tasks & SPA_ASYNC_REBUILD_DONE \|\|
	tasks & SPA_ASYNC_DETACH_SPARE) {
	spa_vdev_resilver_done(spa);
	}

	/*
	* Kick off a resilver.
	*/
	if (tasks & SPA_ASYNC_RESILVER &&
	!vdev_rebuild_active(spa->spa_root_vdev) &&
	(!dsl_scan_resilvering(dp) \|\|
	!spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER)))
	dsl_scan_restart_resilver(dp, 0);

	if (tasks & SPA_ASYNC_INITIALIZE_RESTART) {
	mutex_enter(&spa_namespace_lock);
	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
	vdev_initialize_restart(spa->spa_root_vdev);
	spa_config_exit(spa, SCL_CONFIG, FTAG);
	mutex_exit(&spa_namespace_lock);
	}

	if (tasks & SPA_ASYNC_TRIM_RESTART) {
	mutex_enter(&spa_namespace_lock);
	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
	vdev_trim_restart(spa->spa_root_vdev);
	spa_config_exit(spa, SCL_CONFIG, FTAG);
	mutex_exit(&spa_namespace_lock);
	}

	if (tasks & SPA_ASYNC_AUTOTRIM_RESTART) {
	mutex_enter(&spa_namespace_lock);
	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
	vdev_autotrim_restart(spa);
	spa_config_exit(spa, SCL_CONFIG, FTAG);
	mutex_exit(&spa_namespace_lock);
	}

	/*
	* Kick off L2 cache whole device TRIM.
	*/
	if (tasks & SPA_ASYNC_L2CACHE_TRIM) {
	mutex_enter(&spa_namespace_lock);
	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
	vdev_trim_l2arc(spa);
	spa_config_exit(spa, SCL_CONFIG, FTAG);
	mutex_exit(&spa_namespace_lock);
	}

	/*
	* Kick off L2 cache rebuilding.
	*/
	if (tasks & SPA_ASYNC_L2CACHE_REBUILD) {
	mutex_enter(&spa_namespace_lock);
	spa_config_enter(spa, SCL_L2ARC, FTAG, RW_READER);
	l2arc_spa_rebuild_start(spa);
	spa_config_exit(spa, SCL_L2ARC, FTAG);
	mutex_exit(&spa_namespace_lock);
	}

	/*
	* Let the world know that we're done.
	*/
	mutex_enter(&spa->spa_async_lock);
	spa->spa_async_thread = NULL;
	cv_broadcast(&spa->spa_async_cv);
	mutex_exit(&spa->spa_async_lock);
	thread_exit();
	}

	void
	spa_async_suspend(spa_t *spa)
	{
	mutex_enter(&spa->spa_async_lock);
	spa->spa_async_suspended++;
	while (spa->spa_async_thread != NULL)
	cv_wait(&spa->spa_async_cv, &spa->spa_async_lock);
	mutex_exit(&spa->spa_async_lock);

	spa_vdev_remove_suspend(spa);

	zthr_t *condense_thread = spa->spa_condense_zthr;
	if (condense_thread != NULL)
	zthr_cancel(condense_thread);

	zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr;
	if (discard_thread != NULL)
	zthr_cancel(discard_thread);

	zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr;
	if (ll_delete_thread != NULL)
	zthr_cancel(ll_delete_thread);

	zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr;
	if (ll_condense_thread != NULL)
	zthr_cancel(ll_condense_thread);
	}

	void
	spa_async_resume(spa_t *spa)
	{
	mutex_enter(&spa->spa_async_lock);
	ASSERT(spa->spa_async_suspended != 0);
	spa->spa_async_suspended--;
	mutex_exit(&spa->spa_async_lock);
	spa_restart_removal(spa);

	zthr_t *condense_thread = spa->spa_condense_zthr;
	if (condense_thread != NULL)
	zthr_resume(condense_thread);

	zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr;
	if (discard_thread != NULL)
	zthr_resume(discard_thread);

	zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr;
	if (ll_delete_thread != NULL)
	zthr_resume(ll_delete_thread);

	zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr;
	if (ll_condense_thread != NULL)
	zthr_resume(ll_condense_thread);
	}

	static boolean_t
	spa_async_tasks_pending(spa_t *spa)
	{
	uint_t non_config_tasks;
	uint_t config_task;
	boolean_t config_task_suspended;

	non_config_tasks = spa->spa_async_tasks & ~SPA_ASYNC_CONFIG_UPDATE;
	config_task = spa->spa_async_tasks & SPA_ASYNC_CONFIG_UPDATE;
	if (spa->spa_ccw_fail_time == 0) {
	config_task_suspended = B_FALSE;
	} else {
	config_task_suspended =
	(gethrtime() - spa->spa_ccw_fail_time) <
	((hrtime_t)zfs_ccw_retry_interval * NANOSEC);
	}

	return (non_config_tasks \|\| (config_task && !config_task_suspended));
	}

	static void
	spa_async_dispatch(spa_t *spa)
	{
	mutex_enter(&spa->spa_async_lock);
	if (spa_async_tasks_pending(spa) &&
	!spa->spa_async_suspended &&
	spa->spa_async_thread == NULL)
	spa->spa_async_thread = thread_create(NULL, 0,
	spa_async_thread, spa, 0, &p0, TS_RUN, maxclsyspri);
	mutex_exit(&spa->spa_async_lock);
	}

	void
	spa_async_request(spa_t *spa, int task)
	{
	zfs_dbgmsg("spa=%s async request task=%u", spa->spa_name, task);
	mutex_enter(&spa->spa_async_lock);
	spa->spa_async_tasks \|= task;
	mutex_exit(&spa->spa_async_lock);
	}

	int
	spa_async_tasks(spa_t *spa)
	{
	return (spa->spa_async_tasks);
	}

	/*
	* ==========================================================================
	* SPA syncing routines
	* ==========================================================================
	*/


	static int
	bpobj_enqueue_cb(void arg, const blkptr_t bp, boolean_t bp_freed,
	dmu_tx_t *tx)
	{
	bpobj_t *bpo = arg;
	bpobj_enqueue(bpo, bp, bp_freed, tx);
	return (0);
	}

	int
	bpobj_enqueue_alloc_cb(void arg, const blkptr_t bp, dmu_tx_t *tx)
	{
	return (bpobj_enqueue_cb(arg, bp, B_FALSE, tx));
	}

	int
	bpobj_enqueue_free_cb(void arg, const blkptr_t bp, dmu_tx_t *tx)
	{
	return (bpobj_enqueue_cb(arg, bp, B_TRUE, tx));
	}

	static int
	spa_free_sync_cb(void arg, const blkptr_t bp, dmu_tx_t *tx)
	{
	zio_t *pio = arg;

	zio_nowait(zio_free_sync(pio, pio->io_spa, dmu_tx_get_txg(tx), bp,
	pio->io_flags));
	return (0);
	}

	static int
	bpobj_spa_free_sync_cb(void arg, const blkptr_t bp, boolean_t bp_freed,
	dmu_tx_t *tx)
	{
	ASSERT(!bp_freed);
	return (spa_free_sync_cb(arg, bp, tx));
	}

	/*
	* Note: this simple function is not inlined to make it easier to dtrace the
	* amount of time spent syncing frees.
	*/
	static void
	spa_sync_frees(spa_t spa, bplist_t bpl, dmu_tx_t *tx)
	{
	zio_t *zio = zio_root(spa, NULL, NULL, 0);
	bplist_iterate(bpl, spa_free_sync_cb, zio, tx);
	VERIFY(zio_wait(zio) == 0);
	}

	/*
	* Note: this simple function is not inlined to make it easier to dtrace the
	* amount of time spent syncing deferred frees.
	*/
	static void
	spa_sync_deferred_frees(spa_t spa, dmu_tx_t tx)
	{
	if (spa_sync_pass(spa) != 1)
	return;

	/*
	* Note:
	* If the log space map feature is active, we stop deferring
	* frees to the next TXG and therefore running this function
	* would be considered a no-op as spa_deferred_bpobj should
	* not have any entries.
	*
	* That said we run this function anyway (instead of returning
	* immediately) for the edge-case scenario where we just
	* activated the log space map feature in this TXG but we have
	* deferred frees from the previous TXG.
	*/
	zio_t *zio = zio_root(spa, NULL, NULL, 0);
	VERIFY3U(bpobj_iterate(&spa->spa_deferred_bpobj,
	bpobj_spa_free_sync_cb, zio, tx), ==, 0);
	VERIFY0(zio_wait(zio));
	}

	static void
	spa_sync_nvlist(spa_t spa, uint64_t obj, nvlist_t nv, dmu_tx_t *tx)
	{
	char *packed = NULL;
	size_t bufsize;
	size_t nvsize = 0;
	dmu_buf_t *db;

	VERIFY(nvlist_size(nv, &nvsize, NV_ENCODE_XDR) == 0);

	/*
	* Write full (SPA_CONFIG_BLOCKSIZE) blocks of configuration
	* information. This avoids the dmu_buf_will_dirty() path and
	* saves us a pre-read to get data we don't actually care about.
	*/
	bufsize = P2ROUNDUP((uint64_t)nvsize, SPA_CONFIG_BLOCKSIZE);
	packed = vmem_alloc(bufsize, KM_SLEEP);

	VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR,
	KM_SLEEP) == 0);
	memset(packed + nvsize, 0, bufsize - nvsize);

	dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx);

	vmem_free(packed, bufsize);

	VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db));
	dmu_buf_will_dirty(db, tx);
	(uint64_t )db->db_data = nvsize;
	dmu_buf_rele(db, FTAG);
	}

	static void
	spa_sync_aux_dev(spa_t spa, spa_aux_vdev_t sav, dmu_tx_t *tx,
	const char config, const char entry)
	{
	nvlist_t *nvroot;
	nvlist_t **list;
	int i;

	if (!sav->sav_sync)
	return;

	/*
	* Update the MOS nvlist describing the list of available devices.
	* spa_validate_aux() will have already made sure this nvlist is
	* valid and the vdevs are labeled appropriately.
	*/
	if (sav->sav_object == 0) {
	sav->sav_object = dmu_object_alloc(spa->spa_meta_objset,
	DMU_OT_PACKED_NVLIST, 1 << 14, DMU_OT_PACKED_NVLIST_SIZE,
	sizeof (uint64_t), tx);
	VERIFY(zap_update(spa->spa_meta_objset,
	DMU_POOL_DIRECTORY_OBJECT, entry, sizeof (uint64_t), 1,
	&sav->sav_object, tx) == 0);
	}

	nvroot = fnvlist_alloc();
	if (sav->sav_count == 0) {
	fnvlist_add_nvlist_array(nvroot, config,
	(const nvlist_t * const *)NULL, 0);
	} else {
	list = kmem_alloc(sav->sav_countsizeof (void ), KM_SLEEP);
	for (i = 0; i < sav->sav_count; i++)
	list[i] = vdev_config_generate(spa, sav->sav_vdevs[i],
	B_FALSE, VDEV_CONFIG_L2CACHE);
	fnvlist_add_nvlist_array(nvroot, config,
	(const nvlist_t * const *)list, sav->sav_count);
	for (i = 0; i < sav->sav_count; i++)
	nvlist_free(list[i]);
	kmem_free(list, sav->sav_count * sizeof (void *));
	}

	spa_sync_nvlist(spa, sav->sav_object, nvroot, tx);
	nvlist_free(nvroot);

	sav->sav_sync = B_FALSE;
	}

	/*
	* Rebuild spa's all-vdev ZAP from the vdev ZAPs indicated in each vdev_t.
	* The all-vdev ZAP must be empty.
	*/
	static void
	spa_avz_build(vdev_t vd, uint64_t avz, dmu_tx_t tx)
	{
	spa_t *spa = vd->vdev_spa;

	if (vd->vdev_root_zap != 0 &&
	spa_feature_is_active(spa, SPA_FEATURE_AVZ_V2)) {
	VERIFY0(zap_add_int(spa->spa_meta_objset, avz,
	vd->vdev_root_zap, tx));
	}
	if (vd->vdev_top_zap != 0) {
	VERIFY0(zap_add_int(spa->spa_meta_objset, avz,
	vd->vdev_top_zap, tx));
	}
	if (vd->vdev_leaf_zap != 0) {
	VERIFY0(zap_add_int(spa->spa_meta_objset, avz,
	vd->vdev_leaf_zap, tx));
	}
	for (uint64_t i = 0; i < vd->vdev_children; i++) {
	spa_avz_build(vd->vdev_child[i], avz, tx);
	}
	}

	static void
	spa_sync_config_object(spa_t spa, dmu_tx_t tx)
	{
	nvlist_t *config;

	/*
	* If the pool is being imported from a pre-per-vdev-ZAP version of ZFS,
	* its config may not be dirty but we still need to build per-vdev ZAPs.
	* Similarly, if the pool is being assembled (e.g. after a split), we
	* need to rebuild the AVZ although the config may not be dirty.
	*/
	if (list_is_empty(&spa->spa_config_dirty_list) &&
	spa->spa_avz_action == AVZ_ACTION_NONE)
	return;

	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);

	ASSERT(spa->spa_avz_action == AVZ_ACTION_NONE \|\|
	spa->spa_avz_action == AVZ_ACTION_INITIALIZE \|\|
	spa->spa_all_vdev_zaps != 0);

	if (spa->spa_avz_action == AVZ_ACTION_REBUILD) {
	/* Make and build the new AVZ */
	uint64_t new_avz = zap_create(spa->spa_meta_objset,
	DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx);
	spa_avz_build(spa->spa_root_vdev, new_avz, tx);

	/* Diff old AVZ with new one */
	zap_cursor_t zc;
	zap_attribute_t za;

	for (zap_cursor_init(&zc, spa->spa_meta_objset,
	spa->spa_all_vdev_zaps);
	zap_cursor_retrieve(&zc, &za) == 0;
	zap_cursor_advance(&zc)) {
	uint64_t vdzap = za.za_first_integer;
	if (zap_lookup_int(spa->spa_meta_objset, new_avz,
	vdzap) == ENOENT) {
	/*
	* ZAP is listed in old AVZ but not in new one;
	* destroy it
	*/
	VERIFY0(zap_destroy(spa->spa_meta_objset, vdzap,
	tx));
	}
	}

	zap_cursor_fini(&zc);

	/* Destroy the old AVZ */
	VERIFY0(zap_destroy(spa->spa_meta_objset,
	spa->spa_all_vdev_zaps, tx));

	/* Replace the old AVZ in the dir obj with the new one */
	VERIFY0(zap_update(spa->spa_meta_objset,
	DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP,
	sizeof (new_avz), 1, &new_avz, tx));

	spa->spa_all_vdev_zaps = new_avz;
	} else if (spa->spa_avz_action == AVZ_ACTION_DESTROY) {
	zap_cursor_t zc;
	zap_attribute_t za;

	/* Walk through the AVZ and destroy all listed ZAPs */
	for (zap_cursor_init(&zc, spa->spa_meta_objset,
	spa->spa_all_vdev_zaps);
	zap_cursor_retrieve(&zc, &za) == 0;
	zap_cursor_advance(&zc)) {
	uint64_t zap = za.za_first_integer;
	VERIFY0(zap_destroy(spa->spa_meta_objset, zap, tx));
	}

	zap_cursor_fini(&zc);

	/* Destroy and unlink the AVZ itself */
	VERIFY0(zap_destroy(spa->spa_meta_objset,
	spa->spa_all_vdev_zaps, tx));
	VERIFY0(zap_remove(spa->spa_meta_objset,
	DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, tx));
	spa->spa_all_vdev_zaps = 0;
	}

	if (spa->spa_all_vdev_zaps == 0) {
	spa->spa_all_vdev_zaps = zap_create_link(spa->spa_meta_objset,
	DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT,
	DMU_POOL_VDEV_ZAP_MAP, tx);
	}
	spa->spa_avz_action = AVZ_ACTION_NONE;

	/* Create ZAPs for vdevs that don't have them. */
	vdev_construct_zaps(spa->spa_root_vdev, tx);

	config = spa_config_generate(spa, spa->spa_root_vdev,
	dmu_tx_get_txg(tx), B_FALSE);

	/*
	* If we're upgrading the spa version then make sure that
	* the config object gets updated with the correct version.
	*/
	if (spa->spa_ubsync.ub_version < spa->spa_uberblock.ub_version)
	fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION,
	spa->spa_uberblock.ub_version);

	spa_config_exit(spa, SCL_STATE, FTAG);

	nvlist_free(spa->spa_config_syncing);
	spa->spa_config_syncing = config;

	spa_sync_nvlist(spa, spa->spa_config_object, config, tx);
	}

	static void
	spa_sync_version(void arg, dmu_tx_t tx)
	{
	uint64_t *versionp = arg;
	uint64_t version = *versionp;
	spa_t *spa = dmu_tx_pool(tx)->dp_spa;

	/*
	* Setting the version is special cased when first creating the pool.
	*/
	ASSERT(tx->tx_txg != TXG_INITIAL);

	ASSERT(SPA_VERSION_IS_SUPPORTED(version));
	ASSERT(version >= spa_version(spa));

	spa->spa_uberblock.ub_version = version;
	vdev_config_dirty(spa->spa_root_vdev);
	spa_history_log_internal(spa, "set", tx, "version=%lld",
	(longlong_t)version);
	}

	/*
	* Set zpool properties.
	*/
	static void
	spa_sync_props(void arg, dmu_tx_t tx)
	{
	nvlist_t *nvp = arg;
	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
	objset_t *mos = spa->spa_meta_objset;
	nvpair_t *elem = NULL;

	mutex_enter(&spa->spa_props_lock);

	while ((elem = nvlist_next_nvpair(nvp, elem))) {
	uint64_t intval;
	const char strval, fname;
	zpool_prop_t prop;
	const char *propname;
	const char *elemname = nvpair_name(elem);
	zprop_type_t proptype;
	spa_feature_t fid;

	switch (prop = zpool_name_to_prop(elemname)) {
	case ZPOOL_PROP_VERSION:
	intval = fnvpair_value_uint64(elem);
	/*
	* The version is synced separately before other
	* properties and should be correct by now.
	*/
	ASSERT3U(spa_version(spa), >=, intval);
	break;

	case ZPOOL_PROP_ALTROOT:
	/*
	* 'altroot' is a non-persistent property. It should
	* have been set temporarily at creation or import time.
	*/
	ASSERT(spa->spa_root != NULL);
	break;

	case ZPOOL_PROP_READONLY:
	case ZPOOL_PROP_CACHEFILE:
	/*
	* 'readonly' and 'cachefile' are also non-persistent
	* properties.
	*/
	break;
	case ZPOOL_PROP_COMMENT:
	strval = fnvpair_value_string(elem);
	if (spa->spa_comment != NULL)
	spa_strfree(spa->spa_comment);
	spa->spa_comment = spa_strdup(strval);
	/*
	* We need to dirty the configuration on all the vdevs
	* so that their labels get updated. We also need to
	* update the cache file to keep it in sync with the
	* MOS version. It's unnecessary to do this for pool
	* creation since the vdev's configuration has already
	* been dirtied.
	*/
	if (tx->tx_txg != TXG_INITIAL) {
	vdev_config_dirty(spa->spa_root_vdev);
	spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
	}
	spa_history_log_internal(spa, "set", tx,
	"%s=%s", elemname, strval);
	break;
	case ZPOOL_PROP_COMPATIBILITY:
	strval = fnvpair_value_string(elem);
	if (spa->spa_compatibility != NULL)
	spa_strfree(spa->spa_compatibility);
	spa->spa_compatibility = spa_strdup(strval);
	/*
	* Dirty the configuration on vdevs as above.
	*/
	if (tx->tx_txg != TXG_INITIAL) {
	vdev_config_dirty(spa->spa_root_vdev);
	spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
	}

	spa_history_log_internal(spa, "set", tx,
	"%s=%s", nvpair_name(elem), strval);
	break;

	case ZPOOL_PROP_INVAL:
	if (zpool_prop_feature(elemname)) {
	fname = strchr(elemname, '@') + 1;
	VERIFY0(zfeature_lookup_name(fname, &fid));

	spa_feature_enable(spa, fid, tx);
	spa_history_log_internal(spa, "set", tx,
	"%s=enabled", elemname);
	break;
	} else if (!zfs_prop_user(elemname)) {
	ASSERT(zpool_prop_feature(elemname));
	break;
	}
	zfs_fallthrough;
	default:
	/*
	* Set pool property values in the poolprops mos object.
	*/
	if (spa->spa_pool_props_object == 0) {
	spa->spa_pool_props_object =
	zap_create_link(mos, DMU_OT_POOL_PROPS,
	DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_PROPS,
	tx);
	}

	/* normalize the property name */
	if (prop == ZPOOL_PROP_INVAL) {
	propname = elemname;
	proptype = PROP_TYPE_STRING;
	} else {
	propname = zpool_prop_to_name(prop);
	proptype = zpool_prop_get_type(prop);
	}

	if (nvpair_type(elem) == DATA_TYPE_STRING) {
	ASSERT(proptype == PROP_TYPE_STRING);
	strval = fnvpair_value_string(elem);
	VERIFY0(zap_update(mos,
	spa->spa_pool_props_object, propname,
	1, strlen(strval) + 1, strval, tx));
	spa_history_log_internal(spa, "set", tx,
	"%s=%s", elemname, strval);
	} else if (nvpair_type(elem) == DATA_TYPE_UINT64) {
	intval = fnvpair_value_uint64(elem);

	if (proptype == PROP_TYPE_INDEX) {
	const char *unused;
	VERIFY0(zpool_prop_index_to_string(
	prop, intval, &unused));
	}
	VERIFY0(zap_update(mos,
	spa->spa_pool_props_object, propname,
	8, 1, &intval, tx));
	spa_history_log_internal(spa, "set", tx,
	"%s=%lld", elemname,
	(longlong_t)intval);

	switch (prop) {
	case ZPOOL_PROP_DELEGATION:
	spa->spa_delegation = intval;
	break;
	case ZPOOL_PROP_BOOTFS:
	spa->spa_bootfs = intval;
	break;
	case ZPOOL_PROP_FAILUREMODE:
	spa->spa_failmode = intval;
	break;
	case ZPOOL_PROP_AUTOTRIM:
	spa->spa_autotrim = intval;
	spa_async_request(spa,
	SPA_ASYNC_AUTOTRIM_RESTART);
	break;
	case ZPOOL_PROP_AUTOEXPAND:
	spa->spa_autoexpand = intval;
	if (tx->tx_txg != TXG_INITIAL)
	spa_async_request(spa,
	SPA_ASYNC_AUTOEXPAND);
	break;
	case ZPOOL_PROP_MULTIHOST:
	spa->spa_multihost = intval;
	break;
	default:
	break;
	}
	} else {
	ASSERT(0); /* not allowed */
	}
	}

	}

	mutex_exit(&spa->spa_props_lock);
	}

	/*
	* Perform one-time upgrade on-disk changes. spa_version() does not
	* reflect the new version this txg, so there must be no changes this
	* txg to anything that the upgrade code depends on after it executes.
	* Therefore this must be called after dsl_pool_sync() does the sync
	* tasks.
	*/
	static void
	spa_sync_upgrades(spa_t spa, dmu_tx_t tx)
	{
	if (spa_sync_pass(spa) != 1)
	return;

	dsl_pool_t *dp = spa->spa_dsl_pool;
	rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG);

	if (spa->spa_ubsync.ub_version < SPA_VERSION_ORIGIN &&
	spa->spa_uberblock.ub_version >= SPA_VERSION_ORIGIN) {
	dsl_pool_create_origin(dp, tx);

	/* Keeping the origin open increases spa_minref */
	spa->spa_minref += 3;
	}

	if (spa->spa_ubsync.ub_version < SPA_VERSION_NEXT_CLONES &&
	spa->spa_uberblock.ub_version >= SPA_VERSION_NEXT_CLONES) {
	dsl_pool_upgrade_clones(dp, tx);
	}

	if (spa->spa_ubsync.ub_version < SPA_VERSION_DIR_CLONES &&
	spa->spa_uberblock.ub_version >= SPA_VERSION_DIR_CLONES) {
	dsl_pool_upgrade_dir_clones(dp, tx);

	/* Keeping the freedir open increases spa_minref */
	spa->spa_minref += 3;
	}

	if (spa->spa_ubsync.ub_version < SPA_VERSION_FEATURES &&
	spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) {
	spa_feature_create_zap_objects(spa, tx);
	}

	/*
	* LZ4_COMPRESS feature's behaviour was changed to activate_on_enable
	* when possibility to use lz4 compression for metadata was added
	* Old pools that have this feature enabled must be upgraded to have
	* this feature active
	*/
	if (spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) {
	boolean_t lz4_en = spa_feature_is_enabled(spa,
	SPA_FEATURE_LZ4_COMPRESS);
	boolean_t lz4_ac = spa_feature_is_active(spa,
	SPA_FEATURE_LZ4_COMPRESS);

	if (lz4_en && !lz4_ac)
	spa_feature_incr(spa, SPA_FEATURE_LZ4_COMPRESS, tx);
	}

	/*
	* If we haven't written the salt, do so now. Note that the
	* feature may not be activated yet, but that's fine since
	* the presence of this ZAP entry is backwards compatible.
	*/
	if (zap_contains(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
	DMU_POOL_CHECKSUM_SALT) == ENOENT) {
	VERIFY0(zap_add(spa->spa_meta_objset,
	DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CHECKSUM_SALT, 1,
	sizeof (spa->spa_cksum_salt.zcs_bytes),
	spa->spa_cksum_salt.zcs_bytes, tx));
	}

	rrw_exit(&dp->dp_config_rwlock, FTAG);
	}

	static void
	vdev_indirect_state_sync_verify(vdev_t *vd)
	{
	vdev_indirect_mapping_t *vim __maybe_unused = vd->vdev_indirect_mapping;
	vdev_indirect_births_t *vib __maybe_unused = vd->vdev_indirect_births;

	if (vd->vdev_ops == &vdev_indirect_ops) {
	ASSERT(vim != NULL);
	ASSERT(vib != NULL);
	}

	uint64_t obsolete_sm_object = 0;
	ASSERT0(vdev_obsolete_sm_object(vd, &obsolete_sm_object));
	if (obsolete_sm_object != 0) {
	ASSERT(vd->vdev_obsolete_sm != NULL);
	ASSERT(vd->vdev_removing \|\|
	vd->vdev_ops == &vdev_indirect_ops);
	ASSERT(vdev_indirect_mapping_num_entries(vim) > 0);
	ASSERT(vdev_indirect_mapping_bytes_mapped(vim) > 0);
	ASSERT3U(obsolete_sm_object, ==,
	space_map_object(vd->vdev_obsolete_sm));
	ASSERT3U(vdev_indirect_mapping_bytes_mapped(vim), >=,
	space_map_allocated(vd->vdev_obsolete_sm));
	}
	ASSERT(vd->vdev_obsolete_segments != NULL);

	/*
	* Since frees / remaps to an indirect vdev can only
	* happen in syncing context, the obsolete segments
	* tree must be empty when we start syncing.
	*/
	ASSERT0(range_tree_space(vd->vdev_obsolete_segments));
	}

	/*
	* Set the top-level vdev's max queue depth. Evaluate each top-level's
	* async write queue depth in case it changed. The max queue depth will
	* not change in the middle of syncing out this txg.
	*/
	static void
	spa_sync_adjust_vdev_max_queue_depth(spa_t *spa)
	{
	ASSERT(spa_writeable(spa));

	vdev_t *rvd = spa->spa_root_vdev;
	uint32_t max_queue_depth = zfs_vdev_async_write_max_active *
	zfs_vdev_queue_depth_pct / 100;
	metaslab_class_t *normal = spa_normal_class(spa);
	metaslab_class_t *special = spa_special_class(spa);
	metaslab_class_t *dedup = spa_dedup_class(spa);

	uint64_t slots_per_allocator = 0;
	for (int c = 0; c < rvd->vdev_children; c++) {
	vdev_t *tvd = rvd->vdev_child[c];

	metaslab_group_t *mg = tvd->vdev_mg;
	if (mg == NULL \|\| !metaslab_group_initialized(mg))
	continue;

	metaslab_class_t *mc = mg->mg_class;
	if (mc != normal && mc != special && mc != dedup)
	continue;

	/*
	* It is safe to do a lock-free check here because only async
	* allocations look at mg_max_alloc_queue_depth, and async
	* allocations all happen from spa_sync().
	*/
	for (int i = 0; i < mg->mg_allocators; i++) {
	ASSERT0(zfs_refcount_count(
	&(mg->mg_allocator[i].mga_alloc_queue_depth)));
	}
	mg->mg_max_alloc_queue_depth = max_queue_depth;

	for (int i = 0; i < mg->mg_allocators; i++) {
	mg->mg_allocator[i].mga_cur_max_alloc_queue_depth =
	zfs_vdev_def_queue_depth;
	}
	slots_per_allocator += zfs_vdev_def_queue_depth;
	}

	for (int i = 0; i < spa->spa_alloc_count; i++) {
	ASSERT0(zfs_refcount_count(&normal->mc_allocator[i].
	mca_alloc_slots));
	ASSERT0(zfs_refcount_count(&special->mc_allocator[i].
	mca_alloc_slots));
	ASSERT0(zfs_refcount_count(&dedup->mc_allocator[i].
	mca_alloc_slots));
	normal->mc_allocator[i].mca_alloc_max_slots =
	slots_per_allocator;
	special->mc_allocator[i].mca_alloc_max_slots =
	slots_per_allocator;
	dedup->mc_allocator[i].mca_alloc_max_slots =
	slots_per_allocator;
	}
	normal->mc_alloc_throttle_enabled = zio_dva_throttle_enabled;
	special->mc_alloc_throttle_enabled = zio_dva_throttle_enabled;
	dedup->mc_alloc_throttle_enabled = zio_dva_throttle_enabled;
	}

	static void
	spa_sync_condense_indirect(spa_t spa, dmu_tx_t tx)
	{
	ASSERT(spa_writeable(spa));

	vdev_t *rvd = spa->spa_root_vdev;
	for (int c = 0; c < rvd->vdev_children; c++) {
	vdev_t *vd = rvd->vdev_child[c];
	vdev_indirect_state_sync_verify(vd);

	if (vdev_indirect_should_condense(vd)) {
	spa_condense_indirect_start_sync(vd, tx);
	break;
	}
	}
	}

	static void
	spa_sync_iterate_to_convergence(spa_t spa, dmu_tx_t tx)
	{
	objset_t *mos = spa->spa_meta_objset;
	dsl_pool_t *dp = spa->spa_dsl_pool;
	uint64_t txg = tx->tx_txg;
	bplist_t *free_bpl = &spa->spa_free_bplist[txg & TXG_MASK];

	do {
	int pass = ++spa->spa_sync_pass;

	spa_sync_config_object(spa, tx);
	spa_sync_aux_dev(spa, &spa->spa_spares, tx,
	ZPOOL_CONFIG_SPARES, DMU_POOL_SPARES);
	spa_sync_aux_dev(spa, &spa->spa_l2cache, tx,
	ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE);
	spa_errlog_sync(spa, txg);
	dsl_pool_sync(dp, txg);

	if (pass < zfs_sync_pass_deferred_free \|\|
	spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) {
	/*
	* If the log space map feature is active we don't
	* care about deferred frees and the deferred bpobj
	* as the log space map should effectively have the
	* same results (i.e. appending only to one object).
	*/
	spa_sync_frees(spa, free_bpl, tx);
	} else {
	/*
	* We can not defer frees in pass 1, because
	* we sync the deferred frees later in pass 1.
	*/
	ASSERT3U(pass, >, 1);
	bplist_iterate(free_bpl, bpobj_enqueue_alloc_cb,
	&spa->spa_deferred_bpobj, tx);
	}

	brt_sync(spa, txg);
	ddt_sync(spa, txg);
	dsl_scan_sync(dp, tx);
	dsl_errorscrub_sync(dp, tx);
	svr_sync(spa, tx);
	spa_sync_upgrades(spa, tx);

	spa_flush_metaslabs(spa, tx);

	vdev_t *vd = NULL;
	while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, txg))
	!= NULL)
	vdev_sync(vd, txg);

	/*
	* Note: We need to check if the MOS is dirty because we could
	* have marked the MOS dirty without updating the uberblock
	* (e.g. if we have sync tasks but no dirty user data). We need
	* to check the uberblock's rootbp because it is updated if we
	* have synced out dirty data (though in this case the MOS will
	* most likely also be dirty due to second order effects, we
	* don't want to rely on that here).
	*/
	if (pass == 1 &&
	spa->spa_uberblock.ub_rootbp.blk_birth < txg &&
	!dmu_objset_is_dirty(mos, txg)) {
	/*
	* Nothing changed on the first pass, therefore this
	* TXG is a no-op. Avoid syncing deferred frees, so
	* that we can keep this TXG as a no-op.
	*/
	ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg));
	ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg));
	ASSERT(txg_list_empty(&dp->dp_sync_tasks, txg));
	ASSERT(txg_list_empty(&dp->dp_early_sync_tasks, txg));
	break;
	}

	spa_sync_deferred_frees(spa, tx);
	} while (dmu_objset_is_dirty(mos, txg));
	}

	/*
	* Rewrite the vdev configuration (which includes the uberblock) to
	* commit the transaction group.
	*
	* If there are no dirty vdevs, we sync the uberblock to a few random
	* top-level vdevs that are known to be visible in the config cache
	* (see spa_vdev_add() for a complete description). If there are dirty
	* vdevs, sync the uberblock to all vdevs.
	*/
	static void
	spa_sync_rewrite_vdev_config(spa_t spa, dmu_tx_t tx)
	{
	vdev_t *rvd = spa->spa_root_vdev;
	uint64_t txg = tx->tx_txg;

	for (;;) {
	int error = 0;

	/*
	* We hold SCL_STATE to prevent vdev open/close/etc.
	* while we're attempting to write the vdev labels.
	*/
	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);

	if (list_is_empty(&spa->spa_config_dirty_list)) {
	vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL };
	int svdcount = 0;
	int children = rvd->vdev_children;
	int c0 = random_in_range(children);

	for (int c = 0; c < children; c++) {
	vdev_t *vd =
	rvd->vdev_child[(c0 + c) % children];

	/* Stop when revisiting the first vdev */
	if (c > 0 && svd[0] == vd)
	break;

	if (vd->vdev_ms_array == 0 \|\|
	vd->vdev_islog \|\|
	!vdev_is_concrete(vd))
	continue;

	svd[svdcount++] = vd;
	if (svdcount == SPA_SYNC_MIN_VDEVS)
	break;
	}
	error = vdev_config_sync(svd, svdcount, txg);
	} else {
	error = vdev_config_sync(rvd->vdev_child,
	rvd->vdev_children, txg);
	}

	if (error == 0)
	spa->spa_last_synced_guid = rvd->vdev_guid;

	spa_config_exit(spa, SCL_STATE, FTAG);

	if (error == 0)
	break;
	zio_suspend(spa, NULL, ZIO_SUSPEND_IOERR);
	zio_resume_wait(spa);
	}
	}

	/*
	* Sync the specified transaction group. New blocks may be dirtied as
	* part of the process, so we iterate until it converges.
	*/
	void
	spa_sync(spa_t *spa, uint64_t txg)
	{
	vdev_t *vd = NULL;

	VERIFY(spa_writeable(spa));

	/*
	* Wait for i/os issued in open context that need to complete
	* before this txg syncs.
	*/
	(void) zio_wait(spa->spa_txg_zio[txg & TXG_MASK]);
	spa->spa_txg_zio[txg & TXG_MASK] = zio_root(spa, NULL, NULL,
	ZIO_FLAG_CANFAIL);

	/*
	* Now that there can be no more cloning in this transaction group,
	* but we are still before issuing frees, we can process pending BRT
	* updates.
	*/
	brt_pending_apply(spa, txg);

	/*
	* Lock out configuration changes.
	*/
	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);

	spa->spa_syncing_txg = txg;
	spa->spa_sync_pass = 0;

	for (int i = 0; i < spa->spa_alloc_count; i++) {
	mutex_enter(&spa->spa_allocs[i].spaa_lock);
	VERIFY0(avl_numnodes(&spa->spa_allocs[i].spaa_tree));
	mutex_exit(&spa->spa_allocs[i].spaa_lock);
	}

	/*
	* If there are any pending vdev state changes, convert them
	* into config changes that go out with this transaction group.
	*/
	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
	while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) {
	/* Avoid holding the write lock unless actually necessary */
	if (vd->vdev_aux == NULL) {
	vdev_state_clean(vd);
	vdev_config_dirty(vd);
	continue;
	}
	/*
	* We need the write lock here because, for aux vdevs,
	* calling vdev_config_dirty() modifies sav_config.
	* This is ugly and will become unnecessary when we
	* eliminate the aux vdev wart by integrating all vdevs
	* into the root vdev tree.
	*/
	spa_config_exit(spa, SCL_CONFIG \| SCL_STATE, FTAG);
	spa_config_enter(spa, SCL_CONFIG \| SCL_STATE, FTAG, RW_WRITER);
	while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) {
	vdev_state_clean(vd);
	vdev_config_dirty(vd);
	}
	spa_config_exit(spa, SCL_CONFIG \| SCL_STATE, FTAG);
	spa_config_enter(spa, SCL_CONFIG \| SCL_STATE, FTAG, RW_READER);
	}
	spa_config_exit(spa, SCL_STATE, FTAG);

	dsl_pool_t *dp = spa->spa_dsl_pool;
	dmu_tx_t *tx = dmu_tx_create_assigned(dp, txg);

	spa->spa_sync_starttime = gethrtime();
	taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid);
	spa->spa_deadman_tqid = taskq_dispatch_delay(system_delay_taskq,
	spa_deadman, spa, TQ_SLEEP, ddi_get_lbolt() +
	NSEC_TO_TICK(spa->spa_deadman_synctime));

	/*
	* If we are upgrading to SPA_VERSION_RAIDZ_DEFLATE this txg,
	* set spa_deflate if we have no raid-z vdevs.
	*/
	if (spa->spa_ubsync.ub_version < SPA_VERSION_RAIDZ_DEFLATE &&
	spa->spa_uberblock.ub_version >= SPA_VERSION_RAIDZ_DEFLATE) {
	vdev_t *rvd = spa->spa_root_vdev;

	int i;
	for (i = 0; i < rvd->vdev_children; i++) {
	vd = rvd->vdev_child[i];
	if (vd->vdev_deflate_ratio != SPA_MINBLOCKSIZE)
	break;
	}
	if (i == rvd->vdev_children) {
	spa->spa_deflate = TRUE;
	VERIFY0(zap_add(spa->spa_meta_objset,
	DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE,
	sizeof (uint64_t), 1, &spa->spa_deflate, tx));
	}
	}

	spa_sync_adjust_vdev_max_queue_depth(spa);

	spa_sync_condense_indirect(spa, tx);

	spa_sync_iterate_to_convergence(spa, tx);

	#ifdef ZFS_DEBUG
	if (!list_is_empty(&spa->spa_config_dirty_list)) {
	/*
	* Make sure that the number of ZAPs for all the vdevs matches
	* the number of ZAPs in the per-vdev ZAP list. This only gets
	* called if the config is dirty; otherwise there may be
	* outstanding AVZ operations that weren't completed in
	* spa_sync_config_object.
	*/
	uint64_t all_vdev_zap_entry_count;
	ASSERT0(zap_count(spa->spa_meta_objset,
	spa->spa_all_vdev_zaps, &all_vdev_zap_entry_count));
	ASSERT3U(vdev_count_verify_zaps(spa->spa_root_vdev), ==,
	all_vdev_zap_entry_count);
	}
	#endif

	if (spa->spa_vdev_removal != NULL) {
	ASSERT0(spa->spa_vdev_removal->svr_bytes_done[txg & TXG_MASK]);
	}

	spa_sync_rewrite_vdev_config(spa, tx);
	dmu_tx_commit(tx);

	taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid);
	spa->spa_deadman_tqid = 0;

	/*
	* Clear the dirty config list.
	*/
	while ((vd = list_head(&spa->spa_config_dirty_list)) != NULL)
	vdev_config_clean(vd);

	/*
	* Now that the new config has synced transactionally,
	* let it become visible to the config cache.
	*/
	if (spa->spa_config_syncing != NULL) {
	spa_config_set(spa, spa->spa_config_syncing);
	spa->spa_config_txg = txg;
	spa->spa_config_syncing = NULL;
	}

	dsl_pool_sync_done(dp, txg);

	for (int i = 0; i < spa->spa_alloc_count; i++) {
	mutex_enter(&spa->spa_allocs[i].spaa_lock);
	VERIFY0(avl_numnodes(&spa->spa_allocs[i].spaa_tree));
	mutex_exit(&spa->spa_allocs[i].spaa_lock);
	}

	/*
	* Update usable space statistics.
	*/
	while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, TXG_CLEAN(txg)))
	!= NULL)
	vdev_sync_done(vd, txg);

	metaslab_class_evict_old(spa->spa_normal_class, txg);
	metaslab_class_evict_old(spa->spa_log_class, txg);

	spa_sync_close_syncing_log_sm(spa);

	spa_update_dspace(spa);

	if (spa_get_autotrim(spa) == SPA_AUTOTRIM_ON)
	vdev_autotrim_kick(spa);

	/*
	* It had better be the case that we didn't dirty anything
	* since vdev_config_sync().
	*/
	ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg));
	ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg));
	ASSERT(txg_list_empty(&spa->spa_vdev_txg_list, txg));

	while (zfs_pause_spa_sync)
	delay(1);

	spa->spa_sync_pass = 0;

	/*
	* Update the last synced uberblock here. We want to do this at
	* the end of spa_sync() so that consumers of spa_last_synced_txg()
	* will be guaranteed that all the processing associated with
	* that txg has been completed.
	*/
	spa->spa_ubsync = spa->spa_uberblock;
	spa_config_exit(spa, SCL_CONFIG, FTAG);

	spa_handle_ignored_writes(spa);

	/*
	* If any async tasks have been requested, kick them off.
	*/
	spa_async_dispatch(spa);
	}

	/*
	* Sync all pools. We don't want to hold the namespace lock across these
	* operations, so we take a reference on the spa_t and drop the lock during the
	* sync.
	*/
	void
	spa_sync_allpools(void)
	{
	spa_t *spa = NULL;
	mutex_enter(&spa_namespace_lock);
	while ((spa = spa_next(spa)) != NULL) {
	if (spa_state(spa) != POOL_STATE_ACTIVE \|\|
	!spa_writeable(spa) \|\| spa_suspended(spa))
	continue;
	spa_open_ref(spa, FTAG);
	mutex_exit(&spa_namespace_lock);
	txg_wait_synced(spa_get_dsl(spa), 0);
	mutex_enter(&spa_namespace_lock);
	spa_close(spa, FTAG);
	}
	mutex_exit(&spa_namespace_lock);
	}

	/*
	* ==========================================================================
	* Miscellaneous routines
	* ==========================================================================
	*/

	/*
	* Remove all pools in the system.
	*/
	void
	spa_evict_all(void)
	{
	spa_t *spa;

	/*
	* Remove all cached state. All pools should be closed now,
	* so every spa in the AVL tree should be unreferenced.
	*/
	mutex_enter(&spa_namespace_lock);
	while ((spa = spa_next(NULL)) != NULL) {
	/*
	* Stop async tasks. The async thread may need to detach
	* a device that's been replaced, which requires grabbing
	* spa_namespace_lock, so we must drop it here.
	*/
	spa_open_ref(spa, FTAG);
	mutex_exit(&spa_namespace_lock);
	spa_async_suspend(spa);
	mutex_enter(&spa_namespace_lock);
	spa_close(spa, FTAG);

	if (spa->spa_state != POOL_STATE_UNINITIALIZED) {
	spa_unload(spa);
	spa_deactivate(spa);
	}
	spa_remove(spa);
	}
	mutex_exit(&spa_namespace_lock);
	}

	vdev_t *
	spa_lookup_by_guid(spa_t *spa, uint64_t guid, boolean_t aux)
	{
	vdev_t *vd;
	int i;

	if ((vd = vdev_lookup_by_guid(spa->spa_root_vdev, guid)) != NULL)
	return (vd);

	if (aux) {
	for (i = 0; i < spa->spa_l2cache.sav_count; i++) {
	vd = spa->spa_l2cache.sav_vdevs[i];
	if (vd->vdev_guid == guid)
	return (vd);
	}

	for (i = 0; i < spa->spa_spares.sav_count; i++) {
	vd = spa->spa_spares.sav_vdevs[i];
	if (vd->vdev_guid == guid)
	return (vd);
	}
	}

	return (NULL);
	}

	void
	spa_upgrade(spa_t *spa, uint64_t version)
	{
	ASSERT(spa_writeable(spa));

	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);

	/*
	* This should only be called for a non-faulted pool, and since a
	* future version would result in an unopenable pool, this shouldn't be
	* possible.
	*/
	ASSERT(SPA_VERSION_IS_SUPPORTED(spa->spa_uberblock.ub_version));
	ASSERT3U(version, >=, spa->spa_uberblock.ub_version);

	spa->spa_uberblock.ub_version = version;
	vdev_config_dirty(spa->spa_root_vdev);

	spa_config_exit(spa, SCL_ALL, FTAG);

	txg_wait_synced(spa_get_dsl(spa), 0);
	}

	static boolean_t
	spa_has_aux_vdev(spa_t spa, uint64_t guid, spa_aux_vdev_t sav)
	{
	(void) spa;
	int i;
	uint64_t vdev_guid;

	for (i = 0; i < sav->sav_count; i++)
	if (sav->sav_vdevs[i]->vdev_guid == guid)
	return (B_TRUE);

	for (i = 0; i < sav->sav_npending; i++) {
	if (nvlist_lookup_uint64(sav->sav_pending[i], ZPOOL_CONFIG_GUID,
	&vdev_guid) == 0 && vdev_guid == guid)
	return (B_TRUE);
	}

	return (B_FALSE);
	}

	boolean_t
	spa_has_l2cache(spa_t *spa, uint64_t guid)
	{
	return (spa_has_aux_vdev(spa, guid, &spa->spa_l2cache));
	}

	boolean_t
	spa_has_spare(spa_t *spa, uint64_t guid)
	{
	return (spa_has_aux_vdev(spa, guid, &spa->spa_spares));
	}

	/*
	* Check if a pool has an active shared spare device.
	* Note: reference count of an active spare is 2, as a spare and as a replace
	*/
	static boolean_t
	spa_has_active_shared_spare(spa_t *spa)
	{
	int i, refcnt;
	uint64_t pool;
	spa_aux_vdev_t *sav = &spa->spa_spares;

	for (i = 0; i < sav->sav_count; i++) {
	if (spa_spare_exists(sav->sav_vdevs[i]->vdev_guid, &pool,
	&refcnt) && pool != 0ULL && pool == spa_guid(spa) &&
	refcnt > 2)
	return (B_TRUE);
	}

	return (B_FALSE);
	}

	uint64_t
	spa_total_metaslabs(spa_t *spa)
	{
	vdev_t *rvd = spa->spa_root_vdev;

	uint64_t m = 0;
	for (uint64_t c = 0; c < rvd->vdev_children; c++) {
	vdev_t *vd = rvd->vdev_child[c];
	if (!vdev_is_concrete(vd))
	continue;
	m += vd->vdev_ms_count;
	}
	return (m);
	}

	/*
	* Notify any waiting threads that some activity has switched from being in-
	* progress to not-in-progress so that the thread can wake up and determine
	* whether it is finished waiting.
	*/
	void
	spa_notify_waiters(spa_t *spa)
	{
	/*
	* Acquiring spa_activities_lock here prevents the cv_broadcast from
	* happening between the waiting thread's check and cv_wait.
	*/
	mutex_enter(&spa->spa_activities_lock);
	cv_broadcast(&spa->spa_activities_cv);
	mutex_exit(&spa->spa_activities_lock);
	}

	/*
	* Notify any waiting threads that the pool is exporting, and then block until
	* they are finished using the spa_t.
	*/
	void
	spa_wake_waiters(spa_t *spa)
	{
	mutex_enter(&spa->spa_activities_lock);
	spa->spa_waiters_cancel = B_TRUE;
	cv_broadcast(&spa->spa_activities_cv);
	while (spa->spa_waiters != 0)
	cv_wait(&spa->spa_waiters_cv, &spa->spa_activities_lock);
	spa->spa_waiters_cancel = B_FALSE;
	mutex_exit(&spa->spa_activities_lock);
	}

	/* Whether the vdev or any of its descendants are being initialized/trimmed. */
	static boolean_t
	spa_vdev_activity_in_progress_impl(vdev_t *vd, zpool_wait_activity_t activity)
	{
	spa_t *spa = vd->vdev_spa;

	ASSERT(spa_config_held(spa, SCL_CONFIG \| SCL_STATE, RW_READER));
	ASSERT(MUTEX_HELD(&spa->spa_activities_lock));
	ASSERT(activity == ZPOOL_WAIT_INITIALIZE \|\|
	activity == ZPOOL_WAIT_TRIM);

	kmutex_t *lock = activity == ZPOOL_WAIT_INITIALIZE ?
	&vd->vdev_initialize_lock : &vd->vdev_trim_lock;

	mutex_exit(&spa->spa_activities_lock);
	mutex_enter(lock);
	mutex_enter(&spa->spa_activities_lock);

	boolean_t in_progress = (activity == ZPOOL_WAIT_INITIALIZE) ?
	(vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE) :
	(vd->vdev_trim_state == VDEV_TRIM_ACTIVE);
	mutex_exit(lock);

	if (in_progress)
	return (B_TRUE);

	for (int i = 0; i < vd->vdev_children; i++) {
	if (spa_vdev_activity_in_progress_impl(vd->vdev_child[i],
	activity))
	return (B_TRUE);
	}

	return (B_FALSE);
	}

	/*
	* If use_guid is true, this checks whether the vdev specified by guid is
	* being initialized/trimmed. Otherwise, it checks whether any vdev in the pool
	* is being initialized/trimmed. The caller must hold the config lock and
	* spa_activities_lock.
	*/
	static int
	spa_vdev_activity_in_progress(spa_t *spa, boolean_t use_guid, uint64_t guid,
	zpool_wait_activity_t activity, boolean_t *in_progress)
	{
	mutex_exit(&spa->spa_activities_lock);
	spa_config_enter(spa, SCL_CONFIG \| SCL_STATE, FTAG, RW_READER);
	mutex_enter(&spa->spa_activities_lock);

	vdev_t *vd;
	if (use_guid) {
	vd = spa_lookup_by_guid(spa, guid, B_FALSE);
	if (vd == NULL \|\| !vd->vdev_ops->vdev_op_leaf) {
	spa_config_exit(spa, SCL_CONFIG \| SCL_STATE, FTAG);
	return (EINVAL);
	}
	} else {
	vd = spa->spa_root_vdev;
	}

	*in_progress = spa_vdev_activity_in_progress_impl(vd, activity);

	spa_config_exit(spa, SCL_CONFIG \| SCL_STATE, FTAG);
	return (0);
	}

	/*
	* Locking for waiting threads
	* ---------------------------
	*
	* Waiting threads need a way to check whether a given activity is in progress,
	* and then, if it is, wait for it to complete. Each activity will have some
	* in-memory representation of the relevant on-disk state which can be used to
	* determine whether or not the activity is in progress. The in-memory state and
	* the locking used to protect it will be different for each activity, and may
	* not be suitable for use with a cvar (e.g., some state is protected by the
	* config lock). To allow waiting threads to wait without any races, another
	* lock, spa_activities_lock, is used.
	*
	* When the state is checked, both the activity-specific lock (if there is one)
	* and spa_activities_lock are held. In some cases, the activity-specific lock
	* is acquired explicitly (e.g. the config lock). In others, the locking is
	* internal to some check (e.g. bpobj_is_empty). After checking, the waiting
	* thread releases the activity-specific lock and, if the activity is in
	* progress, then cv_waits using spa_activities_lock.
	*
	* The waiting thread is woken when another thread, one completing some
	* activity, updates the state of the activity and then calls
	* spa_notify_waiters, which will cv_broadcast. This 'completing' thread only
	* needs to hold its activity-specific lock when updating the state, and this
	* lock can (but doesn't have to) be dropped before calling spa_notify_waiters.
	*
	* Because spa_notify_waiters acquires spa_activities_lock before broadcasting,
	* and because it is held when the waiting thread checks the state of the
	* activity, it can never be the case that the completing thread both updates
	* the activity state and cv_broadcasts in between the waiting thread's check
	* and cv_wait. Thus, a waiting thread can never miss a wakeup.
	*
	* In order to prevent deadlock, when the waiting thread does its check, in some
	* cases it will temporarily drop spa_activities_lock in order to acquire the
	* activity-specific lock. The order in which spa_activities_lock and the
	* activity specific lock are acquired in the waiting thread is determined by
	* the order in which they are acquired in the completing thread; if the
	* completing thread calls spa_notify_waiters with the activity-specific lock
	* held, then the waiting thread must also acquire the activity-specific lock
	* first.
	*/

	static int
	spa_activity_in_progress(spa_t *spa, zpool_wait_activity_t activity,
	boolean_t use_tag, uint64_t tag, boolean_t *in_progress)
	{
	int error = 0;

	ASSERT(MUTEX_HELD(&spa->spa_activities_lock));

	switch (activity) {
	case ZPOOL_WAIT_CKPT_DISCARD:
	*in_progress =
	(spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT) &&
	zap_contains(spa_meta_objset(spa),
	DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_ZPOOL_CHECKPOINT) ==
	ENOENT);
	break;
	case ZPOOL_WAIT_FREE:
	*in_progress = ((spa_version(spa) >= SPA_VERSION_DEADLISTS &&
	!bpobj_is_empty(&spa->spa_dsl_pool->dp_free_bpobj)) \|\|
	spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY) \|\|
	spa_livelist_delete_check(spa));
	break;
	case ZPOOL_WAIT_INITIALIZE:
	case ZPOOL_WAIT_TRIM:
	error = spa_vdev_activity_in_progress(spa, use_tag, tag,
	activity, in_progress);
	break;
	case ZPOOL_WAIT_REPLACE:
	mutex_exit(&spa->spa_activities_lock);
	spa_config_enter(spa, SCL_CONFIG \| SCL_STATE, FTAG, RW_READER);
	mutex_enter(&spa->spa_activities_lock);

	*in_progress = vdev_replace_in_progress(spa->spa_root_vdev);
	spa_config_exit(spa, SCL_CONFIG \| SCL_STATE, FTAG);
	break;
	case ZPOOL_WAIT_REMOVE:
	*in_progress = (spa->spa_removing_phys.sr_state ==
	DSS_SCANNING);
	break;
	case ZPOOL_WAIT_RESILVER:
	if ((*in_progress = vdev_rebuild_active(spa->spa_root_vdev)))
	break;
	zfs_fallthrough;
	case ZPOOL_WAIT_SCRUB:
	{
	boolean_t scanning, paused, is_scrub;
	dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan;

	is_scrub = (scn->scn_phys.scn_func == POOL_SCAN_SCRUB);
	scanning = (scn->scn_phys.scn_state == DSS_SCANNING);
	paused = dsl_scan_is_paused_scrub(scn);
	*in_progress = (scanning && !paused &&
	is_scrub == (activity == ZPOOL_WAIT_SCRUB));
	break;
	}
	default:
	panic("unrecognized value for activity %d", activity);
	}

	return (error);
	}

	static int
	spa_wait_common(const char *pool, zpool_wait_activity_t activity,
	boolean_t use_tag, uint64_t tag, boolean_t *waited)
	{
	/*
	* The tag is used to distinguish between instances of an activity.
	* 'initialize' and 'trim' are the only activities that we use this for.
	* The other activities can only have a single instance in progress in a
	* pool at one time, making the tag unnecessary.
	*
	* There can be multiple devices being replaced at once, but since they
	* all finish once resilvering finishes, we don't bother keeping track
	* of them individually, we just wait for them all to finish.
	*/
	if (use_tag && activity != ZPOOL_WAIT_INITIALIZE &&
	activity != ZPOOL_WAIT_TRIM)
	return (EINVAL);

	if (activity < 0 \|\| activity >= ZPOOL_WAIT_NUM_ACTIVITIES)
	return (EINVAL);

	spa_t *spa;
	int error = spa_open(pool, &spa, FTAG);
	if (error != 0)
	return (error);

	/*
	* Increment the spa's waiter count so that we can call spa_close and
	* still ensure that the spa_t doesn't get freed before this thread is
	* finished with it when the pool is exported. We want to call spa_close
	* before we start waiting because otherwise the additional ref would
	* prevent the pool from being exported or destroyed throughout the
	* potentially long wait.
	*/
	mutex_enter(&spa->spa_activities_lock);
	spa->spa_waiters++;
	spa_close(spa, FTAG);

	*waited = B_FALSE;
	for (;;) {
	boolean_t in_progress;
	error = spa_activity_in_progress(spa, activity, use_tag, tag,
	&in_progress);

	if (error \|\| !in_progress \|\| spa->spa_waiters_cancel)
	break;

	*waited = B_TRUE;

	if (cv_wait_sig(&spa->spa_activities_cv,
	&spa->spa_activities_lock) == 0) {
	error = EINTR;
	break;
	}
	}

	spa->spa_waiters--;
	cv_signal(&spa->spa_waiters_cv);
	mutex_exit(&spa->spa_activities_lock);

	return (error);
	}

	/*
	* Wait for a particular instance of the specified activity to complete, where
	* the instance is identified by 'tag'
	*/
	int
	spa_wait_tag(const char *pool, zpool_wait_activity_t activity, uint64_t tag,
	boolean_t *waited)
	{
	return (spa_wait_common(pool, activity, B_TRUE, tag, waited));
	}

	/*
	* Wait for all instances of the specified activity complete
	*/
	int
	spa_wait(const char pool, zpool_wait_activity_t activity, boolean_t waited)
	{

	return (spa_wait_common(pool, activity, B_FALSE, 0, waited));
	}

	sysevent_t *
	spa_event_create(spa_t spa, vdev_t vd, nvlist_t hist_nvl, const char name)
	{
	sysevent_t *ev = NULL;
	#ifdef _KERNEL
	nvlist_t *resource;

	resource = zfs_event_create(spa, vd, FM_SYSEVENT_CLASS, name, hist_nvl);
	if (resource) {
	ev = kmem_alloc(sizeof (sysevent_t), KM_SLEEP);
	ev->resource = resource;
	}
	#else
	(void) spa, (void) vd, (void) hist_nvl, (void) name;
	#endif
	return (ev);
	}

	void
	spa_event_post(sysevent_t *ev)
	{
	#ifdef _KERNEL
	if (ev) {
	zfs_zevent_post(ev->resource, NULL, zfs_zevent_post_cb);
	kmem_free(ev, sizeof (*ev));
	}
	#else
	(void) ev;
	#endif
	}

	/*
	* Post a zevent corresponding to the given sysevent. The 'name' must be one
	* of the event definitions in sys/sysevent/eventdefs.h. The payload will be
	* filled in from the spa and (optionally) the vdev. This doesn't do anything
	* in the userland libzpool, as we don't want consumers to misinterpret ztest
	* or zdb as real changes.
	*/
	void
	spa_event_notify(spa_t spa, vdev_t vd, nvlist_t hist_nvl, const char name)
	{
	spa_event_post(spa_event_create(spa, vd, hist_nvl, name));
	}

	/* state manipulation functions */
	EXPORT_SYMBOL(spa_open);
	EXPORT_SYMBOL(spa_open_rewind);
	EXPORT_SYMBOL(spa_get_stats);
	EXPORT_SYMBOL(spa_create);
	EXPORT_SYMBOL(spa_import);
	EXPORT_SYMBOL(spa_tryimport);
	EXPORT_SYMBOL(spa_destroy);
	EXPORT_SYMBOL(spa_export);
	EXPORT_SYMBOL(spa_reset);
	EXPORT_SYMBOL(spa_async_request);
	EXPORT_SYMBOL(spa_async_suspend);
	EXPORT_SYMBOL(spa_async_resume);
	EXPORT_SYMBOL(spa_inject_addref);
	EXPORT_SYMBOL(spa_inject_delref);
	EXPORT_SYMBOL(spa_scan_stat_init);
	EXPORT_SYMBOL(spa_scan_get_stats);

	/* device manipulation */
	EXPORT_SYMBOL(spa_vdev_add);
	EXPORT_SYMBOL(spa_vdev_attach);
	EXPORT_SYMBOL(spa_vdev_detach);
	EXPORT_SYMBOL(spa_vdev_setpath);
	EXPORT_SYMBOL(spa_vdev_setfru);
	EXPORT_SYMBOL(spa_vdev_split_mirror);

	/* spare statech is global across all pools) */
	EXPORT_SYMBOL(spa_spare_add);
	EXPORT_SYMBOL(spa_spare_remove);
	EXPORT_SYMBOL(spa_spare_exists);
	EXPORT_SYMBOL(spa_spare_activate);

	/* L2ARC statech is global across all pools) */
	EXPORT_SYMBOL(spa_l2cache_add);
	EXPORT_SYMBOL(spa_l2cache_remove);
	EXPORT_SYMBOL(spa_l2cache_exists);
	EXPORT_SYMBOL(spa_l2cache_activate);
	EXPORT_SYMBOL(spa_l2cache_drop);

	/* scanning */
	EXPORT_SYMBOL(spa_scan);
	EXPORT_SYMBOL(spa_scan_stop);

	/* spa syncing */
	EXPORT_SYMBOL(spa_sync); /* only for DMU use */
	EXPORT_SYMBOL(spa_sync_allpools);

	/* properties */
	EXPORT_SYMBOL(spa_prop_set);
	EXPORT_SYMBOL(spa_prop_get);
	EXPORT_SYMBOL(spa_prop_clear_bootfs);

	/* asynchronous event notification */
	EXPORT_SYMBOL(spa_event_notify);

	ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, preload_pct, UINT, ZMOD_RW,
	"Percentage of CPUs to run a metaslab preload taskq");

	/* BEGIN CSTYLED */
	ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_shift, UINT, ZMOD_RW,
	"log2 fraction of arc that can be used by inflight I/Os when "
	"verifying pool during import");
	/* END CSTYLED */

	ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_metadata, INT, ZMOD_RW,
	"Set to traverse metadata on pool import");

	ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_data, INT, ZMOD_RW,
	"Set to traverse data on pool import");

	ZFS_MODULE_PARAM(zfs_spa, spa_, load_print_vdev_tree, INT, ZMOD_RW,
	"Print vdev tree to zfs_dbgmsg during pool import");

	ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_pct, UINT, ZMOD_RD,
	"Percentage of CPUs to run an IO worker thread");

	ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_tpq, UINT, ZMOD_RD,
	"Number of threads per IO worker taskqueue");

	/* BEGIN CSTYLED */
	ZFS_MODULE_PARAM(zfs, zfs_, max_missing_tvds, U64, ZMOD_RW,
	"Allow importing pool with up to this number of missing top-level "
	"vdevs (in read-only mode)");
	/* END CSTYLED */

	ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_pause, INT,
	ZMOD_RW, "Set the livelist condense zthr to pause");

	ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_pause, INT,
	ZMOD_RW, "Set the livelist condense synctask to pause");

	/* BEGIN CSTYLED */
	ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_cancel,
	INT, ZMOD_RW,
	"Whether livelist condensing was canceled in the synctask");

	ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_cancel,
	INT, ZMOD_RW,
	"Whether livelist condensing was canceled in the zthr function");

	ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, new_alloc, INT,
	ZMOD_RW,
	"Whether extra ALLOC blkptrs were added to a livelist entry while it "
	"was being condensed");

	#ifdef _KERNEL
	ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs_zio, zio_, taskq_read,
	spa_taskq_read_param_set, spa_taskq_read_param_get, ZMOD_RD,
	"Configure IO queues for read IO");
	ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs_zio, zio_, taskq_write,
	spa_taskq_write_param_set, spa_taskq_write_param_get, ZMOD_RD,
	"Configure IO queues for write IO");
	#endif
	/* END CSTYLED */
	diff --git a/sys/contrib/openzfs/module/zfs/spa_log_spacemap.c b/sys/contrib/openzfs/module/zfs/spa_log_spacemap.c
	index 2878e68c6e4b..873089a53e34 100644
	--- a/sys/contrib/openzfs/module/zfs/spa_log_spacemap.c
	+++ b/sys/contrib/openzfs/module/zfs/spa_log_spacemap.c
	@@ -1,1401 +1,1407 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2018, 2019 by Delphix. All rights reserved.
	*/

	#include <sys/dmu_objset.h>
	#include <sys/metaslab.h>
	#include <sys/metaslab_impl.h>
	#include <sys/spa.h>
	#include <sys/spa_impl.h>
	#include <sys/spa_log_spacemap.h>
	#include <sys/vdev_impl.h>
	#include <sys/zap.h>

	/*
	* Log Space Maps
	*
	* Log space maps are an optimization in ZFS metadata allocations for pools
	* whose workloads are primarily random-writes. Random-write workloads are also
	* typically random-free, meaning that they are freeing from locations scattered
	* throughout the pool. This means that each TXG we will have to append some
	* FREE records to almost every metaslab. With log space maps, we hold their
	* changes in memory and log them altogether in one pool-wide space map on-disk
	* for persistence. As more blocks are accumulated in the log space maps and
	* more unflushed changes are accounted in memory, we flush a selected group
	* of metaslabs every TXG to relieve memory pressure and potential overheads
	* when loading the pool. Flushing a metaslab to disk relieves memory as we
	* flush any unflushed changes from memory to disk (i.e. the metaslab's space
	* map) and saves import time by making old log space maps obsolete and
	* eventually destroying them. [A log space map is said to be obsolete when all
	* its entries have made it to their corresponding metaslab space maps].
	*
	* == On disk data structures used ==
	*
	* - The pool has a new feature flag and a new entry in the MOS. The feature
	* is activated when we create the first log space map and remains active
	* for the lifetime of the pool. The new entry in the MOS Directory [refer
	* to DMU_POOL_LOG_SPACEMAP_ZAP] is populated with a ZAP whose key-value
	* pairs are of the form <key: txg, value: log space map object for that txg>.
	* This entry is our on-disk reference of the log space maps that exist in
	* the pool for each TXG and it is used during import to load all the
	* metaslab unflushed changes in memory. To see how this structure is first
	* created and later populated refer to spa_generate_syncing_log_sm(). To see
	* how it is used during import time refer to spa_ld_log_sm_metadata().
	*
	* - Each vdev has a new entry in its vdev_top_zap (see field
	* VDEV_TOP_ZAP_MS_UNFLUSHED_PHYS_TXGS) which holds the msp_unflushed_txg of
	* each metaslab in this vdev. This field is the on-disk counterpart of the
	* in-memory field ms_unflushed_txg which tells us from which TXG and onwards
	* the metaslab haven't had its changes flushed. During import, we use this
	* to ignore any entries in the space map log that are for this metaslab but
	* from a TXG before msp_unflushed_txg. At that point, we also populate its
	* in-memory counterpart and from there both fields are updated every time
	* we flush that metaslab.
	*
	* - A space map is created every TXG and, during that TXG, it is used to log
	* all incoming changes (the log space map). When created, the log space map
	* is referenced in memory by spa_syncing_log_sm and its object ID is inserted
	* to the space map ZAP mentioned above. The log space map is closed at the
	* end of the TXG and will be destroyed when it becomes fully obsolete. We
	* know when a log space map has become obsolete by looking at the oldest
	* (and smallest) ms_unflushed_txg in the pool. If the value of that is bigger
	* than the log space map's TXG, then it means that there is no metaslab who
	* doesn't have the changes from that log and we can therefore destroy it.
	* [see spa_cleanup_old_sm_logs()].
	*
	* == Important in-memory structures ==
	*
	* - The per-spa field spa_metaslabs_by_flushed sorts all the metaslabs in
	* the pool by their ms_unflushed_txg field. It is primarily used for three
	* reasons. First of all, it is used during flushing where we try to flush
	* metaslabs in-order from the oldest-flushed to the most recently flushed
	* every TXG. Secondly, it helps us to lookup the ms_unflushed_txg of the
	* oldest flushed metaslab to distinguish which log space maps have become
	* obsolete and which ones are still relevant. Finally it tells us which
	* metaslabs have unflushed changes in a pool where this feature was just
	* enabled, as we don't immediately add all of the pool's metaslabs but we
	* add them over time as they go through metaslab_sync(). The reason that
	* we do that is to ease these pools into the behavior of the flushing
	* algorithm (described later on).
	*
	* - The per-spa field spa_sm_logs_by_txg can be thought as the in-memory
	* counterpart of the space map ZAP mentioned above. It's an AVL tree whose
	* nodes represent the log space maps in the pool. This in-memory
	* representation of log space maps in the pool sorts the log space maps by
	* the TXG that they were created (which is also the TXG of their unflushed
	* changes). It also contains the following extra information for each
	* space map:
	* [1] The number of metaslabs that were last flushed on that TXG. This is
	* important because if that counter is zero and this is the oldest
	* log then it means that it is also obsolete.
	* [2] The number of blocks of that space map. This field is used by the
	* block heuristic of our flushing algorithm (described later on).
	* It represents how many blocks of metadata changes ZFS had to write
	* to disk for that TXG.
	*
	* - The per-spa field spa_log_summary is a list of entries that summarizes
	* the metaslab and block counts of all the nodes of the spa_sm_logs_by_txg
	* AVL tree mentioned above. The reason this exists is that our flushing
	* algorithm (described later) tries to estimate how many metaslabs to flush
	* in each TXG by iterating over all the log space maps and looking at their
	* block counts. Summarizing that information means that don't have to
	* iterate through each space map, minimizing the runtime overhead of the
	* flushing algorithm which would be induced in syncing context. In terms of
	* implementation the log summary is used as a queue:
	* * we modify or pop entries from its head when we flush metaslabs
	* * we modify or append entries to its tail when we sync changes.
	*
	* - Each metaslab has two new range trees that hold its unflushed changes,
	* ms_unflushed_allocs and ms_unflushed_frees. These are always disjoint.
	*
	* == Flushing algorithm ==
	*
	* The decision of how many metaslabs to flush on a give TXG is guided by
	* two heuristics:
	*
	* [1] The memory heuristic -
	* We keep track of the memory used by the unflushed trees from all the
	* metaslabs [see sus_memused of spa_unflushed_stats] and we ensure that it
	* stays below a certain threshold which is determined by an arbitrary hard
	* limit and an arbitrary percentage of the system's memory [see
	* spa_log_exceeds_memlimit()]. When we see that the memory usage of the
	* unflushed changes are passing that threshold, we flush metaslabs, which
	* empties their unflushed range trees, reducing the memory used.
	*
	* [2] The block heuristic -
	* We try to keep the total number of blocks in the log space maps in check
	* so the log doesn't grow indefinitely and we don't induce a lot of overhead
	* when loading the pool. At the same time we don't want to flush a lot of
	* metaslabs too often as this would defeat the purpose of the log space map.
	* As a result we set a limit in the amount of blocks that we think it's
	* acceptable for the log space maps to have and try not to cross it.
	* [see sus_blocklimit from spa_unflushed_stats].
	*
	* In order to stay below the block limit every TXG we have to estimate how
	* many metaslabs we need to flush based on the current rate of incoming blocks
	* and our history of log space map blocks. The main idea here is to answer
	* the question of how many metaslabs do we need to flush in order to get rid
	* at least an X amount of log space map blocks. We can answer this question
	* by iterating backwards from the oldest log space map to the newest one
	* and looking at their metaslab and block counts. At this point the log summary
	* mentioned above comes handy as it reduces the amount of things that we have
	* to iterate (even though it may reduce the preciseness of our estimates due
	* to its aggregation of data). So with that in mind, we project the incoming
	* rate of the current TXG into the future and attempt to approximate how many
	* metaslabs would we need to flush from now in order to avoid exceeding our
	* block limit in different points in the future (granted that we would keep
	* flushing the same number of metaslabs for every TXG). Then we take the
	* maximum number from all these estimates to be on the safe side. For the
	* exact implementation details of algorithm refer to
	* spa_estimate_metaslabs_to_flush.
	*/

	/*
	* This is used as the block size for the space maps used for the
	* log space map feature. These space maps benefit from a bigger
	* block size as we expect to be writing a lot of data to them at
	* once.
	*/
	static const unsigned long zfs_log_sm_blksz = 1ULL << 17;

	/*
	* Percentage of the overall system's memory that ZFS allows to be
	* used for unflushed changes (e.g. the sum of size of all the nodes
	* in the unflushed trees).
	*
	* Note that this value is calculated over 1000000 for finer granularity
	* (thus the _ppm suffix; reads as "parts per million"). As an example,
	* the default of 1000 allows 0.1% of memory to be used.
	*/
	static uint64_t zfs_unflushed_max_mem_ppm = 1000;

	/*
	* Specific hard-limit in memory that ZFS allows to be used for
	* unflushed changes.
	*/
	static uint64_t zfs_unflushed_max_mem_amt = 1ULL << 30;

	/*
	* The following tunable determines the number of blocks that can be used for
	* the log space maps. It is expressed as a percentage of the total number of
	* metaslabs in the pool (i.e. the default of 400 means that the number of log
	* blocks is capped at 4 times the number of metaslabs).
	*
	* This value exists to tune our flushing algorithm, with higher values
	* flushing metaslabs less often (doing less I/Os) per TXG versus lower values
	* flushing metaslabs more aggressively with the upside of saving overheads
	* when loading the pool. Another factor in this tradeoff is that flushing
	* less often can potentially lead to better utilization of the metaslab space
	* map's block size as we accumulate more changes per flush.
	*
	* Given that this tunable indirectly controls the flush rate (metaslabs
	* flushed per txg) and that's why making it a percentage in terms of the
	* number of metaslabs in the pool makes sense here.
	*
	* As a rule of thumb we default this tunable to 400% based on the following:
	*
	* 1] Assuming a constant flush rate and a constant incoming rate of log blocks
	* it is reasonable to expect that the amount of obsolete entries changes
	* linearly from txg to txg (e.g. the oldest log should have the most
	* obsolete entries, and the most recent one the least). With this we could
	* say that, at any given time, about half of the entries in the whole space
	* map log are obsolete. Thus for every two entries for a metaslab in the
	* log space map, only one of them is valid and actually makes it to the
	* metaslab's space map.
	* [factor of 2]
	* 2] Each entry in the log space map is guaranteed to be two words while
	* entries in metaslab space maps are generally single-word.
	* [an extra factor of 2 - 400% overall]
	* 3] Even if [1] and [2] are slightly less than 2 each, we haven't taken into
	* account any consolidation of segments from the log space map to the
	* unflushed range trees nor their history (e.g. a segment being allocated,
	* then freed, then allocated again means 3 log space map entries but 0
	* metaslab space map entries). Depending on the workload, we've seen ~1.8
	* non-obsolete log space map entries per metaslab entry, for a total of
	* ~600%. Since most of these estimates though are workload dependent, we
	* default on 400% to be conservative.
	*
	* Thus we could say that even in the worst
	* case of [1] and [2], the factor should end up being 4.
	*
	* That said, regardless of the number of metaslabs in the pool we need to
	* provide upper and lower bounds for the log block limit.
	* [see zfs_unflushed_log_block_{min,max}]
	*/
	static uint_t zfs_unflushed_log_block_pct = 400;

	/*
	* If the number of metaslabs is small and our incoming rate is high, we could
	* get into a situation that we are flushing all our metaslabs every TXG. Thus
	* we always allow at least this many log blocks.
	*/
	static uint64_t zfs_unflushed_log_block_min = 1000;

	/*
	* If the log becomes too big, the import time of the pool can take a hit in
	* terms of performance. Thus we have a hard limit in the size of the log in
	* terms of blocks.
	*/
	static uint64_t zfs_unflushed_log_block_max = (1ULL << 17);

	/*
	* Also we have a hard limit in the size of the log in terms of dirty TXGs.
	*/
	static uint64_t zfs_unflushed_log_txg_max = 1000;

	/*
	* Max # of rows allowed for the log_summary. The tradeoff here is accuracy and
	* stability of the flushing algorithm (longer summary) vs its runtime overhead
	* (smaller summary is faster to traverse).
	*/
	static uint64_t zfs_max_logsm_summary_length = 10;

	/*
	* Tunable that sets the lower bound on the metaslabs to flush every TXG.
	*
	* Setting this to 0 has no effect since if the pool is idle we won't even be
	* creating log space maps and therefore we won't be flushing. On the other
	* hand if the pool has any incoming workload our block heuristic will start
	* flushing metaslabs anyway.
	*
	* The point of this tunable is to be used in extreme cases where we really
	* want to flush more metaslabs than our adaptable heuristic plans to flush.
	*/
	static uint64_t zfs_min_metaslabs_to_flush = 1;

	/*
	* Tunable that specifies how far in the past do we want to look when trying to
	* estimate the incoming log blocks for the current TXG.
	*
	* Setting this too high may not only increase runtime but also minimize the
	* effect of the incoming rates from the most recent TXGs as we take the
	* average over all the blocks that we walk
	* [see spa_estimate_incoming_log_blocks].
	*/
	static uint64_t zfs_max_log_walking = 5;

	/*
	* This tunable exists solely for testing purposes. It ensures that the log
	* spacemaps are not flushed and destroyed during export in order for the
	* relevant log spacemap import code paths to be tested (effectively simulating
	* a crash).
	*/
	int zfs_keep_log_spacemaps_at_export = 0;

	static uint64_t
	spa_estimate_incoming_log_blocks(spa_t *spa)
	{
	ASSERT3U(spa_sync_pass(spa), ==, 1);
	uint64_t steps = 0, sum = 0;
	for (spa_log_sm_t *sls = avl_last(&spa->spa_sm_logs_by_txg);
	sls != NULL && steps < zfs_max_log_walking;
	sls = AVL_PREV(&spa->spa_sm_logs_by_txg, sls)) {
	if (sls->sls_txg == spa_syncing_txg(spa)) {
	/*
	* skip the log created in this TXG as this would
	* make our estimations inaccurate.
	*/
	continue;
	}
	sum += sls->sls_nblocks;
	steps++;
	}
	return ((steps > 0) ? DIV_ROUND_UP(sum, steps) : 0);
	}

	uint64_t
	spa_log_sm_blocklimit(spa_t *spa)
	{
	return (spa->spa_unflushed_stats.sus_blocklimit);
	}

	void
	spa_log_sm_set_blocklimit(spa_t *spa)
	{
	if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) {
	ASSERT0(spa_log_sm_blocklimit(spa));
	return;
	}

	uint64_t msdcount = 0;
	for (log_summary_entry_t *e = list_head(&spa->spa_log_summary);
	e; e = list_next(&spa->spa_log_summary, e))
	msdcount += e->lse_msdcount;

	uint64_t limit = msdcount * zfs_unflushed_log_block_pct / 100;
	spa->spa_unflushed_stats.sus_blocklimit = MIN(MAX(limit,
	zfs_unflushed_log_block_min), zfs_unflushed_log_block_max);
	}

	uint64_t
	spa_log_sm_nblocks(spa_t *spa)
	{
	return (spa->spa_unflushed_stats.sus_nblocks);
	}

	/*
	* Ensure that the in-memory log space map structures and the summary
	* have the same block and metaslab counts.
	*/
	static void
	spa_log_summary_verify_counts(spa_t *spa)
	{
	ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP));

	if ((zfs_flags & ZFS_DEBUG_LOG_SPACEMAP) == 0)
	return;

	uint64_t ms_in_avl = avl_numnodes(&spa->spa_metaslabs_by_flushed);

	uint64_t ms_in_summary = 0, blk_in_summary = 0;
	for (log_summary_entry_t *e = list_head(&spa->spa_log_summary);
	e; e = list_next(&spa->spa_log_summary, e)) {
	ms_in_summary += e->lse_mscount;
	blk_in_summary += e->lse_blkcount;
	}

	uint64_t ms_in_logs = 0, blk_in_logs = 0;
	for (spa_log_sm_t *sls = avl_first(&spa->spa_sm_logs_by_txg);
	sls; sls = AVL_NEXT(&spa->spa_sm_logs_by_txg, sls)) {
	ms_in_logs += sls->sls_mscount;
	blk_in_logs += sls->sls_nblocks;
	}

	VERIFY3U(ms_in_logs, ==, ms_in_summary);
	VERIFY3U(ms_in_logs, ==, ms_in_avl);
	VERIFY3U(blk_in_logs, ==, blk_in_summary);
	VERIFY3U(blk_in_logs, ==, spa_log_sm_nblocks(spa));
	}

	static boolean_t
	summary_entry_is_full(spa_t spa, log_summary_entry_t e, uint64_t txg)
	{
	if (e->lse_end == txg)
	return (0);
	if (e->lse_txgcount >= DIV_ROUND_UP(zfs_unflushed_log_txg_max,
	zfs_max_logsm_summary_length))
	return (1);
	uint64_t blocks_per_row = MAX(1,
	DIV_ROUND_UP(spa_log_sm_blocklimit(spa),
	zfs_max_logsm_summary_length));
	return (blocks_per_row <= e->lse_blkcount);
	}

	/*
	* Update the log summary information to reflect the fact that a metaslab
	* was flushed or destroyed (e.g due to device removal or pool export/destroy).
	*
	* We typically flush the oldest flushed metaslab so the first (and oldest)
	* entry of the summary is updated. However if that metaslab is getting loaded
	* we may flush the second oldest one which may be part of an entry later in
	* the summary. Moreover, if we call into this function from metaslab_fini()
	* the metaslabs probably won't be ordered by ms_unflushed_txg. Thus we ask
	* for a txg as an argument so we can locate the appropriate summary entry for
	* the metaslab.
	*/
	void
	spa_log_summary_decrement_mscount(spa_t *spa, uint64_t txg, boolean_t dirty)
	{
	/*
	* We don't track summary data for read-only pools and this function
	* can be called from metaslab_fini(). In that case return immediately.
	*/
	if (!spa_writeable(spa))
	return;

	log_summary_entry_t *target = NULL;
	for (log_summary_entry_t *e = list_head(&spa->spa_log_summary);
	e != NULL; e = list_next(&spa->spa_log_summary, e)) {
	if (e->lse_start > txg)
	break;
	target = e;
	}

	if (target == NULL \|\| target->lse_mscount == 0) {
	/*
	* We didn't find a summary entry for this metaslab. We must be
	* at the teardown of a spa_load() attempt that got an error
	* while reading the log space maps.
	*/
	VERIFY3S(spa_load_state(spa), ==, SPA_LOAD_ERROR);
	return;
	}

	target->lse_mscount--;
	if (dirty)
	target->lse_msdcount--;
	}

	/*
	* Update the log summary information to reflect the fact that we destroyed
	* old log space maps. Since we can only destroy the oldest log space maps,
	* we decrement the block count of the oldest summary entry and potentially
	* destroy it when that count hits 0.
	*
	* This function is called after a metaslab is flushed and typically that
	* metaslab is the oldest flushed, which means that this function will
	* typically decrement the block count of the first entry of the summary and
	* potentially free it if the block count gets to zero (its metaslab count
	* should be zero too at that point).
	*
	* There are certain scenarios though that don't work exactly like that so we
	* need to account for them:
	*
	* Scenario [1]: It is possible that after we flushed the oldest flushed
	* metaslab and we destroyed the oldest log space map, more recent logs had 0
	* metaslabs pointing to them so we got rid of them too. This can happen due
	* to metaslabs being destroyed through device removal, or because the oldest
	* flushed metaslab was loading but we kept flushing more recently flushed
	* metaslabs due to the memory pressure of unflushed changes. Because of that,
	* we always iterate from the beginning of the summary and if blocks_gone is
	* bigger than the block_count of the current entry we free that entry (we
	* expect its metaslab count to be zero), we decrement blocks_gone and on to
	* the next entry repeating this procedure until blocks_gone gets decremented
	* to 0. Doing this also works for the typical case mentioned above.
	*
	* Scenario [2]: The oldest flushed metaslab isn't necessarily accounted by
	* the first (and oldest) entry in the summary. If the first few entries of
	* the summary were only accounting metaslabs from a device that was just
	* removed, then the current oldest flushed metaslab could be accounted by an
	* entry somewhere in the middle of the summary. Moreover flushing that
	* metaslab will destroy all the log space maps older than its ms_unflushed_txg
	* because they became obsolete after the removal. Thus, iterating as we did
	* for scenario [1] works out for this case too.
	*
	* Scenario [3]: At times we decide to flush all the metaslabs in the pool
	* in one TXG (either because we are exporting the pool or because our flushing
	* heuristics decided to do so). When that happens all the log space maps get
	* destroyed except the one created for the current TXG which doesn't have
	* any log blocks yet. As log space maps get destroyed with every metaslab that
	* we flush, entries in the summary are also destroyed. This brings a weird
	* corner-case when we flush the last metaslab and the log space map of the
	* current TXG is in the same summary entry with other log space maps that
	* are older. When that happens we are eventually left with this one last
	* summary entry whose blocks are gone (blocks_gone equals the entry's block
	* count) but its metaslab count is non-zero (because it accounts all the
	* metaslabs in the pool as they all got flushed). Under this scenario we can't
	* free this last summary entry as it's referencing all the metaslabs in the
	* pool and its block count will get incremented at the end of this sync (when
	* we close the syncing log space map). Thus we just decrement its current
	* block count and leave it alone. In the case that the pool gets exported,
	* its metaslab count will be decremented over time as we call metaslab_fini()
	* for all the metaslabs in the pool and the entry will be freed at
	* spa_unload_log_sm_metadata().
	*/
	void
	spa_log_summary_decrement_blkcount(spa_t *spa, uint64_t blocks_gone)
	{
	log_summary_entry_t *e = list_head(&spa->spa_log_summary);
	ASSERT3P(e, !=, NULL);
	if (e->lse_txgcount > 0)
	e->lse_txgcount--;
	for (; e != NULL; e = list_head(&spa->spa_log_summary)) {
	if (e->lse_blkcount > blocks_gone) {
	e->lse_blkcount -= blocks_gone;
	blocks_gone = 0;
	break;
	} else if (e->lse_mscount == 0) {
	/* remove obsolete entry */
	blocks_gone -= e->lse_blkcount;
	list_remove(&spa->spa_log_summary, e);
	kmem_free(e, sizeof (log_summary_entry_t));
	} else {
	/* Verify that this is scenario [3] mentioned above. */
	VERIFY3U(blocks_gone, ==, e->lse_blkcount);

	/*
	* Assert that this is scenario [3] further by ensuring
	* that this is the only entry in the summary.
	*/
	VERIFY3P(e, ==, list_tail(&spa->spa_log_summary));
	ASSERT3P(e, ==, list_head(&spa->spa_log_summary));

	blocks_gone = e->lse_blkcount = 0;
	break;
	}
	}

	/*
	* Ensure that there is no way we are trying to remove more blocks
	* than the # of blocks in the summary.
	*/
	ASSERT0(blocks_gone);
	}

	void
	spa_log_sm_decrement_mscount(spa_t *spa, uint64_t txg)
	{
	spa_log_sm_t target = { .sls_txg = txg };
	spa_log_sm_t *sls = avl_find(&spa->spa_sm_logs_by_txg,
	&target, NULL);

	if (sls == NULL) {
	/*
	* We must be at the teardown of a spa_load() attempt that
	* got an error while reading the log space maps.
	*/
	VERIFY3S(spa_load_state(spa), ==, SPA_LOAD_ERROR);
	return;
	}

	ASSERT(sls->sls_mscount > 0);
	sls->sls_mscount--;
	}

	void
	spa_log_sm_increment_current_mscount(spa_t *spa)
	{
	spa_log_sm_t *last_sls = avl_last(&spa->spa_sm_logs_by_txg);
	ASSERT3U(last_sls->sls_txg, ==, spa_syncing_txg(spa));
	last_sls->sls_mscount++;
	}

	static void
	summary_add_data(spa_t *spa, uint64_t txg, uint64_t metaslabs_flushed,
	uint64_t metaslabs_dirty, uint64_t nblocks)
	{
	log_summary_entry_t *e = list_tail(&spa->spa_log_summary);

	if (e == NULL \|\| summary_entry_is_full(spa, e, txg)) {
	e = kmem_zalloc(sizeof (log_summary_entry_t), KM_SLEEP);
	e->lse_start = e->lse_end = txg;
	e->lse_txgcount = 1;
	list_insert_tail(&spa->spa_log_summary, e);
	}

	ASSERT3U(e->lse_start, <=, txg);
	if (e->lse_end < txg) {
	e->lse_end = txg;
	e->lse_txgcount++;
	}
	e->lse_mscount += metaslabs_flushed;
	e->lse_msdcount += metaslabs_dirty;
	e->lse_blkcount += nblocks;
	}

	static void
	spa_log_summary_add_incoming_blocks(spa_t *spa, uint64_t nblocks)
	{
	summary_add_data(spa, spa_syncing_txg(spa), 0, 0, nblocks);
	}

	void
	spa_log_summary_add_flushed_metaslab(spa_t *spa, boolean_t dirty)
	{
	summary_add_data(spa, spa_syncing_txg(spa), 1, dirty ? 1 : 0, 0);
	}

	void
	spa_log_summary_dirty_flushed_metaslab(spa_t *spa, uint64_t txg)
	{
	log_summary_entry_t *target = NULL;
	for (log_summary_entry_t *e = list_head(&spa->spa_log_summary);
	e != NULL; e = list_next(&spa->spa_log_summary, e)) {
	if (e->lse_start > txg)
	break;
	target = e;
	}
	ASSERT3P(target, !=, NULL);
	ASSERT3U(target->lse_mscount, !=, 0);
	target->lse_msdcount++;
	}

	/*
	* This function attempts to estimate how many metaslabs should
	* we flush to satisfy our block heuristic for the log spacemap
	* for the upcoming TXGs.
	*
	* Specifically, it first tries to estimate the number of incoming
	* blocks in this TXG. Then by projecting that incoming rate to
	* future TXGs and using the log summary, it figures out how many
	* flushes we would need to do for future TXGs individually to
	* stay below our block limit and returns the maximum number of
	* flushes from those estimates.
	*/
	static uint64_t
	spa_estimate_metaslabs_to_flush(spa_t *spa)
	{
	ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP));
	ASSERT3U(spa_sync_pass(spa), ==, 1);
	ASSERT(spa_log_sm_blocklimit(spa) != 0);

	/*
	* This variable contains the incoming rate that will be projected
	* and used for our flushing estimates in the future.
	*/
	uint64_t incoming = spa_estimate_incoming_log_blocks(spa);

	/*
	* At any point in time this variable tells us how many
	* TXGs in the future we are so we can make our estimations.
	*/
	uint64_t txgs_in_future = 1;

	/*
	* This variable tells us how much room do we have until we hit
	* our limit. When it goes negative, it means that we've exceeded
	* our limit and we need to flush.
	*
	* Note that since we start at the first TXG in the future (i.e.
	* txgs_in_future starts from 1) we already decrement this
	* variable by the incoming rate.
	*/
	int64_t available_blocks =
	spa_log_sm_blocklimit(spa) - spa_log_sm_nblocks(spa) - incoming;

	int64_t available_txgs = zfs_unflushed_log_txg_max;
	for (log_summary_entry_t *e = list_head(&spa->spa_log_summary);
	e; e = list_next(&spa->spa_log_summary, e))
	available_txgs -= e->lse_txgcount;

	/*
	* This variable tells us the total number of flushes needed to
	* keep the log size within the limit when we reach txgs_in_future.
	*/
	uint64_t total_flushes = 0;

	/* Holds the current maximum of our estimates so far. */
	uint64_t max_flushes_pertxg = zfs_min_metaslabs_to_flush;

	/*
	* For our estimations we only look as far in the future
	* as the summary allows us.
	*/
	for (log_summary_entry_t *e = list_head(&spa->spa_log_summary);
	e; e = list_next(&spa->spa_log_summary, e)) {

	/*
	* If there is still room before we exceed our limit
	* then keep skipping TXGs accumulating more blocks
	* based on the incoming rate until we exceed it.
	*/
	if (available_blocks >= 0 && available_txgs >= 0) {
	uint64_t skip_txgs = (incoming == 0) ?
	available_txgs + 1 : MIN(available_txgs + 1,
	(available_blocks / incoming) + 1);
	available_blocks -= (skip_txgs * incoming);
	available_txgs -= skip_txgs;
	txgs_in_future += skip_txgs;
	ASSERT3S(available_blocks, >=, -incoming);
	ASSERT3S(available_txgs, >=, -1);
	}

	/*
	* At this point we're far enough into the future where
	* the limit was just exceeded and we flush metaslabs
	* based on the current entry in the summary, updating
	* our available_blocks.
	*/
	ASSERT(available_blocks < 0 \|\| available_txgs < 0);
	available_blocks += e->lse_blkcount;
	available_txgs += e->lse_txgcount;
	total_flushes += e->lse_msdcount;

	/*
	* Keep the running maximum of the total_flushes that
	* we've done so far over the number of TXGs in the
	* future that we are. The idea here is to estimate
	* the average number of flushes that we should do
	* every TXG so that when we are that many TXGs in the
	* future we stay under the limit.
	*/
	max_flushes_pertxg = MAX(max_flushes_pertxg,
	DIV_ROUND_UP(total_flushes, txgs_in_future));
	}
	return (max_flushes_pertxg);
	}

	uint64_t
	spa_log_sm_memused(spa_t *spa)
	{
	return (spa->spa_unflushed_stats.sus_memused);
	}

	static boolean_t
	spa_log_exceeds_memlimit(spa_t *spa)
	{
	if (spa_log_sm_memused(spa) > zfs_unflushed_max_mem_amt)
	return (B_TRUE);

	uint64_t system_mem_allowed = ((physmem * PAGESIZE) *
	zfs_unflushed_max_mem_ppm) / 1000000;
	if (spa_log_sm_memused(spa) > system_mem_allowed)
	return (B_TRUE);

	return (B_FALSE);
	}

	boolean_t
	spa_flush_all_logs_requested(spa_t *spa)
	{
	return (spa->spa_log_flushall_txg != 0);
	}

	void
	spa_flush_metaslabs(spa_t spa, dmu_tx_t tx)
	{
	uint64_t txg = dmu_tx_get_txg(tx);

	if (spa_sync_pass(spa) != 1)
	return;

	if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP))
	return;

	/*
	* If we don't have any metaslabs with unflushed changes
	* return immediately.
	*/
	if (avl_numnodes(&spa->spa_metaslabs_by_flushed) == 0)
	return;

	/*
	* During SPA export we leave a few empty TXGs to go by [see
	* spa_final_dirty_txg() to understand why]. For this specific
	* case, it is important to not flush any metaslabs as that
	* would dirty this TXG.
	*
	* That said, during one of these dirty TXGs that is less or
	* equal to spa_final_dirty(), spa_unload() will request that
	* we try to flush all the metaslabs for that TXG before
	* exporting the pool, thus we ensure that we didn't get a
	* request of flushing everything before we attempt to return
	* immediately.
	*/
	if (spa->spa_uberblock.ub_rootbp.blk_birth < txg &&
	!dmu_objset_is_dirty(spa_meta_objset(spa), txg) &&
	!spa_flush_all_logs_requested(spa))
	return;

	/*
	* We need to generate a log space map before flushing because this
	* will set up the in-memory data (i.e. node in spa_sm_logs_by_txg)
	* for this TXG's flushed metaslab count (aka sls_mscount which is
	* manipulated in many ways down the metaslab_flush() codepath).
	*
	* That is not to say that we may generate a log space map when we
	* don't need it. If we are flushing metaslabs, that means that we
	* were going to write changes to disk anyway, so even if we were
	* not flushing, a log space map would have been created anyway in
	* metaslab_sync().
	*/
	spa_generate_syncing_log_sm(spa, tx);

	/*
	* This variable tells us how many metaslabs we want to flush based
	* on the block-heuristic of our flushing algorithm (see block comment
	* of log space map feature). We also decrement this as we flush
	* metaslabs and attempt to destroy old log space maps.
	*/
	uint64_t want_to_flush;
	if (spa_flush_all_logs_requested(spa)) {
	ASSERT3S(spa_state(spa), ==, POOL_STATE_EXPORTED);
	want_to_flush = UINT64_MAX;
	} else {
	want_to_flush = spa_estimate_metaslabs_to_flush(spa);
	}

	/* Used purely for verification purposes */
	uint64_t visited = 0;

	/*
	* Ideally we would only iterate through spa_metaslabs_by_flushed
	* using only one variable (curr). We can't do that because
	* metaslab_flush() mutates position of curr in the AVL when
	* it flushes that metaslab by moving it to the end of the tree.
	* Thus we always keep track of the original next node of the
	* current node (curr) in another variable (next).
	*/
	metaslab_t *next = NULL;
	for (metaslab_t *curr = avl_first(&spa->spa_metaslabs_by_flushed);
	curr != NULL; curr = next) {
	next = AVL_NEXT(&spa->spa_metaslabs_by_flushed, curr);

	/*
	* If this metaslab has been flushed this txg then we've done
	* a full circle over the metaslabs.
	*/
	if (metaslab_unflushed_txg(curr) == txg)
	break;

	/*
	* If we are done flushing for the block heuristic and the
	* unflushed changes don't exceed the memory limit just stop.
	*/
	if (want_to_flush == 0 && !spa_log_exceeds_memlimit(spa))
	break;

	if (metaslab_unflushed_dirty(curr)) {
	mutex_enter(&curr->ms_sync_lock);
	mutex_enter(&curr->ms_lock);
	metaslab_flush(curr, tx);
	mutex_exit(&curr->ms_lock);
	mutex_exit(&curr->ms_sync_lock);
	if (want_to_flush > 0)
	want_to_flush--;
	} else
	metaslab_unflushed_bump(curr, tx, B_FALSE);

	visited++;
	}
	ASSERT3U(avl_numnodes(&spa->spa_metaslabs_by_flushed), >=, visited);

	spa_log_sm_set_blocklimit(spa);
	}

	/*
	* Close the log space map for this TXG and update the block counts
	* for the log's in-memory structure and the summary.
	*/
	void
	spa_sync_close_syncing_log_sm(spa_t *spa)
	{
	if (spa_syncing_log_sm(spa) == NULL)
	return;
	ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP));

	spa_log_sm_t *sls = avl_last(&spa->spa_sm_logs_by_txg);
	ASSERT3U(sls->sls_txg, ==, spa_syncing_txg(spa));

	sls->sls_nblocks = space_map_nblocks(spa_syncing_log_sm(spa));
	spa->spa_unflushed_stats.sus_nblocks += sls->sls_nblocks;

	/*
	* Note that we can't assert that sls_mscount is not 0,
	* because there is the case where the first metaslab
	* in spa_metaslabs_by_flushed is loading and we were
	* not able to flush any metaslabs the current TXG.
	*/
	ASSERT(sls->sls_nblocks != 0);

	spa_log_summary_add_incoming_blocks(spa, sls->sls_nblocks);
	spa_log_summary_verify_counts(spa);

	space_map_close(spa->spa_syncing_log_sm);
	spa->spa_syncing_log_sm = NULL;

	/*
	* At this point we tried to flush as many metaslabs as we
	* can as the pool is getting exported. Reset the "flush all"
	* so the last few TXGs before closing the pool can be empty
	* (e.g. not dirty).
	*/
	if (spa_flush_all_logs_requested(spa)) {
	ASSERT3S(spa_state(spa), ==, POOL_STATE_EXPORTED);
	spa->spa_log_flushall_txg = 0;
	}
	}

	void
	spa_cleanup_old_sm_logs(spa_t spa, dmu_tx_t tx)
	{
	objset_t *mos = spa_meta_objset(spa);

	uint64_t spacemap_zap;
	int error = zap_lookup(mos, DMU_POOL_DIRECTORY_OBJECT,
	DMU_POOL_LOG_SPACEMAP_ZAP, sizeof (spacemap_zap), 1, &spacemap_zap);
	if (error == ENOENT) {
	ASSERT(avl_is_empty(&spa->spa_sm_logs_by_txg));
	return;
	}
	VERIFY0(error);

	metaslab_t *oldest = avl_first(&spa->spa_metaslabs_by_flushed);
	uint64_t oldest_flushed_txg = metaslab_unflushed_txg(oldest);

	/* Free all log space maps older than the oldest_flushed_txg. */
	for (spa_log_sm_t *sls = avl_first(&spa->spa_sm_logs_by_txg);
	sls && sls->sls_txg < oldest_flushed_txg;
	sls = avl_first(&spa->spa_sm_logs_by_txg)) {
	ASSERT0(sls->sls_mscount);
	avl_remove(&spa->spa_sm_logs_by_txg, sls);
	space_map_free_obj(mos, sls->sls_sm_obj, tx);
	VERIFY0(zap_remove_int(mos, spacemap_zap, sls->sls_txg, tx));
	spa_log_summary_decrement_blkcount(spa, sls->sls_nblocks);
	spa->spa_unflushed_stats.sus_nblocks -= sls->sls_nblocks;
	kmem_free(sls, sizeof (spa_log_sm_t));
	}
	}

	static spa_log_sm_t *
	spa_log_sm_alloc(uint64_t sm_obj, uint64_t txg)
	{
	spa_log_sm_t sls = kmem_zalloc(sizeof (sls), KM_SLEEP);
	sls->sls_sm_obj = sm_obj;
	sls->sls_txg = txg;
	return (sls);
	}

	void
	spa_generate_syncing_log_sm(spa_t spa, dmu_tx_t tx)
	{
	uint64_t txg = dmu_tx_get_txg(tx);
	objset_t *mos = spa_meta_objset(spa);

	if (spa_syncing_log_sm(spa) != NULL)
	return;

	if (!spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP))
	return;

	uint64_t spacemap_zap;
	int error = zap_lookup(mos, DMU_POOL_DIRECTORY_OBJECT,
	DMU_POOL_LOG_SPACEMAP_ZAP, sizeof (spacemap_zap), 1, &spacemap_zap);
	if (error == ENOENT) {
	ASSERT(avl_is_empty(&spa->spa_sm_logs_by_txg));

	error = 0;
	spacemap_zap = zap_create(mos,
	DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx);
	VERIFY0(zap_add(mos, DMU_POOL_DIRECTORY_OBJECT,
	DMU_POOL_LOG_SPACEMAP_ZAP, sizeof (spacemap_zap), 1,
	&spacemap_zap, tx));
	spa_feature_incr(spa, SPA_FEATURE_LOG_SPACEMAP, tx);
	}
	VERIFY0(error);

	uint64_t sm_obj;
	ASSERT3U(zap_lookup_int_key(mos, spacemap_zap, txg, &sm_obj),
	==, ENOENT);
	sm_obj = space_map_alloc(mos, zfs_log_sm_blksz, tx);
	VERIFY0(zap_add_int_key(mos, spacemap_zap, txg, sm_obj, tx));
	avl_add(&spa->spa_sm_logs_by_txg, spa_log_sm_alloc(sm_obj, txg));

	/*
	* We pass UINT64_MAX as the space map's representation size
	* and SPA_MINBLOCKSHIFT as the shift, to make the space map
	* accept any sorts of segments since there's no real advantage
	* to being more restrictive (given that we're already going
	* to be using 2-word entries).
	*/
	VERIFY0(space_map_open(&spa->spa_syncing_log_sm, mos, sm_obj,
	0, UINT64_MAX, SPA_MINBLOCKSHIFT));

	spa_log_sm_set_blocklimit(spa);
	}

	/*
	* Find all the log space maps stored in the space map ZAP and sort
	* them by their TXG in spa_sm_logs_by_txg.
	*/
	static int
	spa_ld_log_sm_metadata(spa_t *spa)
	{
	int error;
	uint64_t spacemap_zap;

	ASSERT(avl_is_empty(&spa->spa_sm_logs_by_txg));

	error = zap_lookup(spa_meta_objset(spa), DMU_POOL_DIRECTORY_OBJECT,
	DMU_POOL_LOG_SPACEMAP_ZAP, sizeof (spacemap_zap), 1, &spacemap_zap);
	if (error == ENOENT) {
	/* the space map ZAP doesn't exist yet */
	return (0);
	} else if (error != 0) {
	spa_load_failed(spa, "spa_ld_log_sm_metadata(): failed at "
	"zap_lookup(DMU_POOL_DIRECTORY_OBJECT) [error %d]",
	error);
	return (error);
	}

	zap_cursor_t zc;
	zap_attribute_t za;
	for (zap_cursor_init(&zc, spa_meta_objset(spa), spacemap_zap);
	(error = zap_cursor_retrieve(&zc, &za)) == 0;
	zap_cursor_advance(&zc)) {
	uint64_t log_txg = zfs_strtonum(za.za_name, NULL);
	spa_log_sm_t *sls =
	spa_log_sm_alloc(za.za_first_integer, log_txg);
	avl_add(&spa->spa_sm_logs_by_txg, sls);
	}
	zap_cursor_fini(&zc);
	if (error != ENOENT) {
	spa_load_failed(spa, "spa_ld_log_sm_metadata(): failed at "
	"zap_cursor_retrieve(spacemap_zap) [error %d]",
	error);
	return (error);
	}

	for (metaslab_t *m = avl_first(&spa->spa_metaslabs_by_flushed);
	m; m = AVL_NEXT(&spa->spa_metaslabs_by_flushed, m)) {
	spa_log_sm_t target = { .sls_txg = metaslab_unflushed_txg(m) };
	spa_log_sm_t *sls = avl_find(&spa->spa_sm_logs_by_txg,
	&target, NULL);

	/*
	* At this point if sls is zero it means that a bug occurred
	* in ZFS the last time the pool was open or earlier in the
	* import code path. In general, we would have placed a
	* VERIFY() here or in this case just let the kernel panic
	* with NULL pointer dereference when incrementing sls_mscount,
	* but since this is the import code path we can be a bit more
	* lenient. Thus, for DEBUG bits we always cause a panic, while
	* in production we log the error and just fail the import.
	*/
	ASSERT(sls != NULL);
	if (sls == NULL) {
	spa_load_failed(spa, "spa_ld_log_sm_metadata(): bug "
	"encountered: could not find log spacemap for "
	"TXG %llu [error %d]",
	(u_longlong_t)metaslab_unflushed_txg(m), ENOENT);
	return (ENOENT);
	}
	sls->sls_mscount++;
	}

	return (0);
	}

	typedef struct spa_ld_log_sm_arg {
	spa_t *slls_spa;
	uint64_t slls_txg;
	} spa_ld_log_sm_arg_t;

	static int
	spa_ld_log_sm_cb(space_map_entry_t sme, void arg)
	{
	uint64_t offset = sme->sme_offset;
	uint64_t size = sme->sme_run;
	uint32_t vdev_id = sme->sme_vdev;

	spa_ld_log_sm_arg_t *slls = arg;
	spa_t *spa = slls->slls_spa;

	vdev_t *vd = vdev_lookup_top(spa, vdev_id);

	/*
	* If the vdev has been removed (i.e. it is indirect or a hole)
	* skip this entry. The contents of this vdev have already moved
	* elsewhere.
	*/
	if (!vdev_is_concrete(vd))
	return (0);

	metaslab_t *ms = vd->vdev_ms[offset >> vd->vdev_ms_shift];
	ASSERT(!ms->ms_loaded);

	/*
	* If we have already flushed entries for this TXG to this
	* metaslab's space map, then ignore it. Note that we flush
	* before processing any allocations/frees for that TXG, so
	* the metaslab's space map only has entries from before
	* the unflushed TXG.
	*/
	if (slls->slls_txg < metaslab_unflushed_txg(ms))
	return (0);

	switch (sme->sme_type) {
	case SM_ALLOC:
	range_tree_remove_xor_add_segment(offset, offset + size,
	ms->ms_unflushed_frees, ms->ms_unflushed_allocs);
	break;
	case SM_FREE:
	range_tree_remove_xor_add_segment(offset, offset + size,
	ms->ms_unflushed_allocs, ms->ms_unflushed_frees);
	break;
	default:
	panic("invalid maptype_t");
	break;
	}
	if (!metaslab_unflushed_dirty(ms)) {
	metaslab_set_unflushed_dirty(ms, B_TRUE);
	spa_log_summary_dirty_flushed_metaslab(spa,
	metaslab_unflushed_txg(ms));
	}
	return (0);
	}

	static int
	spa_ld_log_sm_data(spa_t *spa)
	{
	spa_log_sm_t sls, psls;
	int error = 0;

	/*
	* If we are not going to do any writes there is no need
	* to read the log space maps.
	*/
	if (!spa_writeable(spa))
	return (0);

	ASSERT0(spa->spa_unflushed_stats.sus_nblocks);
	ASSERT0(spa->spa_unflushed_stats.sus_memused);

	hrtime_t read_logs_starttime = gethrtime();

	/* Prefetch log spacemaps dnodes. */
	for (sls = avl_first(&spa->spa_sm_logs_by_txg); sls;
	sls = AVL_NEXT(&spa->spa_sm_logs_by_txg, sls)) {
	- dmu_prefetch(spa_meta_objset(spa), sls->sls_sm_obj,
	- 0, 0, 0, ZIO_PRIORITY_SYNC_READ);
	+ dmu_prefetch_dnode(spa_meta_objset(spa), sls->sls_sm_obj,
	+ ZIO_PRIORITY_SYNC_READ);
	}

	uint_t pn = 0;
	uint64_t ps = 0;
	+ uint64_t nsm = 0;
	psls = sls = avl_first(&spa->spa_sm_logs_by_txg);
	while (sls != NULL) {
	/* Prefetch log spacemaps up to 16 TXGs or MBs ahead. */
	if (psls != NULL && pn < 16 &&
	(pn < 2 \|\| ps < 2 * dmu_prefetch_max)) {
	error = space_map_open(&psls->sls_sm,
	spa_meta_objset(spa), psls->sls_sm_obj, 0,
	UINT64_MAX, SPA_MINBLOCKSHIFT);
	if (error != 0) {
	spa_load_failed(spa, "spa_ld_log_sm_data(): "
	"failed at space_map_open(obj=%llu) "
	"[error %d]",
	(u_longlong_t)sls->sls_sm_obj, error);
	goto out;
	}
	dmu_prefetch(spa_meta_objset(spa), psls->sls_sm_obj,
	0, 0, space_map_length(psls->sls_sm),
	ZIO_PRIORITY_ASYNC_READ);
	pn++;
	ps += space_map_length(psls->sls_sm);
	psls = AVL_NEXT(&spa->spa_sm_logs_by_txg, psls);
	continue;
	}

	/* Load TXG log spacemap into ms_unflushed_allocs/frees. */
	kpreempt(KPREEMPT_SYNC);
	ASSERT0(sls->sls_nblocks);
	sls->sls_nblocks = space_map_nblocks(sls->sls_sm);
	spa->spa_unflushed_stats.sus_nblocks += sls->sls_nblocks;
	summary_add_data(spa, sls->sls_txg,
	sls->sls_mscount, 0, sls->sls_nblocks);

	+ spa_import_progress_set_notes_nolog(spa,
	+ "Read %llu of %lu log space maps", (u_longlong_t)nsm,
	+ avl_numnodes(&spa->spa_sm_logs_by_txg));
	+
	struct spa_ld_log_sm_arg vla = {
	.slls_spa = spa,
	.slls_txg = sls->sls_txg
	};
	error = space_map_iterate(sls->sls_sm,
	space_map_length(sls->sls_sm), spa_ld_log_sm_cb, &vla);
	if (error != 0) {
	spa_load_failed(spa, "spa_ld_log_sm_data(): failed "
	"at space_map_iterate(obj=%llu) [error %d]",
	(u_longlong_t)sls->sls_sm_obj, error);
	goto out;
	}

	pn--;
	ps -= space_map_length(sls->sls_sm);
	+ nsm++;
	space_map_close(sls->sls_sm);
	sls->sls_sm = NULL;
	sls = AVL_NEXT(&spa->spa_sm_logs_by_txg, sls);

	/* Update log block limits considering just loaded. */
	spa_log_sm_set_blocklimit(spa);
	}

	hrtime_t read_logs_endtime = gethrtime();
	spa_load_note(spa,
	- "read %llu log space maps (%llu total blocks - blksz = %llu bytes) "
	- "in %lld ms", (u_longlong_t)avl_numnodes(&spa->spa_sm_logs_by_txg),
	+ "Read %lu log space maps (%llu total blocks - blksz = %llu bytes) "
	+ "in %lld ms", avl_numnodes(&spa->spa_sm_logs_by_txg),
	(u_longlong_t)spa_log_sm_nblocks(spa),
	(u_longlong_t)zfs_log_sm_blksz,
	- (longlong_t)((read_logs_endtime - read_logs_starttime) / 1000000));
	+ (longlong_t)NSEC2MSEC(read_logs_endtime - read_logs_starttime));

	out:
	if (error != 0) {
	for (spa_log_sm_t *sls = avl_first(&spa->spa_sm_logs_by_txg);
	sls; sls = AVL_NEXT(&spa->spa_sm_logs_by_txg, sls)) {
	if (sls->sls_sm) {
	space_map_close(sls->sls_sm);
	sls->sls_sm = NULL;
	}
	}
	} else {
	ASSERT0(pn);
	ASSERT0(ps);
	}
	/*
	* Now that the metaslabs contain their unflushed changes:
	* [1] recalculate their actual allocated space
	* [2] recalculate their weights
	* [3] sum up the memory usage of their unflushed range trees
	* [4] optionally load them, if debug_load is set
	*
	* Note that even in the case where we get here because of an
	* error (e.g. error != 0), we still want to update the fields
	* below in order to have a proper teardown in spa_unload().
	*/
	for (metaslab_t *m = avl_first(&spa->spa_metaslabs_by_flushed);
	m != NULL; m = AVL_NEXT(&spa->spa_metaslabs_by_flushed, m)) {
	mutex_enter(&m->ms_lock);
	m->ms_allocated_space = space_map_allocated(m->ms_sm) +
	range_tree_space(m->ms_unflushed_allocs) -
	range_tree_space(m->ms_unflushed_frees);

	vdev_t *vd = m->ms_group->mg_vd;
	metaslab_space_update(vd, m->ms_group->mg_class,
	range_tree_space(m->ms_unflushed_allocs), 0, 0);
	metaslab_space_update(vd, m->ms_group->mg_class,
	-range_tree_space(m->ms_unflushed_frees), 0, 0);

	ASSERT0(m->ms_weight & METASLAB_ACTIVE_MASK);
	metaslab_recalculate_weight_and_sort(m);

	spa->spa_unflushed_stats.sus_memused +=
	metaslab_unflushed_changes_memused(m);

	if (metaslab_debug_load && m->ms_sm != NULL) {
	VERIFY0(metaslab_load(m));
	metaslab_set_selected_txg(m, 0);
	}
	mutex_exit(&m->ms_lock);
	}

	return (error);
	}

	static int
	spa_ld_unflushed_txgs(vdev_t *vd)
	{
	spa_t *spa = vd->vdev_spa;
	objset_t *mos = spa_meta_objset(spa);

	if (vd->vdev_top_zap == 0)
	return (0);

	uint64_t object = 0;
	int error = zap_lookup(mos, vd->vdev_top_zap,
	VDEV_TOP_ZAP_MS_UNFLUSHED_PHYS_TXGS,
	sizeof (uint64_t), 1, &object);
	if (error == ENOENT)
	return (0);
	else if (error != 0) {
	spa_load_failed(spa, "spa_ld_unflushed_txgs(): failed at "
	"zap_lookup(vdev_top_zap=%llu) [error %d]",
	(u_longlong_t)vd->vdev_top_zap, error);
	return (error);
	}

	for (uint64_t m = 0; m < vd->vdev_ms_count; m++) {
	metaslab_t *ms = vd->vdev_ms[m];
	ASSERT(ms != NULL);

	metaslab_unflushed_phys_t entry;
	uint64_t entry_size = sizeof (entry);
	uint64_t entry_offset = ms->ms_id * entry_size;

	error = dmu_read(mos, object,
	entry_offset, entry_size, &entry, 0);
	if (error != 0) {
	spa_load_failed(spa, "spa_ld_unflushed_txgs(): "
	"failed at dmu_read(obj=%llu) [error %d]",
	(u_longlong_t)object, error);
	return (error);
	}

	ms->ms_unflushed_txg = entry.msp_unflushed_txg;
	ms->ms_unflushed_dirty = B_FALSE;
	ASSERT(range_tree_is_empty(ms->ms_unflushed_allocs));
	ASSERT(range_tree_is_empty(ms->ms_unflushed_frees));
	if (ms->ms_unflushed_txg != 0) {
	mutex_enter(&spa->spa_flushed_ms_lock);
	avl_add(&spa->spa_metaslabs_by_flushed, ms);
	mutex_exit(&spa->spa_flushed_ms_lock);
	}
	}
	return (0);
	}

	/*
	* Read all the log space map entries into their respective
	* metaslab unflushed trees and keep them sorted by TXG in the
	* SPA's metadata. In addition, setup all the metadata for the
	* memory and the block heuristics.
	*/
	int
	spa_ld_log_spacemaps(spa_t *spa)
	{
	int error;

	spa_log_sm_set_blocklimit(spa);

	for (uint64_t c = 0; c < spa->spa_root_vdev->vdev_children; c++) {
	vdev_t *vd = spa->spa_root_vdev->vdev_child[c];
	error = spa_ld_unflushed_txgs(vd);
	if (error != 0)
	return (error);
	}

	error = spa_ld_log_sm_metadata(spa);
	if (error != 0)
	return (error);

	/*
	* Note: we don't actually expect anything to change at this point
	* but we grab the config lock so we don't fail any assertions
	* when using vdev_lookup_top().
	*/
	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
	error = spa_ld_log_sm_data(spa);
	spa_config_exit(spa, SCL_CONFIG, FTAG);

	return (error);
	}

	/* BEGIN CSTYLED */
	ZFS_MODULE_PARAM(zfs, zfs_, unflushed_max_mem_amt, U64, ZMOD_RW,
	"Specific hard-limit in memory that ZFS allows to be used for "
	"unflushed changes");

	ZFS_MODULE_PARAM(zfs, zfs_, unflushed_max_mem_ppm, U64, ZMOD_RW,
	"Percentage of the overall system memory that ZFS allows to be "
	"used for unflushed changes (value is calculated over 1000000 for "
	"finer granularity)");

	ZFS_MODULE_PARAM(zfs, zfs_, unflushed_log_block_max, U64, ZMOD_RW,
	"Hard limit (upper-bound) in the size of the space map log "
	"in terms of blocks.");

	ZFS_MODULE_PARAM(zfs, zfs_, unflushed_log_block_min, U64, ZMOD_RW,
	"Lower-bound limit for the maximum amount of blocks allowed in "
	"log spacemap (see zfs_unflushed_log_block_max)");

	ZFS_MODULE_PARAM(zfs, zfs_, unflushed_log_txg_max, U64, ZMOD_RW,
	"Hard limit (upper-bound) in the size of the space map log "
	"in terms of dirty TXGs.");

	ZFS_MODULE_PARAM(zfs, zfs_, unflushed_log_block_pct, UINT, ZMOD_RW,
	"Tunable used to determine the number of blocks that can be used for "
	"the spacemap log, expressed as a percentage of the total number of "
	"metaslabs in the pool (e.g. 400 means the number of log blocks is "
	"capped at 4 times the number of metaslabs)");

	ZFS_MODULE_PARAM(zfs, zfs_, max_log_walking, U64, ZMOD_RW,
	"The number of past TXGs that the flushing algorithm of the log "
	"spacemap feature uses to estimate incoming log blocks");

	ZFS_MODULE_PARAM(zfs, zfs_, keep_log_spacemaps_at_export, INT, ZMOD_RW,
	"Prevent the log spacemaps from being flushed and destroyed "
	"during pool export/destroy");
	/* END CSTYLED */

	ZFS_MODULE_PARAM(zfs, zfs_, max_logsm_summary_length, U64, ZMOD_RW,
	"Maximum number of rows allowed in the summary of the spacemap log");

	ZFS_MODULE_PARAM(zfs, zfs_, min_metaslabs_to_flush, U64, ZMOD_RW,
	"Minimum number of metaslabs to flush per dirty TXG");
	diff --git a/sys/contrib/openzfs/module/zfs/spa_misc.c b/sys/contrib/openzfs/module/zfs/spa_misc.c
	index 24f038ad7f4b..649fe2f634b5 100644
	--- a/sys/contrib/openzfs/module/zfs/spa_misc.c
	+++ b/sys/contrib/openzfs/module/zfs/spa_misc.c
	@@ -1,3006 +1,3072 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2011, 2019 by Delphix. All rights reserved.
	* Copyright 2015 Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
	* Copyright 2013 Saso Kiselkov. All rights reserved.
	* Copyright (c) 2017 Datto Inc.
	* Copyright (c) 2017, Intel Corporation.
	* Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
	* Copyright (c) 2023, Klara Inc.
	*/

	#include <sys/zfs_context.h>
	#include <sys/zfs_chksum.h>
	#include <sys/spa_impl.h>
	#include <sys/zio.h>
	#include <sys/zio_checksum.h>
	#include <sys/zio_compress.h>
	#include <sys/dmu.h>
	#include <sys/dmu_tx.h>
	#include <sys/zap.h>
	#include <sys/zil.h>
	#include <sys/vdev_impl.h>
	#include <sys/vdev_initialize.h>
	#include <sys/vdev_trim.h>
	#include <sys/vdev_file.h>
	#include <sys/vdev_raidz.h>
	#include <sys/metaslab.h>
	#include <sys/uberblock_impl.h>
	#include <sys/txg.h>
	#include <sys/avl.h>
	#include <sys/unique.h>
	#include <sys/dsl_pool.h>
	#include <sys/dsl_dir.h>
	#include <sys/dsl_prop.h>
	#include <sys/fm/util.h>
	#include <sys/dsl_scan.h>
	#include <sys/fs/zfs.h>
	#include <sys/metaslab_impl.h>
	#include <sys/arc.h>
	#include <sys/brt.h>
	#include <sys/ddt.h>
	#include <sys/kstat.h>
	#include "zfs_prop.h"
	#include <sys/btree.h>
	#include <sys/zfeature.h>
	#include <sys/qat.h>
	#include <sys/zstd/zstd.h>

	/*
	* SPA locking
	*
	* There are three basic locks for managing spa_t structures:
	*
	* spa_namespace_lock (global mutex)
	*
	* This lock must be acquired to do any of the following:
	*
	* - Lookup a spa_t by name
	* - Add or remove a spa_t from the namespace
	* - Increase spa_refcount from non-zero
	* - Check if spa_refcount is zero
	* - Rename a spa_t
	* - add/remove/attach/detach devices
	* - Held for the duration of create/destroy/import/export
	*
	* It does not need to handle recursion. A create or destroy may
	* reference objects (files or zvols) in other pools, but by
	* definition they must have an existing reference, and will never need
	* to lookup a spa_t by name.
	*
	* spa_refcount (per-spa zfs_refcount_t protected by mutex)
	*
	* This reference count keep track of any active users of the spa_t. The
	* spa_t cannot be destroyed or freed while this is non-zero. Internally,
	* the refcount is never really 'zero' - opening a pool implicitly keeps
	* some references in the DMU. Internally we check against spa_minref, but
	* present the image of a zero/non-zero value to consumers.
	*
	* spa_config_lock[] (per-spa array of rwlocks)
	*
	* This protects the spa_t from config changes, and must be held in
	* the following circumstances:
	*
	* - RW_READER to perform I/O to the spa
	* - RW_WRITER to change the vdev config
	*
	* The locking order is fairly straightforward:
	*
	* spa_namespace_lock -> spa_refcount
	*
	* The namespace lock must be acquired to increase the refcount from 0
	* or to check if it is zero.
	*
	* spa_refcount -> spa_config_lock[]
	*
	* There must be at least one valid reference on the spa_t to acquire
	* the config lock.
	*
	* spa_namespace_lock -> spa_config_lock[]
	*
	* The namespace lock must always be taken before the config lock.
	*
	*
	* The spa_namespace_lock can be acquired directly and is globally visible.
	*
	* The namespace is manipulated using the following functions, all of which
	* require the spa_namespace_lock to be held.
	*
	* spa_lookup() Lookup a spa_t by name.
	*
	* spa_add() Create a new spa_t in the namespace.
	*
	* spa_remove() Remove a spa_t from the namespace. This also
	* frees up any memory associated with the spa_t.
	*
	* spa_next() Returns the next spa_t in the system, or the
	* first if NULL is passed.
	*
	* spa_evict_all() Shutdown and remove all spa_t structures in
	* the system.
	*
	* spa_guid_exists() Determine whether a pool/device guid exists.
	*
	* The spa_refcount is manipulated using the following functions:
	*
	* spa_open_ref() Adds a reference to the given spa_t. Must be
	* called with spa_namespace_lock held if the
	* refcount is currently zero.
	*
	* spa_close() Remove a reference from the spa_t. This will
	* not free the spa_t or remove it from the
	* namespace. No locking is required.
	*
	* spa_refcount_zero() Returns true if the refcount is currently
	* zero. Must be called with spa_namespace_lock
	* held.
	*
	* The spa_config_lock[] is an array of rwlocks, ordered as follows:
	* SCL_CONFIG > SCL_STATE > SCL_ALLOC > SCL_ZIO > SCL_FREE > SCL_VDEV.
	* spa_config_lock[] is manipulated with spa_config_{enter,exit,held}().
	*
	* To read the configuration, it suffices to hold one of these locks as reader.
	* To modify the configuration, you must hold all locks as writer. To modify
	* vdev state without altering the vdev tree's topology (e.g. online/offline),
	* you must hold SCL_STATE and SCL_ZIO as writer.
	*
	* We use these distinct config locks to avoid recursive lock entry.
	* For example, spa_sync() (which holds SCL_CONFIG as reader) induces
	* block allocations (SCL_ALLOC), which may require reading space maps
	* from disk (dmu_read() -> zio_read() -> SCL_ZIO).
	*
	* The spa config locks cannot be normal rwlocks because we need the
	* ability to hand off ownership. For example, SCL_ZIO is acquired
	* by the issuing thread and later released by an interrupt thread.
	* They do, however, obey the usual write-wanted semantics to prevent
	* writer (i.e. system administrator) starvation.
	*
	* The lock acquisition rules are as follows:
	*
	* SCL_CONFIG
	* Protects changes to the vdev tree topology, such as vdev
	* add/remove/attach/detach. Protects the dirty config list
	* (spa_config_dirty_list) and the set of spares and l2arc devices.
	*
	* SCL_STATE
	* Protects changes to pool state and vdev state, such as vdev
	* online/offline/fault/degrade/clear. Protects the dirty state list
	* (spa_state_dirty_list) and global pool state (spa_state).
	*
	* SCL_ALLOC
	* Protects changes to metaslab groups and classes.
	* Held as reader by metaslab_alloc() and metaslab_claim().
	*
	* SCL_ZIO
	* Held by bp-level zios (those which have no io_vd upon entry)
	* to prevent changes to the vdev tree. The bp-level zio implicitly
	* protects all of its vdev child zios, which do not hold SCL_ZIO.
	*
	* SCL_FREE
	* Protects changes to metaslab groups and classes.
	* Held as reader by metaslab_free(). SCL_FREE is distinct from
	* SCL_ALLOC, and lower than SCL_ZIO, so that we can safely free
	* blocks in zio_done() while another i/o that holds either
	* SCL_ALLOC or SCL_ZIO is waiting for this i/o to complete.
	*
	* SCL_VDEV
	* Held as reader to prevent changes to the vdev tree during trivial
	* inquiries such as bp_get_dsize(). SCL_VDEV is distinct from the
	* other locks, and lower than all of them, to ensure that it's safe
	* to acquire regardless of caller context.
	*
	* In addition, the following rules apply:
	*
	* (a) spa_props_lock protects pool properties, spa_config and spa_config_list.
	* The lock ordering is SCL_CONFIG > spa_props_lock.
	*
	* (b) I/O operations on leaf vdevs. For any zio operation that takes
	* an explicit vdev_t argument -- such as zio_ioctl(), zio_read_phys(),
	* or zio_write_phys() -- the caller must ensure that the config cannot
	* cannot change in the interim, and that the vdev cannot be reopened.
	* SCL_STATE as reader suffices for both.
	*
	* The vdev configuration is protected by spa_vdev_enter() / spa_vdev_exit().
	*
	* spa_vdev_enter() Acquire the namespace lock and the config lock
	* for writing.
	*
	* spa_vdev_exit() Release the config lock, wait for all I/O
	* to complete, sync the updated configs to the
	* cache, and release the namespace lock.
	*
	* vdev state is protected by spa_vdev_state_enter() / spa_vdev_state_exit().
	* Like spa_vdev_enter/exit, these are convenience wrappers -- the actual
	* locking is, always, based on spa_namespace_lock and spa_config_lock[].
	*/

	static avl_tree_t spa_namespace_avl;
	kmutex_t spa_namespace_lock;
	static kcondvar_t spa_namespace_cv;
	static const int spa_max_replication_override = SPA_DVAS_PER_BP;

	static kmutex_t spa_spare_lock;
	static avl_tree_t spa_spare_avl;
	static kmutex_t spa_l2cache_lock;
	static avl_tree_t spa_l2cache_avl;

	spa_mode_t spa_mode_global = SPA_MODE_UNINIT;

	#ifdef ZFS_DEBUG
	/*
	* Everything except dprintf, set_error, spa, and indirect_remap is on
	* by default in debug builds.
	*/
	int zfs_flags = ~(ZFS_DEBUG_DPRINTF \| ZFS_DEBUG_SET_ERROR \|
	ZFS_DEBUG_INDIRECT_REMAP);
	#else
	int zfs_flags = 0;
	#endif

	/*
	* zfs_recover can be set to nonzero to attempt to recover from
	* otherwise-fatal errors, typically caused by on-disk corruption. When
	* set, calls to zfs_panic_recover() will turn into warning messages.
	* This should only be used as a last resort, as it typically results
	* in leaked space, or worse.
	*/
	int zfs_recover = B_FALSE;

	/*
	* If destroy encounters an EIO while reading metadata (e.g. indirect
	* blocks), space referenced by the missing metadata can not be freed.
	* Normally this causes the background destroy to become "stalled", as
	* it is unable to make forward progress. While in this stalled state,
	* all remaining space to free from the error-encountering filesystem is
	* "temporarily leaked". Set this flag to cause it to ignore the EIO,
	* permanently leak the space from indirect blocks that can not be read,
	* and continue to free everything else that it can.
	*
	* The default, "stalling" behavior is useful if the storage partially
	* fails (i.e. some but not all i/os fail), and then later recovers. In
	* this case, we will be able to continue pool operations while it is
	* partially failed, and when it recovers, we can continue to free the
	* space, with no leaks. However, note that this case is actually
	* fairly rare.
	*
	* Typically pools either (a) fail completely (but perhaps temporarily,
	* e.g. a top-level vdev going offline), or (b) have localized,
	* permanent errors (e.g. disk returns the wrong data due to bit flip or
	* firmware bug). In case (a), this setting does not matter because the
	* pool will be suspended and the sync thread will not be able to make
	* forward progress regardless. In case (b), because the error is
	* permanent, the best we can do is leak the minimum amount of space,
	* which is what setting this flag will do. Therefore, it is reasonable
	* for this flag to normally be set, but we chose the more conservative
	* approach of not setting it, so that there is no possibility of
	* leaking space in the "partial temporary" failure case.
	*/
	int zfs_free_leak_on_eio = B_FALSE;

	/*
	* Expiration time in milliseconds. This value has two meanings. First it is
	* used to determine when the spa_deadman() logic should fire. By default the
	* spa_deadman() will fire if spa_sync() has not completed in 600 seconds.
	* Secondly, the value determines if an I/O is considered "hung". Any I/O that
	* has not completed in zfs_deadman_synctime_ms is considered "hung" resulting
	* in one of three behaviors controlled by zfs_deadman_failmode.
	*/
	uint64_t zfs_deadman_synctime_ms = 600000UL; /* 10 min. */

	/*
	* This value controls the maximum amount of time zio_wait() will block for an
	* outstanding IO. By default this is 300 seconds at which point the "hung"
	* behavior will be applied as described for zfs_deadman_synctime_ms.
	*/
	uint64_t zfs_deadman_ziotime_ms = 300000UL; /* 5 min. */

	/*
	* Check time in milliseconds. This defines the frequency at which we check
	* for hung I/O.
	*/
	uint64_t zfs_deadman_checktime_ms = 60000UL; /* 1 min. */

	/*
	* By default the deadman is enabled.
	*/
	int zfs_deadman_enabled = B_TRUE;

	/*
	* Controls the behavior of the deadman when it detects a "hung" I/O.
	* Valid values are zfs_deadman_failmode=<wait\|continue\|panic>.
	*
	* wait - Wait for the "hung" I/O (default)
	* continue - Attempt to recover from a "hung" I/O
	* panic - Panic the system
	*/
	const char *zfs_deadman_failmode = "wait";

	/*
	* The worst case is single-sector max-parity RAID-Z blocks, in which
	* case the space requirement is exactly (VDEV_RAIDZ_MAXPARITY + 1)
	* times the size; so just assume that. Add to this the fact that
	* we can have up to 3 DVAs per bp, and one more factor of 2 because
	* the block may be dittoed with up to 3 DVAs by ddt_sync(). All together,
	* the worst case is:
	* (VDEV_RAIDZ_MAXPARITY + 1) * SPA_DVAS_PER_BP * 2 == 24
	*/
	uint_t spa_asize_inflation = 24;

	/*
	* Normally, we don't allow the last 3.2% (1/(2^spa_slop_shift)) of space in
	* the pool to be consumed (bounded by spa_max_slop). This ensures that we
	* don't run the pool completely out of space, due to unaccounted changes (e.g.
	* to the MOS). It also limits the worst-case time to allocate space. If we
	* have less than this amount of free space, most ZPL operations (e.g. write,
	* create) will return ENOSPC. The ZIL metaslabs (spa_embedded_log_class) are
	* also part of this 3.2% of space which can't be consumed by normal writes;
	* the slop space "proper" (spa_get_slop_space()) is decreased by the embedded
	* log space.
	*
	* Certain operations (e.g. file removal, most administrative actions) can
	* use half the slop space. They will only return ENOSPC if less than half
	* the slop space is free. Typically, once the pool has less than the slop
	* space free, the user will use these operations to free up space in the pool.
	* These are the operations that call dsl_pool_adjustedsize() with the netfree
	* argument set to TRUE.
	*
	* Operations that are almost guaranteed to free up space in the absence of
	* a pool checkpoint can use up to three quarters of the slop space
	* (e.g zfs destroy).
	*
	* A very restricted set of operations are always permitted, regardless of
	* the amount of free space. These are the operations that call
	* dsl_sync_task(ZFS_SPACE_CHECK_NONE). If these operations result in a net
	* increase in the amount of space used, it is possible to run the pool
	* completely out of space, causing it to be permanently read-only.
	*
	* Note that on very small pools, the slop space will be larger than
	* 3.2%, in an effort to have it be at least spa_min_slop (128MB),
	* but we never allow it to be more than half the pool size.
	*
	* Further, on very large pools, the slop space will be smaller than
	* 3.2%, to avoid reserving much more space than we actually need; bounded
	* by spa_max_slop (128GB).
	*
	* See also the comments in zfs_space_check_t.
	*/
	uint_t spa_slop_shift = 5;
	static const uint64_t spa_min_slop = 128ULL * 1024 * 1024;
	static const uint64_t spa_max_slop = 128ULL * 1024 * 1024 * 1024;
	static const int spa_allocators = 4;


	void
	spa_load_failed(spa_t spa, const char fmt, ...)
	{
	va_list adx;
	char buf[256];

	va_start(adx, fmt);
	(void) vsnprintf(buf, sizeof (buf), fmt, adx);
	va_end(adx);

	zfs_dbgmsg("spa_load(%s, config %s): FAILED: %s", spa->spa_name,
	spa->spa_trust_config ? "trusted" : "untrusted", buf);
	}

	void
	spa_load_note(spa_t spa, const char fmt, ...)
	{
	va_list adx;
	char buf[256];

	va_start(adx, fmt);
	(void) vsnprintf(buf, sizeof (buf), fmt, adx);
	va_end(adx);

	zfs_dbgmsg("spa_load(%s, config %s): %s", spa->spa_name,
	spa->spa_trust_config ? "trusted" : "untrusted", buf);
	+
	+ spa_import_progress_set_notes_nolog(spa, "%s", buf);
	}

	/*
	* By default dedup and user data indirects land in the special class
	*/
	static int zfs_ddt_data_is_special = B_TRUE;
	static int zfs_user_indirect_is_special = B_TRUE;

	/*
	* The percentage of special class final space reserved for metadata only.
	* Once we allocate 100 - zfs_special_class_metadata_reserve_pct we only
	* let metadata into the class.
	*/
	static uint_t zfs_special_class_metadata_reserve_pct = 25;

	/*
	* ==========================================================================
	* SPA config locking
	* ==========================================================================
	*/
	static void
	spa_config_lock_init(spa_t *spa)
	{
	for (int i = 0; i < SCL_LOCKS; i++) {
	spa_config_lock_t *scl = &spa->spa_config_lock[i];
	mutex_init(&scl->scl_lock, NULL, MUTEX_DEFAULT, NULL);
	cv_init(&scl->scl_cv, NULL, CV_DEFAULT, NULL);
	scl->scl_writer = NULL;
	scl->scl_write_wanted = 0;
	scl->scl_count = 0;
	}
	}

	static void
	spa_config_lock_destroy(spa_t *spa)
	{
	for (int i = 0; i < SCL_LOCKS; i++) {
	spa_config_lock_t *scl = &spa->spa_config_lock[i];
	mutex_destroy(&scl->scl_lock);
	cv_destroy(&scl->scl_cv);
	ASSERT(scl->scl_writer == NULL);
	ASSERT(scl->scl_write_wanted == 0);
	ASSERT(scl->scl_count == 0);
	}
	}

	int
	spa_config_tryenter(spa_t spa, int locks, const void tag, krw_t rw)
	{
	for (int i = 0; i < SCL_LOCKS; i++) {
	spa_config_lock_t *scl = &spa->spa_config_lock[i];
	if (!(locks & (1 << i)))
	continue;
	mutex_enter(&scl->scl_lock);
	if (rw == RW_READER) {
	if (scl->scl_writer \|\| scl->scl_write_wanted) {
	mutex_exit(&scl->scl_lock);
	spa_config_exit(spa, locks & ((1 << i) - 1),
	tag);
	return (0);
	}
	} else {
	ASSERT(scl->scl_writer != curthread);
	if (scl->scl_count != 0) {
	mutex_exit(&scl->scl_lock);
	spa_config_exit(spa, locks & ((1 << i) - 1),
	tag);
	return (0);
	}
	scl->scl_writer = curthread;
	}
	scl->scl_count++;
	mutex_exit(&scl->scl_lock);
	}
	return (1);
	}

	static void
	spa_config_enter_impl(spa_t spa, int locks, const void tag, krw_t rw,
	int mmp_flag)
	{
	(void) tag;
	int wlocks_held = 0;

	ASSERT3U(SCL_LOCKS, <, sizeof (wlocks_held) * NBBY);

	for (int i = 0; i < SCL_LOCKS; i++) {
	spa_config_lock_t *scl = &spa->spa_config_lock[i];
	if (scl->scl_writer == curthread)
	wlocks_held \|= (1 << i);
	if (!(locks & (1 << i)))
	continue;
	mutex_enter(&scl->scl_lock);
	if (rw == RW_READER) {
	while (scl->scl_writer \|\|
	(!mmp_flag && scl->scl_write_wanted)) {
	cv_wait(&scl->scl_cv, &scl->scl_lock);
	}
	} else {
	ASSERT(scl->scl_writer != curthread);
	while (scl->scl_count != 0) {
	scl->scl_write_wanted++;
	cv_wait(&scl->scl_cv, &scl->scl_lock);
	scl->scl_write_wanted--;
	}
	scl->scl_writer = curthread;
	}
	scl->scl_count++;
	mutex_exit(&scl->scl_lock);
	}
	ASSERT3U(wlocks_held, <=, locks);
	}

	void
	spa_config_enter(spa_t spa, int locks, const void tag, krw_t rw)
	{
	spa_config_enter_impl(spa, locks, tag, rw, 0);
	}

	/*
	* The spa_config_enter_mmp() allows the mmp thread to cut in front of
	* outstanding write lock requests. This is needed since the mmp updates are
	* time sensitive and failure to service them promptly will result in a
	* suspended pool. This pool suspension has been seen in practice when there is
	* a single disk in a pool that is responding slowly and presumably about to
	* fail.
	*/

	void
	spa_config_enter_mmp(spa_t spa, int locks, const void tag, krw_t rw)
	{
	spa_config_enter_impl(spa, locks, tag, rw, 1);
	}

	void
	spa_config_exit(spa_t spa, int locks, const void tag)
	{
	(void) tag;
	for (int i = SCL_LOCKS - 1; i >= 0; i--) {
	spa_config_lock_t *scl = &spa->spa_config_lock[i];
	if (!(locks & (1 << i)))
	continue;
	mutex_enter(&scl->scl_lock);
	ASSERT(scl->scl_count > 0);
	if (--scl->scl_count == 0) {
	ASSERT(scl->scl_writer == NULL \|\|
	scl->scl_writer == curthread);
	scl->scl_writer = NULL; /* OK in either case */
	cv_broadcast(&scl->scl_cv);
	}
	mutex_exit(&scl->scl_lock);
	}
	}

	int
	spa_config_held(spa_t *spa, int locks, krw_t rw)
	{
	int locks_held = 0;

	for (int i = 0; i < SCL_LOCKS; i++) {
	spa_config_lock_t *scl = &spa->spa_config_lock[i];
	if (!(locks & (1 << i)))
	continue;
	if ((rw == RW_READER && scl->scl_count != 0) \|\|
	(rw == RW_WRITER && scl->scl_writer == curthread))
	locks_held \|= 1 << i;
	}

	return (locks_held);
	}

	/*
	* ==========================================================================
	* SPA namespace functions
	* ==========================================================================
	*/

	/*
	* Lookup the named spa_t in the AVL tree. The spa_namespace_lock must be held.
	* Returns NULL if no matching spa_t is found.
	*/
	spa_t *
	spa_lookup(const char *name)
	{
	static spa_t search; /* spa_t is large; don't allocate on stack */
	spa_t *spa;
	avl_index_t where;
	char *cp;

	ASSERT(MUTEX_HELD(&spa_namespace_lock));

	(void) strlcpy(search.spa_name, name, sizeof (search.spa_name));

	/*
	* If it's a full dataset name, figure out the pool name and
	* just use that.
	*/
	cp = strpbrk(search.spa_name, "/@#");
	if (cp != NULL)
	*cp = '\0';

	spa = avl_find(&spa_namespace_avl, &search, &where);

	return (spa);
	}

	/*
	* Fires when spa_sync has not completed within zfs_deadman_synctime_ms.
	* If the zfs_deadman_enabled flag is set then it inspects all vdev queues
	* looking for potentially hung I/Os.
	*/
	void
	spa_deadman(void *arg)
	{
	spa_t *spa = arg;

	/* Disable the deadman if the pool is suspended. */
	if (spa_suspended(spa))
	return;

	zfs_dbgmsg("slow spa_sync: started %llu seconds ago, calls %llu",
	(gethrtime() - spa->spa_sync_starttime) / NANOSEC,
	(u_longlong_t)++spa->spa_deadman_calls);
	if (zfs_deadman_enabled)
	vdev_deadman(spa->spa_root_vdev, FTAG);

	spa->spa_deadman_tqid = taskq_dispatch_delay(system_delay_taskq,
	spa_deadman, spa, TQ_SLEEP, ddi_get_lbolt() +
	MSEC_TO_TICK(zfs_deadman_checktime_ms));
	}

	static int
	spa_log_sm_sort_by_txg(const void va, const void vb)
	{
	const spa_log_sm_t *a = va;
	const spa_log_sm_t *b = vb;

	return (TREE_CMP(a->sls_txg, b->sls_txg));
	}

	/*
	* Create an uninitialized spa_t with the given name. Requires
	* spa_namespace_lock. The caller must ensure that the spa_t doesn't already
	* exist by calling spa_lookup() first.
	*/
	spa_t *
	spa_add(const char name, nvlist_t config, const char *altroot)
	{
	spa_t *spa;
	spa_config_dirent_t *dp;

	ASSERT(MUTEX_HELD(&spa_namespace_lock));

	spa = kmem_zalloc(sizeof (spa_t), KM_SLEEP);

	mutex_init(&spa->spa_async_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&spa->spa_errlist_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&spa->spa_errlog_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&spa->spa_evicting_os_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&spa->spa_history_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&spa->spa_proc_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&spa->spa_props_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&spa->spa_cksum_tmpls_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&spa->spa_scrub_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&spa->spa_suspend_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&spa->spa_vdev_top_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&spa->spa_feat_stats_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&spa->spa_flushed_ms_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&spa->spa_activities_lock, NULL, MUTEX_DEFAULT, NULL);

	cv_init(&spa->spa_async_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&spa->spa_evicting_os_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&spa->spa_proc_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&spa->spa_scrub_io_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&spa->spa_suspend_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&spa->spa_activities_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&spa->spa_waiters_cv, NULL, CV_DEFAULT, NULL);

	for (int t = 0; t < TXG_SIZE; t++)
	bplist_create(&spa->spa_free_bplist[t]);

	(void) strlcpy(spa->spa_name, name, sizeof (spa->spa_name));
	spa->spa_state = POOL_STATE_UNINITIALIZED;
	spa->spa_freeze_txg = UINT64_MAX;
	spa->spa_final_txg = UINT64_MAX;
	spa->spa_load_max_txg = UINT64_MAX;
	spa->spa_proc = &p0;
	spa->spa_proc_state = SPA_PROC_NONE;
	spa->spa_trust_config = B_TRUE;
	spa->spa_hostid = zone_get_hostid(NULL);

	spa->spa_deadman_synctime = MSEC2NSEC(zfs_deadman_synctime_ms);
	spa->spa_deadman_ziotime = MSEC2NSEC(zfs_deadman_ziotime_ms);
	spa_set_deadman_failmode(spa, zfs_deadman_failmode);

	zfs_refcount_create(&spa->spa_refcount);
	spa_config_lock_init(spa);
	spa_stats_init(spa);

	avl_add(&spa_namespace_avl, spa);

	/*
	* Set the alternate root, if there is one.
	*/
	if (altroot)
	spa->spa_root = spa_strdup(altroot);

	spa->spa_alloc_count = spa_allocators;
	spa->spa_allocs = kmem_zalloc(spa->spa_alloc_count *
	sizeof (spa_alloc_t), KM_SLEEP);
	for (int i = 0; i < spa->spa_alloc_count; i++) {
	mutex_init(&spa->spa_allocs[i].spaa_lock, NULL, MUTEX_DEFAULT,
	NULL);
	avl_create(&spa->spa_allocs[i].spaa_tree, zio_bookmark_compare,
	sizeof (zio_t), offsetof(zio_t, io_queue_node.a));
	}
	avl_create(&spa->spa_metaslabs_by_flushed, metaslab_sort_by_flushed,
	sizeof (metaslab_t), offsetof(metaslab_t, ms_spa_txg_node));
	avl_create(&spa->spa_sm_logs_by_txg, spa_log_sm_sort_by_txg,
	sizeof (spa_log_sm_t), offsetof(spa_log_sm_t, sls_node));
	list_create(&spa->spa_log_summary, sizeof (log_summary_entry_t),
	offsetof(log_summary_entry_t, lse_node));

	/*
	* Every pool starts with the default cachefile
	*/
	list_create(&spa->spa_config_list, sizeof (spa_config_dirent_t),
	offsetof(spa_config_dirent_t, scd_link));

	dp = kmem_zalloc(sizeof (spa_config_dirent_t), KM_SLEEP);
	dp->scd_path = altroot ? NULL : spa_strdup(spa_config_path);
	list_insert_head(&spa->spa_config_list, dp);

	VERIFY(nvlist_alloc(&spa->spa_load_info, NV_UNIQUE_NAME,
	KM_SLEEP) == 0);

	if (config != NULL) {
	nvlist_t *features;

	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_FEATURES_FOR_READ,
	&features) == 0) {
	VERIFY(nvlist_dup(features, &spa->spa_label_features,
	0) == 0);
	}

	VERIFY(nvlist_dup(config, &spa->spa_config, 0) == 0);
	}

	if (spa->spa_label_features == NULL) {
	VERIFY(nvlist_alloc(&spa->spa_label_features, NV_UNIQUE_NAME,
	KM_SLEEP) == 0);
	}

	spa->spa_min_ashift = INT_MAX;
	spa->spa_max_ashift = 0;
	spa->spa_min_alloc = INT_MAX;
	spa->spa_gcd_alloc = INT_MAX;

	/* Reset cached value */
	spa->spa_dedup_dspace = ~0ULL;

	/*
	* As a pool is being created, treat all features as disabled by
	* setting SPA_FEATURE_DISABLED for all entries in the feature
	* refcount cache.
	*/
	for (int i = 0; i < SPA_FEATURES; i++) {
	spa->spa_feat_refcount_cache[i] = SPA_FEATURE_DISABLED;
	}

	list_create(&spa->spa_leaf_list, sizeof (vdev_t),
	offsetof(vdev_t, vdev_leaf_node));

	return (spa);
	}

	/*
	* Removes a spa_t from the namespace, freeing up any memory used. Requires
	* spa_namespace_lock. This is called only after the spa_t has been closed and
	* deactivated.
	*/
	void
	spa_remove(spa_t *spa)
	{
	spa_config_dirent_t *dp;

	ASSERT(MUTEX_HELD(&spa_namespace_lock));
	ASSERT(spa_state(spa) == POOL_STATE_UNINITIALIZED);
	ASSERT3U(zfs_refcount_count(&spa->spa_refcount), ==, 0);
	ASSERT0(spa->spa_waiters);

	nvlist_free(spa->spa_config_splitting);

	avl_remove(&spa_namespace_avl, spa);
	cv_broadcast(&spa_namespace_cv);

	if (spa->spa_root)
	spa_strfree(spa->spa_root);

	while ((dp = list_remove_head(&spa->spa_config_list)) != NULL) {
	if (dp->scd_path != NULL)
	spa_strfree(dp->scd_path);
	kmem_free(dp, sizeof (spa_config_dirent_t));
	}

	for (int i = 0; i < spa->spa_alloc_count; i++) {
	avl_destroy(&spa->spa_allocs[i].spaa_tree);
	mutex_destroy(&spa->spa_allocs[i].spaa_lock);
	}
	kmem_free(spa->spa_allocs, spa->spa_alloc_count *
	sizeof (spa_alloc_t));

	avl_destroy(&spa->spa_metaslabs_by_flushed);
	avl_destroy(&spa->spa_sm_logs_by_txg);
	list_destroy(&spa->spa_log_summary);
	list_destroy(&spa->spa_config_list);
	list_destroy(&spa->spa_leaf_list);

	nvlist_free(spa->spa_label_features);
	nvlist_free(spa->spa_load_info);
	nvlist_free(spa->spa_feat_stats);
	spa_config_set(spa, NULL);

	zfs_refcount_destroy(&spa->spa_refcount);

	spa_stats_destroy(spa);
	spa_config_lock_destroy(spa);

	for (int t = 0; t < TXG_SIZE; t++)
	bplist_destroy(&spa->spa_free_bplist[t]);

	zio_checksum_templates_free(spa);

	cv_destroy(&spa->spa_async_cv);
	cv_destroy(&spa->spa_evicting_os_cv);
	cv_destroy(&spa->spa_proc_cv);
	cv_destroy(&spa->spa_scrub_io_cv);
	cv_destroy(&spa->spa_suspend_cv);
	cv_destroy(&spa->spa_activities_cv);
	cv_destroy(&spa->spa_waiters_cv);

	mutex_destroy(&spa->spa_flushed_ms_lock);
	mutex_destroy(&spa->spa_async_lock);
	mutex_destroy(&spa->spa_errlist_lock);
	mutex_destroy(&spa->spa_errlog_lock);
	mutex_destroy(&spa->spa_evicting_os_lock);
	mutex_destroy(&spa->spa_history_lock);
	mutex_destroy(&spa->spa_proc_lock);
	mutex_destroy(&spa->spa_props_lock);
	mutex_destroy(&spa->spa_cksum_tmpls_lock);
	mutex_destroy(&spa->spa_scrub_lock);
	mutex_destroy(&spa->spa_suspend_lock);
	mutex_destroy(&spa->spa_vdev_top_lock);
	mutex_destroy(&spa->spa_feat_stats_lock);
	mutex_destroy(&spa->spa_activities_lock);

	kmem_free(spa, sizeof (spa_t));
	}

	/*
	* Given a pool, return the next pool in the namespace, or NULL if there is
	* none. If 'prev' is NULL, return the first pool.
	*/
	spa_t *
	spa_next(spa_t *prev)
	{
	ASSERT(MUTEX_HELD(&spa_namespace_lock));

	if (prev)
	return (AVL_NEXT(&spa_namespace_avl, prev));
	else
	return (avl_first(&spa_namespace_avl));
	}

	/*
	* ==========================================================================
	* SPA refcount functions
	* ==========================================================================
	*/

	/*
	* Add a reference to the given spa_t. Must have at least one reference, or
	* have the namespace lock held.
	*/
	void
	spa_open_ref(spa_t spa, const void tag)
	{
	ASSERT(zfs_refcount_count(&spa->spa_refcount) >= spa->spa_minref \|\|
	MUTEX_HELD(&spa_namespace_lock));
	(void) zfs_refcount_add(&spa->spa_refcount, tag);
	}

	/*
	* Remove a reference to the given spa_t. Must have at least one reference, or
	* have the namespace lock held.
	*/
	void
	spa_close(spa_t spa, const void tag)
	{
	ASSERT(zfs_refcount_count(&spa->spa_refcount) > spa->spa_minref \|\|
	MUTEX_HELD(&spa_namespace_lock));
	(void) zfs_refcount_remove(&spa->spa_refcount, tag);
	}

	/*
	* Remove a reference to the given spa_t held by a dsl dir that is
	* being asynchronously released. Async releases occur from a taskq
	* performing eviction of dsl datasets and dirs. The namespace lock
	* isn't held and the hold by the object being evicted may contribute to
	* spa_minref (e.g. dataset or directory released during pool export),
	* so the asserts in spa_close() do not apply.
	*/
	void
	spa_async_close(spa_t spa, const void tag)
	{
	(void) zfs_refcount_remove(&spa->spa_refcount, tag);
	}

	/*
	* Check to see if the spa refcount is zero. Must be called with
	* spa_namespace_lock held. We really compare against spa_minref, which is the
	* number of references acquired when opening a pool
	*/
	boolean_t
	spa_refcount_zero(spa_t *spa)
	{
	ASSERT(MUTEX_HELD(&spa_namespace_lock));

	return (zfs_refcount_count(&spa->spa_refcount) == spa->spa_minref);
	}

	/*
	* ==========================================================================
	* SPA spare and l2cache tracking
	* ==========================================================================
	*/

	/*
	* Hot spares and cache devices are tracked using the same code below,
	* for 'auxiliary' devices.
	*/

	typedef struct spa_aux {
	uint64_t aux_guid;
	uint64_t aux_pool;
	avl_node_t aux_avl;
	int aux_count;
	} spa_aux_t;

	static inline int
	spa_aux_compare(const void a, const void b)
	{
	const spa_aux_t sa = (const spa_aux_t )a;
	const spa_aux_t sb = (const spa_aux_t )b;

	return (TREE_CMP(sa->aux_guid, sb->aux_guid));
	}

	static void
	spa_aux_add(vdev_t vd, avl_tree_t avl)
	{
	avl_index_t where;
	spa_aux_t search;
	spa_aux_t *aux;

	search.aux_guid = vd->vdev_guid;
	if ((aux = avl_find(avl, &search, &where)) != NULL) {
	aux->aux_count++;
	} else {
	aux = kmem_zalloc(sizeof (spa_aux_t), KM_SLEEP);
	aux->aux_guid = vd->vdev_guid;
	aux->aux_count = 1;
	avl_insert(avl, aux, where);
	}
	}

	static void
	spa_aux_remove(vdev_t vd, avl_tree_t avl)
	{
	spa_aux_t search;
	spa_aux_t *aux;
	avl_index_t where;

	search.aux_guid = vd->vdev_guid;
	aux = avl_find(avl, &search, &where);

	ASSERT(aux != NULL);

	if (--aux->aux_count == 0) {
	avl_remove(avl, aux);
	kmem_free(aux, sizeof (spa_aux_t));
	} else if (aux->aux_pool == spa_guid(vd->vdev_spa)) {
	aux->aux_pool = 0ULL;
	}
	}

	static boolean_t
	spa_aux_exists(uint64_t guid, uint64_t pool, int refcnt, avl_tree_t *avl)
	{
	spa_aux_t search, *found;

	search.aux_guid = guid;
	found = avl_find(avl, &search, NULL);

	if (pool) {
	if (found)
	*pool = found->aux_pool;
	else
	*pool = 0ULL;
	}

	if (refcnt) {
	if (found)
	*refcnt = found->aux_count;
	else
	*refcnt = 0;
	}

	return (found != NULL);
	}

	static void
	spa_aux_activate(vdev_t vd, avl_tree_t avl)
	{
	spa_aux_t search, *found;
	avl_index_t where;

	search.aux_guid = vd->vdev_guid;
	found = avl_find(avl, &search, &where);
	ASSERT(found != NULL);
	ASSERT(found->aux_pool == 0ULL);

	found->aux_pool = spa_guid(vd->vdev_spa);
	}

	/*
	* Spares are tracked globally due to the following constraints:
	*
	* - A spare may be part of multiple pools.
	* - A spare may be added to a pool even if it's actively in use within
	* another pool.
	* - A spare in use in any pool can only be the source of a replacement if
	* the target is a spare in the same pool.
	*
	* We keep track of all spares on the system through the use of a reference
	* counted AVL tree. When a vdev is added as a spare, or used as a replacement
	* spare, then we bump the reference count in the AVL tree. In addition, we set
	* the 'vdev_isspare' member to indicate that the device is a spare (active or
	* inactive). When a spare is made active (used to replace a device in the
	* pool), we also keep track of which pool its been made a part of.
	*
	* The 'spa_spare_lock' protects the AVL tree. These functions are normally
	* called under the spa_namespace lock as part of vdev reconfiguration. The
	* separate spare lock exists for the status query path, which does not need to
	* be completely consistent with respect to other vdev configuration changes.
	*/

	static int
	spa_spare_compare(const void a, const void b)
	{
	return (spa_aux_compare(a, b));
	}

	void
	spa_spare_add(vdev_t *vd)
	{
	mutex_enter(&spa_spare_lock);
	ASSERT(!vd->vdev_isspare);
	spa_aux_add(vd, &spa_spare_avl);
	vd->vdev_isspare = B_TRUE;
	mutex_exit(&spa_spare_lock);
	}

	void
	spa_spare_remove(vdev_t *vd)
	{
	mutex_enter(&spa_spare_lock);
	ASSERT(vd->vdev_isspare);
	spa_aux_remove(vd, &spa_spare_avl);
	vd->vdev_isspare = B_FALSE;
	mutex_exit(&spa_spare_lock);
	}

	boolean_t
	spa_spare_exists(uint64_t guid, uint64_t pool, int refcnt)
	{
	boolean_t found;

	mutex_enter(&spa_spare_lock);
	found = spa_aux_exists(guid, pool, refcnt, &spa_spare_avl);
	mutex_exit(&spa_spare_lock);

	return (found);
	}

	void
	spa_spare_activate(vdev_t *vd)
	{
	mutex_enter(&spa_spare_lock);
	ASSERT(vd->vdev_isspare);
	spa_aux_activate(vd, &spa_spare_avl);
	mutex_exit(&spa_spare_lock);
	}

	/*
	* Level 2 ARC devices are tracked globally for the same reasons as spares.
	* Cache devices currently only support one pool per cache device, and so
	* for these devices the aux reference count is currently unused beyond 1.
	*/

	static int
	spa_l2cache_compare(const void a, const void b)
	{
	return (spa_aux_compare(a, b));
	}

	void
	spa_l2cache_add(vdev_t *vd)
	{
	mutex_enter(&spa_l2cache_lock);
	ASSERT(!vd->vdev_isl2cache);
	spa_aux_add(vd, &spa_l2cache_avl);
	vd->vdev_isl2cache = B_TRUE;
	mutex_exit(&spa_l2cache_lock);
	}

	void
	spa_l2cache_remove(vdev_t *vd)
	{
	mutex_enter(&spa_l2cache_lock);
	ASSERT(vd->vdev_isl2cache);
	spa_aux_remove(vd, &spa_l2cache_avl);
	vd->vdev_isl2cache = B_FALSE;
	mutex_exit(&spa_l2cache_lock);
	}

	boolean_t
	spa_l2cache_exists(uint64_t guid, uint64_t *pool)
	{
	boolean_t found;

	mutex_enter(&spa_l2cache_lock);
	found = spa_aux_exists(guid, pool, NULL, &spa_l2cache_avl);
	mutex_exit(&spa_l2cache_lock);

	return (found);
	}

	void
	spa_l2cache_activate(vdev_t *vd)
	{
	mutex_enter(&spa_l2cache_lock);
	ASSERT(vd->vdev_isl2cache);
	spa_aux_activate(vd, &spa_l2cache_avl);
	mutex_exit(&spa_l2cache_lock);
	}

	/*
	* ==========================================================================
	* SPA vdev locking
	* ==========================================================================
	*/

	/*
	* Lock the given spa_t for the purpose of adding or removing a vdev.
	* Grabs the global spa_namespace_lock plus the spa config lock for writing.
	* It returns the next transaction group for the spa_t.
	*/
	uint64_t
	spa_vdev_enter(spa_t *spa)
	{
	mutex_enter(&spa->spa_vdev_top_lock);
	mutex_enter(&spa_namespace_lock);

	vdev_autotrim_stop_all(spa);

	return (spa_vdev_config_enter(spa));
	}

	/*
	* The same as spa_vdev_enter() above but additionally takes the guid of
	* the vdev being detached. When there is a rebuild in process it will be
	* suspended while the vdev tree is modified then resumed by spa_vdev_exit().
	* The rebuild is canceled if only a single child remains after the detach.
	*/
	uint64_t
	spa_vdev_detach_enter(spa_t *spa, uint64_t guid)
	{
	mutex_enter(&spa->spa_vdev_top_lock);
	mutex_enter(&spa_namespace_lock);

	vdev_autotrim_stop_all(spa);

	if (guid != 0) {
	vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE);
	if (vd) {
	vdev_rebuild_stop_wait(vd->vdev_top);
	}
	}

	return (spa_vdev_config_enter(spa));
	}

	/*
	* Internal implementation for spa_vdev_enter(). Used when a vdev
	* operation requires multiple syncs (i.e. removing a device) while
	* keeping the spa_namespace_lock held.
	*/
	uint64_t
	spa_vdev_config_enter(spa_t *spa)
	{
	ASSERT(MUTEX_HELD(&spa_namespace_lock));

	spa_config_enter(spa, SCL_ALL, spa, RW_WRITER);

	return (spa_last_synced_txg(spa) + 1);
	}

	/*
	* Used in combination with spa_vdev_config_enter() to allow the syncing
	* of multiple transactions without releasing the spa_namespace_lock.
	*/
	void
	spa_vdev_config_exit(spa_t spa, vdev_t vd, uint64_t txg, int error,
	const char *tag)
	{
	ASSERT(MUTEX_HELD(&spa_namespace_lock));

	int config_changed = B_FALSE;

	ASSERT(txg > spa_last_synced_txg(spa));

	spa->spa_pending_vdev = NULL;

	/*
	* Reassess the DTLs.
	*/
	vdev_dtl_reassess(spa->spa_root_vdev, 0, 0, B_FALSE, B_FALSE);

	if (error == 0 && !list_is_empty(&spa->spa_config_dirty_list)) {
	config_changed = B_TRUE;
	spa->spa_config_generation++;
	}

	/*
	* Verify the metaslab classes.
	*/
	ASSERT(metaslab_class_validate(spa_normal_class(spa)) == 0);
	ASSERT(metaslab_class_validate(spa_log_class(spa)) == 0);
	ASSERT(metaslab_class_validate(spa_embedded_log_class(spa)) == 0);
	ASSERT(metaslab_class_validate(spa_special_class(spa)) == 0);
	ASSERT(metaslab_class_validate(spa_dedup_class(spa)) == 0);

	spa_config_exit(spa, SCL_ALL, spa);

	/*
	* Panic the system if the specified tag requires it. This
	* is useful for ensuring that configurations are updated
	* transactionally.
	*/
	if (zio_injection_enabled)
	zio_handle_panic_injection(spa, tag, 0);

	/*
	* Note: this txg_wait_synced() is important because it ensures
	* that there won't be more than one config change per txg.
	* This allows us to use the txg as the generation number.
	*/
	if (error == 0)
	txg_wait_synced(spa->spa_dsl_pool, txg);

	if (vd != NULL) {
	ASSERT(!vd->vdev_detached \|\| vd->vdev_dtl_sm == NULL);
	if (vd->vdev_ops->vdev_op_leaf) {
	mutex_enter(&vd->vdev_initialize_lock);
	vdev_initialize_stop(vd, VDEV_INITIALIZE_CANCELED,
	NULL);
	mutex_exit(&vd->vdev_initialize_lock);

	mutex_enter(&vd->vdev_trim_lock);
	vdev_trim_stop(vd, VDEV_TRIM_CANCELED, NULL);
	mutex_exit(&vd->vdev_trim_lock);
	}

	/*
	* The vdev may be both a leaf and top-level device.
	*/
	vdev_autotrim_stop_wait(vd);

	spa_config_enter(spa, SCL_STATE_ALL, spa, RW_WRITER);
	vdev_free(vd);
	spa_config_exit(spa, SCL_STATE_ALL, spa);
	}

	/*
	* If the config changed, update the config cache.
	*/
	if (config_changed)
	spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE);
	}

	/*
	* Unlock the spa_t after adding or removing a vdev. Besides undoing the
	* locking of spa_vdev_enter(), we also want make sure the transactions have
	* synced to disk, and then update the global configuration cache with the new
	* information.
	*/
	int
	spa_vdev_exit(spa_t spa, vdev_t vd, uint64_t txg, int error)
	{
	vdev_autotrim_restart(spa);
	vdev_rebuild_restart(spa);

	spa_vdev_config_exit(spa, vd, txg, error, FTAG);
	mutex_exit(&spa_namespace_lock);
	mutex_exit(&spa->spa_vdev_top_lock);

	return (error);
	}

	/*
	* Lock the given spa_t for the purpose of changing vdev state.
	*/
	void
	spa_vdev_state_enter(spa_t *spa, int oplocks)
	{
	int locks = SCL_STATE_ALL \| oplocks;

	/*
	* Root pools may need to read of the underlying devfs filesystem
	* when opening up a vdev. Unfortunately if we're holding the
	* SCL_ZIO lock it will result in a deadlock when we try to issue
	* the read from the root filesystem. Instead we "prefetch"
	* the associated vnodes that we need prior to opening the
	* underlying devices and cache them so that we can prevent
	* any I/O when we are doing the actual open.
	*/
	if (spa_is_root(spa)) {
	int low = locks & ~(SCL_ZIO - 1);
	int high = locks & ~low;

	spa_config_enter(spa, high, spa, RW_WRITER);
	vdev_hold(spa->spa_root_vdev);
	spa_config_enter(spa, low, spa, RW_WRITER);
	} else {
	spa_config_enter(spa, locks, spa, RW_WRITER);
	}
	spa->spa_vdev_locks = locks;
	}

	int
	spa_vdev_state_exit(spa_t spa, vdev_t vd, int error)
	{
	boolean_t config_changed = B_FALSE;
	vdev_t *vdev_top;

	if (vd == NULL \|\| vd == spa->spa_root_vdev) {
	vdev_top = spa->spa_root_vdev;
	} else {
	vdev_top = vd->vdev_top;
	}

	if (vd != NULL \|\| error == 0)
	vdev_dtl_reassess(vdev_top, 0, 0, B_FALSE, B_FALSE);

	if (vd != NULL) {
	if (vd != spa->spa_root_vdev)
	vdev_state_dirty(vdev_top);

	config_changed = B_TRUE;
	spa->spa_config_generation++;
	}

	if (spa_is_root(spa))
	vdev_rele(spa->spa_root_vdev);

	ASSERT3U(spa->spa_vdev_locks, >=, SCL_STATE_ALL);
	spa_config_exit(spa, spa->spa_vdev_locks, spa);

	/*
	* If anything changed, wait for it to sync. This ensures that,
	* from the system administrator's perspective, zpool(8) commands
	* are synchronous. This is important for things like zpool offline:
	* when the command completes, you expect no further I/O from ZFS.
	*/
	if (vd != NULL)
	txg_wait_synced(spa->spa_dsl_pool, 0);

	/*
	* If the config changed, update the config cache.
	*/
	if (config_changed) {
	mutex_enter(&spa_namespace_lock);
	spa_write_cachefile(spa, B_FALSE, B_TRUE, B_FALSE);
	mutex_exit(&spa_namespace_lock);
	}

	return (error);
	}

	/*
	* ==========================================================================
	* Miscellaneous functions
	* ==========================================================================
	*/

	void
	spa_activate_mos_feature(spa_t spa, const char feature, dmu_tx_t *tx)
	{
	if (!nvlist_exists(spa->spa_label_features, feature)) {
	fnvlist_add_boolean(spa->spa_label_features, feature);
	/*
	* When we are creating the pool (tx_txg==TXG_INITIAL), we can't
	* dirty the vdev config because lock SCL_CONFIG is not held.
	* Thankfully, in this case we don't need to dirty the config
	* because it will be written out anyway when we finish
	* creating the pool.
	*/
	if (tx->tx_txg != TXG_INITIAL)
	vdev_config_dirty(spa->spa_root_vdev);
	}
	}

	void
	spa_deactivate_mos_feature(spa_t spa, const char feature)
	{
	if (nvlist_remove_all(spa->spa_label_features, feature) == 0)
	vdev_config_dirty(spa->spa_root_vdev);
	}

	/*
	* Return the spa_t associated with given pool_guid, if it exists. If
	* device_guid is non-zero, determine whether the pool exists and contains
	* a device with the specified device_guid.
	*/
	spa_t *
	spa_by_guid(uint64_t pool_guid, uint64_t device_guid)
	{
	spa_t *spa;
	avl_tree_t *t = &spa_namespace_avl;

	ASSERT(MUTEX_HELD(&spa_namespace_lock));

	for (spa = avl_first(t); spa != NULL; spa = AVL_NEXT(t, spa)) {
	if (spa->spa_state == POOL_STATE_UNINITIALIZED)
	continue;
	if (spa->spa_root_vdev == NULL)
	continue;
	if (spa_guid(spa) == pool_guid) {
	if (device_guid == 0)
	break;

	if (vdev_lookup_by_guid(spa->spa_root_vdev,
	device_guid) != NULL)
	break;

	/*
	* Check any devices we may be in the process of adding.
	*/
	if (spa->spa_pending_vdev) {
	if (vdev_lookup_by_guid(spa->spa_pending_vdev,
	device_guid) != NULL)
	break;
	}
	}
	}

	return (spa);
	}

	/*
	* Determine whether a pool with the given pool_guid exists.
	*/
	boolean_t
	spa_guid_exists(uint64_t pool_guid, uint64_t device_guid)
	{
	return (spa_by_guid(pool_guid, device_guid) != NULL);
	}

	char *
	spa_strdup(const char *s)
	{
	size_t len;
	char *new;

	len = strlen(s);
	new = kmem_alloc(len + 1, KM_SLEEP);
	memcpy(new, s, len + 1);

	return (new);
	}

	void
	spa_strfree(char *s)
	{
	kmem_free(s, strlen(s) + 1);
	}

	uint64_t
	spa_generate_guid(spa_t *spa)
	{
	uint64_t guid;

	if (spa != NULL) {
	do {
	(void) random_get_pseudo_bytes((void *)&guid,
	sizeof (guid));
	} while (guid == 0 \|\| spa_guid_exists(spa_guid(spa), guid));
	} else {
	do {
	(void) random_get_pseudo_bytes((void *)&guid,
	sizeof (guid));
	} while (guid == 0 \|\| spa_guid_exists(guid, 0));
	}

	return (guid);
	}

	void
	snprintf_blkptr(char buf, size_t buflen, const blkptr_t bp)
	{
	char type[256];
	const char *checksum = NULL;
	const char *compress = NULL;

	if (bp != NULL) {
	if (BP_GET_TYPE(bp) & DMU_OT_NEWTYPE) {
	dmu_object_byteswap_t bswap =
	DMU_OT_BYTESWAP(BP_GET_TYPE(bp));
	(void) snprintf(type, sizeof (type), "bswap %s %s",
	DMU_OT_IS_METADATA(BP_GET_TYPE(bp)) ?
	"metadata" : "data",
	dmu_ot_byteswap[bswap].ob_name);
	} else {
	(void) strlcpy(type, dmu_ot[BP_GET_TYPE(bp)].ot_name,
	sizeof (type));
	}
	if (!BP_IS_EMBEDDED(bp)) {
	checksum =
	zio_checksum_table[BP_GET_CHECKSUM(bp)].ci_name;
	}
	compress = zio_compress_table[BP_GET_COMPRESS(bp)].ci_name;
	}

	SNPRINTF_BLKPTR(kmem_scnprintf, ' ', buf, buflen, bp, type, checksum,
	compress);
	}

	void
	spa_freeze(spa_t *spa)
	{
	uint64_t freeze_txg = 0;

	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	if (spa->spa_freeze_txg == UINT64_MAX) {
	freeze_txg = spa_last_synced_txg(spa) + TXG_SIZE;
	spa->spa_freeze_txg = freeze_txg;
	}
	spa_config_exit(spa, SCL_ALL, FTAG);
	if (freeze_txg != 0)
	txg_wait_synced(spa_get_dsl(spa), freeze_txg);
	}

	void
	zfs_panic_recover(const char *fmt, ...)
	{
	va_list adx;

	va_start(adx, fmt);
	vcmn_err(zfs_recover ? CE_WARN : CE_PANIC, fmt, adx);
	va_end(adx);
	}

	/*
	* This is a stripped-down version of strtoull, suitable only for converting
	* lowercase hexadecimal numbers that don't overflow.
	*/
	uint64_t
	zfs_strtonum(const char str, char *nptr)
	{
	uint64_t val = 0;
	char c;
	int digit;

	while ((c = *str) != '\0') {
	if (c >= '0' && c <= '9')
	digit = c - '0';
	else if (c >= 'a' && c <= 'f')
	digit = 10 + c - 'a';
	else
	break;

	val *= 16;
	val += digit;

	str++;
	}

	if (nptr)
	nptr = (char )str;

	return (val);
	}

	void
	spa_activate_allocation_classes(spa_t spa, dmu_tx_t tx)
	{
	/*
	* We bump the feature refcount for each special vdev added to the pool
	*/
	ASSERT(spa_feature_is_enabled(spa, SPA_FEATURE_ALLOCATION_CLASSES));
	spa_feature_incr(spa, SPA_FEATURE_ALLOCATION_CLASSES, tx);
	}

	/*
	* ==========================================================================
	* Accessor functions
	* ==========================================================================
	*/

	boolean_t
	spa_shutting_down(spa_t *spa)
	{
	return (spa->spa_async_suspended);
	}

	dsl_pool_t *
	spa_get_dsl(spa_t *spa)
	{
	return (spa->spa_dsl_pool);
	}

	boolean_t
	spa_is_initializing(spa_t *spa)
	{
	return (spa->spa_is_initializing);
	}

	boolean_t
	spa_indirect_vdevs_loaded(spa_t *spa)
	{
	return (spa->spa_indirect_vdevs_loaded);
	}

	blkptr_t *
	spa_get_rootblkptr(spa_t *spa)
	{
	return (&spa->spa_ubsync.ub_rootbp);
	}

	void
	spa_set_rootblkptr(spa_t spa, const blkptr_t bp)
	{
	spa->spa_uberblock.ub_rootbp = *bp;
	}

	void
	spa_altroot(spa_t spa, char buf, size_t buflen)
	{
	if (spa->spa_root == NULL)
	buf[0] = '\0';
	else
	(void) strlcpy(buf, spa->spa_root, buflen);
	}

	uint32_t
	spa_sync_pass(spa_t *spa)
	{
	return (spa->spa_sync_pass);
	}

	char *
	spa_name(spa_t *spa)
	{
	return (spa->spa_name);
	}

	uint64_t
	spa_guid(spa_t *spa)
	{
	dsl_pool_t *dp = spa_get_dsl(spa);
	uint64_t guid;

	/*
	* If we fail to parse the config during spa_load(), we can go through
	* the error path (which posts an ereport) and end up here with no root
	* vdev. We stash the original pool guid in 'spa_config_guid' to handle
	* this case.
	*/
	if (spa->spa_root_vdev == NULL)
	return (spa->spa_config_guid);

	guid = spa->spa_last_synced_guid != 0 ?
	spa->spa_last_synced_guid : spa->spa_root_vdev->vdev_guid;

	/*
	* Return the most recently synced out guid unless we're
	* in syncing context.
	*/
	if (dp && dsl_pool_sync_context(dp))
	return (spa->spa_root_vdev->vdev_guid);
	else
	return (guid);
	}

	uint64_t
	spa_load_guid(spa_t *spa)
	{
	/*
	* This is a GUID that exists solely as a reference for the
	* purposes of the arc. It is generated at load time, and
	* is never written to persistent storage.
	*/
	return (spa->spa_load_guid);
	}

	uint64_t
	spa_last_synced_txg(spa_t *spa)
	{
	return (spa->spa_ubsync.ub_txg);
	}

	uint64_t
	spa_first_txg(spa_t *spa)
	{
	return (spa->spa_first_txg);
	}

	uint64_t
	spa_syncing_txg(spa_t *spa)
	{
	return (spa->spa_syncing_txg);
	}

	/*
	* Return the last txg where data can be dirtied. The final txgs
	* will be used to just clear out any deferred frees that remain.
	*/
	uint64_t
	spa_final_dirty_txg(spa_t *spa)
	{
	return (spa->spa_final_txg - TXG_DEFER_SIZE);
	}

	pool_state_t
	spa_state(spa_t *spa)
	{
	return (spa->spa_state);
	}

	spa_load_state_t
	spa_load_state(spa_t *spa)
	{
	return (spa->spa_load_state);
	}

	uint64_t
	spa_freeze_txg(spa_t *spa)
	{
	return (spa->spa_freeze_txg);
	}

	/*
	* Return the inflated asize for a logical write in bytes. This is used by the
	* DMU to calculate the space a logical write will require on disk.
	* If lsize is smaller than the largest physical block size allocatable on this
	* pool we use its value instead, since the write will end up using the whole
	* block anyway.
	*/
	uint64_t
	spa_get_worst_case_asize(spa_t *spa, uint64_t lsize)
	{
	if (lsize == 0)
	return (0); /* No inflation needed */
	return (MAX(lsize, 1 << spa->spa_max_ashift) * spa_asize_inflation);
	}

	/*
	* Return the amount of slop space in bytes. It is typically 1/32 of the pool
	* (3.2%), minus the embedded log space. On very small pools, it may be
	* slightly larger than this. On very large pools, it will be capped to
	* the value of spa_max_slop. The embedded log space is not included in
	* spa_dspace. By subtracting it, the usable space (per "zfs list") is a
	* constant 97% of the total space, regardless of metaslab size (assuming the
	* default spa_slop_shift=5 and a non-tiny pool).
	*
	* See the comment above spa_slop_shift for more details.
	*/
	uint64_t
	spa_get_slop_space(spa_t *spa)
	{
	uint64_t space = 0;
	uint64_t slop = 0;

	/*
	* Make sure spa_dedup_dspace has been set.
	*/
	if (spa->spa_dedup_dspace == ~0ULL)
	spa_update_dspace(spa);

	/*
	* spa_get_dspace() includes the space only logically "used" by
	* deduplicated data, so since it's not useful to reserve more
	* space with more deduplicated data, we subtract that out here.
	*/
	space =
	spa_get_dspace(spa) - spa->spa_dedup_dspace - brt_get_dspace(spa);
	slop = MIN(space >> spa_slop_shift, spa_max_slop);

	/*
	* Subtract the embedded log space, but no more than half the (3.2%)
	* unusable space. Note, the "no more than half" is only relevant if
	* zfs_embedded_slog_min_ms >> spa_slop_shift < 2, which is not true by
	* default.
	*/
	uint64_t embedded_log =
	metaslab_class_get_dspace(spa_embedded_log_class(spa));
	slop -= MIN(embedded_log, slop >> 1);

	/*
	* Slop space should be at least spa_min_slop, but no more than half
	* the entire pool.
	*/
	slop = MAX(slop, MIN(space >> 1, spa_min_slop));
	return (slop);
	}

	uint64_t
	spa_get_dspace(spa_t *spa)
	{
	return (spa->spa_dspace);
	}

	uint64_t
	spa_get_checkpoint_space(spa_t *spa)
	{
	return (spa->spa_checkpoint_info.sci_dspace);
	}

	void
	spa_update_dspace(spa_t *spa)
	{
	spa->spa_dspace = metaslab_class_get_dspace(spa_normal_class(spa)) +
	ddt_get_dedup_dspace(spa) + brt_get_dspace(spa);
	if (spa->spa_nonallocating_dspace > 0) {
	/*
	* Subtract the space provided by all non-allocating vdevs that
	* contribute to dspace. If a file is overwritten, its old
	* blocks are freed and new blocks are allocated. If there are
	* no snapshots of the file, the available space should remain
	* the same. The old blocks could be freed from the
	* non-allocating vdev, but the new blocks must be allocated on
	* other (allocating) vdevs. By reserving the entire size of
	* the non-allocating vdevs (including allocated space), we
	* ensure that there will be enough space on the allocating
	* vdevs for this file overwrite to succeed.
	*
	* Note that the DMU/DSL doesn't actually know or care
	* how much space is allocated (it does its own tracking
	* of how much space has been logically used). So it
	* doesn't matter that the data we are moving may be
	* allocated twice (on the old device and the new device).
	*/
	ASSERT3U(spa->spa_dspace, >=, spa->spa_nonallocating_dspace);
	spa->spa_dspace -= spa->spa_nonallocating_dspace;
	}
	}

	/*
	* Return the failure mode that has been set to this pool. The default
	* behavior will be to block all I/Os when a complete failure occurs.
	*/
	uint64_t
	spa_get_failmode(spa_t *spa)
	{
	return (spa->spa_failmode);
	}

	boolean_t
	spa_suspended(spa_t *spa)
	{
	return (spa->spa_suspended != ZIO_SUSPEND_NONE);
	}

	uint64_t
	spa_version(spa_t *spa)
	{
	return (spa->spa_ubsync.ub_version);
	}

	boolean_t
	spa_deflate(spa_t *spa)
	{
	return (spa->spa_deflate);
	}

	metaslab_class_t *
	spa_normal_class(spa_t *spa)
	{
	return (spa->spa_normal_class);
	}

	metaslab_class_t *
	spa_log_class(spa_t *spa)
	{
	return (spa->spa_log_class);
	}

	metaslab_class_t *
	spa_embedded_log_class(spa_t *spa)
	{
	return (spa->spa_embedded_log_class);
	}

	metaslab_class_t *
	spa_special_class(spa_t *spa)
	{
	return (spa->spa_special_class);
	}

	metaslab_class_t *
	spa_dedup_class(spa_t *spa)
	{
	return (spa->spa_dedup_class);
	}

	/*
	* Locate an appropriate allocation class
	*/
	metaslab_class_t *
	spa_preferred_class(spa_t *spa, uint64_t size, dmu_object_type_t objtype,
	uint_t level, uint_t special_smallblk)
	{
	/*
	* ZIL allocations determine their class in zio_alloc_zil().
	*/
	ASSERT(objtype != DMU_OT_INTENT_LOG);

	boolean_t has_special_class = spa->spa_special_class->mc_groups != 0;

	if (DMU_OT_IS_DDT(objtype)) {
	if (spa->spa_dedup_class->mc_groups != 0)
	return (spa_dedup_class(spa));
	else if (has_special_class && zfs_ddt_data_is_special)
	return (spa_special_class(spa));
	else
	return (spa_normal_class(spa));
	}

	/* Indirect blocks for user data can land in special if allowed */
	if (level > 0 && (DMU_OT_IS_FILE(objtype) \|\| objtype == DMU_OT_ZVOL)) {
	if (has_special_class && zfs_user_indirect_is_special)
	return (spa_special_class(spa));
	else
	return (spa_normal_class(spa));
	}

	if (DMU_OT_IS_METADATA(objtype) \|\| level > 0) {
	if (has_special_class)
	return (spa_special_class(spa));
	else
	return (spa_normal_class(spa));
	}

	/*
	* Allow small file blocks in special class in some cases (like
	* for the dRAID vdev feature). But always leave a reserve of
	* zfs_special_class_metadata_reserve_pct exclusively for metadata.
	*/
	if (DMU_OT_IS_FILE(objtype) &&
	has_special_class && size <= special_smallblk) {
	metaslab_class_t *special = spa_special_class(spa);
	uint64_t alloc = metaslab_class_get_alloc(special);
	uint64_t space = metaslab_class_get_space(special);
	uint64_t limit =
	(space * (100 - zfs_special_class_metadata_reserve_pct))
	/ 100;

	if (alloc < limit)
	return (special);
	}

	return (spa_normal_class(spa));
	}

	void
	spa_evicting_os_register(spa_t spa, objset_t os)
	{
	mutex_enter(&spa->spa_evicting_os_lock);
	list_insert_head(&spa->spa_evicting_os_list, os);
	mutex_exit(&spa->spa_evicting_os_lock);
	}

	void
	spa_evicting_os_deregister(spa_t spa, objset_t os)
	{
	mutex_enter(&spa->spa_evicting_os_lock);
	list_remove(&spa->spa_evicting_os_list, os);
	cv_broadcast(&spa->spa_evicting_os_cv);
	mutex_exit(&spa->spa_evicting_os_lock);
	}

	void
	spa_evicting_os_wait(spa_t *spa)
	{
	mutex_enter(&spa->spa_evicting_os_lock);
	while (!list_is_empty(&spa->spa_evicting_os_list))
	cv_wait(&spa->spa_evicting_os_cv, &spa->spa_evicting_os_lock);
	mutex_exit(&spa->spa_evicting_os_lock);

	dmu_buf_user_evict_wait();
	}

	int
	spa_max_replication(spa_t *spa)
	{
	/*
	* As of SPA_VERSION == SPA_VERSION_DITTO_BLOCKS, we are able to
	* handle BPs with more than one DVA allocated. Set our max
	* replication level accordingly.
	*/
	if (spa_version(spa) < SPA_VERSION_DITTO_BLOCKS)
	return (1);
	return (MIN(SPA_DVAS_PER_BP, spa_max_replication_override));
	}

	int
	spa_prev_software_version(spa_t *spa)
	{
	return (spa->spa_prev_software_version);
	}

	uint64_t
	spa_deadman_synctime(spa_t *spa)
	{
	return (spa->spa_deadman_synctime);
	}

	spa_autotrim_t
	spa_get_autotrim(spa_t *spa)
	{
	return (spa->spa_autotrim);
	}

	uint64_t
	spa_deadman_ziotime(spa_t *spa)
	{
	return (spa->spa_deadman_ziotime);
	}

	uint64_t
	spa_get_deadman_failmode(spa_t *spa)
	{
	return (spa->spa_deadman_failmode);
	}

	void
	spa_set_deadman_failmode(spa_t spa, const char failmode)
	{
	if (strcmp(failmode, "wait") == 0)
	spa->spa_deadman_failmode = ZIO_FAILURE_MODE_WAIT;
	else if (strcmp(failmode, "continue") == 0)
	spa->spa_deadman_failmode = ZIO_FAILURE_MODE_CONTINUE;
	else if (strcmp(failmode, "panic") == 0)
	spa->spa_deadman_failmode = ZIO_FAILURE_MODE_PANIC;
	else
	spa->spa_deadman_failmode = ZIO_FAILURE_MODE_WAIT;
	}

	void
	spa_set_deadman_ziotime(hrtime_t ns)
	{
	spa_t *spa = NULL;

	if (spa_mode_global != SPA_MODE_UNINIT) {
	mutex_enter(&spa_namespace_lock);
	while ((spa = spa_next(spa)) != NULL)
	spa->spa_deadman_ziotime = ns;
	mutex_exit(&spa_namespace_lock);
	}
	}

	void
	spa_set_deadman_synctime(hrtime_t ns)
	{
	spa_t *spa = NULL;

	if (spa_mode_global != SPA_MODE_UNINIT) {
	mutex_enter(&spa_namespace_lock);
	while ((spa = spa_next(spa)) != NULL)
	spa->spa_deadman_synctime = ns;
	mutex_exit(&spa_namespace_lock);
	}
	}

	uint64_t
	dva_get_dsize_sync(spa_t spa, const dva_t dva)
	{
	uint64_t asize = DVA_GET_ASIZE(dva);
	uint64_t dsize = asize;

	ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) != 0);

	if (asize != 0 && spa->spa_deflate) {
	vdev_t *vd = vdev_lookup_top(spa, DVA_GET_VDEV(dva));
	if (vd != NULL)
	dsize = (asize >> SPA_MINBLOCKSHIFT) *
	vd->vdev_deflate_ratio;
	}

	return (dsize);
	}

	uint64_t
	bp_get_dsize_sync(spa_t spa, const blkptr_t bp)
	{
	uint64_t dsize = 0;

	for (int d = 0; d < BP_GET_NDVAS(bp); d++)
	dsize += dva_get_dsize_sync(spa, &bp->blk_dva[d]);

	return (dsize);
	}

	uint64_t
	bp_get_dsize(spa_t spa, const blkptr_t bp)
	{
	uint64_t dsize = 0;

	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);

	for (int d = 0; d < BP_GET_NDVAS(bp); d++)
	dsize += dva_get_dsize_sync(spa, &bp->blk_dva[d]);

	spa_config_exit(spa, SCL_VDEV, FTAG);

	return (dsize);
	}

	uint64_t
	spa_dirty_data(spa_t *spa)
	{
	return (spa->spa_dsl_pool->dp_dirty_total);
	}

	/*
	* ==========================================================================
	* SPA Import Progress Routines
	* ==========================================================================
	*/

	typedef struct spa_import_progress {
	uint64_t pool_guid; /* unique id for updates */
	char *pool_name;
	spa_load_state_t spa_load_state;
	+ char *spa_load_notes;
	uint64_t mmp_sec_remaining; /* MMP activity check */
	uint64_t spa_load_max_txg; /* rewind txg */
	procfs_list_node_t smh_node;
	} spa_import_progress_t;

	spa_history_list_t *spa_import_progress_list = NULL;

	static int
	spa_import_progress_show_header(struct seq_file *f)
	{
	- seq_printf(f, "%-20s %-14s %-14s %-12s %s\n", "pool_guid",
	+ seq_printf(f, "%-20s %-14s %-14s %-12s %-16s %s\n", "pool_guid",
	"load_state", "multihost_secs", "max_txg",
	- "pool_name");
	+ "pool_name", "notes");
	return (0);
	}

	static int
	spa_import_progress_show(struct seq_file f, void data)
	{
	spa_import_progress_t sip = (spa_import_progress_t )data;

	- seq_printf(f, "%-20llu %-14llu %-14llu %-12llu %s\n",
	+ seq_printf(f, "%-20llu %-14llu %-14llu %-12llu %-16s %s\n",
	(u_longlong_t)sip->pool_guid, (u_longlong_t)sip->spa_load_state,
	(u_longlong_t)sip->mmp_sec_remaining,
	(u_longlong_t)sip->spa_load_max_txg,
	- (sip->pool_name ? sip->pool_name : "-"));
	+ (sip->pool_name ? sip->pool_name : "-"),
	+ (sip->spa_load_notes ? sip->spa_load_notes : "-"));

	return (0);
	}

	/* Remove oldest elements from list until there are no more than 'size' left */
	static void
	spa_import_progress_truncate(spa_history_list_t *shl, unsigned int size)
	{
	spa_import_progress_t *sip;
	while (shl->size > size) {
	sip = list_remove_head(&shl->procfs_list.pl_list);
	if (sip->pool_name)
	spa_strfree(sip->pool_name);
	+ if (sip->spa_load_notes)
	+ kmem_strfree(sip->spa_load_notes);
	kmem_free(sip, sizeof (spa_import_progress_t));
	shl->size--;
	}

	IMPLY(size == 0, list_is_empty(&shl->procfs_list.pl_list));
	}

	static void
	spa_import_progress_init(void)
	{
	spa_import_progress_list = kmem_zalloc(sizeof (spa_history_list_t),
	KM_SLEEP);

	spa_import_progress_list->size = 0;

	spa_import_progress_list->procfs_list.pl_private =
	spa_import_progress_list;

	procfs_list_install("zfs",
	NULL,
	"import_progress",
	0644,
	&spa_import_progress_list->procfs_list,
	spa_import_progress_show,
	spa_import_progress_show_header,
	NULL,
	offsetof(spa_import_progress_t, smh_node));
	}

	static void
	spa_import_progress_destroy(void)
	{
	spa_history_list_t *shl = spa_import_progress_list;
	procfs_list_uninstall(&shl->procfs_list);
	spa_import_progress_truncate(shl, 0);
	procfs_list_destroy(&shl->procfs_list);
	kmem_free(shl, sizeof (spa_history_list_t));
	}

	int
	spa_import_progress_set_state(uint64_t pool_guid,
	spa_load_state_t load_state)
	{
	spa_history_list_t *shl = spa_import_progress_list;
	spa_import_progress_t *sip;
	int error = ENOENT;

	if (shl->size == 0)
	return (0);

	mutex_enter(&shl->procfs_list.pl_lock);
	for (sip = list_tail(&shl->procfs_list.pl_list); sip != NULL;
	sip = list_prev(&shl->procfs_list.pl_list, sip)) {
	if (sip->pool_guid == pool_guid) {
	sip->spa_load_state = load_state;
	+ if (sip->spa_load_notes != NULL) {
	+ kmem_strfree(sip->spa_load_notes);
	+ sip->spa_load_notes = NULL;
	+ }
	error = 0;
	break;
	}
	}
	mutex_exit(&shl->procfs_list.pl_lock);

	return (error);
	}

	+static void
	+spa_import_progress_set_notes_impl(spa_t *spa, boolean_t log_dbgmsg,
	+ const char *fmt, va_list adx)
	+{
	+ spa_history_list_t *shl = spa_import_progress_list;
	+ spa_import_progress_t *sip;
	+ uint64_t pool_guid = spa_guid(spa);
	+
	+ if (shl->size == 0)
	+ return;
	+
	+ char *notes = kmem_vasprintf(fmt, adx);
	+
	+ mutex_enter(&shl->procfs_list.pl_lock);
	+ for (sip = list_tail(&shl->procfs_list.pl_list); sip != NULL;
	+ sip = list_prev(&shl->procfs_list.pl_list, sip)) {
	+ if (sip->pool_guid == pool_guid) {
	+ if (sip->spa_load_notes != NULL) {
	+ kmem_strfree(sip->spa_load_notes);
	+ sip->spa_load_notes = NULL;
	+ }
	+ sip->spa_load_notes = notes;
	+ if (log_dbgmsg)
	+ zfs_dbgmsg("'%s' %s", sip->pool_name, notes);
	+ notes = NULL;
	+ break;
	+ }
	+ }
	+ mutex_exit(&shl->procfs_list.pl_lock);
	+ if (notes != NULL)
	+ kmem_strfree(notes);
	+}
	+
	+void
	+spa_import_progress_set_notes(spa_t spa, const char fmt, ...)
	+{
	+ va_list adx;
	+
	+ va_start(adx, fmt);
	+ spa_import_progress_set_notes_impl(spa, B_TRUE, fmt, adx);
	+ va_end(adx);
	+}
	+
	+void
	+spa_import_progress_set_notes_nolog(spa_t spa, const char fmt, ...)
	+{
	+ va_list adx;
	+
	+ va_start(adx, fmt);
	+ spa_import_progress_set_notes_impl(spa, B_FALSE, fmt, adx);
	+ va_end(adx);
	+}
	+
	int
	spa_import_progress_set_max_txg(uint64_t pool_guid, uint64_t load_max_txg)
	{
	spa_history_list_t *shl = spa_import_progress_list;
	spa_import_progress_t *sip;
	int error = ENOENT;

	if (shl->size == 0)
	return (0);

	mutex_enter(&shl->procfs_list.pl_lock);
	for (sip = list_tail(&shl->procfs_list.pl_list); sip != NULL;
	sip = list_prev(&shl->procfs_list.pl_list, sip)) {
	if (sip->pool_guid == pool_guid) {
	sip->spa_load_max_txg = load_max_txg;
	error = 0;
	break;
	}
	}
	mutex_exit(&shl->procfs_list.pl_lock);

	return (error);
	}

	int
	spa_import_progress_set_mmp_check(uint64_t pool_guid,
	uint64_t mmp_sec_remaining)
	{
	spa_history_list_t *shl = spa_import_progress_list;
	spa_import_progress_t *sip;
	int error = ENOENT;

	if (shl->size == 0)
	return (0);

	mutex_enter(&shl->procfs_list.pl_lock);
	for (sip = list_tail(&shl->procfs_list.pl_list); sip != NULL;
	sip = list_prev(&shl->procfs_list.pl_list, sip)) {
	if (sip->pool_guid == pool_guid) {
	sip->mmp_sec_remaining = mmp_sec_remaining;
	error = 0;
	break;
	}
	}
	mutex_exit(&shl->procfs_list.pl_lock);

	return (error);
	}

	/*
	* A new import is in progress, add an entry.
	*/
	void
	spa_import_progress_add(spa_t *spa)
	{
	spa_history_list_t *shl = spa_import_progress_list;
	spa_import_progress_t *sip;
	const char *poolname = NULL;

	sip = kmem_zalloc(sizeof (spa_import_progress_t), KM_SLEEP);
	sip->pool_guid = spa_guid(spa);

	(void) nvlist_lookup_string(spa->spa_config, ZPOOL_CONFIG_POOL_NAME,
	&poolname);
	if (poolname == NULL)
	poolname = spa_name(spa);
	sip->pool_name = spa_strdup(poolname);
	sip->spa_load_state = spa_load_state(spa);
	+ sip->spa_load_notes = NULL;

	mutex_enter(&shl->procfs_list.pl_lock);
	procfs_list_add(&shl->procfs_list, sip);
	shl->size++;
	mutex_exit(&shl->procfs_list.pl_lock);
	}

	void
	spa_import_progress_remove(uint64_t pool_guid)
	{
	spa_history_list_t *shl = spa_import_progress_list;
	spa_import_progress_t *sip;

	mutex_enter(&shl->procfs_list.pl_lock);
	for (sip = list_tail(&shl->procfs_list.pl_list); sip != NULL;
	sip = list_prev(&shl->procfs_list.pl_list, sip)) {
	if (sip->pool_guid == pool_guid) {
	if (sip->pool_name)
	spa_strfree(sip->pool_name);
	+ if (sip->spa_load_notes)
	+ spa_strfree(sip->spa_load_notes);
	list_remove(&shl->procfs_list.pl_list, sip);
	shl->size--;
	kmem_free(sip, sizeof (spa_import_progress_t));
	break;
	}
	}
	mutex_exit(&shl->procfs_list.pl_lock);
	}

	/*
	* ==========================================================================
	* Initialization and Termination
	* ==========================================================================
	*/

	static int
	spa_name_compare(const void a1, const void a2)
	{
	const spa_t *s1 = a1;
	const spa_t *s2 = a2;
	int s;

	s = strcmp(s1->spa_name, s2->spa_name);

	return (TREE_ISIGN(s));
	}

	void
	spa_boot_init(void)
	{
	spa_config_load();
	}

	void
	spa_init(spa_mode_t mode)
	{
	mutex_init(&spa_namespace_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&spa_spare_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&spa_l2cache_lock, NULL, MUTEX_DEFAULT, NULL);
	cv_init(&spa_namespace_cv, NULL, CV_DEFAULT, NULL);

	avl_create(&spa_namespace_avl, spa_name_compare, sizeof (spa_t),
	offsetof(spa_t, spa_avl));

	avl_create(&spa_spare_avl, spa_spare_compare, sizeof (spa_aux_t),
	offsetof(spa_aux_t, aux_avl));

	avl_create(&spa_l2cache_avl, spa_l2cache_compare, sizeof (spa_aux_t),
	offsetof(spa_aux_t, aux_avl));

	spa_mode_global = mode;

	#ifndef _KERNEL
	if (spa_mode_global != SPA_MODE_READ && dprintf_find_string("watch")) {
	struct sigaction sa;

	sa.sa_flags = SA_SIGINFO;
	sigemptyset(&sa.sa_mask);
	sa.sa_sigaction = arc_buf_sigsegv;

	if (sigaction(SIGSEGV, &sa, NULL) == -1) {
	perror("could not enable watchpoints: "
	"sigaction(SIGSEGV, ...) = ");
	} else {
	arc_watch = B_TRUE;
	}
	}
	#endif

	fm_init();
	zfs_refcount_init();
	unique_init();
	zfs_btree_init();
	metaslab_stat_init();
	brt_init();
	ddt_init();
	zio_init();
	dmu_init();
	zil_init();
	vdev_mirror_stat_init();
	vdev_raidz_math_init();
	vdev_file_init();
	zfs_prop_init();
	chksum_init();
	zpool_prop_init();
	zpool_feature_init();
	spa_config_load();
	vdev_prop_init();
	l2arc_start();
	scan_init();
	qat_init();
	spa_import_progress_init();
	}

	void
	spa_fini(void)
	{
	l2arc_stop();

	spa_evict_all();

	vdev_file_fini();
	vdev_mirror_stat_fini();
	vdev_raidz_math_fini();
	chksum_fini();
	zil_fini();
	dmu_fini();
	zio_fini();
	ddt_fini();
	brt_fini();
	metaslab_stat_fini();
	zfs_btree_fini();
	unique_fini();
	zfs_refcount_fini();
	fm_fini();
	scan_fini();
	qat_fini();
	spa_import_progress_destroy();

	avl_destroy(&spa_namespace_avl);
	avl_destroy(&spa_spare_avl);
	avl_destroy(&spa_l2cache_avl);

	cv_destroy(&spa_namespace_cv);
	mutex_destroy(&spa_namespace_lock);
	mutex_destroy(&spa_spare_lock);
	mutex_destroy(&spa_l2cache_lock);
	}

	/*
	* Return whether this pool has a dedicated slog device. No locking needed.
	* It's not a problem if the wrong answer is returned as it's only for
	* performance and not correctness.
	*/
	boolean_t
	spa_has_slogs(spa_t *spa)
	{
	return (spa->spa_log_class->mc_groups != 0);
	}

	spa_log_state_t
	spa_get_log_state(spa_t *spa)
	{
	return (spa->spa_log_state);
	}

	void
	spa_set_log_state(spa_t *spa, spa_log_state_t state)
	{
	spa->spa_log_state = state;
	}

	boolean_t
	spa_is_root(spa_t *spa)
	{
	return (spa->spa_is_root);
	}

	boolean_t
	spa_writeable(spa_t *spa)
	{
	return (!!(spa->spa_mode & SPA_MODE_WRITE) && spa->spa_trust_config);
	}

	/*
	* Returns true if there is a pending sync task in any of the current
	* syncing txg, the current quiescing txg, or the current open txg.
	*/
	boolean_t
	spa_has_pending_synctask(spa_t *spa)
	{
	return (!txg_all_lists_empty(&spa->spa_dsl_pool->dp_sync_tasks) \|\|
	!txg_all_lists_empty(&spa->spa_dsl_pool->dp_early_sync_tasks));
	}

	spa_mode_t
	spa_mode(spa_t *spa)
	{
	return (spa->spa_mode);
	}

	uint64_t
	spa_bootfs(spa_t *spa)
	{
	return (spa->spa_bootfs);
	}

	uint64_t
	spa_delegation(spa_t *spa)
	{
	return (spa->spa_delegation);
	}

	objset_t *
	spa_meta_objset(spa_t *spa)
	{
	return (spa->spa_meta_objset);
	}

	enum zio_checksum
	spa_dedup_checksum(spa_t *spa)
	{
	return (spa->spa_dedup_checksum);
	}

	/*
	* Reset pool scan stat per scan pass (or reboot).
	*/
	void
	spa_scan_stat_init(spa_t *spa)
	{
	/* data not stored on disk */
	spa->spa_scan_pass_start = gethrestime_sec();
	if (dsl_scan_is_paused_scrub(spa->spa_dsl_pool->dp_scan))
	spa->spa_scan_pass_scrub_pause = spa->spa_scan_pass_start;
	else
	spa->spa_scan_pass_scrub_pause = 0;

	if (dsl_errorscrub_is_paused(spa->spa_dsl_pool->dp_scan))
	spa->spa_scan_pass_errorscrub_pause = spa->spa_scan_pass_start;
	else
	spa->spa_scan_pass_errorscrub_pause = 0;

	spa->spa_scan_pass_scrub_spent_paused = 0;
	spa->spa_scan_pass_exam = 0;
	spa->spa_scan_pass_issued = 0;

	// error scrub stats
	spa->spa_scan_pass_errorscrub_spent_paused = 0;
	}

	/*
	* Get scan stats for zpool status reports
	*/
	int
	spa_scan_get_stats(spa_t spa, pool_scan_stat_t ps)
	{
	dsl_scan_t *scn = spa->spa_dsl_pool ? spa->spa_dsl_pool->dp_scan : NULL;

	if (scn == NULL \|\| (scn->scn_phys.scn_func == POOL_SCAN_NONE &&
	scn->errorscrub_phys.dep_func == POOL_SCAN_NONE))
	return (SET_ERROR(ENOENT));

	memset(ps, 0, sizeof (pool_scan_stat_t));

	/* data stored on disk */
	ps->pss_func = scn->scn_phys.scn_func;
	ps->pss_state = scn->scn_phys.scn_state;
	ps->pss_start_time = scn->scn_phys.scn_start_time;
	ps->pss_end_time = scn->scn_phys.scn_end_time;
	ps->pss_to_examine = scn->scn_phys.scn_to_examine;
	ps->pss_examined = scn->scn_phys.scn_examined;
	ps->pss_skipped = scn->scn_phys.scn_skipped;
	ps->pss_processed = scn->scn_phys.scn_processed;
	ps->pss_errors = scn->scn_phys.scn_errors;

	/* data not stored on disk */
	ps->pss_pass_exam = spa->spa_scan_pass_exam;
	ps->pss_pass_start = spa->spa_scan_pass_start;
	ps->pss_pass_scrub_pause = spa->spa_scan_pass_scrub_pause;
	ps->pss_pass_scrub_spent_paused = spa->spa_scan_pass_scrub_spent_paused;
	ps->pss_pass_issued = spa->spa_scan_pass_issued;
	ps->pss_issued =
	scn->scn_issued_before_pass + spa->spa_scan_pass_issued;

	/* error scrub data stored on disk */
	ps->pss_error_scrub_func = scn->errorscrub_phys.dep_func;
	ps->pss_error_scrub_state = scn->errorscrub_phys.dep_state;
	ps->pss_error_scrub_start = scn->errorscrub_phys.dep_start_time;
	ps->pss_error_scrub_end = scn->errorscrub_phys.dep_end_time;
	ps->pss_error_scrub_examined = scn->errorscrub_phys.dep_examined;
	ps->pss_error_scrub_to_be_examined =
	scn->errorscrub_phys.dep_to_examine;

	/* error scrub data not stored on disk */
	ps->pss_pass_error_scrub_pause = spa->spa_scan_pass_errorscrub_pause;

	return (0);
	}

	int
	spa_maxblocksize(spa_t *spa)
	{
	if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS))
	return (SPA_MAXBLOCKSIZE);
	else
	return (SPA_OLD_MAXBLOCKSIZE);
	}


	/*
	* Returns the txg that the last device removal completed. No indirect mappings
	* have been added since this txg.
	*/
	uint64_t
	spa_get_last_removal_txg(spa_t *spa)
	{
	uint64_t vdevid;
	uint64_t ret = -1ULL;

	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
	/*
	* sr_prev_indirect_vdev is only modified while holding all the
	* config locks, so it is sufficient to hold SCL_VDEV as reader when
	* examining it.
	*/
	vdevid = spa->spa_removing_phys.sr_prev_indirect_vdev;

	while (vdevid != -1ULL) {
	vdev_t *vd = vdev_lookup_top(spa, vdevid);
	vdev_indirect_births_t *vib = vd->vdev_indirect_births;

	ASSERT3P(vd->vdev_ops, ==, &vdev_indirect_ops);

	/*
	* If the removal did not remap any data, we don't care.
	*/
	if (vdev_indirect_births_count(vib) != 0) {
	ret = vdev_indirect_births_last_entry_txg(vib);
	break;
	}

	vdevid = vd->vdev_indirect_config.vic_prev_indirect_vdev;
	}
	spa_config_exit(spa, SCL_VDEV, FTAG);

	IMPLY(ret != -1ULL,
	spa_feature_is_active(spa, SPA_FEATURE_DEVICE_REMOVAL));

	return (ret);
	}

	int
	spa_maxdnodesize(spa_t *spa)
	{
	if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE))
	return (DNODE_MAX_SIZE);
	else
	return (DNODE_MIN_SIZE);
	}

	boolean_t
	spa_multihost(spa_t *spa)
	{
	return (spa->spa_multihost ? B_TRUE : B_FALSE);
	}

	uint32_t
	spa_get_hostid(spa_t *spa)
	{
	return (spa->spa_hostid);
	}

	boolean_t
	spa_trust_config(spa_t *spa)
	{
	return (spa->spa_trust_config);
	}

	uint64_t
	spa_missing_tvds_allowed(spa_t *spa)
	{
	return (spa->spa_missing_tvds_allowed);
	}

	space_map_t *
	spa_syncing_log_sm(spa_t *spa)
	{
	return (spa->spa_syncing_log_sm);
	}

	void
	spa_set_missing_tvds(spa_t *spa, uint64_t missing)
	{
	spa->spa_missing_tvds = missing;
	}

	/*
	* Return the pool state string ("ONLINE", "DEGRADED", "SUSPENDED", etc).
	*/
	const char *
	spa_state_to_name(spa_t *spa)
	{
	ASSERT3P(spa, !=, NULL);

	/*
	* it is possible for the spa to exist, without root vdev
	* as the spa transitions during import/export
	*/
	vdev_t *rvd = spa->spa_root_vdev;
	if (rvd == NULL) {
	return ("TRANSITIONING");
	}
	vdev_state_t state = rvd->vdev_state;
	vdev_aux_t aux = rvd->vdev_stat.vs_aux;

	if (spa_suspended(spa))
	return ("SUSPENDED");

	switch (state) {
	case VDEV_STATE_CLOSED:
	case VDEV_STATE_OFFLINE:
	return ("OFFLINE");
	case VDEV_STATE_REMOVED:
	return ("REMOVED");
	case VDEV_STATE_CANT_OPEN:
	if (aux == VDEV_AUX_CORRUPT_DATA \|\| aux == VDEV_AUX_BAD_LOG)
	return ("FAULTED");
	else if (aux == VDEV_AUX_SPLIT_POOL)
	return ("SPLIT");
	else
	return ("UNAVAIL");
	case VDEV_STATE_FAULTED:
	return ("FAULTED");
	case VDEV_STATE_DEGRADED:
	return ("DEGRADED");
	case VDEV_STATE_HEALTHY:
	return ("ONLINE");
	default:
	break;
	}

	return ("UNKNOWN");
	}

	boolean_t
	spa_top_vdevs_spacemap_addressable(spa_t *spa)
	{
	vdev_t *rvd = spa->spa_root_vdev;
	for (uint64_t c = 0; c < rvd->vdev_children; c++) {
	if (!vdev_is_spacemap_addressable(rvd->vdev_child[c]))
	return (B_FALSE);
	}
	return (B_TRUE);
	}

	boolean_t
	spa_has_checkpoint(spa_t *spa)
	{
	return (spa->spa_checkpoint_txg != 0);
	}

	boolean_t
	spa_importing_readonly_checkpoint(spa_t *spa)
	{
	return ((spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT) &&
	spa->spa_mode == SPA_MODE_READ);
	}

	uint64_t
	spa_min_claim_txg(spa_t *spa)
	{
	uint64_t checkpoint_txg = spa->spa_uberblock.ub_checkpoint_txg;

	if (checkpoint_txg != 0)
	return (checkpoint_txg + 1);

	return (spa->spa_first_txg);
	}

	/*
	* If there is a checkpoint, async destroys may consume more space from
	* the pool instead of freeing it. In an attempt to save the pool from
	* getting suspended when it is about to run out of space, we stop
	* processing async destroys.
	*/
	boolean_t
	spa_suspend_async_destroy(spa_t *spa)
	{
	dsl_pool_t *dp = spa_get_dsl(spa);

	uint64_t unreserved = dsl_pool_unreserved_space(dp,
	ZFS_SPACE_CHECK_EXTRA_RESERVED);
	uint64_t used = dsl_dir_phys(dp->dp_root_dir)->dd_used_bytes;
	uint64_t avail = (unreserved > used) ? (unreserved - used) : 0;

	if (spa_has_checkpoint(spa) && avail == 0)
	return (B_TRUE);

	return (B_FALSE);
	}

	#if defined(_KERNEL)

	int
	param_set_deadman_failmode_common(const char *val)
	{
	spa_t *spa = NULL;
	char *p;

	if (val == NULL)
	return (SET_ERROR(EINVAL));

	if ((p = strchr(val, '\n')) != NULL)
	*p = '\0';

	if (strcmp(val, "wait") != 0 && strcmp(val, "continue") != 0 &&
	strcmp(val, "panic"))
	return (SET_ERROR(EINVAL));

	if (spa_mode_global != SPA_MODE_UNINIT) {
	mutex_enter(&spa_namespace_lock);
	while ((spa = spa_next(spa)) != NULL)
	spa_set_deadman_failmode(spa, val);
	mutex_exit(&spa_namespace_lock);
	}

	return (0);
	}
	#endif

	/* Namespace manipulation */
	EXPORT_SYMBOL(spa_lookup);
	EXPORT_SYMBOL(spa_add);
	EXPORT_SYMBOL(spa_remove);
	EXPORT_SYMBOL(spa_next);

	/* Refcount functions */
	EXPORT_SYMBOL(spa_open_ref);
	EXPORT_SYMBOL(spa_close);
	EXPORT_SYMBOL(spa_refcount_zero);

	/* Pool configuration lock */
	EXPORT_SYMBOL(spa_config_tryenter);
	EXPORT_SYMBOL(spa_config_enter);
	EXPORT_SYMBOL(spa_config_exit);
	EXPORT_SYMBOL(spa_config_held);

	/* Pool vdev add/remove lock */
	EXPORT_SYMBOL(spa_vdev_enter);
	EXPORT_SYMBOL(spa_vdev_exit);

	/* Pool vdev state change lock */
	EXPORT_SYMBOL(spa_vdev_state_enter);
	EXPORT_SYMBOL(spa_vdev_state_exit);

	/* Accessor functions */
	EXPORT_SYMBOL(spa_shutting_down);
	EXPORT_SYMBOL(spa_get_dsl);
	EXPORT_SYMBOL(spa_get_rootblkptr);
	EXPORT_SYMBOL(spa_set_rootblkptr);
	EXPORT_SYMBOL(spa_altroot);
	EXPORT_SYMBOL(spa_sync_pass);
	EXPORT_SYMBOL(spa_name);
	EXPORT_SYMBOL(spa_guid);
	EXPORT_SYMBOL(spa_last_synced_txg);
	EXPORT_SYMBOL(spa_first_txg);
	EXPORT_SYMBOL(spa_syncing_txg);
	EXPORT_SYMBOL(spa_version);
	EXPORT_SYMBOL(spa_state);
	EXPORT_SYMBOL(spa_load_state);
	EXPORT_SYMBOL(spa_freeze_txg);
	EXPORT_SYMBOL(spa_get_dspace);
	EXPORT_SYMBOL(spa_update_dspace);
	EXPORT_SYMBOL(spa_deflate);
	EXPORT_SYMBOL(spa_normal_class);
	EXPORT_SYMBOL(spa_log_class);
	EXPORT_SYMBOL(spa_special_class);
	EXPORT_SYMBOL(spa_preferred_class);
	EXPORT_SYMBOL(spa_max_replication);
	EXPORT_SYMBOL(spa_prev_software_version);
	EXPORT_SYMBOL(spa_get_failmode);
	EXPORT_SYMBOL(spa_suspended);
	EXPORT_SYMBOL(spa_bootfs);
	EXPORT_SYMBOL(spa_delegation);
	EXPORT_SYMBOL(spa_meta_objset);
	EXPORT_SYMBOL(spa_maxblocksize);
	EXPORT_SYMBOL(spa_maxdnodesize);

	/* Miscellaneous support routines */
	EXPORT_SYMBOL(spa_guid_exists);
	EXPORT_SYMBOL(spa_strdup);
	EXPORT_SYMBOL(spa_strfree);
	EXPORT_SYMBOL(spa_generate_guid);
	EXPORT_SYMBOL(snprintf_blkptr);
	EXPORT_SYMBOL(spa_freeze);
	EXPORT_SYMBOL(spa_upgrade);
	EXPORT_SYMBOL(spa_evict_all);
	EXPORT_SYMBOL(spa_lookup_by_guid);
	EXPORT_SYMBOL(spa_has_spare);
	EXPORT_SYMBOL(dva_get_dsize_sync);
	EXPORT_SYMBOL(bp_get_dsize_sync);
	EXPORT_SYMBOL(bp_get_dsize);
	EXPORT_SYMBOL(spa_has_slogs);
	EXPORT_SYMBOL(spa_is_root);
	EXPORT_SYMBOL(spa_writeable);
	EXPORT_SYMBOL(spa_mode);
	EXPORT_SYMBOL(spa_namespace_lock);
	EXPORT_SYMBOL(spa_trust_config);
	EXPORT_SYMBOL(spa_missing_tvds_allowed);
	EXPORT_SYMBOL(spa_set_missing_tvds);
	EXPORT_SYMBOL(spa_state_to_name);
	EXPORT_SYMBOL(spa_importing_readonly_checkpoint);
	EXPORT_SYMBOL(spa_min_claim_txg);
	EXPORT_SYMBOL(spa_suspend_async_destroy);
	EXPORT_SYMBOL(spa_has_checkpoint);
	EXPORT_SYMBOL(spa_top_vdevs_spacemap_addressable);

	ZFS_MODULE_PARAM(zfs, zfs_, flags, UINT, ZMOD_RW,
	"Set additional debugging flags");

	ZFS_MODULE_PARAM(zfs, zfs_, recover, INT, ZMOD_RW,
	"Set to attempt to recover from fatal errors");

	ZFS_MODULE_PARAM(zfs, zfs_, free_leak_on_eio, INT, ZMOD_RW,
	"Set to ignore IO errors during free and permanently leak the space");

	ZFS_MODULE_PARAM(zfs_deadman, zfs_deadman_, checktime_ms, U64, ZMOD_RW,
	"Dead I/O check interval in milliseconds");

	ZFS_MODULE_PARAM(zfs_deadman, zfs_deadman_, enabled, INT, ZMOD_RW,
	"Enable deadman timer");

	ZFS_MODULE_PARAM(zfs_spa, spa_, asize_inflation, UINT, ZMOD_RW,
	"SPA size estimate multiplication factor");

	ZFS_MODULE_PARAM(zfs, zfs_, ddt_data_is_special, INT, ZMOD_RW,
	"Place DDT data into the special class");

	ZFS_MODULE_PARAM(zfs, zfs_, user_indirect_is_special, INT, ZMOD_RW,
	"Place user data indirect blocks into the special class");

	/* BEGIN CSTYLED */
	ZFS_MODULE_PARAM_CALL(zfs_deadman, zfs_deadman_, failmode,
	param_set_deadman_failmode, param_get_charp, ZMOD_RW,
	"Failmode for deadman timer");

	ZFS_MODULE_PARAM_CALL(zfs_deadman, zfs_deadman_, synctime_ms,
	param_set_deadman_synctime, spl_param_get_u64, ZMOD_RW,
	"Pool sync expiration time in milliseconds");

	ZFS_MODULE_PARAM_CALL(zfs_deadman, zfs_deadman_, ziotime_ms,
	param_set_deadman_ziotime, spl_param_get_u64, ZMOD_RW,
	"IO expiration time in milliseconds");

	ZFS_MODULE_PARAM(zfs, zfs_, special_class_metadata_reserve_pct, UINT, ZMOD_RW,
	"Small file blocks in special vdevs depends on this much "
	"free space available");
	/* END CSTYLED */

	ZFS_MODULE_PARAM_CALL(zfs_spa, spa_, slop_shift, param_set_slop_shift,
	param_get_uint, ZMOD_RW, "Reserved free space in pool");
	diff --git a/sys/contrib/openzfs/module/zfs/txg.c b/sys/contrib/openzfs/module/zfs/txg.c
	index a67c043446f5..5ce6be69be14 100644
	--- a/sys/contrib/openzfs/module/zfs/txg.c
	+++ b/sys/contrib/openzfs/module/zfs/txg.c
	@@ -1,1068 +1,1077 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Portions Copyright 2011 Martin Matuska
	* Copyright (c) 2012, 2019 by Delphix. All rights reserved.
	*/

	#include <sys/zfs_context.h>
	#include <sys/txg_impl.h>
	#include <sys/dmu_impl.h>
	#include <sys/spa_impl.h>
	#include <sys/dmu_tx.h>
	#include <sys/dsl_pool.h>
	#include <sys/dsl_scan.h>
	#include <sys/zil.h>
	#include <sys/callb.h>
	#include <sys/trace_zfs.h>

	/*
	* ZFS Transaction Groups
	* ----------------------
	*
	* ZFS transaction groups are, as the name implies, groups of transactions
	* that act on persistent state. ZFS asserts consistency at the granularity of
	* these transaction groups. Each successive transaction group (txg) is
	* assigned a 64-bit consecutive identifier. There are three active
	* transaction group states: open, quiescing, or syncing. At any given time,
	* there may be an active txg associated with each state; each active txg may
	* either be processing, or blocked waiting to enter the next state. There may
	* be up to three active txgs, and there is always a txg in the open state
	* (though it may be blocked waiting to enter the quiescing state). In broad
	* strokes, transactions -- operations that change in-memory structures -- are
	* accepted into the txg in the open state, and are completed while the txg is
	* in the open or quiescing states. The accumulated changes are written to
	* disk in the syncing state.
	*
	* Open
	*
	* When a new txg becomes active, it first enters the open state. New
	* transactions -- updates to in-memory structures -- are assigned to the
	* currently open txg. There is always a txg in the open state so that ZFS can
	* accept new changes (though the txg may refuse new changes if it has hit
	* some limit). ZFS advances the open txg to the next state for a variety of
	* reasons such as it hitting a time or size threshold, or the execution of an
	* administrative action that must be completed in the syncing state.
	*
	* Quiescing
	*
	* After a txg exits the open state, it enters the quiescing state. The
	* quiescing state is intended to provide a buffer between accepting new
	* transactions in the open state and writing them out to stable storage in
	* the syncing state. While quiescing, transactions can continue their
	* operation without delaying either of the other states. Typically, a txg is
	* in the quiescing state very briefly since the operations are bounded by
	* software latencies rather than, say, slower I/O latencies. After all
	* transactions complete, the txg is ready to enter the next state.
	*
	* Syncing
	*
	* In the syncing state, the in-memory state built up during the open and (to
	* a lesser degree) the quiescing states is written to stable storage. The
	* process of writing out modified data can, in turn modify more data. For
	* example when we write new blocks, we need to allocate space for them; those
	* allocations modify metadata (space maps)... which themselves must be
	* written to stable storage. During the sync state, ZFS iterates, writing out
	* data until it converges and all in-memory changes have been written out.
	* The first such pass is the largest as it encompasses all the modified user
	* data (as opposed to filesystem metadata). Subsequent passes typically have
	* far less data to write as they consist exclusively of filesystem metadata.
	*
	* To ensure convergence, after a certain number of passes ZFS begins
	* overwriting locations on stable storage that had been allocated earlier in
	* the syncing state (and subsequently freed). ZFS usually allocates new
	* blocks to optimize for large, continuous, writes. For the syncing state to
	* converge however it must complete a pass where no new blocks are allocated
	* since each allocation requires a modification of persistent metadata.
	* Further, to hasten convergence, after a prescribed number of passes, ZFS
	* also defers frees, and stops compressing.
	*
	* In addition to writing out user data, we must also execute synctasks during
	* the syncing context. A synctask is the mechanism by which some
	* administrative activities work such as creating and destroying snapshots or
	* datasets. Note that when a synctask is initiated it enters the open txg,
	* and ZFS then pushes that txg as quickly as possible to completion of the
	* syncing state in order to reduce the latency of the administrative
	* activity. To complete the syncing state, ZFS writes out a new uberblock,
	* the root of the tree of blocks that comprise all state stored on the ZFS
	* pool. Finally, if there is a quiesced txg waiting, we signal that it can
	* now transition to the syncing state.
	*/

	static __attribute__((noreturn)) void txg_sync_thread(void *arg);
	static __attribute__((noreturn)) void txg_quiesce_thread(void *arg);

	uint_t zfs_txg_timeout = 5; /* max seconds worth of delta per txg */

	/*
	* Prepare the txg subsystem.
	*/
	void
	txg_init(dsl_pool_t *dp, uint64_t txg)
	{
	tx_state_t *tx = &dp->dp_tx;
	int c;
	memset(tx, 0, sizeof (tx_state_t));

	tx->tx_cpu = vmem_zalloc(max_ncpus * sizeof (tx_cpu_t), KM_SLEEP);

	for (c = 0; c < max_ncpus; c++) {
	int i;

	mutex_init(&tx->tx_cpu[c].tc_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&tx->tx_cpu[c].tc_open_lock, NULL, MUTEX_NOLOCKDEP,
	NULL);
	for (i = 0; i < TXG_SIZE; i++) {
	cv_init(&tx->tx_cpu[c].tc_cv[i], NULL, CV_DEFAULT,
	NULL);
	list_create(&tx->tx_cpu[c].tc_callbacks[i],
	sizeof (dmu_tx_callback_t),
	offsetof(dmu_tx_callback_t, dcb_node));
	}
	}

	mutex_init(&tx->tx_sync_lock, NULL, MUTEX_DEFAULT, NULL);

	cv_init(&tx->tx_sync_more_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&tx->tx_sync_done_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&tx->tx_quiesce_more_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&tx->tx_quiesce_done_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&tx->tx_exit_cv, NULL, CV_DEFAULT, NULL);

	tx->tx_open_txg = txg;
	}

	/*
	* Close down the txg subsystem.
	*/
	void
	txg_fini(dsl_pool_t *dp)
	{
	tx_state_t *tx = &dp->dp_tx;
	int c;

	ASSERT0(tx->tx_threads);

	mutex_destroy(&tx->tx_sync_lock);

	cv_destroy(&tx->tx_sync_more_cv);
	cv_destroy(&tx->tx_sync_done_cv);
	cv_destroy(&tx->tx_quiesce_more_cv);
	cv_destroy(&tx->tx_quiesce_done_cv);
	cv_destroy(&tx->tx_exit_cv);

	for (c = 0; c < max_ncpus; c++) {
	int i;

	mutex_destroy(&tx->tx_cpu[c].tc_open_lock);
	mutex_destroy(&tx->tx_cpu[c].tc_lock);
	for (i = 0; i < TXG_SIZE; i++) {
	cv_destroy(&tx->tx_cpu[c].tc_cv[i]);
	list_destroy(&tx->tx_cpu[c].tc_callbacks[i]);
	}
	}

	if (tx->tx_commit_cb_taskq != NULL)
	taskq_destroy(tx->tx_commit_cb_taskq);

	vmem_free(tx->tx_cpu, max_ncpus * sizeof (tx_cpu_t));

	memset(tx, 0, sizeof (tx_state_t));
	}

	/*
	* Start syncing transaction groups.
	*/
	void
	txg_sync_start(dsl_pool_t *dp)
	{
	tx_state_t *tx = &dp->dp_tx;

	mutex_enter(&tx->tx_sync_lock);

	dprintf("pool %p\n", dp);

	ASSERT0(tx->tx_threads);

	tx->tx_threads = 2;

	tx->tx_quiesce_thread = thread_create(NULL, 0, txg_quiesce_thread,
	dp, 0, &p0, TS_RUN, defclsyspri);

	/*
	* The sync thread can need a larger-than-default stack size on
	* 32-bit x86. This is due in part to nested pools and
	* scrub_visitbp() recursion.
	*/
	tx->tx_sync_thread = thread_create(NULL, 0, txg_sync_thread,
	dp, 0, &p0, TS_RUN, defclsyspri);

	mutex_exit(&tx->tx_sync_lock);
	}

	static void
	txg_thread_enter(tx_state_t tx, callb_cpr_t cpr)
	{
	CALLB_CPR_INIT(cpr, &tx->tx_sync_lock, callb_generic_cpr, FTAG);
	mutex_enter(&tx->tx_sync_lock);
	}

	static void
	txg_thread_exit(tx_state_t tx, callb_cpr_t cpr, kthread_t **tpp)
	{
	ASSERT(*tpp != NULL);
	*tpp = NULL;
	tx->tx_threads--;
	cv_broadcast(&tx->tx_exit_cv);
	CALLB_CPR_EXIT(cpr); /* drops &tx->tx_sync_lock */
	thread_exit();
	}

	static void
	txg_thread_wait(tx_state_t tx, callb_cpr_t cpr, kcondvar_t *cv, clock_t time)
	{
	CALLB_CPR_SAFE_BEGIN(cpr);

	if (time) {
	(void) cv_timedwait_idle(cv, &tx->tx_sync_lock,
	ddi_get_lbolt() + time);
	} else {
	cv_wait_idle(cv, &tx->tx_sync_lock);
	}

	CALLB_CPR_SAFE_END(cpr, &tx->tx_sync_lock);
	}

	/*
	* Stop syncing transaction groups.
	*/
	void
	txg_sync_stop(dsl_pool_t *dp)
	{
	tx_state_t *tx = &dp->dp_tx;

	dprintf("pool %p\n", dp);
	/*
	* Finish off any work in progress.
	*/
	ASSERT3U(tx->tx_threads, ==, 2);

	/*
	* We need to ensure that we've vacated the deferred metaslab trees.
	*/
	txg_wait_synced(dp, tx->tx_open_txg + TXG_DEFER_SIZE);

	/*
	* Wake all sync threads and wait for them to die.
	*/
	mutex_enter(&tx->tx_sync_lock);

	ASSERT3U(tx->tx_threads, ==, 2);

	tx->tx_exiting = 1;

	cv_broadcast(&tx->tx_quiesce_more_cv);
	cv_broadcast(&tx->tx_quiesce_done_cv);
	cv_broadcast(&tx->tx_sync_more_cv);

	while (tx->tx_threads != 0)
	cv_wait(&tx->tx_exit_cv, &tx->tx_sync_lock);

	tx->tx_exiting = 0;

	mutex_exit(&tx->tx_sync_lock);
	}

	/*
	* Get a handle on the currently open txg and keep it open.
	*
	* The txg is guaranteed to stay open until txg_rele_to_quiesce() is called for
	* the handle. Once txg_rele_to_quiesce() has been called, the txg stays
	* in quiescing state until txg_rele_to_sync() is called for the handle.
	*
	* It is guaranteed that subsequent calls return monotonically increasing
	* txgs for the same dsl_pool_t. Of course this is not strong monotonicity,
	* because the same txg can be returned multiple times in a row. This
	* guarantee holds both for subsequent calls from one thread and for multiple
	* threads. For example, it is impossible to observe the following sequence
	* of events:
	*
	* Thread 1 Thread 2
	*
	* 1 <- txg_hold_open(P, ...)
	* 2 <- txg_hold_open(P, ...)
	* 1 <- txg_hold_open(P, ...)
	*
	*/
	uint64_t
	txg_hold_open(dsl_pool_t dp, txg_handle_t th)
	{
	tx_state_t *tx = &dp->dp_tx;
	tx_cpu_t *tc;
	uint64_t txg;

	/*
	* It appears the processor id is simply used as a "random"
	* number to index into the array, and there isn't any other
	* significance to the chosen tx_cpu. Because.. Why not use
	* the current cpu to index into the array?
	*/
	tc = &tx->tx_cpu[CPU_SEQID_UNSTABLE];

	mutex_enter(&tc->tc_open_lock);
	txg = tx->tx_open_txg;

	mutex_enter(&tc->tc_lock);
	tc->tc_count[txg & TXG_MASK]++;
	mutex_exit(&tc->tc_lock);

	th->th_cpu = tc;
	th->th_txg = txg;

	return (txg);
	}

	void
	txg_rele_to_quiesce(txg_handle_t *th)
	{
	tx_cpu_t *tc = th->th_cpu;

	ASSERT(!MUTEX_HELD(&tc->tc_lock));
	mutex_exit(&tc->tc_open_lock);
	}

	void
	txg_register_callbacks(txg_handle_t th, list_t tx_callbacks)
	{
	tx_cpu_t *tc = th->th_cpu;
	int g = th->th_txg & TXG_MASK;

	mutex_enter(&tc->tc_lock);
	list_move_tail(&tc->tc_callbacks[g], tx_callbacks);
	mutex_exit(&tc->tc_lock);
	}

	void
	txg_rele_to_sync(txg_handle_t *th)
	{
	tx_cpu_t *tc = th->th_cpu;
	int g = th->th_txg & TXG_MASK;

	mutex_enter(&tc->tc_lock);
	ASSERT(tc->tc_count[g] != 0);
	if (--tc->tc_count[g] == 0)
	cv_broadcast(&tc->tc_cv[g]);
	mutex_exit(&tc->tc_lock);

	th->th_cpu = NULL; /* defensive */
	}

	/*
	* Blocks until all transactions in the group are committed.
	*
	* On return, the transaction group has reached a stable state in which it can
	* then be passed off to the syncing context.
	*/
	static void
	txg_quiesce(dsl_pool_t *dp, uint64_t txg)
	{
	tx_state_t *tx = &dp->dp_tx;
	uint64_t tx_open_time;
	int g = txg & TXG_MASK;
	int c;

	/*
	* Grab all tc_open_locks so nobody else can get into this txg.
	*/
	for (c = 0; c < max_ncpus; c++)
	mutex_enter(&tx->tx_cpu[c].tc_open_lock);

	ASSERT(txg == tx->tx_open_txg);
	tx->tx_open_txg++;
	tx->tx_open_time = tx_open_time = gethrtime();

	DTRACE_PROBE2(txg__quiescing, dsl_pool_t *, dp, uint64_t, txg);
	DTRACE_PROBE2(txg__opened, dsl_pool_t *, dp, uint64_t, tx->tx_open_txg);

	/*
	* Now that we've incremented tx_open_txg, we can let threads
	* enter the next transaction group.
	*/
	for (c = 0; c < max_ncpus; c++)
	mutex_exit(&tx->tx_cpu[c].tc_open_lock);

	spa_txg_history_set(dp->dp_spa, txg, TXG_STATE_OPEN, tx_open_time);
	spa_txg_history_add(dp->dp_spa, txg + 1, tx_open_time);

	/*
	* Quiesce the transaction group by waiting for everyone to
	* call txg_rele_to_sync() for their open transaction handles.
	*/
	for (c = 0; c < max_ncpus; c++) {
	tx_cpu_t *tc = &tx->tx_cpu[c];
	mutex_enter(&tc->tc_lock);
	while (tc->tc_count[g] != 0)
	cv_wait(&tc->tc_cv[g], &tc->tc_lock);
	mutex_exit(&tc->tc_lock);
	}

	spa_txg_history_set(dp->dp_spa, txg, TXG_STATE_QUIESCED, gethrtime());
	}

	static void
	txg_do_callbacks(void *cb_list)
	{
	dmu_tx_do_callbacks(cb_list, 0);

	list_destroy(cb_list);

	kmem_free(cb_list, sizeof (list_t));
	}

	/*
	* Dispatch the commit callbacks registered on this txg to worker threads.
	*
	* If no callbacks are registered for a given TXG, nothing happens.
	* This function creates a taskq for the associated pool, if needed.
	*/
	static void
	txg_dispatch_callbacks(dsl_pool_t *dp, uint64_t txg)
	{
	int c;
	tx_state_t *tx = &dp->dp_tx;
	list_t *cb_list;

	for (c = 0; c < max_ncpus; c++) {
	tx_cpu_t *tc = &tx->tx_cpu[c];
	/*
	* No need to lock tx_cpu_t at this point, since this can
	* only be called once a txg has been synced.
	*/

	int g = txg & TXG_MASK;

	if (list_is_empty(&tc->tc_callbacks[g]))
	continue;

	if (tx->tx_commit_cb_taskq == NULL) {
	/*
	* Commit callback taskq hasn't been created yet.
	*/
	tx->tx_commit_cb_taskq = taskq_create("tx_commit_cb",
	100, defclsyspri, boot_ncpus, boot_ncpus * 2,
	TASKQ_PREPOPULATE \| TASKQ_DYNAMIC \|
	TASKQ_THREADS_CPU_PCT);
	}

	cb_list = kmem_alloc(sizeof (list_t), KM_SLEEP);
	list_create(cb_list, sizeof (dmu_tx_callback_t),
	offsetof(dmu_tx_callback_t, dcb_node));

	list_move_tail(cb_list, &tc->tc_callbacks[g]);

	(void) taskq_dispatch(tx->tx_commit_cb_taskq,
	txg_do_callbacks, cb_list, TQ_SLEEP);
	}
	}

	/*
	* Wait for pending commit callbacks of already-synced transactions to finish
	* processing.
	* Calling this function from within a commit callback will deadlock.
	*/
	void
	txg_wait_callbacks(dsl_pool_t *dp)
	{
	tx_state_t *tx = &dp->dp_tx;

	if (tx->tx_commit_cb_taskq != NULL)
	taskq_wait_outstanding(tx->tx_commit_cb_taskq, 0);
	}

	static boolean_t
	txg_is_quiescing(dsl_pool_t *dp)
	{
	tx_state_t *tx = &dp->dp_tx;
	ASSERT(MUTEX_HELD(&tx->tx_sync_lock));
	return (tx->tx_quiescing_txg != 0);
	}

	static boolean_t
	txg_has_quiesced_to_sync(dsl_pool_t *dp)
	{
	tx_state_t *tx = &dp->dp_tx;
	ASSERT(MUTEX_HELD(&tx->tx_sync_lock));
	return (tx->tx_quiesced_txg != 0);
	}

	static __attribute__((noreturn)) void
	txg_sync_thread(void *arg)
	{
	dsl_pool_t *dp = arg;
	spa_t *spa = dp->dp_spa;
	tx_state_t *tx = &dp->dp_tx;
	callb_cpr_t cpr;
	clock_t start, delta;

	(void) spl_fstrans_mark();
	txg_thread_enter(tx, &cpr);

	start = delta = 0;
	for (;;) {
	clock_t timeout = zfs_txg_timeout * hz;
	clock_t timer;
	uint64_t txg;

	/*
	* We sync when we're scanning, there's someone waiting
	* on us, or the quiesce thread has handed off a txg to
	* us, or we have reached our timeout.
	*/
	timer = (delta >= timeout ? 0 : timeout - delta);
	while (!dsl_scan_active(dp->dp_scan) &&
	!tx->tx_exiting && timer > 0 &&
	tx->tx_synced_txg >= tx->tx_sync_txg_waiting &&
	!txg_has_quiesced_to_sync(dp)) {
	dprintf("waiting; tx_synced=%llu waiting=%llu dp=%p\n",
	(u_longlong_t)tx->tx_synced_txg,
	(u_longlong_t)tx->tx_sync_txg_waiting, dp);
	txg_thread_wait(tx, &cpr, &tx->tx_sync_more_cv, timer);
	delta = ddi_get_lbolt() - start;
	timer = (delta > timeout ? 0 : timeout - delta);
	}

	+ /*
	+ * When we're suspended, nothing should be changing and for
	+ * MMP we don't want to bump anything that would make it
	+ * harder to detect if another host is changing it when
	+ * resuming after a MMP suspend.
	+ */
	+ if (spa_suspended(spa))
	+ continue;
	+
	/*
	* Wait until the quiesce thread hands off a txg to us,
	* prompting it to do so if necessary.
	*/
	while (!tx->tx_exiting && !txg_has_quiesced_to_sync(dp)) {
	if (txg_is_quiescing(dp)) {
	txg_thread_wait(tx, &cpr,
	&tx->tx_quiesce_done_cv, 0);
	continue;
	}
	if (tx->tx_quiesce_txg_waiting < tx->tx_open_txg+1)
	tx->tx_quiesce_txg_waiting = tx->tx_open_txg+1;
	cv_broadcast(&tx->tx_quiesce_more_cv);
	txg_thread_wait(tx, &cpr, &tx->tx_quiesce_done_cv, 0);
	}

	if (tx->tx_exiting)
	txg_thread_exit(tx, &cpr, &tx->tx_sync_thread);

	/*
	* Consume the quiesced txg which has been handed off to
	* us. This may cause the quiescing thread to now be
	* able to quiesce another txg, so we must signal it.
	*/
	ASSERT(tx->tx_quiesced_txg != 0);
	txg = tx->tx_quiesced_txg;
	tx->tx_quiesced_txg = 0;
	tx->tx_syncing_txg = txg;
	DTRACE_PROBE2(txg__syncing, dsl_pool_t *, dp, uint64_t, txg);
	cv_broadcast(&tx->tx_quiesce_more_cv);

	dprintf("txg=%llu quiesce_txg=%llu sync_txg=%llu\n",
	(u_longlong_t)txg, (u_longlong_t)tx->tx_quiesce_txg_waiting,
	(u_longlong_t)tx->tx_sync_txg_waiting);
	mutex_exit(&tx->tx_sync_lock);

	txg_stat_t *ts = spa_txg_history_init_io(spa, txg, dp);
	start = ddi_get_lbolt();
	spa_sync(spa, txg);
	delta = ddi_get_lbolt() - start;
	spa_txg_history_fini_io(spa, ts);

	mutex_enter(&tx->tx_sync_lock);
	tx->tx_synced_txg = txg;
	tx->tx_syncing_txg = 0;
	DTRACE_PROBE2(txg__synced, dsl_pool_t *, dp, uint64_t, txg);
	cv_broadcast(&tx->tx_sync_done_cv);

	/*
	* Dispatch commit callbacks to worker threads.
	*/
	txg_dispatch_callbacks(dp, txg);
	}
	}

	static __attribute__((noreturn)) void
	txg_quiesce_thread(void *arg)
	{
	dsl_pool_t *dp = arg;
	tx_state_t *tx = &dp->dp_tx;
	callb_cpr_t cpr;

	txg_thread_enter(tx, &cpr);

	for (;;) {
	uint64_t txg;

	/*
	* We quiesce when there's someone waiting on us.
	* However, we can only have one txg in "quiescing" or
	* "quiesced, waiting to sync" state. So we wait until
	* the "quiesced, waiting to sync" txg has been consumed
	* by the sync thread.
	*/
	while (!tx->tx_exiting &&
	(tx->tx_open_txg >= tx->tx_quiesce_txg_waiting \|\|
	txg_has_quiesced_to_sync(dp)))
	txg_thread_wait(tx, &cpr, &tx->tx_quiesce_more_cv, 0);

	if (tx->tx_exiting)
	txg_thread_exit(tx, &cpr, &tx->tx_quiesce_thread);

	txg = tx->tx_open_txg;
	dprintf("txg=%llu quiesce_txg=%llu sync_txg=%llu\n",
	(u_longlong_t)txg,
	(u_longlong_t)tx->tx_quiesce_txg_waiting,
	(u_longlong_t)tx->tx_sync_txg_waiting);
	tx->tx_quiescing_txg = txg;

	mutex_exit(&tx->tx_sync_lock);
	txg_quiesce(dp, txg);
	mutex_enter(&tx->tx_sync_lock);

	/*
	* Hand this txg off to the sync thread.
	*/
	dprintf("quiesce done, handing off txg %llu\n",
	(u_longlong_t)txg);
	tx->tx_quiescing_txg = 0;
	tx->tx_quiesced_txg = txg;
	DTRACE_PROBE2(txg__quiesced, dsl_pool_t *, dp, uint64_t, txg);
	cv_broadcast(&tx->tx_sync_more_cv);
	cv_broadcast(&tx->tx_quiesce_done_cv);
	}
	}

	/*
	* Delay this thread by delay nanoseconds if we are still in the open
	* transaction group and there is already a waiting txg quiescing or quiesced.
	* Abort the delay if this txg stalls or enters the quiescing state.
	*/
	void
	txg_delay(dsl_pool_t *dp, uint64_t txg, hrtime_t delay, hrtime_t resolution)
	{
	tx_state_t *tx = &dp->dp_tx;
	hrtime_t start = gethrtime();

	/* don't delay if this txg could transition to quiescing immediately */
	if (tx->tx_open_txg > txg \|\|
	tx->tx_syncing_txg == txg-1 \|\| tx->tx_synced_txg == txg-1)
	return;

	mutex_enter(&tx->tx_sync_lock);
	if (tx->tx_open_txg > txg \|\| tx->tx_synced_txg == txg-1) {
	mutex_exit(&tx->tx_sync_lock);
	return;
	}

	while (gethrtime() - start < delay &&
	tx->tx_syncing_txg < txg-1 && !txg_stalled(dp)) {
	(void) cv_timedwait_hires(&tx->tx_quiesce_more_cv,
	&tx->tx_sync_lock, delay, resolution, 0);
	}

	DMU_TX_STAT_BUMP(dmu_tx_delay);

	mutex_exit(&tx->tx_sync_lock);
	}

	static boolean_t
	txg_wait_synced_impl(dsl_pool_t *dp, uint64_t txg, boolean_t wait_sig)
	{
	tx_state_t *tx = &dp->dp_tx;

	ASSERT(!dsl_pool_config_held(dp));

	mutex_enter(&tx->tx_sync_lock);
	ASSERT3U(tx->tx_threads, ==, 2);
	if (txg == 0)
	txg = tx->tx_open_txg + TXG_DEFER_SIZE;
	if (tx->tx_sync_txg_waiting < txg)
	tx->tx_sync_txg_waiting = txg;
	dprintf("txg=%llu quiesce_txg=%llu sync_txg=%llu\n",
	(u_longlong_t)txg, (u_longlong_t)tx->tx_quiesce_txg_waiting,
	(u_longlong_t)tx->tx_sync_txg_waiting);
	while (tx->tx_synced_txg < txg) {
	dprintf("broadcasting sync more "
	"tx_synced=%llu waiting=%llu dp=%px\n",
	(u_longlong_t)tx->tx_synced_txg,
	(u_longlong_t)tx->tx_sync_txg_waiting, dp);
	cv_broadcast(&tx->tx_sync_more_cv);
	if (wait_sig) {
	/*
	* Condition wait here but stop if the thread receives a
	* signal. The caller may call txg_wait_synced*() again
	* to resume waiting for this txg.
	*/
	if (cv_wait_io_sig(&tx->tx_sync_done_cv,
	&tx->tx_sync_lock) == 0) {
	mutex_exit(&tx->tx_sync_lock);
	return (B_TRUE);
	}
	} else {
	cv_wait_io(&tx->tx_sync_done_cv, &tx->tx_sync_lock);
	}
	}
	mutex_exit(&tx->tx_sync_lock);
	return (B_FALSE);
	}

	void
	txg_wait_synced(dsl_pool_t *dp, uint64_t txg)
	{
	VERIFY0(txg_wait_synced_impl(dp, txg, B_FALSE));
	}

	/*
	* Similar to a txg_wait_synced but it can be interrupted from a signal.
	* Returns B_TRUE if the thread was signaled while waiting.
	*/
	boolean_t
	txg_wait_synced_sig(dsl_pool_t *dp, uint64_t txg)
	{
	return (txg_wait_synced_impl(dp, txg, B_TRUE));
	}

	/*
	* Wait for the specified open transaction group. Set should_quiesce
	* when the current open txg should be quiesced immediately.
	*/
	void
	txg_wait_open(dsl_pool_t *dp, uint64_t txg, boolean_t should_quiesce)
	{
	tx_state_t *tx = &dp->dp_tx;

	ASSERT(!dsl_pool_config_held(dp));

	mutex_enter(&tx->tx_sync_lock);
	ASSERT3U(tx->tx_threads, ==, 2);
	if (txg == 0)
	txg = tx->tx_open_txg + 1;
	if (tx->tx_quiesce_txg_waiting < txg && should_quiesce)
	tx->tx_quiesce_txg_waiting = txg;
	dprintf("txg=%llu quiesce_txg=%llu sync_txg=%llu\n",
	(u_longlong_t)txg, (u_longlong_t)tx->tx_quiesce_txg_waiting,
	(u_longlong_t)tx->tx_sync_txg_waiting);
	while (tx->tx_open_txg < txg) {
	cv_broadcast(&tx->tx_quiesce_more_cv);
	/*
	* Callers setting should_quiesce will use cv_wait_io() and
	* be accounted for as iowait time. Otherwise, the caller is
	* understood to be idle and cv_wait_sig() is used to prevent
	* incorrectly inflating the system load average.
	*/
	if (should_quiesce == B_TRUE) {
	cv_wait_io(&tx->tx_quiesce_done_cv, &tx->tx_sync_lock);
	} else {
	cv_wait_idle(&tx->tx_quiesce_done_cv,
	&tx->tx_sync_lock);
	}
	}
	mutex_exit(&tx->tx_sync_lock);
	}

	/*
	* Pass in the txg number that should be synced.
	*/
	void
	txg_kick(dsl_pool_t *dp, uint64_t txg)
	{
	tx_state_t *tx = &dp->dp_tx;

	ASSERT(!dsl_pool_config_held(dp));

	if (tx->tx_sync_txg_waiting >= txg)
	return;

	mutex_enter(&tx->tx_sync_lock);
	if (tx->tx_sync_txg_waiting < txg) {
	tx->tx_sync_txg_waiting = txg;
	cv_broadcast(&tx->tx_sync_more_cv);
	}
	mutex_exit(&tx->tx_sync_lock);
	}

	boolean_t
	txg_stalled(dsl_pool_t *dp)
	{
	tx_state_t *tx = &dp->dp_tx;
	return (tx->tx_quiesce_txg_waiting > tx->tx_open_txg);
	}

	boolean_t
	txg_sync_waiting(dsl_pool_t *dp)
	{
	tx_state_t *tx = &dp->dp_tx;

	return (tx->tx_syncing_txg <= tx->tx_sync_txg_waiting \|\|
	tx->tx_quiesced_txg != 0);
	}

	/*
	* Verify that this txg is active (open, quiescing, syncing). Non-active
	* txg's should not be manipulated.
	*/
	#ifdef ZFS_DEBUG
	void
	txg_verify(spa_t *spa, uint64_t txg)
	{
	dsl_pool_t *dp __maybe_unused = spa_get_dsl(spa);
	if (txg <= TXG_INITIAL \|\| txg == ZILTEST_TXG)
	return;
	ASSERT3U(txg, <=, dp->dp_tx.tx_open_txg);
	ASSERT3U(txg, >=, dp->dp_tx.tx_synced_txg);
	ASSERT3U(txg, >=, dp->dp_tx.tx_open_txg - TXG_CONCURRENT_STATES);
	}
	#endif

	/*
	* Per-txg object lists.
	*/
	void
	txg_list_create(txg_list_t tl, spa_t spa, size_t offset)
	{
	int t;

	mutex_init(&tl->tl_lock, NULL, MUTEX_DEFAULT, NULL);

	tl->tl_offset = offset;
	tl->tl_spa = spa;

	for (t = 0; t < TXG_SIZE; t++)
	tl->tl_head[t] = NULL;
	}

	static boolean_t
	txg_list_empty_impl(txg_list_t *tl, uint64_t txg)
	{
	ASSERT(MUTEX_HELD(&tl->tl_lock));
	TXG_VERIFY(tl->tl_spa, txg);
	return (tl->tl_head[txg & TXG_MASK] == NULL);
	}

	boolean_t
	txg_list_empty(txg_list_t *tl, uint64_t txg)
	{
	mutex_enter(&tl->tl_lock);
	boolean_t ret = txg_list_empty_impl(tl, txg);
	mutex_exit(&tl->tl_lock);

	return (ret);
	}

	void
	txg_list_destroy(txg_list_t *tl)
	{
	int t;

	mutex_enter(&tl->tl_lock);
	for (t = 0; t < TXG_SIZE; t++)
	ASSERT(txg_list_empty_impl(tl, t));
	mutex_exit(&tl->tl_lock);

	mutex_destroy(&tl->tl_lock);
	}

	/*
	* Returns true if all txg lists are empty.
	*
	* Warning: this is inherently racy (an item could be added immediately
	* after this function returns).
	*/
	boolean_t
	txg_all_lists_empty(txg_list_t *tl)
	{
	boolean_t res = B_TRUE;
	for (int i = 0; i < TXG_SIZE; i++)
	res &= (tl->tl_head[i] == NULL);
	return (res);
	}

	/*
	* Add an entry to the list (unless it's already on the list).
	* Returns B_TRUE if it was actually added.
	*/
	boolean_t
	txg_list_add(txg_list_t tl, void p, uint64_t txg)
	{
	int t = txg & TXG_MASK;
	txg_node_t tn = (txg_node_t )((char *)p + tl->tl_offset);
	boolean_t add;

	TXG_VERIFY(tl->tl_spa, txg);
	mutex_enter(&tl->tl_lock);
	add = (tn->tn_member[t] == 0);
	if (add) {
	tn->tn_member[t] = 1;
	tn->tn_next[t] = tl->tl_head[t];
	tl->tl_head[t] = tn;
	}
	mutex_exit(&tl->tl_lock);

	return (add);
	}

	/*
	* Add an entry to the end of the list, unless it's already on the list.
	* (walks list to find end)
	* Returns B_TRUE if it was actually added.
	*/
	boolean_t
	txg_list_add_tail(txg_list_t tl, void p, uint64_t txg)
	{
	int t = txg & TXG_MASK;
	txg_node_t tn = (txg_node_t )((char *)p + tl->tl_offset);
	boolean_t add;

	TXG_VERIFY(tl->tl_spa, txg);
	mutex_enter(&tl->tl_lock);
	add = (tn->tn_member[t] == 0);
	if (add) {
	txg_node_t **tp;

	for (tp = &tl->tl_head[t]; tp != NULL; tp = &(tp)->tn_next[t])
	continue;

	tn->tn_member[t] = 1;
	tn->tn_next[t] = NULL;
	*tp = tn;
	}
	mutex_exit(&tl->tl_lock);

	return (add);
	}

	/*
	* Remove the head of the list and return it.
	*/
	void *
	txg_list_remove(txg_list_t *tl, uint64_t txg)
	{
	int t = txg & TXG_MASK;
	txg_node_t *tn;
	void *p = NULL;

	TXG_VERIFY(tl->tl_spa, txg);
	mutex_enter(&tl->tl_lock);
	if ((tn = tl->tl_head[t]) != NULL) {
	ASSERT(tn->tn_member[t]);
	ASSERT(tn->tn_next[t] == NULL \|\| tn->tn_next[t]->tn_member[t]);
	p = (char *)tn - tl->tl_offset;
	tl->tl_head[t] = tn->tn_next[t];
	tn->tn_next[t] = NULL;
	tn->tn_member[t] = 0;
	}
	mutex_exit(&tl->tl_lock);

	return (p);
	}

	/*
	* Remove a specific item from the list and return it.
	*/
	void *
	txg_list_remove_this(txg_list_t tl, void p, uint64_t txg)
	{
	int t = txg & TXG_MASK;
	txg_node_t tn, *tp;

	TXG_VERIFY(tl->tl_spa, txg);
	mutex_enter(&tl->tl_lock);

	for (tp = &tl->tl_head[t]; (tn = *tp) != NULL; tp = &tn->tn_next[t]) {
	if ((char *)tn - tl->tl_offset == p) {
	*tp = tn->tn_next[t];
	tn->tn_next[t] = NULL;
	tn->tn_member[t] = 0;
	mutex_exit(&tl->tl_lock);
	return (p);
	}
	}

	mutex_exit(&tl->tl_lock);

	return (NULL);
	}

	boolean_t
	txg_list_member(txg_list_t tl, void p, uint64_t txg)
	{
	int t = txg & TXG_MASK;
	txg_node_t tn = (txg_node_t )((char *)p + tl->tl_offset);

	TXG_VERIFY(tl->tl_spa, txg);
	return (tn->tn_member[t] != 0);
	}

	/*
	* Walk a txg list
	*/
	void *
	txg_list_head(txg_list_t *tl, uint64_t txg)
	{
	int t = txg & TXG_MASK;
	txg_node_t *tn;

	mutex_enter(&tl->tl_lock);
	tn = tl->tl_head[t];
	mutex_exit(&tl->tl_lock);

	TXG_VERIFY(tl->tl_spa, txg);
	return (tn == NULL ? NULL : (char *)tn - tl->tl_offset);
	}

	void *
	txg_list_next(txg_list_t tl, void p, uint64_t txg)
	{
	int t = txg & TXG_MASK;
	txg_node_t tn = (txg_node_t )((char *)p + tl->tl_offset);

	TXG_VERIFY(tl->tl_spa, txg);

	mutex_enter(&tl->tl_lock);
	tn = tn->tn_next[t];
	mutex_exit(&tl->tl_lock);

	return (tn == NULL ? NULL : (char *)tn - tl->tl_offset);
	}

	EXPORT_SYMBOL(txg_init);
	EXPORT_SYMBOL(txg_fini);
	EXPORT_SYMBOL(txg_sync_start);
	EXPORT_SYMBOL(txg_sync_stop);
	EXPORT_SYMBOL(txg_hold_open);
	EXPORT_SYMBOL(txg_rele_to_quiesce);
	EXPORT_SYMBOL(txg_rele_to_sync);
	EXPORT_SYMBOL(txg_register_callbacks);
	EXPORT_SYMBOL(txg_delay);
	EXPORT_SYMBOL(txg_wait_synced);
	EXPORT_SYMBOL(txg_wait_open);
	EXPORT_SYMBOL(txg_wait_callbacks);
	EXPORT_SYMBOL(txg_stalled);
	EXPORT_SYMBOL(txg_sync_waiting);

	ZFS_MODULE_PARAM(zfs_txg, zfs_txg_, timeout, UINT, ZMOD_RW,
	"Max seconds worth of delta per txg");
	diff --git a/sys/contrib/openzfs/module/zfs/vdev.c b/sys/contrib/openzfs/module/zfs/vdev.c
	index e1ca1aecc900..981da4e986c4 100644
	--- a/sys/contrib/openzfs/module/zfs/vdev.c
	+++ b/sys/contrib/openzfs/module/zfs/vdev.c
	@@ -1,6427 +1,6461 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2011, 2021 by Delphix. All rights reserved.
	* Copyright 2017 Nexenta Systems, Inc.
	* Copyright (c) 2014 Integros [integros.com]
	* Copyright 2016 Toomas Soome <tsoome@me.com>
	* Copyright 2017 Joyent, Inc.
	* Copyright (c) 2017, Intel Corporation.
	* Copyright (c) 2019, Datto Inc. All rights reserved.
	* Copyright (c) 2021, Klara Inc.
	* Copyright (c) 2021, 2023 Hewlett Packard Enterprise Development LP.
	*/

	#include <sys/zfs_context.h>
	#include <sys/fm/fs/zfs.h>
	#include <sys/spa.h>
	#include <sys/spa_impl.h>
	#include <sys/bpobj.h>
	#include <sys/dmu.h>
	#include <sys/dmu_tx.h>
	#include <sys/dsl_dir.h>
	#include <sys/vdev_impl.h>
	#include <sys/vdev_rebuild.h>
	#include <sys/vdev_draid.h>
	#include <sys/uberblock_impl.h>
	#include <sys/metaslab.h>
	#include <sys/metaslab_impl.h>
	#include <sys/space_map.h>
	#include <sys/space_reftree.h>
	#include <sys/zio.h>
	#include <sys/zap.h>
	#include <sys/fs/zfs.h>
	#include <sys/arc.h>
	#include <sys/zil.h>
	#include <sys/dsl_scan.h>
	#include <sys/vdev_raidz.h>
	#include <sys/abd.h>
	#include <sys/vdev_initialize.h>
	#include <sys/vdev_trim.h>
	#include <sys/zvol.h>
	#include <sys/zfs_ratelimit.h>
	#include "zfs_prop.h"

	/*
	* One metaslab from each (normal-class) vdev is used by the ZIL. These are
	* called "embedded slog metaslabs", are referenced by vdev_log_mg, and are
	* part of the spa_embedded_log_class. The metaslab with the most free space
	* in each vdev is selected for this purpose when the pool is opened (or a
	* vdev is added). See vdev_metaslab_init().
	*
	* Log blocks can be allocated from the following locations. Each one is tried
	* in order until the allocation succeeds:
	* 1. dedicated log vdevs, aka "slog" (spa_log_class)
	* 2. embedded slog metaslabs (spa_embedded_log_class)
	* 3. other metaslabs in normal vdevs (spa_normal_class)
	*
	* zfs_embedded_slog_min_ms disables the embedded slog if there are fewer
	* than this number of metaslabs in the vdev. This ensures that we don't set
	* aside an unreasonable amount of space for the ZIL. If set to less than
	* 1 << (spa_slop_shift + 1), on small pools the usable space may be reduced
	* (by more than 1<<spa_slop_shift) due to the embedded slog metaslab.
	*/
	static uint_t zfs_embedded_slog_min_ms = 64;

	/* default target for number of metaslabs per top-level vdev */
	static uint_t zfs_vdev_default_ms_count = 200;

	/* minimum number of metaslabs per top-level vdev */
	static uint_t zfs_vdev_min_ms_count = 16;

	/* practical upper limit of total metaslabs per top-level vdev */
	static uint_t zfs_vdev_ms_count_limit = 1ULL << 17;

	/* lower limit for metaslab size (512M) */
	static uint_t zfs_vdev_default_ms_shift = 29;

	/* upper limit for metaslab size (16G) */
	static uint_t zfs_vdev_max_ms_shift = 34;

	int vdev_validate_skip = B_FALSE;

	/*
	* Since the DTL space map of a vdev is not expected to have a lot of
	* entries, we default its block size to 4K.
	*/
	int zfs_vdev_dtl_sm_blksz = (1 << 12);

	/*
	* Rate limit slow IO (delay) events to this many per second.
	*/
	static unsigned int zfs_slow_io_events_per_second = 20;

	/*
	* Rate limit checksum events after this many checksum errors per second.
	*/
	static unsigned int zfs_checksum_events_per_second = 20;

	/*
	* Ignore errors during scrub/resilver. Allows to work around resilver
	* upon import when there are pool errors.
	*/
	static int zfs_scan_ignore_errors = 0;

	/*
	* vdev-wide space maps that have lots of entries written to them at
	* the end of each transaction can benefit from a higher I/O bandwidth
	* (e.g. vdev_obsolete_sm), thus we default their block size to 128K.
	*/
	int zfs_vdev_standard_sm_blksz = (1 << 17);

	/*
	* Tunable parameter for debugging or performance analysis. Setting this
	* will cause pool corruption on power loss if a volatile out-of-order
	* write cache is enabled.
	*/
	int zfs_nocacheflush = 0;

	/*
	* Maximum and minimum ashift values that can be automatically set based on
	* vdev's physical ashift (disk's physical sector size). While ASHIFT_MAX
	* is higher than the maximum value, it is intentionally limited here to not
	* excessively impact pool space efficiency. Higher ashift values may still
	* be forced by vdev logical ashift or by user via ashift property, but won't
	* be set automatically as a performance optimization.
	*/
	uint_t zfs_vdev_max_auto_ashift = 14;
	uint_t zfs_vdev_min_auto_ashift = ASHIFT_MIN;

	void
	vdev_dbgmsg(vdev_t vd, const char fmt, ...)
	{
	va_list adx;
	char buf[256];

	va_start(adx, fmt);
	(void) vsnprintf(buf, sizeof (buf), fmt, adx);
	va_end(adx);

	if (vd->vdev_path != NULL) {
	zfs_dbgmsg("%s vdev '%s': %s", vd->vdev_ops->vdev_op_type,
	vd->vdev_path, buf);
	} else {
	zfs_dbgmsg("%s-%llu vdev (guid %llu): %s",
	vd->vdev_ops->vdev_op_type,
	(u_longlong_t)vd->vdev_id,
	(u_longlong_t)vd->vdev_guid, buf);
	}
	}

	void
	vdev_dbgmsg_print_tree(vdev_t *vd, int indent)
	{
	char state[20];

	if (vd->vdev_ishole \|\| vd->vdev_ops == &vdev_missing_ops) {
	zfs_dbgmsg("%*svdev %llu: %s", indent, "",
	(u_longlong_t)vd->vdev_id,
	vd->vdev_ops->vdev_op_type);
	return;
	}

	switch (vd->vdev_state) {
	case VDEV_STATE_UNKNOWN:
	(void) snprintf(state, sizeof (state), "unknown");
	break;
	case VDEV_STATE_CLOSED:
	(void) snprintf(state, sizeof (state), "closed");
	break;
	case VDEV_STATE_OFFLINE:
	(void) snprintf(state, sizeof (state), "offline");
	break;
	case VDEV_STATE_REMOVED:
	(void) snprintf(state, sizeof (state), "removed");
	break;
	case VDEV_STATE_CANT_OPEN:
	(void) snprintf(state, sizeof (state), "can't open");
	break;
	case VDEV_STATE_FAULTED:
	(void) snprintf(state, sizeof (state), "faulted");
	break;
	case VDEV_STATE_DEGRADED:
	(void) snprintf(state, sizeof (state), "degraded");
	break;
	case VDEV_STATE_HEALTHY:
	(void) snprintf(state, sizeof (state), "healthy");
	break;
	default:
	(void) snprintf(state, sizeof (state), "<state %u>",
	(uint_t)vd->vdev_state);
	}

	zfs_dbgmsg("%*svdev %u: %s%s, guid: %llu, path: %s, %s", indent,
	"", (int)vd->vdev_id, vd->vdev_ops->vdev_op_type,
	vd->vdev_islog ? " (log)" : "",
	(u_longlong_t)vd->vdev_guid,
	vd->vdev_path ? vd->vdev_path : "N/A", state);

	for (uint64_t i = 0; i < vd->vdev_children; i++)
	vdev_dbgmsg_print_tree(vd->vdev_child[i], indent + 2);
	}

	/*
	* Virtual device management.
	*/

	static vdev_ops_t *const vdev_ops_table[] = {
	&vdev_root_ops,
	&vdev_raidz_ops,
	&vdev_draid_ops,
	&vdev_draid_spare_ops,
	&vdev_mirror_ops,
	&vdev_replacing_ops,
	&vdev_spare_ops,
	&vdev_disk_ops,
	&vdev_file_ops,
	&vdev_missing_ops,
	&vdev_hole_ops,
	&vdev_indirect_ops,
	NULL
	};

	/*
	* Given a vdev type, return the appropriate ops vector.
	*/
	static vdev_ops_t *
	vdev_getops(const char *type)
	{
	vdev_ops_t ops, const *opspp;

	for (opspp = vdev_ops_table; (ops = *opspp) != NULL; opspp++)
	if (strcmp(ops->vdev_op_type, type) == 0)
	break;

	return (ops);
	}

	/*
	* Given a vdev and a metaslab class, find which metaslab group we're
	* interested in. All vdevs may belong to two different metaslab classes.
	* Dedicated slog devices use only the primary metaslab group, rather than a
	* separate log group. For embedded slogs, the vdev_log_mg will be non-NULL.
	*/
	metaslab_group_t *
	vdev_get_mg(vdev_t vd, metaslab_class_t mc)
	{
	if (mc == spa_embedded_log_class(vd->vdev_spa) &&
	vd->vdev_log_mg != NULL)
	return (vd->vdev_log_mg);
	else
	return (vd->vdev_mg);
	}

	void
	vdev_default_xlate(vdev_t vd, const range_seg64_t logical_rs,
	range_seg64_t physical_rs, range_seg64_t remain_rs)
	{
	(void) vd, (void) remain_rs;

	physical_rs->rs_start = logical_rs->rs_start;
	physical_rs->rs_end = logical_rs->rs_end;
	}

	/*
	* Derive the enumerated allocation bias from string input.
	* String origin is either the per-vdev zap or zpool(8).
	*/
	static vdev_alloc_bias_t
	vdev_derive_alloc_bias(const char *bias)
	{
	vdev_alloc_bias_t alloc_bias = VDEV_BIAS_NONE;

	if (strcmp(bias, VDEV_ALLOC_BIAS_LOG) == 0)
	alloc_bias = VDEV_BIAS_LOG;
	else if (strcmp(bias, VDEV_ALLOC_BIAS_SPECIAL) == 0)
	alloc_bias = VDEV_BIAS_SPECIAL;
	else if (strcmp(bias, VDEV_ALLOC_BIAS_DEDUP) == 0)
	alloc_bias = VDEV_BIAS_DEDUP;

	return (alloc_bias);
	}

	/*
	* Default asize function: return the MAX of psize with the asize of
	* all children. This is what's used by anything other than RAID-Z.
	*/
	uint64_t
	vdev_default_asize(vdev_t *vd, uint64_t psize)
	{
	uint64_t asize = P2ROUNDUP(psize, 1ULL << vd->vdev_top->vdev_ashift);
	uint64_t csize;

	for (int c = 0; c < vd->vdev_children; c++) {
	csize = vdev_psize_to_asize(vd->vdev_child[c], psize);
	asize = MAX(asize, csize);
	}

	return (asize);
	}

	uint64_t
	vdev_default_min_asize(vdev_t *vd)
	{
	return (vd->vdev_min_asize);
	}

	/*
	* Get the minimum allocatable size. We define the allocatable size as
	* the vdev's asize rounded to the nearest metaslab. This allows us to
	* replace or attach devices which don't have the same physical size but
	* can still satisfy the same number of allocations.
	*/
	uint64_t
	vdev_get_min_asize(vdev_t *vd)
	{
	vdev_t *pvd = vd->vdev_parent;

	/*
	* If our parent is NULL (inactive spare or cache) or is the root,
	* just return our own asize.
	*/
	if (pvd == NULL)
	return (vd->vdev_asize);

	/*
	* The top-level vdev just returns the allocatable size rounded
	* to the nearest metaslab.
	*/
	if (vd == vd->vdev_top)
	return (P2ALIGN(vd->vdev_asize, 1ULL << vd->vdev_ms_shift));

	return (pvd->vdev_ops->vdev_op_min_asize(pvd));
	}

	void
	vdev_set_min_asize(vdev_t *vd)
	{
	vd->vdev_min_asize = vdev_get_min_asize(vd);

	for (int c = 0; c < vd->vdev_children; c++)
	vdev_set_min_asize(vd->vdev_child[c]);
	}

	/*
	* Get the minimal allocation size for the top-level vdev.
	*/
	uint64_t
	vdev_get_min_alloc(vdev_t *vd)
	{
	uint64_t min_alloc = 1ULL << vd->vdev_ashift;

	if (vd->vdev_ops->vdev_op_min_alloc != NULL)
	min_alloc = vd->vdev_ops->vdev_op_min_alloc(vd);

	return (min_alloc);
	}

	/*
	* Get the parity level for a top-level vdev.
	*/
	uint64_t
	vdev_get_nparity(vdev_t *vd)
	{
	uint64_t nparity = 0;

	if (vd->vdev_ops->vdev_op_nparity != NULL)
	nparity = vd->vdev_ops->vdev_op_nparity(vd);

	return (nparity);
	}

	static int
	vdev_prop_get_int(vdev_t vd, vdev_prop_t prop, uint64_t value)
	{
	spa_t *spa = vd->vdev_spa;
	objset_t *mos = spa->spa_meta_objset;
	uint64_t objid;
	int err;

	if (vd->vdev_root_zap != 0) {
	objid = vd->vdev_root_zap;
	} else if (vd->vdev_top_zap != 0) {
	objid = vd->vdev_top_zap;
	} else if (vd->vdev_leaf_zap != 0) {
	objid = vd->vdev_leaf_zap;
	} else {
	return (EINVAL);
	}

	err = zap_lookup(mos, objid, vdev_prop_to_name(prop),
	sizeof (uint64_t), 1, value);

	if (err == ENOENT)
	*value = vdev_prop_default_numeric(prop);

	return (err);
	}

	/*
	* Get the number of data disks for a top-level vdev.
	*/
	uint64_t
	vdev_get_ndisks(vdev_t *vd)
	{
	uint64_t ndisks = 1;

	if (vd->vdev_ops->vdev_op_ndisks != NULL)
	ndisks = vd->vdev_ops->vdev_op_ndisks(vd);

	return (ndisks);
	}

	vdev_t *
	vdev_lookup_top(spa_t *spa, uint64_t vdev)
	{
	vdev_t *rvd = spa->spa_root_vdev;

	ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) != 0);

	if (vdev < rvd->vdev_children) {
	ASSERT(rvd->vdev_child[vdev] != NULL);
	return (rvd->vdev_child[vdev]);
	}

	return (NULL);
	}

	vdev_t *
	vdev_lookup_by_guid(vdev_t *vd, uint64_t guid)
	{
	vdev_t *mvd;

	if (vd->vdev_guid == guid)
	return (vd);

	for (int c = 0; c < vd->vdev_children; c++)
	if ((mvd = vdev_lookup_by_guid(vd->vdev_child[c], guid)) !=
	NULL)
	return (mvd);

	return (NULL);
	}

	static int
	vdev_count_leaves_impl(vdev_t *vd)
	{
	int n = 0;

	if (vd->vdev_ops->vdev_op_leaf)
	return (1);

	for (int c = 0; c < vd->vdev_children; c++)
	n += vdev_count_leaves_impl(vd->vdev_child[c]);

	return (n);
	}

	int
	vdev_count_leaves(spa_t *spa)
	{
	int rc;

	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
	rc = vdev_count_leaves_impl(spa->spa_root_vdev);
	spa_config_exit(spa, SCL_VDEV, FTAG);

	return (rc);
	}

	void
	vdev_add_child(vdev_t pvd, vdev_t cvd)
	{
	size_t oldsize, newsize;
	uint64_t id = cvd->vdev_id;
	vdev_t **newchild;

	ASSERT(spa_config_held(cvd->vdev_spa, SCL_ALL, RW_WRITER) == SCL_ALL);
	ASSERT(cvd->vdev_parent == NULL);

	cvd->vdev_parent = pvd;

	if (pvd == NULL)
	return;

	ASSERT(id >= pvd->vdev_children \|\| pvd->vdev_child[id] == NULL);

	oldsize = pvd->vdev_children * sizeof (vdev_t *);
	pvd->vdev_children = MAX(pvd->vdev_children, id + 1);
	newsize = pvd->vdev_children * sizeof (vdev_t *);

	newchild = kmem_alloc(newsize, KM_SLEEP);
	if (pvd->vdev_child != NULL) {
	memcpy(newchild, pvd->vdev_child, oldsize);
	kmem_free(pvd->vdev_child, oldsize);
	}

	pvd->vdev_child = newchild;
	pvd->vdev_child[id] = cvd;

	cvd->vdev_top = (pvd->vdev_top ? pvd->vdev_top: cvd);
	ASSERT(cvd->vdev_top->vdev_parent->vdev_parent == NULL);

	/*
	* Walk up all ancestors to update guid sum.
	*/
	for (; pvd != NULL; pvd = pvd->vdev_parent)
	pvd->vdev_guid_sum += cvd->vdev_guid_sum;

	if (cvd->vdev_ops->vdev_op_leaf) {
	list_insert_head(&cvd->vdev_spa->spa_leaf_list, cvd);
	cvd->vdev_spa->spa_leaf_list_gen++;
	}
	}

	void
	vdev_remove_child(vdev_t pvd, vdev_t cvd)
	{
	int c;
	uint_t id = cvd->vdev_id;

	ASSERT(cvd->vdev_parent == pvd);

	if (pvd == NULL)
	return;

	ASSERT(id < pvd->vdev_children);
	ASSERT(pvd->vdev_child[id] == cvd);

	pvd->vdev_child[id] = NULL;
	cvd->vdev_parent = NULL;

	for (c = 0; c < pvd->vdev_children; c++)
	if (pvd->vdev_child[c])
	break;

	if (c == pvd->vdev_children) {
	kmem_free(pvd->vdev_child, c * sizeof (vdev_t *));
	pvd->vdev_child = NULL;
	pvd->vdev_children = 0;
	}

	if (cvd->vdev_ops->vdev_op_leaf) {
	spa_t *spa = cvd->vdev_spa;
	list_remove(&spa->spa_leaf_list, cvd);
	spa->spa_leaf_list_gen++;
	}

	/*
	* Walk up all ancestors to update guid sum.
	*/
	for (; pvd != NULL; pvd = pvd->vdev_parent)
	pvd->vdev_guid_sum -= cvd->vdev_guid_sum;
	}

	/*
	* Remove any holes in the child array.
	*/
	void
	vdev_compact_children(vdev_t *pvd)
	{
	vdev_t *newchild, cvd;
	int oldc = pvd->vdev_children;
	int newc;

	ASSERT(spa_config_held(pvd->vdev_spa, SCL_ALL, RW_WRITER) == SCL_ALL);

	if (oldc == 0)
	return;

	for (int c = newc = 0; c < oldc; c++)
	if (pvd->vdev_child[c])
	newc++;

	if (newc > 0) {
	newchild = kmem_zalloc(newc * sizeof (vdev_t *), KM_SLEEP);

	for (int c = newc = 0; c < oldc; c++) {
	if ((cvd = pvd->vdev_child[c]) != NULL) {
	newchild[newc] = cvd;
	cvd->vdev_id = newc++;
	}
	}
	} else {
	newchild = NULL;
	}

	kmem_free(pvd->vdev_child, oldc * sizeof (vdev_t *));
	pvd->vdev_child = newchild;
	pvd->vdev_children = newc;
	}

	/*
	* Allocate and minimally initialize a vdev_t.
	*/
	vdev_t *
	vdev_alloc_common(spa_t spa, uint_t id, uint64_t guid, vdev_ops_t ops)
	{
	vdev_t *vd;
	vdev_indirect_config_t *vic;

	vd = kmem_zalloc(sizeof (vdev_t), KM_SLEEP);
	vic = &vd->vdev_indirect_config;

	if (spa->spa_root_vdev == NULL) {
	ASSERT(ops == &vdev_root_ops);
	spa->spa_root_vdev = vd;
	spa->spa_load_guid = spa_generate_guid(NULL);
	}

	if (guid == 0 && ops != &vdev_hole_ops) {
	if (spa->spa_root_vdev == vd) {
	/*
	* The root vdev's guid will also be the pool guid,
	* which must be unique among all pools.
	*/
	guid = spa_generate_guid(NULL);
	} else {
	/*
	* Any other vdev's guid must be unique within the pool.
	*/
	guid = spa_generate_guid(spa);
	}
	ASSERT(!spa_guid_exists(spa_guid(spa), guid));
	}

	vd->vdev_spa = spa;
	vd->vdev_id = id;
	vd->vdev_guid = guid;
	vd->vdev_guid_sum = guid;
	vd->vdev_ops = ops;
	vd->vdev_state = VDEV_STATE_CLOSED;
	vd->vdev_ishole = (ops == &vdev_hole_ops);
	vic->vic_prev_indirect_vdev = UINT64_MAX;

	rw_init(&vd->vdev_indirect_rwlock, NULL, RW_DEFAULT, NULL);
	mutex_init(&vd->vdev_obsolete_lock, NULL, MUTEX_DEFAULT, NULL);
	vd->vdev_obsolete_segments = range_tree_create(NULL, RANGE_SEG64, NULL,
	0, 0);

	/*
	* Initialize rate limit structs for events. We rate limit ZIO delay
	* and checksum events so that we don't overwhelm ZED with thousands
	* of events when a disk is acting up.
	*/
	zfs_ratelimit_init(&vd->vdev_delay_rl, &zfs_slow_io_events_per_second,
	1);
	zfs_ratelimit_init(&vd->vdev_deadman_rl, &zfs_slow_io_events_per_second,
	1);
	zfs_ratelimit_init(&vd->vdev_checksum_rl,
	&zfs_checksum_events_per_second, 1);

	/*
	* Default Thresholds for tuning ZED
	*/
	vd->vdev_checksum_n = vdev_prop_default_numeric(VDEV_PROP_CHECKSUM_N);
	vd->vdev_checksum_t = vdev_prop_default_numeric(VDEV_PROP_CHECKSUM_T);
	vd->vdev_io_n = vdev_prop_default_numeric(VDEV_PROP_IO_N);
	vd->vdev_io_t = vdev_prop_default_numeric(VDEV_PROP_IO_T);
	+ vd->vdev_slow_io_n = vdev_prop_default_numeric(VDEV_PROP_SLOW_IO_N);
	+ vd->vdev_slow_io_t = vdev_prop_default_numeric(VDEV_PROP_SLOW_IO_T);

	list_link_init(&vd->vdev_config_dirty_node);
	list_link_init(&vd->vdev_state_dirty_node);
	list_link_init(&vd->vdev_initialize_node);
	list_link_init(&vd->vdev_leaf_node);
	list_link_init(&vd->vdev_trim_node);

	mutex_init(&vd->vdev_dtl_lock, NULL, MUTEX_NOLOCKDEP, NULL);
	mutex_init(&vd->vdev_stat_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&vd->vdev_probe_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&vd->vdev_scan_io_queue_lock, NULL, MUTEX_DEFAULT, NULL);

	mutex_init(&vd->vdev_initialize_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&vd->vdev_initialize_io_lock, NULL, MUTEX_DEFAULT, NULL);
	cv_init(&vd->vdev_initialize_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&vd->vdev_initialize_io_cv, NULL, CV_DEFAULT, NULL);

	mutex_init(&vd->vdev_trim_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&vd->vdev_autotrim_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&vd->vdev_trim_io_lock, NULL, MUTEX_DEFAULT, NULL);
	cv_init(&vd->vdev_trim_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&vd->vdev_autotrim_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&vd->vdev_autotrim_kick_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&vd->vdev_trim_io_cv, NULL, CV_DEFAULT, NULL);

	mutex_init(&vd->vdev_rebuild_lock, NULL, MUTEX_DEFAULT, NULL);
	cv_init(&vd->vdev_rebuild_cv, NULL, CV_DEFAULT, NULL);

	for (int t = 0; t < DTL_TYPES; t++) {
	vd->vdev_dtl[t] = range_tree_create(NULL, RANGE_SEG64, NULL, 0,
	0);
	}

	txg_list_create(&vd->vdev_ms_list, spa,
	offsetof(struct metaslab, ms_txg_node));
	txg_list_create(&vd->vdev_dtl_list, spa,
	offsetof(struct vdev, vdev_dtl_node));
	vd->vdev_stat.vs_timestamp = gethrtime();
	vdev_queue_init(vd);

	return (vd);
	}

	/*
	* Allocate a new vdev. The 'alloctype' is used to control whether we are
	* creating a new vdev or loading an existing one - the behavior is slightly
	* different for each case.
	*/
	int
	vdev_alloc(spa_t spa, vdev_t vdp, nvlist_t nv, vdev_t *parent, uint_t id,
	int alloctype)
	{
	vdev_ops_t *ops;
	const char *type;
	uint64_t guid = 0, islog;
	vdev_t *vd;
	vdev_indirect_config_t *vic;
	const char *tmp = NULL;
	int rc;
	vdev_alloc_bias_t alloc_bias = VDEV_BIAS_NONE;
	boolean_t top_level = (parent && !parent->vdev_parent);

	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);

	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) != 0)
	return (SET_ERROR(EINVAL));

	if ((ops = vdev_getops(type)) == NULL)
	return (SET_ERROR(EINVAL));

	/*
	* If this is a load, get the vdev guid from the nvlist.
	* Otherwise, vdev_alloc_common() will generate one for us.
	*/
	if (alloctype == VDEV_ALLOC_LOAD) {
	uint64_t label_id;

	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_ID, &label_id) \|\|
	label_id != id)
	return (SET_ERROR(EINVAL));

	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
	return (SET_ERROR(EINVAL));
	} else if (alloctype == VDEV_ALLOC_SPARE) {
	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
	return (SET_ERROR(EINVAL));
	} else if (alloctype == VDEV_ALLOC_L2CACHE) {
	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
	return (SET_ERROR(EINVAL));
	} else if (alloctype == VDEV_ALLOC_ROOTPOOL) {
	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
	return (SET_ERROR(EINVAL));
	}

	/*
	* The first allocated vdev must be of type 'root'.
	*/
	if (ops != &vdev_root_ops && spa->spa_root_vdev == NULL)
	return (SET_ERROR(EINVAL));

	/*
	* Determine whether we're a log vdev.
	*/
	islog = 0;
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_IS_LOG, &islog);
	if (islog && spa_version(spa) < SPA_VERSION_SLOGS)
	return (SET_ERROR(ENOTSUP));

	if (ops == &vdev_hole_ops && spa_version(spa) < SPA_VERSION_HOLES)
	return (SET_ERROR(ENOTSUP));

	if (top_level && alloctype == VDEV_ALLOC_ADD) {
	const char *bias;

	/*
	* If creating a top-level vdev, check for allocation
	* classes input.
	*/
	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_ALLOCATION_BIAS,
	&bias) == 0) {
	alloc_bias = vdev_derive_alloc_bias(bias);

	/* spa_vdev_add() expects feature to be enabled */
	if (spa->spa_load_state != SPA_LOAD_CREATE &&
	!spa_feature_is_enabled(spa,
	SPA_FEATURE_ALLOCATION_CLASSES)) {
	return (SET_ERROR(ENOTSUP));
	}
	}

	/* spa_vdev_add() expects feature to be enabled */
	if (ops == &vdev_draid_ops &&
	spa->spa_load_state != SPA_LOAD_CREATE &&
	!spa_feature_is_enabled(spa, SPA_FEATURE_DRAID)) {
	return (SET_ERROR(ENOTSUP));
	}
	}

	/*
	* Initialize the vdev specific data. This is done before calling
	* vdev_alloc_common() since it may fail and this simplifies the
	* error reporting and cleanup code paths.
	*/
	void *tsd = NULL;
	if (ops->vdev_op_init != NULL) {
	rc = ops->vdev_op_init(spa, nv, &tsd);
	if (rc != 0) {
	return (rc);
	}
	}

	vd = vdev_alloc_common(spa, id, guid, ops);
	vd->vdev_tsd = tsd;
	vd->vdev_islog = islog;

	if (top_level && alloc_bias != VDEV_BIAS_NONE)
	vd->vdev_alloc_bias = alloc_bias;

	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &tmp) == 0)
	vd->vdev_path = spa_strdup(tmp);

	/*
	* ZPOOL_CONFIG_AUX_STATE = "external" means we previously forced a
	* fault on a vdev and want it to persist across imports (like with
	* zpool offline -f).
	*/
	rc = nvlist_lookup_string(nv, ZPOOL_CONFIG_AUX_STATE, &tmp);
	if (rc == 0 && tmp != NULL && strcmp(tmp, "external") == 0) {
	vd->vdev_stat.vs_aux = VDEV_AUX_EXTERNAL;
	vd->vdev_faulted = 1;
	vd->vdev_label_aux = VDEV_AUX_EXTERNAL;
	}

	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_DEVID, &tmp) == 0)
	vd->vdev_devid = spa_strdup(tmp);
	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PHYS_PATH, &tmp) == 0)
	vd->vdev_physpath = spa_strdup(tmp);

	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH,
	&tmp) == 0)
	vd->vdev_enc_sysfs_path = spa_strdup(tmp);

	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_FRU, &tmp) == 0)
	vd->vdev_fru = spa_strdup(tmp);

	/*
	* Set the whole_disk property. If it's not specified, leave the value
	* as -1.
	*/
	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_WHOLE_DISK,
	&vd->vdev_wholedisk) != 0)
	vd->vdev_wholedisk = -1ULL;

	vic = &vd->vdev_indirect_config;

	ASSERT0(vic->vic_mapping_object);
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_INDIRECT_OBJECT,
	&vic->vic_mapping_object);
	ASSERT0(vic->vic_births_object);
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_INDIRECT_BIRTHS,
	&vic->vic_births_object);
	ASSERT3U(vic->vic_prev_indirect_vdev, ==, UINT64_MAX);
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_PREV_INDIRECT_VDEV,
	&vic->vic_prev_indirect_vdev);

	/*
	* Look for the 'not present' flag. This will only be set if the device
	* was not present at the time of import.
	*/
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
	&vd->vdev_not_present);

	/*
	* Get the alignment requirement. Ignore pool ashift for vdev
	* attach case.
	*/
	if (alloctype != VDEV_ALLOC_ATTACH) {
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_ASHIFT,
	&vd->vdev_ashift);
	} else {
	vd->vdev_attaching = B_TRUE;
	}

	/*
	* Retrieve the vdev creation time.
	*/
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_CREATE_TXG,
	&vd->vdev_crtxg);

	if (vd->vdev_ops == &vdev_root_ops &&
	(alloctype == VDEV_ALLOC_LOAD \|\|
	alloctype == VDEV_ALLOC_SPLIT \|\|
	alloctype == VDEV_ALLOC_ROOTPOOL)) {
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_VDEV_ROOT_ZAP,
	&vd->vdev_root_zap);
	}

	/*
	* If we're a top-level vdev, try to load the allocation parameters.
	*/
	if (top_level &&
	(alloctype == VDEV_ALLOC_LOAD \|\| alloctype == VDEV_ALLOC_SPLIT)) {
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_METASLAB_ARRAY,
	&vd->vdev_ms_array);
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_METASLAB_SHIFT,
	&vd->vdev_ms_shift);
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_ASIZE,
	&vd->vdev_asize);
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NONALLOCATING,
	&vd->vdev_noalloc);
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_REMOVING,
	&vd->vdev_removing);
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_VDEV_TOP_ZAP,
	&vd->vdev_top_zap);
	} else {
	ASSERT0(vd->vdev_top_zap);
	}

	if (top_level && alloctype != VDEV_ALLOC_ATTACH) {
	ASSERT(alloctype == VDEV_ALLOC_LOAD \|\|
	alloctype == VDEV_ALLOC_ADD \|\|
	alloctype == VDEV_ALLOC_SPLIT \|\|
	alloctype == VDEV_ALLOC_ROOTPOOL);
	/* Note: metaslab_group_create() is now deferred */
	}

	if (vd->vdev_ops->vdev_op_leaf &&
	(alloctype == VDEV_ALLOC_LOAD \|\| alloctype == VDEV_ALLOC_SPLIT)) {
	(void) nvlist_lookup_uint64(nv,
	ZPOOL_CONFIG_VDEV_LEAF_ZAP, &vd->vdev_leaf_zap);
	} else {
	ASSERT0(vd->vdev_leaf_zap);
	}

	/*
	* If we're a leaf vdev, try to load the DTL object and other state.
	*/

	if (vd->vdev_ops->vdev_op_leaf &&
	(alloctype == VDEV_ALLOC_LOAD \|\| alloctype == VDEV_ALLOC_L2CACHE \|\|
	alloctype == VDEV_ALLOC_ROOTPOOL)) {
	if (alloctype == VDEV_ALLOC_LOAD) {
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_DTL,
	&vd->vdev_dtl_object);
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_UNSPARE,
	&vd->vdev_unspare);
	}

	if (alloctype == VDEV_ALLOC_ROOTPOOL) {
	uint64_t spare = 0;

	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_IS_SPARE,
	&spare) == 0 && spare)
	spa_spare_add(vd);
	}

	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_OFFLINE,
	&vd->vdev_offline);

	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_RESILVER_TXG,
	&vd->vdev_resilver_txg);

	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_REBUILD_TXG,
	&vd->vdev_rebuild_txg);

	if (nvlist_exists(nv, ZPOOL_CONFIG_RESILVER_DEFER))
	vdev_defer_resilver(vd);

	/*
	* In general, when importing a pool we want to ignore the
	* persistent fault state, as the diagnosis made on another
	* system may not be valid in the current context. The only
	* exception is if we forced a vdev to a persistently faulted
	* state with 'zpool offline -f'. The persistent fault will
	* remain across imports until cleared.
	*
	* Local vdevs will remain in the faulted state.
	*/
	if (spa_load_state(spa) == SPA_LOAD_OPEN \|\|
	spa_load_state(spa) == SPA_LOAD_IMPORT) {
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_FAULTED,
	&vd->vdev_faulted);
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_DEGRADED,
	&vd->vdev_degraded);
	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_REMOVED,
	&vd->vdev_removed);

	if (vd->vdev_faulted \|\| vd->vdev_degraded) {
	const char *aux;

	vd->vdev_label_aux =
	VDEV_AUX_ERR_EXCEEDED;
	if (nvlist_lookup_string(nv,
	ZPOOL_CONFIG_AUX_STATE, &aux) == 0 &&
	strcmp(aux, "external") == 0)
	vd->vdev_label_aux = VDEV_AUX_EXTERNAL;
	else
	vd->vdev_faulted = 0ULL;
	}
	}
	}

	/*
	* Add ourselves to the parent's list of children.
	*/
	vdev_add_child(parent, vd);

	*vdp = vd;

	return (0);
	}

	void
	vdev_free(vdev_t *vd)
	{
	spa_t *spa = vd->vdev_spa;

	ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
	ASSERT3P(vd->vdev_trim_thread, ==, NULL);
	ASSERT3P(vd->vdev_autotrim_thread, ==, NULL);
	ASSERT3P(vd->vdev_rebuild_thread, ==, NULL);

	/*
	* Scan queues are normally destroyed at the end of a scan. If the
	* queue exists here, that implies the vdev is being removed while
	* the scan is still running.
	*/
	if (vd->vdev_scan_io_queue != NULL) {
	mutex_enter(&vd->vdev_scan_io_queue_lock);
	dsl_scan_io_queue_destroy(vd->vdev_scan_io_queue);
	vd->vdev_scan_io_queue = NULL;
	mutex_exit(&vd->vdev_scan_io_queue_lock);
	}

	/*
	* vdev_free() implies closing the vdev first. This is simpler than
	* trying to ensure complicated semantics for all callers.
	*/
	vdev_close(vd);

	ASSERT(!list_link_active(&vd->vdev_config_dirty_node));
	ASSERT(!list_link_active(&vd->vdev_state_dirty_node));

	/*
	* Free all children.
	*/
	for (int c = 0; c < vd->vdev_children; c++)
	vdev_free(vd->vdev_child[c]);

	ASSERT(vd->vdev_child == NULL);
	ASSERT(vd->vdev_guid_sum == vd->vdev_guid);

	if (vd->vdev_ops->vdev_op_fini != NULL)
	vd->vdev_ops->vdev_op_fini(vd);

	/*
	* Discard allocation state.
	*/
	if (vd->vdev_mg != NULL) {
	vdev_metaslab_fini(vd);
	metaslab_group_destroy(vd->vdev_mg);
	vd->vdev_mg = NULL;
	}
	if (vd->vdev_log_mg != NULL) {
	ASSERT0(vd->vdev_ms_count);
	metaslab_group_destroy(vd->vdev_log_mg);
	vd->vdev_log_mg = NULL;
	}

	ASSERT0(vd->vdev_stat.vs_space);
	ASSERT0(vd->vdev_stat.vs_dspace);
	ASSERT0(vd->vdev_stat.vs_alloc);

	/*
	* Remove this vdev from its parent's child list.
	*/
	vdev_remove_child(vd->vdev_parent, vd);

	ASSERT(vd->vdev_parent == NULL);
	ASSERT(!list_link_active(&vd->vdev_leaf_node));

	/*
	* Clean up vdev structure.
	*/
	vdev_queue_fini(vd);

	if (vd->vdev_path)
	spa_strfree(vd->vdev_path);
	if (vd->vdev_devid)
	spa_strfree(vd->vdev_devid);
	if (vd->vdev_physpath)
	spa_strfree(vd->vdev_physpath);

	if (vd->vdev_enc_sysfs_path)
	spa_strfree(vd->vdev_enc_sysfs_path);

	if (vd->vdev_fru)
	spa_strfree(vd->vdev_fru);

	if (vd->vdev_isspare)
	spa_spare_remove(vd);
	if (vd->vdev_isl2cache)
	spa_l2cache_remove(vd);

	txg_list_destroy(&vd->vdev_ms_list);
	txg_list_destroy(&vd->vdev_dtl_list);

	mutex_enter(&vd->vdev_dtl_lock);
	space_map_close(vd->vdev_dtl_sm);
	for (int t = 0; t < DTL_TYPES; t++) {
	range_tree_vacate(vd->vdev_dtl[t], NULL, NULL);
	range_tree_destroy(vd->vdev_dtl[t]);
	}
	mutex_exit(&vd->vdev_dtl_lock);

	EQUIV(vd->vdev_indirect_births != NULL,
	vd->vdev_indirect_mapping != NULL);
	if (vd->vdev_indirect_births != NULL) {
	vdev_indirect_mapping_close(vd->vdev_indirect_mapping);
	vdev_indirect_births_close(vd->vdev_indirect_births);
	}

	if (vd->vdev_obsolete_sm != NULL) {
	ASSERT(vd->vdev_removing \|\|
	vd->vdev_ops == &vdev_indirect_ops);
	space_map_close(vd->vdev_obsolete_sm);
	vd->vdev_obsolete_sm = NULL;
	}
	range_tree_destroy(vd->vdev_obsolete_segments);
	rw_destroy(&vd->vdev_indirect_rwlock);
	mutex_destroy(&vd->vdev_obsolete_lock);

	mutex_destroy(&vd->vdev_dtl_lock);
	mutex_destroy(&vd->vdev_stat_lock);
	mutex_destroy(&vd->vdev_probe_lock);
	mutex_destroy(&vd->vdev_scan_io_queue_lock);

	mutex_destroy(&vd->vdev_initialize_lock);
	mutex_destroy(&vd->vdev_initialize_io_lock);
	cv_destroy(&vd->vdev_initialize_io_cv);
	cv_destroy(&vd->vdev_initialize_cv);

	mutex_destroy(&vd->vdev_trim_lock);
	mutex_destroy(&vd->vdev_autotrim_lock);
	mutex_destroy(&vd->vdev_trim_io_lock);
	cv_destroy(&vd->vdev_trim_cv);
	cv_destroy(&vd->vdev_autotrim_cv);
	cv_destroy(&vd->vdev_autotrim_kick_cv);
	cv_destroy(&vd->vdev_trim_io_cv);

	mutex_destroy(&vd->vdev_rebuild_lock);
	cv_destroy(&vd->vdev_rebuild_cv);

	zfs_ratelimit_fini(&vd->vdev_delay_rl);
	zfs_ratelimit_fini(&vd->vdev_deadman_rl);
	zfs_ratelimit_fini(&vd->vdev_checksum_rl);

	if (vd == spa->spa_root_vdev)
	spa->spa_root_vdev = NULL;

	kmem_free(vd, sizeof (vdev_t));
	}

	/*
	* Transfer top-level vdev state from svd to tvd.
	*/
	static void
	vdev_top_transfer(vdev_t svd, vdev_t tvd)
	{
	spa_t *spa = svd->vdev_spa;
	metaslab_t *msp;
	vdev_t *vd;
	int t;

	ASSERT(tvd == tvd->vdev_top);

	tvd->vdev_ms_array = svd->vdev_ms_array;
	tvd->vdev_ms_shift = svd->vdev_ms_shift;
	tvd->vdev_ms_count = svd->vdev_ms_count;
	tvd->vdev_top_zap = svd->vdev_top_zap;

	svd->vdev_ms_array = 0;
	svd->vdev_ms_shift = 0;
	svd->vdev_ms_count = 0;
	svd->vdev_top_zap = 0;

	if (tvd->vdev_mg)
	ASSERT3P(tvd->vdev_mg, ==, svd->vdev_mg);
	if (tvd->vdev_log_mg)
	ASSERT3P(tvd->vdev_log_mg, ==, svd->vdev_log_mg);
	tvd->vdev_mg = svd->vdev_mg;
	tvd->vdev_log_mg = svd->vdev_log_mg;
	tvd->vdev_ms = svd->vdev_ms;

	svd->vdev_mg = NULL;
	svd->vdev_log_mg = NULL;
	svd->vdev_ms = NULL;

	if (tvd->vdev_mg != NULL)
	tvd->vdev_mg->mg_vd = tvd;
	if (tvd->vdev_log_mg != NULL)
	tvd->vdev_log_mg->mg_vd = tvd;

	tvd->vdev_checkpoint_sm = svd->vdev_checkpoint_sm;
	svd->vdev_checkpoint_sm = NULL;

	tvd->vdev_alloc_bias = svd->vdev_alloc_bias;
	svd->vdev_alloc_bias = VDEV_BIAS_NONE;

	tvd->vdev_stat.vs_alloc = svd->vdev_stat.vs_alloc;
	tvd->vdev_stat.vs_space = svd->vdev_stat.vs_space;
	tvd->vdev_stat.vs_dspace = svd->vdev_stat.vs_dspace;

	svd->vdev_stat.vs_alloc = 0;
	svd->vdev_stat.vs_space = 0;
	svd->vdev_stat.vs_dspace = 0;

	/*
	* State which may be set on a top-level vdev that's in the
	* process of being removed.
	*/
	ASSERT0(tvd->vdev_indirect_config.vic_births_object);
	ASSERT0(tvd->vdev_indirect_config.vic_mapping_object);
	ASSERT3U(tvd->vdev_indirect_config.vic_prev_indirect_vdev, ==, -1ULL);
	ASSERT3P(tvd->vdev_indirect_mapping, ==, NULL);
	ASSERT3P(tvd->vdev_indirect_births, ==, NULL);
	ASSERT3P(tvd->vdev_obsolete_sm, ==, NULL);
	ASSERT0(tvd->vdev_noalloc);
	ASSERT0(tvd->vdev_removing);
	ASSERT0(tvd->vdev_rebuilding);
	tvd->vdev_noalloc = svd->vdev_noalloc;
	tvd->vdev_removing = svd->vdev_removing;
	tvd->vdev_rebuilding = svd->vdev_rebuilding;
	tvd->vdev_rebuild_config = svd->vdev_rebuild_config;
	tvd->vdev_indirect_config = svd->vdev_indirect_config;
	tvd->vdev_indirect_mapping = svd->vdev_indirect_mapping;
	tvd->vdev_indirect_births = svd->vdev_indirect_births;
	range_tree_swap(&svd->vdev_obsolete_segments,
	&tvd->vdev_obsolete_segments);
	tvd->vdev_obsolete_sm = svd->vdev_obsolete_sm;
	svd->vdev_indirect_config.vic_mapping_object = 0;
	svd->vdev_indirect_config.vic_births_object = 0;
	svd->vdev_indirect_config.vic_prev_indirect_vdev = -1ULL;
	svd->vdev_indirect_mapping = NULL;
	svd->vdev_indirect_births = NULL;
	svd->vdev_obsolete_sm = NULL;
	svd->vdev_noalloc = 0;
	svd->vdev_removing = 0;
	svd->vdev_rebuilding = 0;

	for (t = 0; t < TXG_SIZE; t++) {
	while ((msp = txg_list_remove(&svd->vdev_ms_list, t)) != NULL)
	(void) txg_list_add(&tvd->vdev_ms_list, msp, t);
	while ((vd = txg_list_remove(&svd->vdev_dtl_list, t)) != NULL)
	(void) txg_list_add(&tvd->vdev_dtl_list, vd, t);
	if (txg_list_remove_this(&spa->spa_vdev_txg_list, svd, t))
	(void) txg_list_add(&spa->spa_vdev_txg_list, tvd, t);
	}

	if (list_link_active(&svd->vdev_config_dirty_node)) {
	vdev_config_clean(svd);
	vdev_config_dirty(tvd);
	}

	if (list_link_active(&svd->vdev_state_dirty_node)) {
	vdev_state_clean(svd);
	vdev_state_dirty(tvd);
	}

	tvd->vdev_deflate_ratio = svd->vdev_deflate_ratio;
	svd->vdev_deflate_ratio = 0;

	tvd->vdev_islog = svd->vdev_islog;
	svd->vdev_islog = 0;

	dsl_scan_io_queue_vdev_xfer(svd, tvd);
	}

	static void
	vdev_top_update(vdev_t tvd, vdev_t vd)
	{
	if (vd == NULL)
	return;

	vd->vdev_top = tvd;

	for (int c = 0; c < vd->vdev_children; c++)
	vdev_top_update(tvd, vd->vdev_child[c]);
	}

	/*
	* Add a mirror/replacing vdev above an existing vdev. There is no need to
	* call .vdev_op_init() since mirror/replacing vdevs do not have private state.
	*/
	vdev_t *
	vdev_add_parent(vdev_t cvd, vdev_ops_t ops)
	{
	spa_t *spa = cvd->vdev_spa;
	vdev_t *pvd = cvd->vdev_parent;
	vdev_t *mvd;

	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);

	mvd = vdev_alloc_common(spa, cvd->vdev_id, 0, ops);

	mvd->vdev_asize = cvd->vdev_asize;
	mvd->vdev_min_asize = cvd->vdev_min_asize;
	mvd->vdev_max_asize = cvd->vdev_max_asize;
	mvd->vdev_psize = cvd->vdev_psize;
	mvd->vdev_ashift = cvd->vdev_ashift;
	mvd->vdev_logical_ashift = cvd->vdev_logical_ashift;
	mvd->vdev_physical_ashift = cvd->vdev_physical_ashift;
	mvd->vdev_state = cvd->vdev_state;
	mvd->vdev_crtxg = cvd->vdev_crtxg;

	vdev_remove_child(pvd, cvd);
	vdev_add_child(pvd, mvd);
	cvd->vdev_id = mvd->vdev_children;
	vdev_add_child(mvd, cvd);
	vdev_top_update(cvd->vdev_top, cvd->vdev_top);

	if (mvd == mvd->vdev_top)
	vdev_top_transfer(cvd, mvd);

	return (mvd);
	}

	/*
	* Remove a 1-way mirror/replacing vdev from the tree.
	*/
	void
	vdev_remove_parent(vdev_t *cvd)
	{
	vdev_t *mvd = cvd->vdev_parent;
	vdev_t *pvd = mvd->vdev_parent;

	ASSERT(spa_config_held(cvd->vdev_spa, SCL_ALL, RW_WRITER) == SCL_ALL);

	ASSERT(mvd->vdev_children == 1);
	ASSERT(mvd->vdev_ops == &vdev_mirror_ops \|\|
	mvd->vdev_ops == &vdev_replacing_ops \|\|
	mvd->vdev_ops == &vdev_spare_ops);
	cvd->vdev_ashift = mvd->vdev_ashift;
	cvd->vdev_logical_ashift = mvd->vdev_logical_ashift;
	cvd->vdev_physical_ashift = mvd->vdev_physical_ashift;
	vdev_remove_child(mvd, cvd);
	vdev_remove_child(pvd, mvd);

	/*
	* If cvd will replace mvd as a top-level vdev, preserve mvd's guid.
	* Otherwise, we could have detached an offline device, and when we
	* go to import the pool we'll think we have two top-level vdevs,
	* instead of a different version of the same top-level vdev.
	*/
	if (mvd->vdev_top == mvd) {
	uint64_t guid_delta = mvd->vdev_guid - cvd->vdev_guid;
	cvd->vdev_orig_guid = cvd->vdev_guid;
	cvd->vdev_guid += guid_delta;
	cvd->vdev_guid_sum += guid_delta;

	/*
	* If pool not set for autoexpand, we need to also preserve
	* mvd's asize to prevent automatic expansion of cvd.
	* Otherwise if we are adjusting the mirror by attaching and
	* detaching children of non-uniform sizes, the mirror could
	* autoexpand, unexpectedly requiring larger devices to
	* re-establish the mirror.
	*/
	if (!cvd->vdev_spa->spa_autoexpand)
	cvd->vdev_asize = mvd->vdev_asize;
	}
	cvd->vdev_id = mvd->vdev_id;
	vdev_add_child(pvd, cvd);
	vdev_top_update(cvd->vdev_top, cvd->vdev_top);

	if (cvd == cvd->vdev_top)
	vdev_top_transfer(mvd, cvd);

	ASSERT(mvd->vdev_children == 0);
	vdev_free(mvd);
	}

	/*
	* Choose GCD for spa_gcd_alloc.
	*/
	static uint64_t
	vdev_gcd(uint64_t a, uint64_t b)
	{
	while (b != 0) {
	uint64_t t = b;
	b = a % b;
	a = t;
	}
	return (a);
	}

	/*
	* Set spa_min_alloc and spa_gcd_alloc.
	*/
	static void
	vdev_spa_set_alloc(spa_t *spa, uint64_t min_alloc)
	{
	if (min_alloc < spa->spa_min_alloc)
	spa->spa_min_alloc = min_alloc;
	if (spa->spa_gcd_alloc == INT_MAX) {
	spa->spa_gcd_alloc = min_alloc;
	} else {
	spa->spa_gcd_alloc = vdev_gcd(min_alloc,
	spa->spa_gcd_alloc);
	}
	}

	void
	vdev_metaslab_group_create(vdev_t *vd)
	{
	spa_t *spa = vd->vdev_spa;

	/*
	* metaslab_group_create was delayed until allocation bias was available
	*/
	if (vd->vdev_mg == NULL) {
	metaslab_class_t *mc;

	if (vd->vdev_islog && vd->vdev_alloc_bias == VDEV_BIAS_NONE)
	vd->vdev_alloc_bias = VDEV_BIAS_LOG;

	ASSERT3U(vd->vdev_islog, ==,
	(vd->vdev_alloc_bias == VDEV_BIAS_LOG));

	switch (vd->vdev_alloc_bias) {
	case VDEV_BIAS_LOG:
	mc = spa_log_class(spa);
	break;
	case VDEV_BIAS_SPECIAL:
	mc = spa_special_class(spa);
	break;
	case VDEV_BIAS_DEDUP:
	mc = spa_dedup_class(spa);
	break;
	default:
	mc = spa_normal_class(spa);
	}

	vd->vdev_mg = metaslab_group_create(mc, vd,
	spa->spa_alloc_count);

	if (!vd->vdev_islog) {
	vd->vdev_log_mg = metaslab_group_create(
	spa_embedded_log_class(spa), vd, 1);
	}

	/*
	* The spa ashift min/max only apply for the normal metaslab
	* class. Class destination is late binding so ashift boundary
	* setting had to wait until now.
	*/
	if (vd->vdev_top == vd && vd->vdev_ashift != 0 &&
	mc == spa_normal_class(spa) && vd->vdev_aux == NULL) {
	if (vd->vdev_ashift > spa->spa_max_ashift)
	spa->spa_max_ashift = vd->vdev_ashift;
	if (vd->vdev_ashift < spa->spa_min_ashift)
	spa->spa_min_ashift = vd->vdev_ashift;

	uint64_t min_alloc = vdev_get_min_alloc(vd);
	vdev_spa_set_alloc(spa, min_alloc);
	}
	}
	}

	int
	vdev_metaslab_init(vdev_t *vd, uint64_t txg)
	{
	spa_t *spa = vd->vdev_spa;
	uint64_t oldc = vd->vdev_ms_count;
	uint64_t newc = vd->vdev_asize >> vd->vdev_ms_shift;
	metaslab_t **mspp;
	int error;
	boolean_t expanding = (oldc != 0);

	ASSERT(txg == 0 \|\| spa_config_held(spa, SCL_ALLOC, RW_WRITER));

	/*
	* This vdev is not being allocated from yet or is a hole.
	*/
	if (vd->vdev_ms_shift == 0)
	return (0);

	ASSERT(!vd->vdev_ishole);

	ASSERT(oldc <= newc);

	mspp = vmem_zalloc(newc * sizeof (*mspp), KM_SLEEP);

	if (expanding) {
	memcpy(mspp, vd->vdev_ms, oldc * sizeof (*mspp));
	vmem_free(vd->vdev_ms, oldc * sizeof (*mspp));
	}

	vd->vdev_ms = mspp;
	vd->vdev_ms_count = newc;

	for (uint64_t m = oldc; m < newc; m++) {
	uint64_t object = 0;
	/*
	* vdev_ms_array may be 0 if we are creating the "fake"
	* metaslabs for an indirect vdev for zdb's leak detection.
	* See zdb_leak_init().
	*/
	if (txg == 0 && vd->vdev_ms_array != 0) {
	error = dmu_read(spa->spa_meta_objset,
	vd->vdev_ms_array,
	m * sizeof (uint64_t), sizeof (uint64_t), &object,
	DMU_READ_PREFETCH);
	if (error != 0) {
	vdev_dbgmsg(vd, "unable to read the metaslab "
	"array [error=%d]", error);
	return (error);
	}
	}

	error = metaslab_init(vd->vdev_mg, m, object, txg,
	&(vd->vdev_ms[m]));
	if (error != 0) {
	vdev_dbgmsg(vd, "metaslab_init failed [error=%d]",
	error);
	return (error);
	}
	}

	/*
	* Find the emptiest metaslab on the vdev and mark it for use for
	* embedded slog by moving it from the regular to the log metaslab
	* group.
	*/
	if (vd->vdev_mg->mg_class == spa_normal_class(spa) &&
	vd->vdev_ms_count > zfs_embedded_slog_min_ms &&
	avl_is_empty(&vd->vdev_log_mg->mg_metaslab_tree)) {
	uint64_t slog_msid = 0;
	uint64_t smallest = UINT64_MAX;

	/*
	* Note, we only search the new metaslabs, because the old
	* (pre-existing) ones may be active (e.g. have non-empty
	* range_tree's), and we don't move them to the new
	* metaslab_t.
	*/
	for (uint64_t m = oldc; m < newc; m++) {
	uint64_t alloc =
	space_map_allocated(vd->vdev_ms[m]->ms_sm);
	if (alloc < smallest) {
	slog_msid = m;
	smallest = alloc;
	}
	}
	metaslab_t *slog_ms = vd->vdev_ms[slog_msid];
	/*
	* The metaslab was marked as dirty at the end of
	* metaslab_init(). Remove it from the dirty list so that we
	* can uninitialize and reinitialize it to the new class.
	*/
	if (txg != 0) {
	(void) txg_list_remove_this(&vd->vdev_ms_list,
	slog_ms, txg);
	}
	uint64_t sm_obj = space_map_object(slog_ms->ms_sm);
	metaslab_fini(slog_ms);
	VERIFY0(metaslab_init(vd->vdev_log_mg, slog_msid, sm_obj, txg,
	&vd->vdev_ms[slog_msid]));
	}

	if (txg == 0)
	spa_config_enter(spa, SCL_ALLOC, FTAG, RW_WRITER);

	/*
	* If the vdev is marked as non-allocating then don't
	* activate the metaslabs since we want to ensure that
	* no allocations are performed on this device.
	*/
	if (vd->vdev_noalloc) {
	/* track non-allocating vdev space */
	spa->spa_nonallocating_dspace += spa_deflate(spa) ?
	vd->vdev_stat.vs_dspace : vd->vdev_stat.vs_space;
	} else if (!expanding) {
	metaslab_group_activate(vd->vdev_mg);
	if (vd->vdev_log_mg != NULL)
	metaslab_group_activate(vd->vdev_log_mg);
	}

	if (txg == 0)
	spa_config_exit(spa, SCL_ALLOC, FTAG);

	return (0);
	}

	void
	vdev_metaslab_fini(vdev_t *vd)
	{
	if (vd->vdev_checkpoint_sm != NULL) {
	ASSERT(spa_feature_is_active(vd->vdev_spa,
	SPA_FEATURE_POOL_CHECKPOINT));
	space_map_close(vd->vdev_checkpoint_sm);
	/*
	* Even though we close the space map, we need to set its
	* pointer to NULL. The reason is that vdev_metaslab_fini()
	* may be called multiple times for certain operations
	* (i.e. when destroying a pool) so we need to ensure that
	* this clause never executes twice. This logic is similar
	* to the one used for the vdev_ms clause below.
	*/
	vd->vdev_checkpoint_sm = NULL;
	}

	if (vd->vdev_ms != NULL) {
	metaslab_group_t *mg = vd->vdev_mg;

	metaslab_group_passivate(mg);
	if (vd->vdev_log_mg != NULL) {
	ASSERT(!vd->vdev_islog);
	metaslab_group_passivate(vd->vdev_log_mg);
	}

	uint64_t count = vd->vdev_ms_count;
	for (uint64_t m = 0; m < count; m++) {
	metaslab_t *msp = vd->vdev_ms[m];
	if (msp != NULL)
	metaslab_fini(msp);
	}
	vmem_free(vd->vdev_ms, count * sizeof (metaslab_t *));
	vd->vdev_ms = NULL;
	vd->vdev_ms_count = 0;

	for (int i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++) {
	ASSERT0(mg->mg_histogram[i]);
	if (vd->vdev_log_mg != NULL)
	ASSERT0(vd->vdev_log_mg->mg_histogram[i]);
	}
	}
	ASSERT0(vd->vdev_ms_count);
	}

	typedef struct vdev_probe_stats {
	boolean_t vps_readable;
	boolean_t vps_writeable;
	+ boolean_t vps_zio_done_probe;
	int vps_flags;
	} vdev_probe_stats_t;

	static void
	vdev_probe_done(zio_t *zio)
	{
	spa_t *spa = zio->io_spa;
	vdev_t *vd = zio->io_vd;
	vdev_probe_stats_t *vps = zio->io_private;

	ASSERT(vd->vdev_probe_zio != NULL);

	if (zio->io_type == ZIO_TYPE_READ) {
	if (zio->io_error == 0)
	vps->vps_readable = 1;
	if (zio->io_error == 0 && spa_writeable(spa)) {
	zio_nowait(zio_write_phys(vd->vdev_probe_zio, vd,
	zio->io_offset, zio->io_size, zio->io_abd,
	ZIO_CHECKSUM_OFF, vdev_probe_done, vps,
	ZIO_PRIORITY_SYNC_WRITE, vps->vps_flags, B_TRUE));
	} else {
	abd_free(zio->io_abd);
	}
	} else if (zio->io_type == ZIO_TYPE_WRITE) {
	if (zio->io_error == 0)
	vps->vps_writeable = 1;
	abd_free(zio->io_abd);
	} else if (zio->io_type == ZIO_TYPE_NULL) {
	zio_t *pio;
	zio_link_t *zl;

	vd->vdev_cant_read \|= !vps->vps_readable;
	vd->vdev_cant_write \|= !vps->vps_writeable;

	if (vdev_readable(vd) &&
	(vdev_writeable(vd) \|\| !spa_writeable(spa))) {
	zio->io_error = 0;
	} else {
	ASSERT(zio->io_error != 0);
	vdev_dbgmsg(vd, "failed probe");
	(void) zfs_ereport_post(FM_EREPORT_ZFS_PROBE_FAILURE,
	spa, vd, NULL, NULL, 0);
	zio->io_error = SET_ERROR(ENXIO);
	+
	+ /*
	+ * If this probe was initiated from zio pipeline, then
	+ * change the state in a spa_async_request. Probes that
	+ * were initiated from a vdev_open can change the state
	+ * as part of the open call.
	+ */
	+ if (vps->vps_zio_done_probe) {
	+ vd->vdev_fault_wanted = B_TRUE;
	+ spa_async_request(spa, SPA_ASYNC_FAULT_VDEV);
	+ }
	}

	mutex_enter(&vd->vdev_probe_lock);
	ASSERT(vd->vdev_probe_zio == zio);
	vd->vdev_probe_zio = NULL;
	mutex_exit(&vd->vdev_probe_lock);

	zl = NULL;
	while ((pio = zio_walk_parents(zio, &zl)) != NULL)
	if (!vdev_accessible(vd, pio))
	pio->io_error = SET_ERROR(ENXIO);

	kmem_free(vps, sizeof (*vps));
	}
	}

	/*
	* Determine whether this device is accessible.
	*
	* Read and write to several known locations: the pad regions of each
	* vdev label but the first, which we leave alone in case it contains
	* a VTOC.
	*/
	zio_t *
	vdev_probe(vdev_t vd, zio_t zio)
	{
	spa_t *spa = vd->vdev_spa;
	vdev_probe_stats_t *vps = NULL;
	zio_t *pio;

	ASSERT(vd->vdev_ops->vdev_op_leaf);

	/*
	* Don't probe the probe.
	*/
	if (zio && (zio->io_flags & ZIO_FLAG_PROBE))
	return (NULL);

	/*
	* To prevent 'probe storms' when a device fails, we create
	* just one probe i/o at a time. All zios that want to probe
	* this vdev will become parents of the probe io.
	*/
	mutex_enter(&vd->vdev_probe_lock);

	if ((pio = vd->vdev_probe_zio) == NULL) {
	vps = kmem_zalloc(sizeof (*vps), KM_SLEEP);

	vps->vps_flags = ZIO_FLAG_CANFAIL \| ZIO_FLAG_PROBE \|
	ZIO_FLAG_DONT_AGGREGATE \| ZIO_FLAG_TRYHARD;
	+ vps->vps_zio_done_probe = (zio != NULL);

	if (spa_config_held(spa, SCL_ZIO, RW_WRITER)) {
	/*
	* vdev_cant_read and vdev_cant_write can only
	* transition from TRUE to FALSE when we have the
	* SCL_ZIO lock as writer; otherwise they can only
	* transition from FALSE to TRUE. This ensures that
	* any zio looking at these values can assume that
	* failures persist for the life of the I/O. That's
	* important because when a device has intermittent
	* connectivity problems, we want to ensure that
	* they're ascribed to the device (ENXIO) and not
	* the zio (EIO).
	*
	* Since we hold SCL_ZIO as writer here, clear both
	* values so the probe can reevaluate from first
	* principles.
	*/
	vps->vps_flags \|= ZIO_FLAG_CONFIG_WRITER;
	vd->vdev_cant_read = B_FALSE;
	vd->vdev_cant_write = B_FALSE;
	}

	vd->vdev_probe_zio = pio = zio_null(NULL, spa, vd,
	vdev_probe_done, vps,
	vps->vps_flags \| ZIO_FLAG_DONT_PROPAGATE);
	-
	- /*
	- * We can't change the vdev state in this context, so we
	- * kick off an async task to do it on our behalf.
	- */
	- if (zio != NULL) {
	- vd->vdev_probe_wanted = B_TRUE;
	- spa_async_request(spa, SPA_ASYNC_PROBE);
	- }
	}

	if (zio != NULL)
	zio_add_child(zio, pio);

	mutex_exit(&vd->vdev_probe_lock);

	if (vps == NULL) {
	ASSERT(zio != NULL);
	return (NULL);
	}

	for (int l = 1; l < VDEV_LABELS; l++) {
	zio_nowait(zio_read_phys(pio, vd,
	vdev_label_offset(vd->vdev_psize, l,
	offsetof(vdev_label_t, vl_be)), VDEV_PAD_SIZE,
	abd_alloc_for_io(VDEV_PAD_SIZE, B_TRUE),
	ZIO_CHECKSUM_OFF, vdev_probe_done, vps,
	ZIO_PRIORITY_SYNC_READ, vps->vps_flags, B_TRUE));
	}

	if (zio == NULL)
	return (pio);

	zio_nowait(pio);
	return (NULL);
	}

	static void
	vdev_load_child(void *arg)
	{
	vdev_t *vd = arg;

	vd->vdev_load_error = vdev_load(vd);
	}

	static void
	vdev_open_child(void *arg)
	{
	vdev_t *vd = arg;

	vd->vdev_open_thread = curthread;
	vd->vdev_open_error = vdev_open(vd);
	vd->vdev_open_thread = NULL;
	}

	static boolean_t
	vdev_uses_zvols(vdev_t *vd)
	{
	#ifdef _KERNEL
	if (zvol_is_zvol(vd->vdev_path))
	return (B_TRUE);
	#endif

	for (int c = 0; c < vd->vdev_children; c++)
	if (vdev_uses_zvols(vd->vdev_child[c]))
	return (B_TRUE);

	return (B_FALSE);
	}

	/*
	* Returns B_TRUE if the passed child should be opened.
	*/
	static boolean_t
	vdev_default_open_children_func(vdev_t *vd)
	{
	(void) vd;
	return (B_TRUE);
	}

	/*
	* Open the requested child vdevs. If any of the leaf vdevs are using
	* a ZFS volume then do the opens in a single thread. This avoids a
	* deadlock when the current thread is holding the spa_namespace_lock.
	*/
	static void
	vdev_open_children_impl(vdev_t vd, vdev_open_children_func_t open_func)
	{
	int children = vd->vdev_children;

	taskq_t *tq = taskq_create("vdev_open", children, minclsyspri,
	children, children, TASKQ_PREPOPULATE);
	vd->vdev_nonrot = B_TRUE;

	for (int c = 0; c < children; c++) {
	vdev_t *cvd = vd->vdev_child[c];

	if (open_func(cvd) == B_FALSE)
	continue;

	if (tq == NULL \|\| vdev_uses_zvols(vd)) {
	cvd->vdev_open_error = vdev_open(cvd);
	} else {
	VERIFY(taskq_dispatch(tq, vdev_open_child,
	cvd, TQ_SLEEP) != TASKQID_INVALID);
	}

	vd->vdev_nonrot &= cvd->vdev_nonrot;
	}

	if (tq != NULL) {
	taskq_wait(tq);
	taskq_destroy(tq);
	}
	}

	/*
	* Open all child vdevs.
	*/
	void
	vdev_open_children(vdev_t *vd)
	{
	vdev_open_children_impl(vd, vdev_default_open_children_func);
	}

	/*
	* Conditionally open a subset of child vdevs.
	*/
	void
	vdev_open_children_subset(vdev_t vd, vdev_open_children_func_t open_func)
	{
	vdev_open_children_impl(vd, open_func);
	}

	/*
	* Compute the raidz-deflation ratio. Note, we hard-code
	* in 128k (1 << 17) because it is the "typical" blocksize.
	* Even though SPA_MAXBLOCKSIZE changed, this algorithm can not change,
	* otherwise it would inconsistently account for existing bp's.
	*/
	static void
	vdev_set_deflate_ratio(vdev_t *vd)
	{
	if (vd == vd->vdev_top && !vd->vdev_ishole && vd->vdev_ashift != 0) {
	vd->vdev_deflate_ratio = (1 << 17) /
	(vdev_psize_to_asize(vd, 1 << 17) >> SPA_MINBLOCKSHIFT);
	}
	}

	/*
	* Choose the best of two ashifts, preferring one between logical ashift
	* (absolute minimum) and administrator defined maximum, otherwise take
	* the biggest of the two.
	*/
	uint64_t
	vdev_best_ashift(uint64_t logical, uint64_t a, uint64_t b)
	{
	if (a > logical && a <= zfs_vdev_max_auto_ashift) {
	if (b <= logical \|\| b > zfs_vdev_max_auto_ashift)
	return (a);
	else
	return (MAX(a, b));
	} else if (b <= logical \|\| b > zfs_vdev_max_auto_ashift)
	return (MAX(a, b));
	return (b);
	}

	/*
	* Maximize performance by inflating the configured ashift for top level
	* vdevs to be as close to the physical ashift as possible while maintaining
	* administrator defined limits and ensuring it doesn't go below the
	* logical ashift.
	*/
	static void
	vdev_ashift_optimize(vdev_t *vd)
	{
	ASSERT(vd == vd->vdev_top);

	if (vd->vdev_ashift < vd->vdev_physical_ashift &&
	vd->vdev_physical_ashift <= zfs_vdev_max_auto_ashift) {
	vd->vdev_ashift = MIN(
	MAX(zfs_vdev_max_auto_ashift, vd->vdev_ashift),
	MAX(zfs_vdev_min_auto_ashift,
	vd->vdev_physical_ashift));
	} else {
	/*
	* If the logical and physical ashifts are the same, then
	* we ensure that the top-level vdev's ashift is not smaller
	* than our minimum ashift value. For the unusual case
	* where logical ashift > physical ashift, we can't cap
	* the calculated ashift based on max ashift as that
	* would cause failures.
	* We still check if we need to increase it to match
	* the min ashift.
	*/
	vd->vdev_ashift = MAX(zfs_vdev_min_auto_ashift,
	vd->vdev_ashift);
	}
	}

	/*
	* Prepare a virtual device for access.
	*/
	int
	vdev_open(vdev_t *vd)
	{
	spa_t *spa = vd->vdev_spa;
	int error;
	uint64_t osize = 0;
	uint64_t max_osize = 0;
	uint64_t asize, max_asize, psize;
	uint64_t logical_ashift = 0;
	uint64_t physical_ashift = 0;

	ASSERT(vd->vdev_open_thread == curthread \|\|
	spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);
	ASSERT(vd->vdev_state == VDEV_STATE_CLOSED \|\|
	vd->vdev_state == VDEV_STATE_CANT_OPEN \|\|
	vd->vdev_state == VDEV_STATE_OFFLINE);

	vd->vdev_stat.vs_aux = VDEV_AUX_NONE;
	vd->vdev_cant_read = B_FALSE;
	vd->vdev_cant_write = B_FALSE;
	vd->vdev_min_asize = vdev_get_min_asize(vd);

	/*
	* If this vdev is not removed, check its fault status. If it's
	* faulted, bail out of the open.
	*/
	if (!vd->vdev_removed && vd->vdev_faulted) {
	ASSERT(vd->vdev_children == 0);
	ASSERT(vd->vdev_label_aux == VDEV_AUX_ERR_EXCEEDED \|\|
	vd->vdev_label_aux == VDEV_AUX_EXTERNAL);
	vdev_set_state(vd, B_TRUE, VDEV_STATE_FAULTED,
	vd->vdev_label_aux);
	return (SET_ERROR(ENXIO));
	} else if (vd->vdev_offline) {
	ASSERT(vd->vdev_children == 0);
	vdev_set_state(vd, B_TRUE, VDEV_STATE_OFFLINE, VDEV_AUX_NONE);
	return (SET_ERROR(ENXIO));
	}

	error = vd->vdev_ops->vdev_op_open(vd, &osize, &max_osize,
	&logical_ashift, &physical_ashift);

	/* Keep the device in removed state if unplugged */
	if (error == ENOENT && vd->vdev_removed) {
	vdev_set_state(vd, B_TRUE, VDEV_STATE_REMOVED,
	VDEV_AUX_NONE);
	return (error);
	}

	/*
	* Physical volume size should never be larger than its max size, unless
	* the disk has shrunk while we were reading it or the device is buggy
	* or damaged: either way it's not safe for use, bail out of the open.
	*/
	if (osize > max_osize) {
	vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_OPEN_FAILED);
	return (SET_ERROR(ENXIO));
	}

	/*
	* Reset the vdev_reopening flag so that we actually close
	* the vdev on error.
	*/
	vd->vdev_reopening = B_FALSE;
	if (zio_injection_enabled && error == 0)
	error = zio_handle_device_injection(vd, NULL, SET_ERROR(ENXIO));

	if (error) {
	if (vd->vdev_removed &&
	vd->vdev_stat.vs_aux != VDEV_AUX_OPEN_FAILED)
	vd->vdev_removed = B_FALSE;

	if (vd->vdev_stat.vs_aux == VDEV_AUX_CHILDREN_OFFLINE) {
	vdev_set_state(vd, B_TRUE, VDEV_STATE_OFFLINE,
	vd->vdev_stat.vs_aux);
	} else {
	vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
	vd->vdev_stat.vs_aux);
	}
	return (error);
	}

	vd->vdev_removed = B_FALSE;

	/*
	* Recheck the faulted flag now that we have confirmed that
	* the vdev is accessible. If we're faulted, bail.
	*/
	if (vd->vdev_faulted) {
	ASSERT(vd->vdev_children == 0);
	ASSERT(vd->vdev_label_aux == VDEV_AUX_ERR_EXCEEDED \|\|
	vd->vdev_label_aux == VDEV_AUX_EXTERNAL);
	vdev_set_state(vd, B_TRUE, VDEV_STATE_FAULTED,
	vd->vdev_label_aux);
	return (SET_ERROR(ENXIO));
	}

	if (vd->vdev_degraded) {
	ASSERT(vd->vdev_children == 0);
	vdev_set_state(vd, B_TRUE, VDEV_STATE_DEGRADED,
	VDEV_AUX_ERR_EXCEEDED);
	} else {
	vdev_set_state(vd, B_TRUE, VDEV_STATE_HEALTHY, 0);
	}

	/*
	* For hole or missing vdevs we just return success.
	*/
	if (vd->vdev_ishole \|\| vd->vdev_ops == &vdev_missing_ops)
	return (0);

	for (int c = 0; c < vd->vdev_children; c++) {
	if (vd->vdev_child[c]->vdev_state != VDEV_STATE_HEALTHY) {
	vdev_set_state(vd, B_TRUE, VDEV_STATE_DEGRADED,
	VDEV_AUX_NONE);
	break;
	}
	}

	osize = P2ALIGN(osize, (uint64_t)sizeof (vdev_label_t));
	max_osize = P2ALIGN(max_osize, (uint64_t)sizeof (vdev_label_t));

	if (vd->vdev_children == 0) {
	if (osize < SPA_MINDEVSIZE) {
	vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_TOO_SMALL);
	return (SET_ERROR(EOVERFLOW));
	}
	psize = osize;
	asize = osize - (VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE);
	max_asize = max_osize - (VDEV_LABEL_START_SIZE +
	VDEV_LABEL_END_SIZE);
	} else {
	if (vd->vdev_parent != NULL && osize < SPA_MINDEVSIZE -
	(VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE)) {
	vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_TOO_SMALL);
	return (SET_ERROR(EOVERFLOW));
	}
	psize = 0;
	asize = osize;
	max_asize = max_osize;
	}

	/*
	* If the vdev was expanded, record this so that we can re-create the
	* uberblock rings in labels {2,3}, during the next sync.
	*/
	if ((psize > vd->vdev_psize) && (vd->vdev_psize != 0))
	vd->vdev_copy_uberblocks = B_TRUE;

	vd->vdev_psize = psize;

	/*
	* Make sure the allocatable size hasn't shrunk too much.
	*/
	if (asize < vd->vdev_min_asize) {
	vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_BAD_LABEL);
	return (SET_ERROR(EINVAL));
	}

	/*
	* We can always set the logical/physical ashift members since
	* their values are only used to calculate the vdev_ashift when
	* the device is first added to the config. These values should
	* not be used for anything else since they may change whenever
	* the device is reopened and we don't store them in the label.
	*/
	vd->vdev_physical_ashift =
	MAX(physical_ashift, vd->vdev_physical_ashift);
	vd->vdev_logical_ashift = MAX(logical_ashift,
	vd->vdev_logical_ashift);

	if (vd->vdev_asize == 0) {
	/*
	* This is the first-ever open, so use the computed values.
	* For compatibility, a different ashift can be requested.
	*/
	vd->vdev_asize = asize;
	vd->vdev_max_asize = max_asize;

	/*
	* If the vdev_ashift was not overridden at creation time,
	* then set it the logical ashift and optimize the ashift.
	*/
	if (vd->vdev_ashift == 0) {
	vd->vdev_ashift = vd->vdev_logical_ashift;

	if (vd->vdev_logical_ashift > ASHIFT_MAX) {
	vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_ASHIFT_TOO_BIG);
	return (SET_ERROR(EDOM));
	}

	if (vd->vdev_top == vd && vd->vdev_attaching == B_FALSE)
	vdev_ashift_optimize(vd);
	vd->vdev_attaching = B_FALSE;
	}
	if (vd->vdev_ashift != 0 && (vd->vdev_ashift < ASHIFT_MIN \|\|
	vd->vdev_ashift > ASHIFT_MAX)) {
	vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_BAD_ASHIFT);
	return (SET_ERROR(EDOM));
	}
	} else {
	/*
	* Make sure the alignment required hasn't increased.
	*/
	if (vd->vdev_ashift > vd->vdev_top->vdev_ashift &&
	vd->vdev_ops->vdev_op_leaf) {
	(void) zfs_ereport_post(
	FM_EREPORT_ZFS_DEVICE_BAD_ASHIFT,
	spa, vd, NULL, NULL, 0);
	vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_BAD_LABEL);
	return (SET_ERROR(EDOM));
	}
	vd->vdev_max_asize = max_asize;
	}

	/*
	* If all children are healthy we update asize if either:
	* The asize has increased, due to a device expansion caused by dynamic
	* LUN growth or vdev replacement, and automatic expansion is enabled;
	* making the additional space available.
	*
	* The asize has decreased, due to a device shrink usually caused by a
	* vdev replace with a smaller device. This ensures that calculations
	* based of max_asize and asize e.g. esize are always valid. It's safe
	* to do this as we've already validated that asize is greater than
	* vdev_min_asize.
	*/
	if (vd->vdev_state == VDEV_STATE_HEALTHY &&
	((asize > vd->vdev_asize &&
	(vd->vdev_expanding \|\| spa->spa_autoexpand)) \|\|
	(asize < vd->vdev_asize)))
	vd->vdev_asize = asize;

	vdev_set_min_asize(vd);

	/*
	* Ensure we can issue some IO before declaring the
	* vdev open for business.
	*/
	if (vd->vdev_ops->vdev_op_leaf &&
	(error = zio_wait(vdev_probe(vd, NULL))) != 0) {
	vdev_set_state(vd, B_TRUE, VDEV_STATE_FAULTED,
	VDEV_AUX_ERR_EXCEEDED);
	return (error);
	}

	/*
	* Track the minimum allocation size.
	*/
	if (vd->vdev_top == vd && vd->vdev_ashift != 0 &&
	vd->vdev_islog == 0 && vd->vdev_aux == NULL) {
	uint64_t min_alloc = vdev_get_min_alloc(vd);
	vdev_spa_set_alloc(spa, min_alloc);
	}

	/*
	* If this is a leaf vdev, assess whether a resilver is needed.
	* But don't do this if we are doing a reopen for a scrub, since
	* this would just restart the scrub we are already doing.
	*/
	if (vd->vdev_ops->vdev_op_leaf && !spa->spa_scrub_reopen)
	dsl_scan_assess_vdev(spa->spa_dsl_pool, vd);

	return (0);
	}

	static void
	vdev_validate_child(void *arg)
	{
	vdev_t *vd = arg;

	vd->vdev_validate_thread = curthread;
	vd->vdev_validate_error = vdev_validate(vd);
	vd->vdev_validate_thread = NULL;
	}

	/*
	* Called once the vdevs are all opened, this routine validates the label
	* contents. This needs to be done before vdev_load() so that we don't
	* inadvertently do repair I/Os to the wrong device.
	*
	* This function will only return failure if one of the vdevs indicates that it
	* has since been destroyed or exported. This is only possible if
	* /etc/zfs/zpool.cache was readonly at the time. Otherwise, the vdev state
	* will be updated but the function will return 0.
	*/
	int
	vdev_validate(vdev_t *vd)
	{
	spa_t *spa = vd->vdev_spa;
	taskq_t *tq = NULL;
	nvlist_t *label;
	uint64_t guid = 0, aux_guid = 0, top_guid;
	uint64_t state;
	nvlist_t *nvl;
	uint64_t txg;
	int children = vd->vdev_children;

	if (vdev_validate_skip)
	return (0);

	if (children > 0) {
	tq = taskq_create("vdev_validate", children, minclsyspri,
	children, children, TASKQ_PREPOPULATE);
	}

	for (uint64_t c = 0; c < children; c++) {
	vdev_t *cvd = vd->vdev_child[c];

	if (tq == NULL \|\| vdev_uses_zvols(cvd)) {
	vdev_validate_child(cvd);
	} else {
	VERIFY(taskq_dispatch(tq, vdev_validate_child, cvd,
	TQ_SLEEP) != TASKQID_INVALID);
	}
	}
	if (tq != NULL) {
	taskq_wait(tq);
	taskq_destroy(tq);
	}
	for (int c = 0; c < children; c++) {
	int error = vd->vdev_child[c]->vdev_validate_error;

	if (error != 0)
	return (SET_ERROR(EBADF));
	}


	/*
	* If the device has already failed, or was marked offline, don't do
	* any further validation. Otherwise, label I/O will fail and we will
	* overwrite the previous state.
	*/
	if (!vd->vdev_ops->vdev_op_leaf \|\| !vdev_readable(vd))
	return (0);

	/*
	* If we are performing an extreme rewind, we allow for a label that
	* was modified at a point after the current txg.
	* If config lock is not held do not check for the txg. spa_sync could
	* be updating the vdev's label before updating spa_last_synced_txg.
	*/
	if (spa->spa_extreme_rewind \|\| spa_last_synced_txg(spa) == 0 \|\|
	spa_config_held(spa, SCL_CONFIG, RW_WRITER) != SCL_CONFIG)
	txg = UINT64_MAX;
	else
	txg = spa_last_synced_txg(spa);

	if ((label = vdev_label_read_config(vd, txg)) == NULL) {
	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_BAD_LABEL);
	vdev_dbgmsg(vd, "vdev_validate: failed reading config for "
	"txg %llu", (u_longlong_t)txg);
	return (0);
	}

	/*
	* Determine if this vdev has been split off into another
	* pool. If so, then refuse to open it.
	*/
	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_SPLIT_GUID,
	&aux_guid) == 0 && aux_guid == spa_guid(spa)) {
	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_SPLIT_POOL);
	nvlist_free(label);
	vdev_dbgmsg(vd, "vdev_validate: vdev split into other pool");
	return (0);
	}

	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_GUID, &guid) != 0) {
	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_CORRUPT_DATA);
	nvlist_free(label);
	vdev_dbgmsg(vd, "vdev_validate: '%s' missing from label",
	ZPOOL_CONFIG_POOL_GUID);
	return (0);
	}

	/*
	* If config is not trusted then ignore the spa guid check. This is
	* necessary because if the machine crashed during a re-guid the new
	* guid might have been written to all of the vdev labels, but not the
	* cached config. The check will be performed again once we have the
	* trusted config from the MOS.
	*/
	if (spa->spa_trust_config && guid != spa_guid(spa)) {
	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_CORRUPT_DATA);
	nvlist_free(label);
	vdev_dbgmsg(vd, "vdev_validate: vdev label pool_guid doesn't "
	"match config (%llu != %llu)", (u_longlong_t)guid,
	(u_longlong_t)spa_guid(spa));
	return (0);
	}

	if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_VDEV_TREE, &nvl)
	!= 0 \|\| nvlist_lookup_uint64(nvl, ZPOOL_CONFIG_ORIG_GUID,
	&aux_guid) != 0)
	aux_guid = 0;

	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_GUID, &guid) != 0) {
	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_CORRUPT_DATA);
	nvlist_free(label);
	vdev_dbgmsg(vd, "vdev_validate: '%s' missing from label",
	ZPOOL_CONFIG_GUID);
	return (0);
	}

	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_TOP_GUID, &top_guid)
	!= 0) {
	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_CORRUPT_DATA);
	nvlist_free(label);
	vdev_dbgmsg(vd, "vdev_validate: '%s' missing from label",
	ZPOOL_CONFIG_TOP_GUID);
	return (0);
	}

	/*
	* If this vdev just became a top-level vdev because its sibling was
	* detached, it will have adopted the parent's vdev guid -- but the
	* label may or may not be on disk yet. Fortunately, either version
	* of the label will have the same top guid, so if we're a top-level
	* vdev, we can safely compare to that instead.
	* However, if the config comes from a cachefile that failed to update
	* after the detach, a top-level vdev will appear as a non top-level
	* vdev in the config. Also relax the constraints if we perform an
	* extreme rewind.
	*
	* If we split this vdev off instead, then we also check the
	* original pool's guid. We don't want to consider the vdev
	* corrupt if it is partway through a split operation.
	*/
	if (vd->vdev_guid != guid && vd->vdev_guid != aux_guid) {
	boolean_t mismatch = B_FALSE;
	if (spa->spa_trust_config && !spa->spa_extreme_rewind) {
	if (vd != vd->vdev_top \|\| vd->vdev_guid != top_guid)
	mismatch = B_TRUE;
	} else {
	if (vd->vdev_guid != top_guid &&
	vd->vdev_top->vdev_guid != guid)
	mismatch = B_TRUE;
	}

	if (mismatch) {
	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_CORRUPT_DATA);
	nvlist_free(label);
	vdev_dbgmsg(vd, "vdev_validate: config guid "
	"doesn't match label guid");
	vdev_dbgmsg(vd, "CONFIG: guid %llu, top_guid %llu",
	(u_longlong_t)vd->vdev_guid,
	(u_longlong_t)vd->vdev_top->vdev_guid);
	vdev_dbgmsg(vd, "LABEL: guid %llu, top_guid %llu, "
	"aux_guid %llu", (u_longlong_t)guid,
	(u_longlong_t)top_guid, (u_longlong_t)aux_guid);
	return (0);
	}
	}

	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_STATE,
	&state) != 0) {
	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_CORRUPT_DATA);
	nvlist_free(label);
	vdev_dbgmsg(vd, "vdev_validate: '%s' missing from label",
	ZPOOL_CONFIG_POOL_STATE);
	return (0);
	}

	nvlist_free(label);

	/*
	* If this is a verbatim import, no need to check the
	* state of the pool.
	*/
	if (!(spa->spa_import_flags & ZFS_IMPORT_VERBATIM) &&
	spa_load_state(spa) == SPA_LOAD_OPEN &&
	state != POOL_STATE_ACTIVE) {
	vdev_dbgmsg(vd, "vdev_validate: invalid pool state (%llu) "
	"for spa %s", (u_longlong_t)state, spa->spa_name);
	return (SET_ERROR(EBADF));
	}

	/*
	* If we were able to open and validate a vdev that was
	* previously marked permanently unavailable, clear that state
	* now.
	*/
	if (vd->vdev_not_present)
	vd->vdev_not_present = 0;

	return (0);
	}

	static void
	vdev_update_path(const char prefix, char svd, char **dvd, uint64_t guid)
	{
	if (svd != NULL && *dvd != NULL) {
	if (strcmp(svd, *dvd) != 0) {
	zfs_dbgmsg("vdev_copy_path: vdev %llu: %s changed "
	"from '%s' to '%s'", (u_longlong_t)guid, prefix,
	*dvd, svd);
	spa_strfree(*dvd);
	*dvd = spa_strdup(svd);
	}
	} else if (svd != NULL) {
	*dvd = spa_strdup(svd);
	zfs_dbgmsg("vdev_copy_path: vdev %llu: path set to '%s'",
	(u_longlong_t)guid, *dvd);
	}
	}

	static void
	vdev_copy_path_impl(vdev_t svd, vdev_t dvd)
	{
	char old, new;

	vdev_update_path("vdev_path", svd->vdev_path, &dvd->vdev_path,
	dvd->vdev_guid);

	vdev_update_path("vdev_devid", svd->vdev_devid, &dvd->vdev_devid,
	dvd->vdev_guid);

	vdev_update_path("vdev_physpath", svd->vdev_physpath,
	&dvd->vdev_physpath, dvd->vdev_guid);

	/*
	* Our enclosure sysfs path may have changed between imports
	*/
	old = dvd->vdev_enc_sysfs_path;
	new = svd->vdev_enc_sysfs_path;
	if ((old != NULL && new == NULL) \|\|
	(old == NULL && new != NULL) \|\|
	((old != NULL && new != NULL) && strcmp(new, old) != 0)) {
	zfs_dbgmsg("vdev_copy_path: vdev %llu: vdev_enc_sysfs_path "
	"changed from '%s' to '%s'", (u_longlong_t)dvd->vdev_guid,
	old, new);

	if (dvd->vdev_enc_sysfs_path)
	spa_strfree(dvd->vdev_enc_sysfs_path);

	if (svd->vdev_enc_sysfs_path) {
	dvd->vdev_enc_sysfs_path = spa_strdup(
	svd->vdev_enc_sysfs_path);
	} else {
	dvd->vdev_enc_sysfs_path = NULL;
	}
	}
	}

	/*
	* Recursively copy vdev paths from one vdev to another. Source and destination
	* vdev trees must have same geometry otherwise return error. Intended to copy
	* paths from userland config into MOS config.
	*/
	int
	vdev_copy_path_strict(vdev_t svd, vdev_t dvd)
	{
	if ((svd->vdev_ops == &vdev_missing_ops) \|\|
	(svd->vdev_ishole && dvd->vdev_ishole) \|\|
	(dvd->vdev_ops == &vdev_indirect_ops))
	return (0);

	if (svd->vdev_ops != dvd->vdev_ops) {
	vdev_dbgmsg(svd, "vdev_copy_path: vdev type mismatch: %s != %s",
	svd->vdev_ops->vdev_op_type, dvd->vdev_ops->vdev_op_type);
	return (SET_ERROR(EINVAL));
	}

	if (svd->vdev_guid != dvd->vdev_guid) {
	vdev_dbgmsg(svd, "vdev_copy_path: guids mismatch (%llu != "
	"%llu)", (u_longlong_t)svd->vdev_guid,
	(u_longlong_t)dvd->vdev_guid);
	return (SET_ERROR(EINVAL));
	}

	if (svd->vdev_children != dvd->vdev_children) {
	vdev_dbgmsg(svd, "vdev_copy_path: children count mismatch: "
	"%llu != %llu", (u_longlong_t)svd->vdev_children,
	(u_longlong_t)dvd->vdev_children);
	return (SET_ERROR(EINVAL));
	}

	for (uint64_t i = 0; i < svd->vdev_children; i++) {
	int error = vdev_copy_path_strict(svd->vdev_child[i],
	dvd->vdev_child[i]);
	if (error != 0)
	return (error);
	}

	if (svd->vdev_ops->vdev_op_leaf)
	vdev_copy_path_impl(svd, dvd);

	return (0);
	}

	static void
	vdev_copy_path_search(vdev_t stvd, vdev_t dvd)
	{
	ASSERT(stvd->vdev_top == stvd);
	ASSERT3U(stvd->vdev_id, ==, dvd->vdev_top->vdev_id);

	for (uint64_t i = 0; i < dvd->vdev_children; i++) {
	vdev_copy_path_search(stvd, dvd->vdev_child[i]);
	}

	if (!dvd->vdev_ops->vdev_op_leaf \|\| !vdev_is_concrete(dvd))
	return;

	/*
	* The idea here is that while a vdev can shift positions within
	* a top vdev (when replacing, attaching mirror, etc.) it cannot
	* step outside of it.
	*/
	vdev_t *vd = vdev_lookup_by_guid(stvd, dvd->vdev_guid);

	if (vd == NULL \|\| vd->vdev_ops != dvd->vdev_ops)
	return;

	ASSERT(vd->vdev_ops->vdev_op_leaf);

	vdev_copy_path_impl(vd, dvd);
	}

	/*
	* Recursively copy vdev paths from one root vdev to another. Source and
	* destination vdev trees may differ in geometry. For each destination leaf
	* vdev, search a vdev with the same guid and top vdev id in the source.
	* Intended to copy paths from userland config into MOS config.
	*/
	void
	vdev_copy_path_relaxed(vdev_t srvd, vdev_t drvd)
	{
	uint64_t children = MIN(srvd->vdev_children, drvd->vdev_children);
	ASSERT(srvd->vdev_ops == &vdev_root_ops);
	ASSERT(drvd->vdev_ops == &vdev_root_ops);

	for (uint64_t i = 0; i < children; i++) {
	vdev_copy_path_search(srvd->vdev_child[i],
	drvd->vdev_child[i]);
	}
	}

	/*
	* Close a virtual device.
	*/
	void
	vdev_close(vdev_t *vd)
	{
	vdev_t *pvd = vd->vdev_parent;
	spa_t *spa __maybe_unused = vd->vdev_spa;

	ASSERT(vd != NULL);
	ASSERT(vd->vdev_open_thread == curthread \|\|
	spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);

	/*
	* If our parent is reopening, then we are as well, unless we are
	* going offline.
	*/
	if (pvd != NULL && pvd->vdev_reopening)
	vd->vdev_reopening = (pvd->vdev_reopening && !vd->vdev_offline);

	vd->vdev_ops->vdev_op_close(vd);

	/*
	* We record the previous state before we close it, so that if we are
	* doing a reopen(), we don't generate FMA ereports if we notice that
	* it's still faulted.
	*/
	vd->vdev_prevstate = vd->vdev_state;

	if (vd->vdev_offline)
	vd->vdev_state = VDEV_STATE_OFFLINE;
	else
	vd->vdev_state = VDEV_STATE_CLOSED;
	vd->vdev_stat.vs_aux = VDEV_AUX_NONE;
	}

	void
	vdev_hold(vdev_t *vd)
	{
	spa_t *spa = vd->vdev_spa;

	ASSERT(spa_is_root(spa));
	if (spa->spa_state == POOL_STATE_UNINITIALIZED)
	return;

	for (int c = 0; c < vd->vdev_children; c++)
	vdev_hold(vd->vdev_child[c]);

	if (vd->vdev_ops->vdev_op_leaf && vd->vdev_ops->vdev_op_hold != NULL)
	vd->vdev_ops->vdev_op_hold(vd);
	}

	void
	vdev_rele(vdev_t *vd)
	{
	ASSERT(spa_is_root(vd->vdev_spa));
	for (int c = 0; c < vd->vdev_children; c++)
	vdev_rele(vd->vdev_child[c]);

	if (vd->vdev_ops->vdev_op_leaf && vd->vdev_ops->vdev_op_rele != NULL)
	vd->vdev_ops->vdev_op_rele(vd);
	}

	/*
	* Reopen all interior vdevs and any unopened leaves. We don't actually
	* reopen leaf vdevs which had previously been opened as they might deadlock
	* on the spa_config_lock. Instead we only obtain the leaf's physical size.
	* If the leaf has never been opened then open it, as usual.
	*/
	void
	vdev_reopen(vdev_t *vd)
	{
	spa_t *spa = vd->vdev_spa;

	ASSERT(spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);

	/* set the reopening flag unless we're taking the vdev offline */
	vd->vdev_reopening = !vd->vdev_offline;
	vdev_close(vd);
	(void) vdev_open(vd);

	/*
	* Call vdev_validate() here to make sure we have the same device.
	* Otherwise, a device with an invalid label could be successfully
	* opened in response to vdev_reopen().
	*/
	if (vd->vdev_aux) {
	(void) vdev_validate_aux(vd);
	if (vdev_readable(vd) && vdev_writeable(vd) &&
	vd->vdev_aux == &spa->spa_l2cache) {
	/*
	* In case the vdev is present we should evict all ARC
	* buffers and pointers to log blocks and reclaim their
	* space before restoring its contents to L2ARC.
	*/
	if (l2arc_vdev_present(vd)) {
	l2arc_rebuild_vdev(vd, B_TRUE);
	} else {
	l2arc_add_vdev(spa, vd);
	}
	spa_async_request(spa, SPA_ASYNC_L2CACHE_REBUILD);
	spa_async_request(spa, SPA_ASYNC_L2CACHE_TRIM);
	}
	} else {
	(void) vdev_validate(vd);
	}

	/*
	* Recheck if resilver is still needed and cancel any
	* scheduled resilver if resilver is unneeded.
	*/
	if (!vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL) &&
	spa->spa_async_tasks & SPA_ASYNC_RESILVER) {
	mutex_enter(&spa->spa_async_lock);
	spa->spa_async_tasks &= ~SPA_ASYNC_RESILVER;
	mutex_exit(&spa->spa_async_lock);
	}

	/*
	* Reassess parent vdev's health.
	*/
	vdev_propagate_state(vd);
	}

	int
	vdev_create(vdev_t *vd, uint64_t txg, boolean_t isreplacing)
	{
	int error;

	/*
	* Normally, partial opens (e.g. of a mirror) are allowed.
	* For a create, however, we want to fail the request if
	* there are any components we can't open.
	*/
	error = vdev_open(vd);

	if (error \|\| vd->vdev_state != VDEV_STATE_HEALTHY) {
	vdev_close(vd);
	return (error ? error : SET_ERROR(ENXIO));
	}

	/*
	* Recursively load DTLs and initialize all labels.
	*/
	if ((error = vdev_dtl_load(vd)) != 0 \|\|
	(error = vdev_label_init(vd, txg, isreplacing ?
	VDEV_LABEL_REPLACE : VDEV_LABEL_CREATE)) != 0) {
	vdev_close(vd);
	return (error);
	}

	return (0);
	}

	void
	vdev_metaslab_set_size(vdev_t *vd)
	{
	uint64_t asize = vd->vdev_asize;
	uint64_t ms_count = asize >> zfs_vdev_default_ms_shift;
	uint64_t ms_shift;

	/*
	* There are two dimensions to the metaslab sizing calculation:
	* the size of the metaslab and the count of metaslabs per vdev.
	*
	* The default values used below are a good balance between memory
	* usage (larger metaslab size means more memory needed for loaded
	* metaslabs; more metaslabs means more memory needed for the
	* metaslab_t structs), metaslab load time (larger metaslabs take
	* longer to load), and metaslab sync time (more metaslabs means
	* more time spent syncing all of them).
	*
	* In general, we aim for zfs_vdev_default_ms_count (200) metaslabs.
	* The range of the dimensions are as follows:
	*
	* 2^29 <= ms_size <= 2^34
	* 16 <= ms_count <= 131,072
	*
	* On the lower end of vdev sizes, we aim for metaslabs sizes of
	* at least 512MB (2^29) to minimize fragmentation effects when
	* testing with smaller devices. However, the count constraint
	* of at least 16 metaslabs will override this minimum size goal.
	*
	* On the upper end of vdev sizes, we aim for a maximum metaslab
	* size of 16GB. However, we will cap the total count to 2^17
	* metaslabs to keep our memory footprint in check and let the
	* metaslab size grow from there if that limit is hit.
	*
	* The net effect of applying above constrains is summarized below.
	*
	* vdev size metaslab count
	* --------------\|-----------------
	* < 8GB ~16
	* 8GB - 100GB one per 512MB
	* 100GB - 3TB ~200
	* 3TB - 2PB one per 16GB
	* > 2PB ~131,072
	* --------------------------------
	*
	* Finally, note that all of the above calculate the initial
	* number of metaslabs. Expanding a top-level vdev will result
	* in additional metaslabs being allocated making it possible
	* to exceed the zfs_vdev_ms_count_limit.
	*/

	if (ms_count < zfs_vdev_min_ms_count)
	ms_shift = highbit64(asize / zfs_vdev_min_ms_count);
	else if (ms_count > zfs_vdev_default_ms_count)
	ms_shift = highbit64(asize / zfs_vdev_default_ms_count);
	else
	ms_shift = zfs_vdev_default_ms_shift;

	if (ms_shift < SPA_MAXBLOCKSHIFT) {
	ms_shift = SPA_MAXBLOCKSHIFT;
	} else if (ms_shift > zfs_vdev_max_ms_shift) {
	ms_shift = zfs_vdev_max_ms_shift;
	/* cap the total count to constrain memory footprint */
	if ((asize >> ms_shift) > zfs_vdev_ms_count_limit)
	ms_shift = highbit64(asize / zfs_vdev_ms_count_limit);
	}

	vd->vdev_ms_shift = ms_shift;
	ASSERT3U(vd->vdev_ms_shift, >=, SPA_MAXBLOCKSHIFT);
	}

	void
	vdev_dirty(vdev_t vd, int flags, void arg, uint64_t txg)
	{
	ASSERT(vd == vd->vdev_top);
	/* indirect vdevs don't have metaslabs or dtls */
	ASSERT(vdev_is_concrete(vd) \|\| flags == 0);
	ASSERT(ISP2(flags));
	ASSERT(spa_writeable(vd->vdev_spa));

	if (flags & VDD_METASLAB)
	(void) txg_list_add(&vd->vdev_ms_list, arg, txg);

	if (flags & VDD_DTL)
	(void) txg_list_add(&vd->vdev_dtl_list, arg, txg);

	(void) txg_list_add(&vd->vdev_spa->spa_vdev_txg_list, vd, txg);
	}

	void
	vdev_dirty_leaves(vdev_t *vd, int flags, uint64_t txg)
	{
	for (int c = 0; c < vd->vdev_children; c++)
	vdev_dirty_leaves(vd->vdev_child[c], flags, txg);

	if (vd->vdev_ops->vdev_op_leaf)
	vdev_dirty(vd->vdev_top, flags, vd, txg);
	}

	/*
	* DTLs.
	*
	* A vdev's DTL (dirty time log) is the set of transaction groups for which
	* the vdev has less than perfect replication. There are four kinds of DTL:
	*
	* DTL_MISSING: txgs for which the vdev has no valid copies of the data
	*
	* DTL_PARTIAL: txgs for which data is available, but not fully replicated
	*
	* DTL_SCRUB: the txgs that could not be repaired by the last scrub; upon
	* scrub completion, DTL_SCRUB replaces DTL_MISSING in the range of
	* txgs that was scrubbed.
	*
	* DTL_OUTAGE: txgs which cannot currently be read, whether due to
	* persistent errors or just some device being offline.
	* Unlike the other three, the DTL_OUTAGE map is not generally
	* maintained; it's only computed when needed, typically to
	* determine whether a device can be detached.
	*
	* For leaf vdevs, DTL_MISSING and DTL_PARTIAL are identical: the device
	* either has the data or it doesn't.
	*
	* For interior vdevs such as mirror and RAID-Z the picture is more complex.
	* A vdev's DTL_PARTIAL is the union of its children's DTL_PARTIALs, because
	* if any child is less than fully replicated, then so is its parent.
	* A vdev's DTL_MISSING is a modified union of its children's DTL_MISSINGs,
	* comprising only those txgs which appear in 'maxfaults' or more children;
	* those are the txgs we don't have enough replication to read. For example,
	* double-parity RAID-Z can tolerate up to two missing devices (maxfaults == 2);
	* thus, its DTL_MISSING consists of the set of txgs that appear in more than
	* two child DTL_MISSING maps.
	*
	* It should be clear from the above that to compute the DTLs and outage maps
	* for all vdevs, it suffices to know just the leaf vdevs' DTL_MISSING maps.
	* Therefore, that is all we keep on disk. When loading the pool, or after
	* a configuration change, we generate all other DTLs from first principles.
	*/
	void
	vdev_dtl_dirty(vdev_t *vd, vdev_dtl_type_t t, uint64_t txg, uint64_t size)
	{
	range_tree_t *rt = vd->vdev_dtl[t];

	ASSERT(t < DTL_TYPES);
	ASSERT(vd != vd->vdev_spa->spa_root_vdev);
	ASSERT(spa_writeable(vd->vdev_spa));

	mutex_enter(&vd->vdev_dtl_lock);
	if (!range_tree_contains(rt, txg, size))
	range_tree_add(rt, txg, size);
	mutex_exit(&vd->vdev_dtl_lock);
	}

	boolean_t
	vdev_dtl_contains(vdev_t *vd, vdev_dtl_type_t t, uint64_t txg, uint64_t size)
	{
	range_tree_t *rt = vd->vdev_dtl[t];
	boolean_t dirty = B_FALSE;

	ASSERT(t < DTL_TYPES);
	ASSERT(vd != vd->vdev_spa->spa_root_vdev);

	/*
	* While we are loading the pool, the DTLs have not been loaded yet.
	* This isn't a problem but it can result in devices being tried
	* which are known to not have the data. In which case, the import
	* is relying on the checksum to ensure that we get the right data.
	* Note that while importing we are only reading the MOS, which is
	* always checksummed.
	*/
	mutex_enter(&vd->vdev_dtl_lock);
	if (!range_tree_is_empty(rt))
	dirty = range_tree_contains(rt, txg, size);
	mutex_exit(&vd->vdev_dtl_lock);

	return (dirty);
	}

	boolean_t
	vdev_dtl_empty(vdev_t *vd, vdev_dtl_type_t t)
	{
	range_tree_t *rt = vd->vdev_dtl[t];
	boolean_t empty;

	mutex_enter(&vd->vdev_dtl_lock);
	empty = range_tree_is_empty(rt);
	mutex_exit(&vd->vdev_dtl_lock);

	return (empty);
	}

	/*
	* Check if the txg falls within the range which must be
	* resilvered. DVAs outside this range can always be skipped.
	*/
	boolean_t
	vdev_default_need_resilver(vdev_t vd, const dva_t dva, size_t psize,
	uint64_t phys_birth)
	{
	(void) dva, (void) psize;

	/* Set by sequential resilver. */
	if (phys_birth == TXG_UNKNOWN)
	return (B_TRUE);

	return (vdev_dtl_contains(vd, DTL_PARTIAL, phys_birth, 1));
	}

	/*
	* Returns B_TRUE if the vdev determines the DVA needs to be resilvered.
	*/
	boolean_t
	vdev_dtl_need_resilver(vdev_t vd, const dva_t dva, size_t psize,
	uint64_t phys_birth)
	{
	ASSERT(vd != vd->vdev_spa->spa_root_vdev);

	if (vd->vdev_ops->vdev_op_need_resilver == NULL \|\|
	vd->vdev_ops->vdev_op_leaf)
	return (B_TRUE);

	return (vd->vdev_ops->vdev_op_need_resilver(vd, dva, psize,
	phys_birth));
	}

	/*
	* Returns the lowest txg in the DTL range.
	*/
	static uint64_t
	vdev_dtl_min(vdev_t *vd)
	{
	ASSERT(MUTEX_HELD(&vd->vdev_dtl_lock));
	ASSERT3U(range_tree_space(vd->vdev_dtl[DTL_MISSING]), !=, 0);
	ASSERT0(vd->vdev_children);

	return (range_tree_min(vd->vdev_dtl[DTL_MISSING]) - 1);
	}

	/*
	* Returns the highest txg in the DTL.
	*/
	static uint64_t
	vdev_dtl_max(vdev_t *vd)
	{
	ASSERT(MUTEX_HELD(&vd->vdev_dtl_lock));
	ASSERT3U(range_tree_space(vd->vdev_dtl[DTL_MISSING]), !=, 0);
	ASSERT0(vd->vdev_children);

	return (range_tree_max(vd->vdev_dtl[DTL_MISSING]));
	}

	/*
	* Determine if a resilvering vdev should remove any DTL entries from
	* its range. If the vdev was resilvering for the entire duration of the
	* scan then it should excise that range from its DTLs. Otherwise, this
	* vdev is considered partially resilvered and should leave its DTL
	* entries intact. The comment in vdev_dtl_reassess() describes how we
	* excise the DTLs.
	*/
	static boolean_t
	vdev_dtl_should_excise(vdev_t *vd, boolean_t rebuild_done)
	{
	ASSERT0(vd->vdev_children);

	if (vd->vdev_state < VDEV_STATE_DEGRADED)
	return (B_FALSE);

	if (vd->vdev_resilver_deferred)
	return (B_FALSE);

	if (range_tree_is_empty(vd->vdev_dtl[DTL_MISSING]))
	return (B_TRUE);

	if (rebuild_done) {
	vdev_rebuild_t *vr = &vd->vdev_top->vdev_rebuild_config;
	vdev_rebuild_phys_t *vrp = &vr->vr_rebuild_phys;

	/* Rebuild not initiated by attach */
	if (vd->vdev_rebuild_txg == 0)
	return (B_TRUE);

	/*
	* When a rebuild completes without error then all missing data
	* up to the rebuild max txg has been reconstructed and the DTL
	* is eligible for excision.
	*/
	if (vrp->vrp_rebuild_state == VDEV_REBUILD_COMPLETE &&
	vdev_dtl_max(vd) <= vrp->vrp_max_txg) {
	ASSERT3U(vrp->vrp_min_txg, <=, vdev_dtl_min(vd));
	ASSERT3U(vrp->vrp_min_txg, <, vd->vdev_rebuild_txg);
	ASSERT3U(vd->vdev_rebuild_txg, <=, vrp->vrp_max_txg);
	return (B_TRUE);
	}
	} else {
	dsl_scan_t *scn = vd->vdev_spa->spa_dsl_pool->dp_scan;
	dsl_scan_phys_t *scnp __maybe_unused = &scn->scn_phys;

	/* Resilver not initiated by attach */
	if (vd->vdev_resilver_txg == 0)
	return (B_TRUE);

	/*
	* When a resilver is initiated the scan will assign the
	* scn_max_txg value to the highest txg value that exists
	* in all DTLs. If this device's max DTL is not part of this
	* scan (i.e. it is not in the range (scn_min_txg, scn_max_txg]
	* then it is not eligible for excision.
	*/
	if (vdev_dtl_max(vd) <= scn->scn_phys.scn_max_txg) {
	ASSERT3U(scnp->scn_min_txg, <=, vdev_dtl_min(vd));
	ASSERT3U(scnp->scn_min_txg, <, vd->vdev_resilver_txg);
	ASSERT3U(vd->vdev_resilver_txg, <=, scnp->scn_max_txg);
	return (B_TRUE);
	}
	}

	return (B_FALSE);
	}

	/*
	* Reassess DTLs after a config change or scrub completion. If txg == 0 no
	* write operations will be issued to the pool.
	*/
	void
	vdev_dtl_reassess(vdev_t *vd, uint64_t txg, uint64_t scrub_txg,
	boolean_t scrub_done, boolean_t rebuild_done)
	{
	spa_t *spa = vd->vdev_spa;
	avl_tree_t reftree;
	int minref;

	ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) != 0);

	for (int c = 0; c < vd->vdev_children; c++)
	vdev_dtl_reassess(vd->vdev_child[c], txg,
	scrub_txg, scrub_done, rebuild_done);

	if (vd == spa->spa_root_vdev \|\| !vdev_is_concrete(vd) \|\| vd->vdev_aux)
	return;

	if (vd->vdev_ops->vdev_op_leaf) {
	dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan;
	vdev_rebuild_t *vr = &vd->vdev_top->vdev_rebuild_config;
	boolean_t check_excise = B_FALSE;
	boolean_t wasempty = B_TRUE;

	mutex_enter(&vd->vdev_dtl_lock);

	/*
	* If requested, pretend the scan or rebuild completed cleanly.
	*/
	if (zfs_scan_ignore_errors) {
	if (scn != NULL)
	scn->scn_phys.scn_errors = 0;
	if (vr != NULL)
	vr->vr_rebuild_phys.vrp_errors = 0;
	}

	if (scrub_txg != 0 &&
	!range_tree_is_empty(vd->vdev_dtl[DTL_MISSING])) {
	wasempty = B_FALSE;
	zfs_dbgmsg("guid:%llu txg:%llu scrub:%llu started:%d "
	"dtl:%llu/%llu errors:%llu",
	(u_longlong_t)vd->vdev_guid, (u_longlong_t)txg,
	(u_longlong_t)scrub_txg, spa->spa_scrub_started,
	(u_longlong_t)vdev_dtl_min(vd),
	(u_longlong_t)vdev_dtl_max(vd),
	(u_longlong_t)(scn ? scn->scn_phys.scn_errors : 0));
	}

	/*
	* If we've completed a scrub/resilver or a rebuild cleanly
	* then determine if this vdev should remove any DTLs. We
	* only want to excise regions on vdevs that were available
	* during the entire duration of this scan.
	*/
	if (rebuild_done &&
	vr != NULL && vr->vr_rebuild_phys.vrp_errors == 0) {
	check_excise = B_TRUE;
	} else {
	if (spa->spa_scrub_started \|\|
	(scn != NULL && scn->scn_phys.scn_errors == 0)) {
	check_excise = B_TRUE;
	}
	}

	if (scrub_txg && check_excise &&
	vdev_dtl_should_excise(vd, rebuild_done)) {
	/*
	* We completed a scrub, resilver or rebuild up to
	* scrub_txg. If we did it without rebooting, then
	* the scrub dtl will be valid, so excise the old
	* region and fold in the scrub dtl. Otherwise,
	* leave the dtl as-is if there was an error.
	*
	* There's little trick here: to excise the beginning
	* of the DTL_MISSING map, we put it into a reference
	* tree and then add a segment with refcnt -1 that
	* covers the range [0, scrub_txg). This means
	* that each txg in that range has refcnt -1 or 0.
	* We then add DTL_SCRUB with a refcnt of 2, so that
	* entries in the range [0, scrub_txg) will have a
	* positive refcnt -- either 1 or 2. We then convert
	* the reference tree into the new DTL_MISSING map.
	*/
	space_reftree_create(&reftree);
	space_reftree_add_map(&reftree,
	vd->vdev_dtl[DTL_MISSING], 1);
	space_reftree_add_seg(&reftree, 0, scrub_txg, -1);
	space_reftree_add_map(&reftree,
	vd->vdev_dtl[DTL_SCRUB], 2);
	space_reftree_generate_map(&reftree,
	vd->vdev_dtl[DTL_MISSING], 1);
	space_reftree_destroy(&reftree);

	if (!range_tree_is_empty(vd->vdev_dtl[DTL_MISSING])) {
	zfs_dbgmsg("update DTL_MISSING:%llu/%llu",
	(u_longlong_t)vdev_dtl_min(vd),
	(u_longlong_t)vdev_dtl_max(vd));
	} else if (!wasempty) {
	zfs_dbgmsg("DTL_MISSING is now empty");
	}
	}
	range_tree_vacate(vd->vdev_dtl[DTL_PARTIAL], NULL, NULL);
	range_tree_walk(vd->vdev_dtl[DTL_MISSING],
	range_tree_add, vd->vdev_dtl[DTL_PARTIAL]);
	if (scrub_done)
	range_tree_vacate(vd->vdev_dtl[DTL_SCRUB], NULL, NULL);
	range_tree_vacate(vd->vdev_dtl[DTL_OUTAGE], NULL, NULL);
	if (!vdev_readable(vd))
	range_tree_add(vd->vdev_dtl[DTL_OUTAGE], 0, -1ULL);
	else
	range_tree_walk(vd->vdev_dtl[DTL_MISSING],
	range_tree_add, vd->vdev_dtl[DTL_OUTAGE]);

	/*
	* If the vdev was resilvering or rebuilding and no longer
	* has any DTLs then reset the appropriate flag and dirty
	* the top level so that we persist the change.
	*/
	if (txg != 0 &&
	range_tree_is_empty(vd->vdev_dtl[DTL_MISSING]) &&
	range_tree_is_empty(vd->vdev_dtl[DTL_OUTAGE])) {
	if (vd->vdev_rebuild_txg != 0) {
	vd->vdev_rebuild_txg = 0;
	vdev_config_dirty(vd->vdev_top);
	} else if (vd->vdev_resilver_txg != 0) {
	vd->vdev_resilver_txg = 0;
	vdev_config_dirty(vd->vdev_top);
	}
	}

	mutex_exit(&vd->vdev_dtl_lock);

	if (txg != 0)
	vdev_dirty(vd->vdev_top, VDD_DTL, vd, txg);
	return;
	}

	mutex_enter(&vd->vdev_dtl_lock);
	for (int t = 0; t < DTL_TYPES; t++) {
	/* account for child's outage in parent's missing map */
	int s = (t == DTL_MISSING) ? DTL_OUTAGE: t;
	if (t == DTL_SCRUB)
	continue; /* leaf vdevs only */
	if (t == DTL_PARTIAL)
	minref = 1; /* i.e. non-zero */
	else if (vdev_get_nparity(vd) != 0)
	minref = vdev_get_nparity(vd) + 1; /* RAID-Z, dRAID */
	else
	minref = vd->vdev_children; /* any kind of mirror */
	space_reftree_create(&reftree);
	for (int c = 0; c < vd->vdev_children; c++) {
	vdev_t *cvd = vd->vdev_child[c];
	mutex_enter(&cvd->vdev_dtl_lock);
	space_reftree_add_map(&reftree, cvd->vdev_dtl[s], 1);
	mutex_exit(&cvd->vdev_dtl_lock);
	}
	space_reftree_generate_map(&reftree, vd->vdev_dtl[t], minref);
	space_reftree_destroy(&reftree);
	}
	mutex_exit(&vd->vdev_dtl_lock);
	}

	/*
	* Iterate over all the vdevs except spare, and post kobj events
	*/
	void
	vdev_post_kobj_evt(vdev_t *vd)
	{
	if (vd->vdev_ops->vdev_op_kobj_evt_post &&
	vd->vdev_kobj_flag == B_FALSE) {
	vd->vdev_kobj_flag = B_TRUE;
	vd->vdev_ops->vdev_op_kobj_evt_post(vd);
	}

	for (int c = 0; c < vd->vdev_children; c++)
	vdev_post_kobj_evt(vd->vdev_child[c]);
	}

	/*
	* Iterate over all the vdevs except spare, and clear kobj events
	*/
	void
	vdev_clear_kobj_evt(vdev_t *vd)
	{
	vd->vdev_kobj_flag = B_FALSE;

	for (int c = 0; c < vd->vdev_children; c++)
	vdev_clear_kobj_evt(vd->vdev_child[c]);
	}

	int
	vdev_dtl_load(vdev_t *vd)
	{
	spa_t *spa = vd->vdev_spa;
	objset_t *mos = spa->spa_meta_objset;
	range_tree_t *rt;
	int error = 0;

	if (vd->vdev_ops->vdev_op_leaf && vd->vdev_dtl_object != 0) {
	ASSERT(vdev_is_concrete(vd));

	/*
	* If the dtl cannot be sync'd there is no need to open it.
	*/
	if (spa->spa_mode == SPA_MODE_READ && !spa->spa_read_spacemaps)
	return (0);

	error = space_map_open(&vd->vdev_dtl_sm, mos,
	vd->vdev_dtl_object, 0, -1ULL, 0);
	if (error)
	return (error);
	ASSERT(vd->vdev_dtl_sm != NULL);

	rt = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0);
	error = space_map_load(vd->vdev_dtl_sm, rt, SM_ALLOC);
	if (error == 0) {
	mutex_enter(&vd->vdev_dtl_lock);
	range_tree_walk(rt, range_tree_add,
	vd->vdev_dtl[DTL_MISSING]);
	mutex_exit(&vd->vdev_dtl_lock);
	}

	range_tree_vacate(rt, NULL, NULL);
	range_tree_destroy(rt);

	return (error);
	}

	for (int c = 0; c < vd->vdev_children; c++) {
	error = vdev_dtl_load(vd->vdev_child[c]);
	if (error != 0)
	break;
	}

	return (error);
	}

	static void
	vdev_zap_allocation_data(vdev_t vd, dmu_tx_t tx)
	{
	spa_t *spa = vd->vdev_spa;
	objset_t *mos = spa->spa_meta_objset;
	vdev_alloc_bias_t alloc_bias = vd->vdev_alloc_bias;
	const char *string;

	ASSERT(alloc_bias != VDEV_BIAS_NONE);

	string =
	(alloc_bias == VDEV_BIAS_LOG) ? VDEV_ALLOC_BIAS_LOG :
	(alloc_bias == VDEV_BIAS_SPECIAL) ? VDEV_ALLOC_BIAS_SPECIAL :
	(alloc_bias == VDEV_BIAS_DEDUP) ? VDEV_ALLOC_BIAS_DEDUP : NULL;

	ASSERT(string != NULL);
	VERIFY0(zap_add(mos, vd->vdev_top_zap, VDEV_TOP_ZAP_ALLOCATION_BIAS,
	1, strlen(string) + 1, string, tx));

	if (alloc_bias == VDEV_BIAS_SPECIAL \|\| alloc_bias == VDEV_BIAS_DEDUP) {
	spa_activate_allocation_classes(spa, tx);
	}
	}

	void
	vdev_destroy_unlink_zap(vdev_t vd, uint64_t zapobj, dmu_tx_t tx)
	{
	spa_t *spa = vd->vdev_spa;

	VERIFY0(zap_destroy(spa->spa_meta_objset, zapobj, tx));
	VERIFY0(zap_remove_int(spa->spa_meta_objset, spa->spa_all_vdev_zaps,
	zapobj, tx));
	}

	uint64_t
	vdev_create_link_zap(vdev_t vd, dmu_tx_t tx)
	{
	spa_t *spa = vd->vdev_spa;
	uint64_t zap = zap_create(spa->spa_meta_objset, DMU_OTN_ZAP_METADATA,
	DMU_OT_NONE, 0, tx);

	ASSERT(zap != 0);
	VERIFY0(zap_add_int(spa->spa_meta_objset, spa->spa_all_vdev_zaps,
	zap, tx));

	return (zap);
	}

	void
	vdev_construct_zaps(vdev_t vd, dmu_tx_t tx)
	{
	if (vd->vdev_ops != &vdev_hole_ops &&
	vd->vdev_ops != &vdev_missing_ops &&
	vd->vdev_ops != &vdev_root_ops &&
	!vd->vdev_top->vdev_removing) {
	if (vd->vdev_ops->vdev_op_leaf && vd->vdev_leaf_zap == 0) {
	vd->vdev_leaf_zap = vdev_create_link_zap(vd, tx);
	}
	if (vd == vd->vdev_top && vd->vdev_top_zap == 0) {
	vd->vdev_top_zap = vdev_create_link_zap(vd, tx);
	if (vd->vdev_alloc_bias != VDEV_BIAS_NONE)
	vdev_zap_allocation_data(vd, tx);
	}
	}
	if (vd->vdev_ops == &vdev_root_ops && vd->vdev_root_zap == 0 &&
	spa_feature_is_enabled(vd->vdev_spa, SPA_FEATURE_AVZ_V2)) {
	if (!spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_AVZ_V2))
	spa_feature_incr(vd->vdev_spa, SPA_FEATURE_AVZ_V2, tx);
	vd->vdev_root_zap = vdev_create_link_zap(vd, tx);
	}

	for (uint64_t i = 0; i < vd->vdev_children; i++) {
	vdev_construct_zaps(vd->vdev_child[i], tx);
	}
	}

	static void
	vdev_dtl_sync(vdev_t *vd, uint64_t txg)
	{
	spa_t *spa = vd->vdev_spa;
	range_tree_t *rt = vd->vdev_dtl[DTL_MISSING];
	objset_t *mos = spa->spa_meta_objset;
	range_tree_t *rtsync;
	dmu_tx_t *tx;
	uint64_t object = space_map_object(vd->vdev_dtl_sm);

	ASSERT(vdev_is_concrete(vd));
	ASSERT(vd->vdev_ops->vdev_op_leaf);

	tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);

	if (vd->vdev_detached \|\| vd->vdev_top->vdev_removing) {
	mutex_enter(&vd->vdev_dtl_lock);
	space_map_free(vd->vdev_dtl_sm, tx);
	space_map_close(vd->vdev_dtl_sm);
	vd->vdev_dtl_sm = NULL;
	mutex_exit(&vd->vdev_dtl_lock);

	/*
	* We only destroy the leaf ZAP for detached leaves or for
	* removed log devices. Removed data devices handle leaf ZAP
	* cleanup later, once cancellation is no longer possible.
	*/
	if (vd->vdev_leaf_zap != 0 && (vd->vdev_detached \|\|
	vd->vdev_top->vdev_islog)) {
	vdev_destroy_unlink_zap(vd, vd->vdev_leaf_zap, tx);
	vd->vdev_leaf_zap = 0;
	}

	dmu_tx_commit(tx);
	return;
	}

	if (vd->vdev_dtl_sm == NULL) {
	uint64_t new_object;

	new_object = space_map_alloc(mos, zfs_vdev_dtl_sm_blksz, tx);
	VERIFY3U(new_object, !=, 0);

	VERIFY0(space_map_open(&vd->vdev_dtl_sm, mos, new_object,
	0, -1ULL, 0));
	ASSERT(vd->vdev_dtl_sm != NULL);
	}

	rtsync = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0);

	mutex_enter(&vd->vdev_dtl_lock);
	range_tree_walk(rt, range_tree_add, rtsync);
	mutex_exit(&vd->vdev_dtl_lock);

	space_map_truncate(vd->vdev_dtl_sm, zfs_vdev_dtl_sm_blksz, tx);
	space_map_write(vd->vdev_dtl_sm, rtsync, SM_ALLOC, SM_NO_VDEVID, tx);
	range_tree_vacate(rtsync, NULL, NULL);

	range_tree_destroy(rtsync);

	/*
	* If the object for the space map has changed then dirty
	* the top level so that we update the config.
	*/
	if (object != space_map_object(vd->vdev_dtl_sm)) {
	vdev_dbgmsg(vd, "txg %llu, spa %s, DTL old object %llu, "
	"new object %llu", (u_longlong_t)txg, spa_name(spa),
	(u_longlong_t)object,
	(u_longlong_t)space_map_object(vd->vdev_dtl_sm));
	vdev_config_dirty(vd->vdev_top);
	}

	dmu_tx_commit(tx);
	}

	/*
	* Determine whether the specified vdev can be offlined/detached/removed
	* without losing data.
	*/
	boolean_t
	vdev_dtl_required(vdev_t *vd)
	{
	spa_t *spa = vd->vdev_spa;
	vdev_t *tvd = vd->vdev_top;
	uint8_t cant_read = vd->vdev_cant_read;
	boolean_t required;

	ASSERT(spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);

	if (vd == spa->spa_root_vdev \|\| vd == tvd)
	return (B_TRUE);

	/*
	* Temporarily mark the device as unreadable, and then determine
	* whether this results in any DTL outages in the top-level vdev.
	* If not, we can safely offline/detach/remove the device.
	*/
	vd->vdev_cant_read = B_TRUE;
	vdev_dtl_reassess(tvd, 0, 0, B_FALSE, B_FALSE);
	required = !vdev_dtl_empty(tvd, DTL_OUTAGE);
	vd->vdev_cant_read = cant_read;
	vdev_dtl_reassess(tvd, 0, 0, B_FALSE, B_FALSE);

	if (!required && zio_injection_enabled) {
	required = !!zio_handle_device_injection(vd, NULL,
	SET_ERROR(ECHILD));
	}

	return (required);
	}

	/*
	* Determine if resilver is needed, and if so the txg range.
	*/
	boolean_t
	vdev_resilver_needed(vdev_t vd, uint64_t minp, uint64_t *maxp)
	{
	boolean_t needed = B_FALSE;
	uint64_t thismin = UINT64_MAX;
	uint64_t thismax = 0;

	if (vd->vdev_children == 0) {
	mutex_enter(&vd->vdev_dtl_lock);
	if (!range_tree_is_empty(vd->vdev_dtl[DTL_MISSING]) &&
	vdev_writeable(vd)) {

	thismin = vdev_dtl_min(vd);
	thismax = vdev_dtl_max(vd);
	needed = B_TRUE;
	}
	mutex_exit(&vd->vdev_dtl_lock);
	} else {
	for (int c = 0; c < vd->vdev_children; c++) {
	vdev_t *cvd = vd->vdev_child[c];
	uint64_t cmin, cmax;

	if (vdev_resilver_needed(cvd, &cmin, &cmax)) {
	thismin = MIN(thismin, cmin);
	thismax = MAX(thismax, cmax);
	needed = B_TRUE;
	}
	}
	}

	if (needed && minp) {
	*minp = thismin;
	*maxp = thismax;
	}
	return (needed);
	}

	/*
	* Gets the checkpoint space map object from the vdev's ZAP. On success sm_obj
	* will contain either the checkpoint spacemap object or zero if none exists.
	* All other errors are returned to the caller.
	*/
	int
	vdev_checkpoint_sm_object(vdev_t vd, uint64_t sm_obj)
	{
	ASSERT0(spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));

	if (vd->vdev_top_zap == 0) {
	*sm_obj = 0;
	return (0);
	}

	int error = zap_lookup(spa_meta_objset(vd->vdev_spa), vd->vdev_top_zap,
	VDEV_TOP_ZAP_POOL_CHECKPOINT_SM, sizeof (uint64_t), 1, sm_obj);
	if (error == ENOENT) {
	*sm_obj = 0;
	error = 0;
	}

	return (error);
	}

	int
	vdev_load(vdev_t *vd)
	{
	int children = vd->vdev_children;
	int error = 0;
	taskq_t *tq = NULL;

	/*
	* It's only worthwhile to use the taskq for the root vdev, because the
	* slow part is metaslab_init, and that only happens for top-level
	* vdevs.
	*/
	if (vd->vdev_ops == &vdev_root_ops && vd->vdev_children > 0) {
	tq = taskq_create("vdev_load", children, minclsyspri,
	children, children, TASKQ_PREPOPULATE);
	}

	/*
	* Recursively load all children.
	*/
	for (int c = 0; c < vd->vdev_children; c++) {
	vdev_t *cvd = vd->vdev_child[c];

	if (tq == NULL \|\| vdev_uses_zvols(cvd)) {
	cvd->vdev_load_error = vdev_load(cvd);
	} else {
	VERIFY(taskq_dispatch(tq, vdev_load_child,
	cvd, TQ_SLEEP) != TASKQID_INVALID);
	}
	}

	if (tq != NULL) {
	taskq_wait(tq);
	taskq_destroy(tq);
	}

	for (int c = 0; c < vd->vdev_children; c++) {
	int error = vd->vdev_child[c]->vdev_load_error;

	if (error != 0)
	return (error);
	}

	vdev_set_deflate_ratio(vd);

	/*
	* On spa_load path, grab the allocation bias from our zap
	*/
	if (vd == vd->vdev_top && vd->vdev_top_zap != 0) {
	spa_t *spa = vd->vdev_spa;
	char bias_str[64];

	error = zap_lookup(spa->spa_meta_objset, vd->vdev_top_zap,
	VDEV_TOP_ZAP_ALLOCATION_BIAS, 1, sizeof (bias_str),
	bias_str);
	if (error == 0) {
	ASSERT(vd->vdev_alloc_bias == VDEV_BIAS_NONE);
	vd->vdev_alloc_bias = vdev_derive_alloc_bias(bias_str);
	} else if (error != ENOENT) {
	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_CORRUPT_DATA);
	vdev_dbgmsg(vd, "vdev_load: zap_lookup(top_zap=%llu) "
	"failed [error=%d]",
	(u_longlong_t)vd->vdev_top_zap, error);
	return (error);
	}
	}

	if (vd == vd->vdev_top && vd->vdev_top_zap != 0) {
	spa_t *spa = vd->vdev_spa;
	uint64_t failfast;

	error = zap_lookup(spa->spa_meta_objset, vd->vdev_top_zap,
	vdev_prop_to_name(VDEV_PROP_FAILFAST), sizeof (failfast),
	1, &failfast);
	if (error == 0) {
	vd->vdev_failfast = failfast & 1;
	} else if (error == ENOENT) {
	vd->vdev_failfast = vdev_prop_default_numeric(
	VDEV_PROP_FAILFAST);
	} else {
	vdev_dbgmsg(vd,
	"vdev_load: zap_lookup(top_zap=%llu) "
	"failed [error=%d]",
	(u_longlong_t)vd->vdev_top_zap, error);
	}
	}

	/*
	* Load any rebuild state from the top-level vdev zap.
	*/
	if (vd == vd->vdev_top && vd->vdev_top_zap != 0) {
	error = vdev_rebuild_load(vd);
	if (error && error != ENOTSUP) {
	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_CORRUPT_DATA);
	vdev_dbgmsg(vd, "vdev_load: vdev_rebuild_load "
	"failed [error=%d]", error);
	return (error);
	}
	}

	if (vd->vdev_top_zap != 0 \|\| vd->vdev_leaf_zap != 0) {
	uint64_t zapobj;

	if (vd->vdev_top_zap != 0)
	zapobj = vd->vdev_top_zap;
	else
	zapobj = vd->vdev_leaf_zap;

	error = vdev_prop_get_int(vd, VDEV_PROP_CHECKSUM_N,
	&vd->vdev_checksum_n);
	if (error && error != ENOENT)
	vdev_dbgmsg(vd, "vdev_load: zap_lookup(zap=%llu) "
	"failed [error=%d]", (u_longlong_t)zapobj, error);

	error = vdev_prop_get_int(vd, VDEV_PROP_CHECKSUM_T,
	&vd->vdev_checksum_t);
	if (error && error != ENOENT)
	vdev_dbgmsg(vd, "vdev_load: zap_lookup(zap=%llu) "
	"failed [error=%d]", (u_longlong_t)zapobj, error);

	error = vdev_prop_get_int(vd, VDEV_PROP_IO_N,
	&vd->vdev_io_n);
	if (error && error != ENOENT)
	vdev_dbgmsg(vd, "vdev_load: zap_lookup(zap=%llu) "
	"failed [error=%d]", (u_longlong_t)zapobj, error);

	error = vdev_prop_get_int(vd, VDEV_PROP_IO_T,
	&vd->vdev_io_t);
	if (error && error != ENOENT)
	vdev_dbgmsg(vd, "vdev_load: zap_lookup(zap=%llu) "
	"failed [error=%d]", (u_longlong_t)zapobj, error);
	+
	+ error = vdev_prop_get_int(vd, VDEV_PROP_SLOW_IO_N,
	+ &vd->vdev_slow_io_n);
	+ if (error && error != ENOENT)
	+ vdev_dbgmsg(vd, "vdev_load: zap_lookup(zap=%llu) "
	+ "failed [error=%d]", (u_longlong_t)zapobj, error);
	+
	+ error = vdev_prop_get_int(vd, VDEV_PROP_SLOW_IO_T,
	+ &vd->vdev_slow_io_t);
	+ if (error && error != ENOENT)
	+ vdev_dbgmsg(vd, "vdev_load: zap_lookup(zap=%llu) "
	+ "failed [error=%d]", (u_longlong_t)zapobj, error);
	}

	/*
	* If this is a top-level vdev, initialize its metaslabs.
	*/
	if (vd == vd->vdev_top && vdev_is_concrete(vd)) {
	vdev_metaslab_group_create(vd);

	if (vd->vdev_ashift == 0 \|\| vd->vdev_asize == 0) {
	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_CORRUPT_DATA);
	vdev_dbgmsg(vd, "vdev_load: invalid size. ashift=%llu, "
	"asize=%llu", (u_longlong_t)vd->vdev_ashift,
	(u_longlong_t)vd->vdev_asize);
	return (SET_ERROR(ENXIO));
	}

	error = vdev_metaslab_init(vd, 0);
	if (error != 0) {
	vdev_dbgmsg(vd, "vdev_load: metaslab_init failed "
	"[error=%d]", error);
	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_CORRUPT_DATA);
	return (error);
	}

	uint64_t checkpoint_sm_obj;
	error = vdev_checkpoint_sm_object(vd, &checkpoint_sm_obj);
	if (error == 0 && checkpoint_sm_obj != 0) {
	objset_t *mos = spa_meta_objset(vd->vdev_spa);
	ASSERT(vd->vdev_asize != 0);
	ASSERT3P(vd->vdev_checkpoint_sm, ==, NULL);

	error = space_map_open(&vd->vdev_checkpoint_sm,
	mos, checkpoint_sm_obj, 0, vd->vdev_asize,
	vd->vdev_ashift);
	if (error != 0) {
	vdev_dbgmsg(vd, "vdev_load: space_map_open "
	"failed for checkpoint spacemap (obj %llu) "
	"[error=%d]",
	(u_longlong_t)checkpoint_sm_obj, error);
	return (error);
	}
	ASSERT3P(vd->vdev_checkpoint_sm, !=, NULL);

	/*
	* Since the checkpoint_sm contains free entries
	* exclusively we can use space_map_allocated() to
	* indicate the cumulative checkpointed space that
	* has been freed.
	*/
	vd->vdev_stat.vs_checkpoint_space =
	-space_map_allocated(vd->vdev_checkpoint_sm);
	vd->vdev_spa->spa_checkpoint_info.sci_dspace +=
	vd->vdev_stat.vs_checkpoint_space;
	} else if (error != 0) {
	vdev_dbgmsg(vd, "vdev_load: failed to retrieve "
	"checkpoint space map object from vdev ZAP "
	"[error=%d]", error);
	return (error);
	}
	}

	/*
	* If this is a leaf vdev, load its DTL.
	*/
	if (vd->vdev_ops->vdev_op_leaf && (error = vdev_dtl_load(vd)) != 0) {
	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_CORRUPT_DATA);
	vdev_dbgmsg(vd, "vdev_load: vdev_dtl_load failed "
	"[error=%d]", error);
	return (error);
	}

	uint64_t obsolete_sm_object;
	error = vdev_obsolete_sm_object(vd, &obsolete_sm_object);
	if (error == 0 && obsolete_sm_object != 0) {
	objset_t *mos = vd->vdev_spa->spa_meta_objset;
	ASSERT(vd->vdev_asize != 0);
	ASSERT3P(vd->vdev_obsolete_sm, ==, NULL);

	if ((error = space_map_open(&vd->vdev_obsolete_sm, mos,
	obsolete_sm_object, 0, vd->vdev_asize, 0))) {
	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_CORRUPT_DATA);
	vdev_dbgmsg(vd, "vdev_load: space_map_open failed for "
	"obsolete spacemap (obj %llu) [error=%d]",
	(u_longlong_t)obsolete_sm_object, error);
	return (error);
	}
	} else if (error != 0) {
	vdev_dbgmsg(vd, "vdev_load: failed to retrieve obsolete "
	"space map object from vdev ZAP [error=%d]", error);
	return (error);
	}

	return (0);
	}

	/*
	* The special vdev case is used for hot spares and l2cache devices. Its
	* sole purpose it to set the vdev state for the associated vdev. To do this,
	* we make sure that we can open the underlying device, then try to read the
	* label, and make sure that the label is sane and that it hasn't been
	* repurposed to another pool.
	*/
	int
	vdev_validate_aux(vdev_t *vd)
	{
	nvlist_t *label;
	uint64_t guid, version;
	uint64_t state;

	if (!vdev_readable(vd))
	return (0);

	if ((label = vdev_label_read_config(vd, -1ULL)) == NULL) {
	vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_CORRUPT_DATA);
	return (-1);
	}

	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_VERSION, &version) != 0 \|\|
	!SPA_VERSION_IS_SUPPORTED(version) \|\|
	nvlist_lookup_uint64(label, ZPOOL_CONFIG_GUID, &guid) != 0 \|\|
	guid != vd->vdev_guid \|\|
	nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_STATE, &state) != 0) {
	vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_CORRUPT_DATA);
	nvlist_free(label);
	return (-1);
	}

	/*
	* We don't actually check the pool state here. If it's in fact in
	* use by another pool, we update this fact on the fly when requested.
	*/
	nvlist_free(label);
	return (0);
	}

	static void
	vdev_destroy_ms_flush_data(vdev_t vd, dmu_tx_t tx)
	{
	objset_t *mos = spa_meta_objset(vd->vdev_spa);

	if (vd->vdev_top_zap == 0)
	return;

	uint64_t object = 0;
	int err = zap_lookup(mos, vd->vdev_top_zap,
	VDEV_TOP_ZAP_MS_UNFLUSHED_PHYS_TXGS, sizeof (uint64_t), 1, &object);
	if (err == ENOENT)
	return;
	VERIFY0(err);

	VERIFY0(dmu_object_free(mos, object, tx));
	VERIFY0(zap_remove(mos, vd->vdev_top_zap,
	VDEV_TOP_ZAP_MS_UNFLUSHED_PHYS_TXGS, tx));
	}

	/*
	* Free the objects used to store this vdev's spacemaps, and the array
	* that points to them.
	*/
	void
	vdev_destroy_spacemaps(vdev_t vd, dmu_tx_t tx)
	{
	if (vd->vdev_ms_array == 0)
	return;

	objset_t *mos = vd->vdev_spa->spa_meta_objset;
	uint64_t array_count = vd->vdev_asize >> vd->vdev_ms_shift;
	size_t array_bytes = array_count * sizeof (uint64_t);
	uint64_t *smobj_array = kmem_alloc(array_bytes, KM_SLEEP);
	VERIFY0(dmu_read(mos, vd->vdev_ms_array, 0,
	array_bytes, smobj_array, 0));

	for (uint64_t i = 0; i < array_count; i++) {
	uint64_t smobj = smobj_array[i];
	if (smobj == 0)
	continue;

	space_map_free_obj(mos, smobj, tx);
	}

	kmem_free(smobj_array, array_bytes);
	VERIFY0(dmu_object_free(mos, vd->vdev_ms_array, tx));
	vdev_destroy_ms_flush_data(vd, tx);
	vd->vdev_ms_array = 0;
	}

	static void
	vdev_remove_empty_log(vdev_t *vd, uint64_t txg)
	{
	spa_t *spa = vd->vdev_spa;

	ASSERT(vd->vdev_islog);
	ASSERT(vd == vd->vdev_top);
	ASSERT3U(txg, ==, spa_syncing_txg(spa));

	dmu_tx_t *tx = dmu_tx_create_assigned(spa_get_dsl(spa), txg);

	vdev_destroy_spacemaps(vd, tx);
	if (vd->vdev_top_zap != 0) {
	vdev_destroy_unlink_zap(vd, vd->vdev_top_zap, tx);
	vd->vdev_top_zap = 0;
	}

	dmu_tx_commit(tx);
	}

	void
	vdev_sync_done(vdev_t *vd, uint64_t txg)
	{
	metaslab_t *msp;
	boolean_t reassess = !txg_list_empty(&vd->vdev_ms_list, TXG_CLEAN(txg));

	ASSERT(vdev_is_concrete(vd));

	while ((msp = txg_list_remove(&vd->vdev_ms_list, TXG_CLEAN(txg)))
	!= NULL)
	metaslab_sync_done(msp, txg);

	if (reassess) {
	metaslab_sync_reassess(vd->vdev_mg);
	if (vd->vdev_log_mg != NULL)
	metaslab_sync_reassess(vd->vdev_log_mg);
	}
	}

	void
	vdev_sync(vdev_t *vd, uint64_t txg)
	{
	spa_t *spa = vd->vdev_spa;
	vdev_t *lvd;
	metaslab_t *msp;

	ASSERT3U(txg, ==, spa->spa_syncing_txg);
	dmu_tx_t *tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
	if (range_tree_space(vd->vdev_obsolete_segments) > 0) {
	ASSERT(vd->vdev_removing \|\|
	vd->vdev_ops == &vdev_indirect_ops);

	vdev_indirect_sync_obsolete(vd, tx);

	/*
	* If the vdev is indirect, it can't have dirty
	* metaslabs or DTLs.
	*/
	if (vd->vdev_ops == &vdev_indirect_ops) {
	ASSERT(txg_list_empty(&vd->vdev_ms_list, txg));
	ASSERT(txg_list_empty(&vd->vdev_dtl_list, txg));
	dmu_tx_commit(tx);
	return;
	}
	}

	ASSERT(vdev_is_concrete(vd));

	if (vd->vdev_ms_array == 0 && vd->vdev_ms_shift != 0 &&
	!vd->vdev_removing) {
	ASSERT(vd == vd->vdev_top);
	ASSERT0(vd->vdev_indirect_config.vic_mapping_object);
	vd->vdev_ms_array = dmu_object_alloc(spa->spa_meta_objset,
	DMU_OT_OBJECT_ARRAY, 0, DMU_OT_NONE, 0, tx);
	ASSERT(vd->vdev_ms_array != 0);
	vdev_config_dirty(vd);
	}

	while ((msp = txg_list_remove(&vd->vdev_ms_list, txg)) != NULL) {
	metaslab_sync(msp, txg);
	(void) txg_list_add(&vd->vdev_ms_list, msp, TXG_CLEAN(txg));
	}

	while ((lvd = txg_list_remove(&vd->vdev_dtl_list, txg)) != NULL)
	vdev_dtl_sync(lvd, txg);

	/*
	* If this is an empty log device being removed, destroy the
	* metadata associated with it.
	*/
	if (vd->vdev_islog && vd->vdev_stat.vs_alloc == 0 && vd->vdev_removing)
	vdev_remove_empty_log(vd, txg);

	(void) txg_list_add(&spa->spa_vdev_txg_list, vd, TXG_CLEAN(txg));
	dmu_tx_commit(tx);
	}

	uint64_t
	vdev_psize_to_asize(vdev_t *vd, uint64_t psize)
	{
	return (vd->vdev_ops->vdev_op_asize(vd, psize));
	}

	/*
	* Mark the given vdev faulted. A faulted vdev behaves as if the device could
	* not be opened, and no I/O is attempted.
	*/
	int
	vdev_fault(spa_t *spa, uint64_t guid, vdev_aux_t aux)
	{
	vdev_t vd, tvd;

	spa_vdev_state_enter(spa, SCL_NONE);

	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
	return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENODEV)));

	if (!vd->vdev_ops->vdev_op_leaf)
	return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENOTSUP)));

	tvd = vd->vdev_top;

	/*
	* If user did a 'zpool offline -f' then make the fault persist across
	* reboots.
	*/
	if (aux == VDEV_AUX_EXTERNAL_PERSIST) {
	/*
	* There are two kinds of forced faults: temporary and
	* persistent. Temporary faults go away at pool import, while
	* persistent faults stay set. Both types of faults can be
	* cleared with a zpool clear.
	*
	* We tell if a vdev is persistently faulted by looking at the
	* ZPOOL_CONFIG_AUX_STATE nvpair. If it's set to "external" at
	* import then it's a persistent fault. Otherwise, it's
	* temporary. We get ZPOOL_CONFIG_AUX_STATE set to "external"
	* by setting vd.vdev_stat.vs_aux to VDEV_AUX_EXTERNAL. This
	* tells vdev_config_generate() (which gets run later) to set
	* ZPOOL_CONFIG_AUX_STATE to "external" in the nvlist.
	*/
	vd->vdev_stat.vs_aux = VDEV_AUX_EXTERNAL;
	vd->vdev_tmpoffline = B_FALSE;
	aux = VDEV_AUX_EXTERNAL;
	} else {
	vd->vdev_tmpoffline = B_TRUE;
	}

	/*
	* We don't directly use the aux state here, but if we do a
	* vdev_reopen(), we need this value to be present to remember why we
	* were faulted.
	*/
	vd->vdev_label_aux = aux;

	/*
	* Faulted state takes precedence over degraded.
	*/
	vd->vdev_delayed_close = B_FALSE;
	vd->vdev_faulted = 1ULL;
	vd->vdev_degraded = 0ULL;
	vdev_set_state(vd, B_FALSE, VDEV_STATE_FAULTED, aux);

	/*
	* If this device has the only valid copy of the data, then
	* back off and simply mark the vdev as degraded instead.
	*/
	if (!tvd->vdev_islog && vd->vdev_aux == NULL && vdev_dtl_required(vd)) {
	vd->vdev_degraded = 1ULL;
	vd->vdev_faulted = 0ULL;

	/*
	* If we reopen the device and it's not dead, only then do we
	* mark it degraded.
	*/
	vdev_reopen(tvd);

	if (vdev_readable(vd))
	vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED, aux);
	}

	return (spa_vdev_state_exit(spa, vd, 0));
	}

	/*
	* Mark the given vdev degraded. A degraded vdev is purely an indication to the
	* user that something is wrong. The vdev continues to operate as normal as far
	* as I/O is concerned.
	*/
	int
	vdev_degrade(spa_t *spa, uint64_t guid, vdev_aux_t aux)
	{
	vdev_t *vd;

	spa_vdev_state_enter(spa, SCL_NONE);

	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
	return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENODEV)));

	if (!vd->vdev_ops->vdev_op_leaf)
	return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENOTSUP)));

	/*
	* If the vdev is already faulted, then don't do anything.
	*/
	if (vd->vdev_faulted \|\| vd->vdev_degraded)
	return (spa_vdev_state_exit(spa, NULL, 0));

	vd->vdev_degraded = 1ULL;
	if (!vdev_is_dead(vd))
	vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED,
	aux);

	return (spa_vdev_state_exit(spa, vd, 0));
	}

	int
	vdev_remove_wanted(spa_t *spa, uint64_t guid)
	{
	vdev_t *vd;

	spa_vdev_state_enter(spa, SCL_NONE);

	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
	return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENODEV)));

	/*
	* If the vdev is already removed, or expanding which can trigger
	* repartition add/remove events, then don't do anything.
	*/
	if (vd->vdev_removed \|\| vd->vdev_expanding)
	return (spa_vdev_state_exit(spa, NULL, 0));

	/*
	* Confirm the vdev has been removed, otherwise don't do anything.
	*/
	if (vd->vdev_ops->vdev_op_leaf && !zio_wait(vdev_probe(vd, NULL)))
	return (spa_vdev_state_exit(spa, NULL, SET_ERROR(EEXIST)));

	vd->vdev_remove_wanted = B_TRUE;
	spa_async_request(spa, SPA_ASYNC_REMOVE);

	return (spa_vdev_state_exit(spa, vd, 0));
	}


	/*
	* Online the given vdev.
	*
	* If 'ZFS_ONLINE_UNSPARE' is set, it implies two things. First, any attached
	* spare device should be detached when the device finishes resilvering.
	* Second, the online should be treated like a 'test' online case, so no FMA
	* events are generated if the device fails to open.
	*/
	int
	vdev_online(spa_t spa, uint64_t guid, uint64_t flags, vdev_state_t newstate)
	{
	vdev_t vd, tvd, pvd, rvd = spa->spa_root_vdev;
	boolean_t wasoffline;
	vdev_state_t oldstate;

	spa_vdev_state_enter(spa, SCL_NONE);

	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
	return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENODEV)));

	if (!vd->vdev_ops->vdev_op_leaf)
	return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENOTSUP)));

	wasoffline = (vd->vdev_offline \|\| vd->vdev_tmpoffline);
	oldstate = vd->vdev_state;

	tvd = vd->vdev_top;
	vd->vdev_offline = B_FALSE;
	vd->vdev_tmpoffline = B_FALSE;
	vd->vdev_checkremove = !!(flags & ZFS_ONLINE_CHECKREMOVE);
	vd->vdev_forcefault = !!(flags & ZFS_ONLINE_FORCEFAULT);

	/* XXX - L2ARC 1.0 does not support expansion */
	if (!vd->vdev_aux) {
	for (pvd = vd; pvd != rvd; pvd = pvd->vdev_parent)
	pvd->vdev_expanding = !!((flags & ZFS_ONLINE_EXPAND) \|\|
	spa->spa_autoexpand);
	vd->vdev_expansion_time = gethrestime_sec();
	}

	vdev_reopen(tvd);
	vd->vdev_checkremove = vd->vdev_forcefault = B_FALSE;

	if (!vd->vdev_aux) {
	for (pvd = vd; pvd != rvd; pvd = pvd->vdev_parent)
	pvd->vdev_expanding = B_FALSE;
	}

	if (newstate)
	*newstate = vd->vdev_state;
	if ((flags & ZFS_ONLINE_UNSPARE) &&
	!vdev_is_dead(vd) && vd->vdev_parent &&
	vd->vdev_parent->vdev_ops == &vdev_spare_ops &&
	vd->vdev_parent->vdev_child[0] == vd)
	vd->vdev_unspare = B_TRUE;

	if ((flags & ZFS_ONLINE_EXPAND) \|\| spa->spa_autoexpand) {

	/* XXX - L2ARC 1.0 does not support expansion */
	if (vd->vdev_aux)
	return (spa_vdev_state_exit(spa, vd, ENOTSUP));
	spa->spa_ccw_fail_time = 0;
	spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
	}

	/* Restart initializing if necessary */
	mutex_enter(&vd->vdev_initialize_lock);
	if (vdev_writeable(vd) &&
	vd->vdev_initialize_thread == NULL &&
	vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE) {
	(void) vdev_initialize(vd);
	}
	mutex_exit(&vd->vdev_initialize_lock);

	/*
	* Restart trimming if necessary. We do not restart trimming for cache
	* devices here. This is triggered by l2arc_rebuild_vdev()
	* asynchronously for the whole device or in l2arc_evict() as it evicts
	* space for upcoming writes.
	*/
	mutex_enter(&vd->vdev_trim_lock);
	if (vdev_writeable(vd) && !vd->vdev_isl2cache &&
	vd->vdev_trim_thread == NULL &&
	vd->vdev_trim_state == VDEV_TRIM_ACTIVE) {
	(void) vdev_trim(vd, vd->vdev_trim_rate, vd->vdev_trim_partial,
	vd->vdev_trim_secure);
	}
	mutex_exit(&vd->vdev_trim_lock);

	if (wasoffline \|\|
	(oldstate < VDEV_STATE_DEGRADED &&
	vd->vdev_state >= VDEV_STATE_DEGRADED)) {
	spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_ONLINE);

	/*
	* Asynchronously detach spare vdev if resilver or
	* rebuild is not required
	*/
	if (vd->vdev_unspare &&
	!dsl_scan_resilvering(spa->spa_dsl_pool) &&
	!dsl_scan_resilver_scheduled(spa->spa_dsl_pool) &&
	!vdev_rebuild_active(tvd))
	spa_async_request(spa, SPA_ASYNC_DETACH_SPARE);
	}
	return (spa_vdev_state_exit(spa, vd, 0));
	}

	static int
	vdev_offline_locked(spa_t *spa, uint64_t guid, uint64_t flags)
	{
	vdev_t vd, tvd;
	int error = 0;
	uint64_t generation;
	metaslab_group_t *mg;

	top:
	spa_vdev_state_enter(spa, SCL_ALLOC);

	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
	return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENODEV)));

	if (!vd->vdev_ops->vdev_op_leaf)
	return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENOTSUP)));

	if (vd->vdev_ops == &vdev_draid_spare_ops)
	return (spa_vdev_state_exit(spa, NULL, ENOTSUP));

	tvd = vd->vdev_top;
	mg = tvd->vdev_mg;
	generation = spa->spa_config_generation + 1;

	/*
	* If the device isn't already offline, try to offline it.
	*/
	if (!vd->vdev_offline) {
	/*
	* If this device has the only valid copy of some data,
	* don't allow it to be offlined. Log devices are always
	* expendable.
	*/
	if (!tvd->vdev_islog && vd->vdev_aux == NULL &&
	vdev_dtl_required(vd))
	return (spa_vdev_state_exit(spa, NULL,
	SET_ERROR(EBUSY)));

	/*
	* If the top-level is a slog and it has had allocations
	* then proceed. We check that the vdev's metaslab group
	* is not NULL since it's possible that we may have just
	* added this vdev but not yet initialized its metaslabs.
	*/
	if (tvd->vdev_islog && mg != NULL) {
	/*
	* Prevent any future allocations.
	*/
	ASSERT3P(tvd->vdev_log_mg, ==, NULL);
	metaslab_group_passivate(mg);
	(void) spa_vdev_state_exit(spa, vd, 0);

	error = spa_reset_logs(spa);

	/*
	* If the log device was successfully reset but has
	* checkpointed data, do not offline it.
	*/
	if (error == 0 &&
	tvd->vdev_checkpoint_sm != NULL) {
	ASSERT3U(space_map_allocated(
	tvd->vdev_checkpoint_sm), !=, 0);
	error = ZFS_ERR_CHECKPOINT_EXISTS;
	}

	spa_vdev_state_enter(spa, SCL_ALLOC);

	/*
	* Check to see if the config has changed.
	*/
	if (error \|\| generation != spa->spa_config_generation) {
	metaslab_group_activate(mg);
	if (error)
	return (spa_vdev_state_exit(spa,
	vd, error));
	(void) spa_vdev_state_exit(spa, vd, 0);
	goto top;
	}
	ASSERT0(tvd->vdev_stat.vs_alloc);
	}

	/*
	* Offline this device and reopen its top-level vdev.
	* If the top-level vdev is a log device then just offline
	* it. Otherwise, if this action results in the top-level
	* vdev becoming unusable, undo it and fail the request.
	*/
	vd->vdev_offline = B_TRUE;
	vdev_reopen(tvd);

	if (!tvd->vdev_islog && vd->vdev_aux == NULL &&
	vdev_is_dead(tvd)) {
	vd->vdev_offline = B_FALSE;
	vdev_reopen(tvd);
	return (spa_vdev_state_exit(spa, NULL,
	SET_ERROR(EBUSY)));
	}

	/*
	* Add the device back into the metaslab rotor so that
	* once we online the device it's open for business.
	*/
	if (tvd->vdev_islog && mg != NULL)
	metaslab_group_activate(mg);
	}

	vd->vdev_tmpoffline = !!(flags & ZFS_OFFLINE_TEMPORARY);

	return (spa_vdev_state_exit(spa, vd, 0));
	}

	int
	vdev_offline(spa_t *spa, uint64_t guid, uint64_t flags)
	{
	int error;

	mutex_enter(&spa->spa_vdev_top_lock);
	error = vdev_offline_locked(spa, guid, flags);
	mutex_exit(&spa->spa_vdev_top_lock);

	return (error);
	}

	/*
	* Clear the error counts associated with this vdev. Unlike vdev_online() and
	* vdev_offline(), we assume the spa config is locked. We also clear all
	* children. If 'vd' is NULL, then the user wants to clear all vdevs.
	*/
	void
	vdev_clear(spa_t spa, vdev_t vd)
	{
	vdev_t *rvd = spa->spa_root_vdev;

	ASSERT(spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);

	if (vd == NULL)
	vd = rvd;

	vd->vdev_stat.vs_read_errors = 0;
	vd->vdev_stat.vs_write_errors = 0;
	vd->vdev_stat.vs_checksum_errors = 0;
	vd->vdev_stat.vs_slow_ios = 0;

	for (int c = 0; c < vd->vdev_children; c++)
	vdev_clear(spa, vd->vdev_child[c]);

	/*
	* It makes no sense to "clear" an indirect or removed vdev.
	*/
	if (!vdev_is_concrete(vd) \|\| vd->vdev_removed)
	return;

	/*
	* If we're in the FAULTED state or have experienced failed I/O, then
	* clear the persistent state and attempt to reopen the device. We
	* also mark the vdev config dirty, so that the new faulted state is
	* written out to disk.
	*/
	if (vd->vdev_faulted \|\| vd->vdev_degraded \|\|
	!vdev_readable(vd) \|\| !vdev_writeable(vd)) {
	/*
	* When reopening in response to a clear event, it may be due to
	* a fmadm repair request. In this case, if the device is
	* still broken, we want to still post the ereport again.
	*/
	vd->vdev_forcefault = B_TRUE;

	vd->vdev_faulted = vd->vdev_degraded = 0ULL;
	vd->vdev_cant_read = B_FALSE;
	vd->vdev_cant_write = B_FALSE;
	vd->vdev_stat.vs_aux = 0;

	vdev_reopen(vd == rvd ? rvd : vd->vdev_top);

	vd->vdev_forcefault = B_FALSE;

	if (vd != rvd && vdev_writeable(vd->vdev_top))
	vdev_state_dirty(vd->vdev_top);

	/* If a resilver isn't required, check if vdevs can be culled */
	if (vd->vdev_aux == NULL && !vdev_is_dead(vd) &&
	!dsl_scan_resilvering(spa->spa_dsl_pool) &&
	!dsl_scan_resilver_scheduled(spa->spa_dsl_pool))
	spa_async_request(spa, SPA_ASYNC_RESILVER_DONE);

	spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_CLEAR);
	}

	/*
	* When clearing a FMA-diagnosed fault, we always want to
	* unspare the device, as we assume that the original spare was
	* done in response to the FMA fault.
	*/
	if (!vdev_is_dead(vd) && vd->vdev_parent != NULL &&
	vd->vdev_parent->vdev_ops == &vdev_spare_ops &&
	vd->vdev_parent->vdev_child[0] == vd)
	vd->vdev_unspare = B_TRUE;

	/* Clear recent error events cache (i.e. duplicate events tracking) */
	zfs_ereport_clear(spa, vd);
	}

	boolean_t
	vdev_is_dead(vdev_t *vd)
	{
	/*
	* Holes and missing devices are always considered "dead".
	* This simplifies the code since we don't have to check for
	* these types of devices in the various code paths.
	* Instead we rely on the fact that we skip over dead devices
	* before issuing I/O to them.
	*/
	return (vd->vdev_state < VDEV_STATE_DEGRADED \|\|
	vd->vdev_ops == &vdev_hole_ops \|\|
	vd->vdev_ops == &vdev_missing_ops);
	}

	boolean_t
	vdev_readable(vdev_t *vd)
	{
	return (!vdev_is_dead(vd) && !vd->vdev_cant_read);
	}

	boolean_t
	vdev_writeable(vdev_t *vd)
	{
	return (!vdev_is_dead(vd) && !vd->vdev_cant_write &&
	vdev_is_concrete(vd));
	}

	boolean_t
	vdev_allocatable(vdev_t *vd)
	{
	uint64_t state = vd->vdev_state;

	/*
	* We currently allow allocations from vdevs which may be in the
	* process of reopening (i.e. VDEV_STATE_CLOSED). If the device
	* fails to reopen then we'll catch it later when we're holding
	* the proper locks. Note that we have to get the vdev state
	* in a local variable because although it changes atomically,
	* we're asking two separate questions about it.
	*/
	return (!(state < VDEV_STATE_DEGRADED && state != VDEV_STATE_CLOSED) &&
	!vd->vdev_cant_write && vdev_is_concrete(vd) &&
	vd->vdev_mg->mg_initialized);
	}

	boolean_t
	vdev_accessible(vdev_t vd, zio_t zio)
	{
	ASSERT(zio->io_vd == vd);

	if (vdev_is_dead(vd) \|\| vd->vdev_remove_wanted)
	return (B_FALSE);

	if (zio->io_type == ZIO_TYPE_READ)
	return (!vd->vdev_cant_read);

	if (zio->io_type == ZIO_TYPE_WRITE)
	return (!vd->vdev_cant_write);

	return (B_TRUE);
	}

	static void
	vdev_get_child_stat(vdev_t cvd, vdev_stat_t vs, vdev_stat_t *cvs)
	{
	/*
	* Exclude the dRAID spare when aggregating to avoid double counting
	* the ops and bytes. These IOs are counted by the physical leaves.
	*/
	if (cvd->vdev_ops == &vdev_draid_spare_ops)
	return;

	for (int t = 0; t < VS_ZIO_TYPES; t++) {
	vs->vs_ops[t] += cvs->vs_ops[t];
	vs->vs_bytes[t] += cvs->vs_bytes[t];
	}

	cvs->vs_scan_removing = cvd->vdev_removing;
	}

	/*
	* Get extended stats
	*/
	static void
	vdev_get_child_stat_ex(vdev_t cvd, vdev_stat_ex_t vsx, vdev_stat_ex_t *cvsx)
	{
	(void) cvd;

	int t, b;
	for (t = 0; t < ZIO_TYPES; t++) {
	for (b = 0; b < ARRAY_SIZE(vsx->vsx_disk_histo[0]); b++)
	vsx->vsx_disk_histo[t][b] += cvsx->vsx_disk_histo[t][b];

	for (b = 0; b < ARRAY_SIZE(vsx->vsx_total_histo[0]); b++) {
	vsx->vsx_total_histo[t][b] +=
	cvsx->vsx_total_histo[t][b];
	}
	}

	for (t = 0; t < ZIO_PRIORITY_NUM_QUEUEABLE; t++) {
	for (b = 0; b < ARRAY_SIZE(vsx->vsx_queue_histo[0]); b++) {
	vsx->vsx_queue_histo[t][b] +=
	cvsx->vsx_queue_histo[t][b];
	}
	vsx->vsx_active_queue[t] += cvsx->vsx_active_queue[t];
	vsx->vsx_pend_queue[t] += cvsx->vsx_pend_queue[t];

	for (b = 0; b < ARRAY_SIZE(vsx->vsx_ind_histo[0]); b++)
	vsx->vsx_ind_histo[t][b] += cvsx->vsx_ind_histo[t][b];

	for (b = 0; b < ARRAY_SIZE(vsx->vsx_agg_histo[0]); b++)
	vsx->vsx_agg_histo[t][b] += cvsx->vsx_agg_histo[t][b];
	}

	}

	boolean_t
	vdev_is_spacemap_addressable(vdev_t *vd)
	{
	if (spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_SPACEMAP_V2))
	return (B_TRUE);

	/*
	* If double-word space map entries are not enabled we assume
	* 47 bits of the space map entry are dedicated to the entry's
	* offset (see SM_OFFSET_BITS in space_map.h). We then use that
	* to calculate the maximum address that can be described by a
	* space map entry for the given device.
	*/
	uint64_t shift = vd->vdev_ashift + SM_OFFSET_BITS;

	if (shift >= 63) /* detect potential overflow */
	return (B_TRUE);

	return (vd->vdev_asize < (1ULL << shift));
	}

	/*
	* Get statistics for the given vdev.
	*/
	static void
	vdev_get_stats_ex_impl(vdev_t vd, vdev_stat_t vs, vdev_stat_ex_t *vsx)
	{
	int t;
	/*
	* If we're getting stats on the root vdev, aggregate the I/O counts
	* over all top-level vdevs (i.e. the direct children of the root).
	*/
	if (!vd->vdev_ops->vdev_op_leaf) {
	if (vs) {
	memset(vs->vs_ops, 0, sizeof (vs->vs_ops));
	memset(vs->vs_bytes, 0, sizeof (vs->vs_bytes));
	}
	if (vsx)
	memset(vsx, 0, sizeof (*vsx));

	for (int c = 0; c < vd->vdev_children; c++) {
	vdev_t *cvd = vd->vdev_child[c];
	vdev_stat_t *cvs = &cvd->vdev_stat;
	vdev_stat_ex_t *cvsx = &cvd->vdev_stat_ex;

	vdev_get_stats_ex_impl(cvd, cvs, cvsx);
	if (vs)
	vdev_get_child_stat(cvd, vs, cvs);
	if (vsx)
	vdev_get_child_stat_ex(cvd, vsx, cvsx);
	}
	} else {
	/*
	* We're a leaf. Just copy our ZIO active queue stats in. The
	* other leaf stats are updated in vdev_stat_update().
	*/
	if (!vsx)
	return;

	memcpy(vsx, &vd->vdev_stat_ex, sizeof (vd->vdev_stat_ex));

	for (t = 0; t < ZIO_PRIORITY_NUM_QUEUEABLE; t++) {
	vsx->vsx_active_queue[t] = vd->vdev_queue.vq_cactive[t];
	vsx->vsx_pend_queue[t] = vdev_queue_class_length(vd, t);
	}
	}
	}

	void
	vdev_get_stats_ex(vdev_t vd, vdev_stat_t vs, vdev_stat_ex_t *vsx)
	{
	vdev_t *tvd = vd->vdev_top;
	mutex_enter(&vd->vdev_stat_lock);
	if (vs) {
	memcpy(vs, &vd->vdev_stat, sizeof (*vs));
	vs->vs_timestamp = gethrtime() - vs->vs_timestamp;
	vs->vs_state = vd->vdev_state;
	vs->vs_rsize = vdev_get_min_asize(vd);

	if (vd->vdev_ops->vdev_op_leaf) {
	vs->vs_pspace = vd->vdev_psize;
	vs->vs_rsize += VDEV_LABEL_START_SIZE +
	VDEV_LABEL_END_SIZE;
	/*
	* Report initializing progress. Since we don't
	* have the initializing locks held, this is only
	* an estimate (although a fairly accurate one).
	*/
	vs->vs_initialize_bytes_done =
	vd->vdev_initialize_bytes_done;
	vs->vs_initialize_bytes_est =
	vd->vdev_initialize_bytes_est;
	vs->vs_initialize_state = vd->vdev_initialize_state;
	vs->vs_initialize_action_time =
	vd->vdev_initialize_action_time;

	/*
	* Report manual TRIM progress. Since we don't have
	* the manual TRIM locks held, this is only an
	* estimate (although fairly accurate one).
	*/
	vs->vs_trim_notsup = !vd->vdev_has_trim;
	vs->vs_trim_bytes_done = vd->vdev_trim_bytes_done;
	vs->vs_trim_bytes_est = vd->vdev_trim_bytes_est;
	vs->vs_trim_state = vd->vdev_trim_state;
	vs->vs_trim_action_time = vd->vdev_trim_action_time;

	/* Set when there is a deferred resilver. */
	vs->vs_resilver_deferred = vd->vdev_resilver_deferred;
	}

	/*
	* Report expandable space on top-level, non-auxiliary devices
	* only. The expandable space is reported in terms of metaslab
	* sized units since that determines how much space the pool
	* can expand.
	*/
	if (vd->vdev_aux == NULL && tvd != NULL) {
	vs->vs_esize = P2ALIGN(
	vd->vdev_max_asize - vd->vdev_asize,
	1ULL << tvd->vdev_ms_shift);
	}

	vs->vs_configured_ashift = vd->vdev_top != NULL
	? vd->vdev_top->vdev_ashift : vd->vdev_ashift;
	vs->vs_logical_ashift = vd->vdev_logical_ashift;
	if (vd->vdev_physical_ashift <= ASHIFT_MAX)
	vs->vs_physical_ashift = vd->vdev_physical_ashift;
	else
	vs->vs_physical_ashift = 0;

	/*
	* Report fragmentation and rebuild progress for top-level,
	* non-auxiliary, concrete devices.
	*/
	if (vd->vdev_aux == NULL && vd == vd->vdev_top &&
	vdev_is_concrete(vd)) {
	/*
	* The vdev fragmentation rating doesn't take into
	* account the embedded slog metaslab (vdev_log_mg).
	* Since it's only one metaslab, it would have a tiny
	* impact on the overall fragmentation.
	*/
	vs->vs_fragmentation = (vd->vdev_mg != NULL) ?
	vd->vdev_mg->mg_fragmentation : 0;
	}
	vs->vs_noalloc = MAX(vd->vdev_noalloc,
	tvd ? tvd->vdev_noalloc : 0);
	}

	vdev_get_stats_ex_impl(vd, vs, vsx);
	mutex_exit(&vd->vdev_stat_lock);
	}

	void
	vdev_get_stats(vdev_t vd, vdev_stat_t vs)
	{
	return (vdev_get_stats_ex(vd, vs, NULL));
	}

	void
	vdev_clear_stats(vdev_t *vd)
	{
	mutex_enter(&vd->vdev_stat_lock);
	vd->vdev_stat.vs_space = 0;
	vd->vdev_stat.vs_dspace = 0;
	vd->vdev_stat.vs_alloc = 0;
	mutex_exit(&vd->vdev_stat_lock);
	}

	void
	vdev_scan_stat_init(vdev_t *vd)
	{
	vdev_stat_t *vs = &vd->vdev_stat;

	for (int c = 0; c < vd->vdev_children; c++)
	vdev_scan_stat_init(vd->vdev_child[c]);

	mutex_enter(&vd->vdev_stat_lock);
	vs->vs_scan_processed = 0;
	mutex_exit(&vd->vdev_stat_lock);
	}

	void
	vdev_stat_update(zio_t *zio, uint64_t psize)
	{
	spa_t *spa = zio->io_spa;
	vdev_t *rvd = spa->spa_root_vdev;
	vdev_t *vd = zio->io_vd ? zio->io_vd : rvd;
	vdev_t *pvd;
	uint64_t txg = zio->io_txg;
	/* Suppress ASAN false positive */
	#ifdef __SANITIZE_ADDRESS__
	vdev_stat_t *vs = vd ? &vd->vdev_stat : NULL;
	vdev_stat_ex_t *vsx = vd ? &vd->vdev_stat_ex : NULL;
	#else
	vdev_stat_t *vs = &vd->vdev_stat;
	vdev_stat_ex_t *vsx = &vd->vdev_stat_ex;
	#endif
	zio_type_t type = zio->io_type;
	int flags = zio->io_flags;

	/*
	* If this i/o is a gang leader, it didn't do any actual work.
	*/
	if (zio->io_gang_tree)
	return;

	if (zio->io_error == 0) {
	/*
	* If this is a root i/o, don't count it -- we've already
	* counted the top-level vdevs, and vdev_get_stats() will
	* aggregate them when asked. This reduces contention on
	* the root vdev_stat_lock and implicitly handles blocks
	* that compress away to holes, for which there is no i/o.
	* (Holes never create vdev children, so all the counters
	* remain zero, which is what we want.)
	*
	* Note: this only applies to successful i/o (io_error == 0)
	* because unlike i/o counts, errors are not additive.
	* When reading a ditto block, for example, failure of
	* one top-level vdev does not imply a root-level error.
	*/
	if (vd == rvd)
	return;

	ASSERT(vd == zio->io_vd);

	if (flags & ZIO_FLAG_IO_BYPASS)
	return;

	mutex_enter(&vd->vdev_stat_lock);

	if (flags & ZIO_FLAG_IO_REPAIR) {
	/*
	* Repair is the result of a resilver issued by the
	* scan thread (spa_sync).
	*/
	if (flags & ZIO_FLAG_SCAN_THREAD) {
	dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan;
	dsl_scan_phys_t *scn_phys = &scn->scn_phys;
	uint64_t *processed = &scn_phys->scn_processed;

	if (vd->vdev_ops->vdev_op_leaf)
	atomic_add_64(processed, psize);
	vs->vs_scan_processed += psize;
	}

	/*
	* Repair is the result of a rebuild issued by the
	* rebuild thread (vdev_rebuild_thread). To avoid
	* double counting repaired bytes the virtual dRAID
	* spare vdev is excluded from the processed bytes.
	*/
	if (zio->io_priority == ZIO_PRIORITY_REBUILD) {
	vdev_t *tvd = vd->vdev_top;
	vdev_rebuild_t *vr = &tvd->vdev_rebuild_config;
	vdev_rebuild_phys_t *vrp = &vr->vr_rebuild_phys;
	uint64_t *rebuilt = &vrp->vrp_bytes_rebuilt;

	if (vd->vdev_ops->vdev_op_leaf &&
	vd->vdev_ops != &vdev_draid_spare_ops) {
	atomic_add_64(rebuilt, psize);
	}
	vs->vs_rebuild_processed += psize;
	}

	if (flags & ZIO_FLAG_SELF_HEAL)
	vs->vs_self_healed += psize;
	}

	/*
	* The bytes/ops/histograms are recorded at the leaf level and
	* aggregated into the higher level vdevs in vdev_get_stats().
	*/
	if (vd->vdev_ops->vdev_op_leaf &&
	(zio->io_priority < ZIO_PRIORITY_NUM_QUEUEABLE)) {
	zio_type_t vs_type = type;
	zio_priority_t priority = zio->io_priority;

	/*
	* TRIM ops and bytes are reported to user space as
	* ZIO_TYPE_IOCTL. This is done to preserve the
	* vdev_stat_t structure layout for user space.
	*/
	if (type == ZIO_TYPE_TRIM)
	vs_type = ZIO_TYPE_IOCTL;

	/*
	* Solely for the purposes of 'zpool iostat -lqrw'
	* reporting use the priority to categorize the IO.
	* Only the following are reported to user space:
	*
	* ZIO_PRIORITY_SYNC_READ,
	* ZIO_PRIORITY_SYNC_WRITE,
	* ZIO_PRIORITY_ASYNC_READ,
	* ZIO_PRIORITY_ASYNC_WRITE,
	* ZIO_PRIORITY_SCRUB,
	* ZIO_PRIORITY_TRIM,
	* ZIO_PRIORITY_REBUILD.
	*/
	if (priority == ZIO_PRIORITY_INITIALIZING) {
	ASSERT3U(type, ==, ZIO_TYPE_WRITE);
	priority = ZIO_PRIORITY_ASYNC_WRITE;
	} else if (priority == ZIO_PRIORITY_REMOVAL) {
	priority = ((type == ZIO_TYPE_WRITE) ?
	ZIO_PRIORITY_ASYNC_WRITE :
	ZIO_PRIORITY_ASYNC_READ);
	}

	vs->vs_ops[vs_type]++;
	vs->vs_bytes[vs_type] += psize;

	if (flags & ZIO_FLAG_DELEGATED) {
	vsx->vsx_agg_histo[priority]
	[RQ_HISTO(zio->io_size)]++;
	} else {
	vsx->vsx_ind_histo[priority]
	[RQ_HISTO(zio->io_size)]++;
	}

	if (zio->io_delta && zio->io_delay) {
	vsx->vsx_queue_histo[priority]
	[L_HISTO(zio->io_delta - zio->io_delay)]++;
	vsx->vsx_disk_histo[type]
	[L_HISTO(zio->io_delay)]++;
	vsx->vsx_total_histo[type]
	[L_HISTO(zio->io_delta)]++;
	}
	}

	mutex_exit(&vd->vdev_stat_lock);
	return;
	}

	if (flags & ZIO_FLAG_SPECULATIVE)
	return;

	/*
	* If this is an I/O error that is going to be retried, then ignore the
	* error. Otherwise, the user may interpret B_FAILFAST I/O errors as
	* hard errors, when in reality they can happen for any number of
	* innocuous reasons (bus resets, MPxIO link failure, etc).
	*/
	if (zio->io_error == EIO &&
	!(zio->io_flags & ZIO_FLAG_IO_RETRY))
	return;

	/*
	* Intent logs writes won't propagate their error to the root
	* I/O so don't mark these types of failures as pool-level
	* errors.
	*/
	if (zio->io_vd == NULL && (zio->io_flags & ZIO_FLAG_DONT_PROPAGATE))
	return;

	if (type == ZIO_TYPE_WRITE && txg != 0 &&
	(!(flags & ZIO_FLAG_IO_REPAIR) \|\|
	(flags & ZIO_FLAG_SCAN_THREAD) \|\|
	spa->spa_claiming)) {
	/*
	* This is either a normal write (not a repair), or it's
	* a repair induced by the scrub thread, or it's a repair
	* made by zil_claim() during spa_load() in the first txg.
	* In the normal case, we commit the DTL change in the same
	* txg as the block was born. In the scrub-induced repair
	* case, we know that scrubs run in first-pass syncing context,
	* so we commit the DTL change in spa_syncing_txg(spa).
	* In the zil_claim() case, we commit in spa_first_txg(spa).
	*
	* We currently do not make DTL entries for failed spontaneous
	* self-healing writes triggered by normal (non-scrubbing)
	* reads, because we have no transactional context in which to
	* do so -- and it's not clear that it'd be desirable anyway.
	*/
	if (vd->vdev_ops->vdev_op_leaf) {
	uint64_t commit_txg = txg;
	if (flags & ZIO_FLAG_SCAN_THREAD) {
	ASSERT(flags & ZIO_FLAG_IO_REPAIR);
	ASSERT(spa_sync_pass(spa) == 1);
	vdev_dtl_dirty(vd, DTL_SCRUB, txg, 1);
	commit_txg = spa_syncing_txg(spa);
	} else if (spa->spa_claiming) {
	ASSERT(flags & ZIO_FLAG_IO_REPAIR);
	commit_txg = spa_first_txg(spa);
	}
	ASSERT(commit_txg >= spa_syncing_txg(spa));
	if (vdev_dtl_contains(vd, DTL_MISSING, txg, 1))
	return;
	for (pvd = vd; pvd != rvd; pvd = pvd->vdev_parent)
	vdev_dtl_dirty(pvd, DTL_PARTIAL, txg, 1);
	vdev_dirty(vd->vdev_top, VDD_DTL, vd, commit_txg);
	}
	if (vd != rvd)
	vdev_dtl_dirty(vd, DTL_MISSING, txg, 1);
	}
	}

	int64_t
	vdev_deflated_space(vdev_t *vd, int64_t space)
	{
	ASSERT((space & (SPA_MINBLOCKSIZE-1)) == 0);
	ASSERT(vd->vdev_deflate_ratio != 0 \|\| vd->vdev_isl2cache);

	return ((space >> SPA_MINBLOCKSHIFT) * vd->vdev_deflate_ratio);
	}

	/*
	* Update the in-core space usage stats for this vdev, its metaslab class,
	* and the root vdev.
	*/
	void
	vdev_space_update(vdev_t *vd, int64_t alloc_delta, int64_t defer_delta,
	int64_t space_delta)
	{
	(void) defer_delta;
	int64_t dspace_delta;
	spa_t *spa = vd->vdev_spa;
	vdev_t *rvd = spa->spa_root_vdev;

	ASSERT(vd == vd->vdev_top);

	/*
	* Apply the inverse of the psize-to-asize (ie. RAID-Z) space-expansion
	* factor. We must calculate this here and not at the root vdev
	* because the root vdev's psize-to-asize is simply the max of its
	* children's, thus not accurate enough for us.
	*/
	dspace_delta = vdev_deflated_space(vd, space_delta);

	mutex_enter(&vd->vdev_stat_lock);
	/* ensure we won't underflow */
	if (alloc_delta < 0) {
	ASSERT3U(vd->vdev_stat.vs_alloc, >=, -alloc_delta);
	}

	vd->vdev_stat.vs_alloc += alloc_delta;
	vd->vdev_stat.vs_space += space_delta;
	vd->vdev_stat.vs_dspace += dspace_delta;
	mutex_exit(&vd->vdev_stat_lock);

	/* every class but log contributes to root space stats */
	if (vd->vdev_mg != NULL && !vd->vdev_islog) {
	ASSERT(!vd->vdev_isl2cache);
	mutex_enter(&rvd->vdev_stat_lock);
	rvd->vdev_stat.vs_alloc += alloc_delta;
	rvd->vdev_stat.vs_space += space_delta;
	rvd->vdev_stat.vs_dspace += dspace_delta;
	mutex_exit(&rvd->vdev_stat_lock);
	}
	/* Note: metaslab_class_space_update moved to metaslab_space_update */
	}

	/*
	* Mark a top-level vdev's config as dirty, placing it on the dirty list
	* so that it will be written out next time the vdev configuration is synced.
	* If the root vdev is specified (vdev_top == NULL), dirty all top-level vdevs.
	*/
	void
	vdev_config_dirty(vdev_t *vd)
	{
	spa_t *spa = vd->vdev_spa;
	vdev_t *rvd = spa->spa_root_vdev;
	int c;

	ASSERT(spa_writeable(spa));

	/*
	* If this is an aux vdev (as with l2cache and spare devices), then we
	* update the vdev config manually and set the sync flag.
	*/
	if (vd->vdev_aux != NULL) {
	spa_aux_vdev_t *sav = vd->vdev_aux;
	nvlist_t **aux;
	uint_t naux;

	for (c = 0; c < sav->sav_count; c++) {
	if (sav->sav_vdevs[c] == vd)
	break;
	}

	if (c == sav->sav_count) {
	/*
	* We're being removed. There's nothing more to do.
	*/
	ASSERT(sav->sav_sync == B_TRUE);
	return;
	}

	sav->sav_sync = B_TRUE;

	if (nvlist_lookup_nvlist_array(sav->sav_config,
	ZPOOL_CONFIG_L2CACHE, &aux, &naux) != 0) {
	VERIFY(nvlist_lookup_nvlist_array(sav->sav_config,
	ZPOOL_CONFIG_SPARES, &aux, &naux) == 0);
	}

	ASSERT(c < naux);

	/*
	* Setting the nvlist in the middle if the array is a little
	* sketchy, but it will work.
	*/
	nvlist_free(aux[c]);
	aux[c] = vdev_config_generate(spa, vd, B_TRUE, 0);

	return;
	}

	/*
	* The dirty list is protected by the SCL_CONFIG lock. The caller
	* must either hold SCL_CONFIG as writer, or must be the sync thread
	* (which holds SCL_CONFIG as reader). There's only one sync thread,
	* so this is sufficient to ensure mutual exclusion.
	*/
	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_WRITER) \|\|
	(dsl_pool_sync_context(spa_get_dsl(spa)) &&
	spa_config_held(spa, SCL_CONFIG, RW_READER)));

	if (vd == rvd) {
	for (c = 0; c < rvd->vdev_children; c++)
	vdev_config_dirty(rvd->vdev_child[c]);
	} else {
	ASSERT(vd == vd->vdev_top);

	if (!list_link_active(&vd->vdev_config_dirty_node) &&
	vdev_is_concrete(vd)) {
	list_insert_head(&spa->spa_config_dirty_list, vd);
	}
	}
	}

	void
	vdev_config_clean(vdev_t *vd)
	{
	spa_t *spa = vd->vdev_spa;

	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_WRITER) \|\|
	(dsl_pool_sync_context(spa_get_dsl(spa)) &&
	spa_config_held(spa, SCL_CONFIG, RW_READER)));

	ASSERT(list_link_active(&vd->vdev_config_dirty_node));
	list_remove(&spa->spa_config_dirty_list, vd);
	}

	/*
	* Mark a top-level vdev's state as dirty, so that the next pass of
	* spa_sync() can convert this into vdev_config_dirty(). We distinguish
	* the state changes from larger config changes because they require
	* much less locking, and are often needed for administrative actions.
	*/
	void
	vdev_state_dirty(vdev_t *vd)
	{
	spa_t *spa = vd->vdev_spa;

	ASSERT(spa_writeable(spa));
	ASSERT(vd == vd->vdev_top);

	/*
	* The state list is protected by the SCL_STATE lock. The caller
	* must either hold SCL_STATE as writer, or must be the sync thread
	* (which holds SCL_STATE as reader). There's only one sync thread,
	* so this is sufficient to ensure mutual exclusion.
	*/
	ASSERT(spa_config_held(spa, SCL_STATE, RW_WRITER) \|\|
	(dsl_pool_sync_context(spa_get_dsl(spa)) &&
	spa_config_held(spa, SCL_STATE, RW_READER)));

	if (!list_link_active(&vd->vdev_state_dirty_node) &&
	vdev_is_concrete(vd))
	list_insert_head(&spa->spa_state_dirty_list, vd);
	}

	void
	vdev_state_clean(vdev_t *vd)
	{
	spa_t *spa = vd->vdev_spa;

	ASSERT(spa_config_held(spa, SCL_STATE, RW_WRITER) \|\|
	(dsl_pool_sync_context(spa_get_dsl(spa)) &&
	spa_config_held(spa, SCL_STATE, RW_READER)));

	ASSERT(list_link_active(&vd->vdev_state_dirty_node));
	list_remove(&spa->spa_state_dirty_list, vd);
	}

	/*
	* Propagate vdev state up from children to parent.
	*/
	void
	vdev_propagate_state(vdev_t *vd)
	{
	spa_t *spa = vd->vdev_spa;
	vdev_t *rvd = spa->spa_root_vdev;
	int degraded = 0, faulted = 0;
	int corrupted = 0;
	vdev_t *child;

	if (vd->vdev_children > 0) {
	for (int c = 0; c < vd->vdev_children; c++) {
	child = vd->vdev_child[c];

	/*
	* Don't factor holes or indirect vdevs into the
	* decision.
	*/
	if (!vdev_is_concrete(child))
	continue;

	if (!vdev_readable(child) \|\|
	(!vdev_writeable(child) && spa_writeable(spa))) {
	/*
	* Root special: if there is a top-level log
	* device, treat the root vdev as if it were
	* degraded.
	*/
	if (child->vdev_islog && vd == rvd)
	degraded++;
	else
	faulted++;
	} else if (child->vdev_state <= VDEV_STATE_DEGRADED) {
	degraded++;
	}

	if (child->vdev_stat.vs_aux == VDEV_AUX_CORRUPT_DATA)
	corrupted++;
	}

	vd->vdev_ops->vdev_op_state_change(vd, faulted, degraded);

	/*
	* Root special: if there is a top-level vdev that cannot be
	* opened due to corrupted metadata, then propagate the root
	* vdev's aux state as 'corrupt' rather than 'insufficient
	* replicas'.
	*/
	if (corrupted && vd == rvd &&
	rvd->vdev_state == VDEV_STATE_CANT_OPEN)
	vdev_set_state(rvd, B_FALSE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_CORRUPT_DATA);
	}

	if (vd->vdev_parent)
	vdev_propagate_state(vd->vdev_parent);
	}

	/*
	* Set a vdev's state. If this is during an open, we don't update the parent
	* state, because we're in the process of opening children depth-first.
	* Otherwise, we propagate the change to the parent.
	*
	* If this routine places a device in a faulted state, an appropriate ereport is
	* generated.
	*/
	void
	vdev_set_state(vdev_t *vd, boolean_t isopen, vdev_state_t state, vdev_aux_t aux)
	{
	uint64_t save_state;
	spa_t *spa = vd->vdev_spa;

	if (state == vd->vdev_state) {
	/*
	* Since vdev_offline() code path is already in an offline
	* state we can miss a statechange event to OFFLINE. Check
	* the previous state to catch this condition.
	*/
	if (vd->vdev_ops->vdev_op_leaf &&
	(state == VDEV_STATE_OFFLINE) &&
	(vd->vdev_prevstate >= VDEV_STATE_FAULTED)) {
	/* post an offline state change */
	zfs_post_state_change(spa, vd, vd->vdev_prevstate);
	}
	vd->vdev_stat.vs_aux = aux;
	return;
	}

	save_state = vd->vdev_state;

	vd->vdev_state = state;
	vd->vdev_stat.vs_aux = aux;

	/*
	* If we are setting the vdev state to anything but an open state, then
	* always close the underlying device unless the device has requested
	* a delayed close (i.e. we're about to remove or fault the device).
	* Otherwise, we keep accessible but invalid devices open forever.
	* We don't call vdev_close() itself, because that implies some extra
	* checks (offline, etc) that we don't want here. This is limited to
	* leaf devices, because otherwise closing the device will affect other
	* children.
	*/
	if (!vd->vdev_delayed_close && vdev_is_dead(vd) &&
	vd->vdev_ops->vdev_op_leaf)
	vd->vdev_ops->vdev_op_close(vd);

	if (vd->vdev_removed &&
	state == VDEV_STATE_CANT_OPEN &&
	(aux == VDEV_AUX_OPEN_FAILED \|\| vd->vdev_checkremove)) {
	/*
	* If the previous state is set to VDEV_STATE_REMOVED, then this
	* device was previously marked removed and someone attempted to
	* reopen it. If this failed due to a nonexistent device, then
	* keep the device in the REMOVED state. We also let this be if
	* it is one of our special test online cases, which is only
	* attempting to online the device and shouldn't generate an FMA
	* fault.
	*/
	vd->vdev_state = VDEV_STATE_REMOVED;
	vd->vdev_stat.vs_aux = VDEV_AUX_NONE;
	} else if (state == VDEV_STATE_REMOVED) {
	vd->vdev_removed = B_TRUE;
	} else if (state == VDEV_STATE_CANT_OPEN) {
	/*
	* If we fail to open a vdev during an import or recovery, we
	* mark it as "not available", which signifies that it was
	* never there to begin with. Failure to open such a device
	* is not considered an error.
	*/
	if ((spa_load_state(spa) == SPA_LOAD_IMPORT \|\|
	spa_load_state(spa) == SPA_LOAD_RECOVER) &&
	vd->vdev_ops->vdev_op_leaf)
	vd->vdev_not_present = 1;

	/*
	* Post the appropriate ereport. If the 'prevstate' field is
	* set to something other than VDEV_STATE_UNKNOWN, it indicates
	* that this is part of a vdev_reopen(). In this case, we don't
	* want to post the ereport if the device was already in the
	* CANT_OPEN state beforehand.
	*
	* If the 'checkremove' flag is set, then this is an attempt to
	* online the device in response to an insertion event. If we
	* hit this case, then we have detected an insertion event for a
	* faulted or offline device that wasn't in the removed state.
	* In this scenario, we don't post an ereport because we are
	* about to replace the device, or attempt an online with
	* vdev_forcefault, which will generate the fault for us.
	*/
	if ((vd->vdev_prevstate != state \|\| vd->vdev_forcefault) &&
	!vd->vdev_not_present && !vd->vdev_checkremove &&
	vd != spa->spa_root_vdev) {
	const char *class;

	switch (aux) {
	case VDEV_AUX_OPEN_FAILED:
	class = FM_EREPORT_ZFS_DEVICE_OPEN_FAILED;
	break;
	case VDEV_AUX_CORRUPT_DATA:
	class = FM_EREPORT_ZFS_DEVICE_CORRUPT_DATA;
	break;
	case VDEV_AUX_NO_REPLICAS:
	class = FM_EREPORT_ZFS_DEVICE_NO_REPLICAS;
	break;
	case VDEV_AUX_BAD_GUID_SUM:
	class = FM_EREPORT_ZFS_DEVICE_BAD_GUID_SUM;
	break;
	case VDEV_AUX_TOO_SMALL:
	class = FM_EREPORT_ZFS_DEVICE_TOO_SMALL;
	break;
	case VDEV_AUX_BAD_LABEL:
	class = FM_EREPORT_ZFS_DEVICE_BAD_LABEL;
	break;
	case VDEV_AUX_BAD_ASHIFT:
	class = FM_EREPORT_ZFS_DEVICE_BAD_ASHIFT;
	break;
	default:
	class = FM_EREPORT_ZFS_DEVICE_UNKNOWN;
	}

	(void) zfs_ereport_post(class, spa, vd, NULL, NULL,
	save_state);
	}

	/* Erase any notion of persistent removed state */
	vd->vdev_removed = B_FALSE;
	} else {
	vd->vdev_removed = B_FALSE;
	}

	/*
	* Notify ZED of any significant state-change on a leaf vdev.
	*
	*/
	if (vd->vdev_ops->vdev_op_leaf) {
	/* preserve original state from a vdev_reopen() */
	if ((vd->vdev_prevstate != VDEV_STATE_UNKNOWN) &&
	(vd->vdev_prevstate != vd->vdev_state) &&
	(save_state <= VDEV_STATE_CLOSED))
	save_state = vd->vdev_prevstate;

	/* filter out state change due to initial vdev_open */
	if (save_state > VDEV_STATE_CLOSED)
	zfs_post_state_change(spa, vd, save_state);
	}

	if (!isopen && vd->vdev_parent)
	vdev_propagate_state(vd->vdev_parent);
	}

	boolean_t
	vdev_children_are_offline(vdev_t *vd)
	{
	ASSERT(!vd->vdev_ops->vdev_op_leaf);

	for (uint64_t i = 0; i < vd->vdev_children; i++) {
	if (vd->vdev_child[i]->vdev_state != VDEV_STATE_OFFLINE)
	return (B_FALSE);
	}

	return (B_TRUE);
	}

	/*
	* Check the vdev configuration to ensure that it's capable of supporting
	* a root pool. We do not support partial configuration.
	*/
	boolean_t
	vdev_is_bootable(vdev_t *vd)
	{
	if (!vd->vdev_ops->vdev_op_leaf) {
	const char *vdev_type = vd->vdev_ops->vdev_op_type;

	if (strcmp(vdev_type, VDEV_TYPE_MISSING) == 0)
	return (B_FALSE);
	}

	for (int c = 0; c < vd->vdev_children; c++) {
	if (!vdev_is_bootable(vd->vdev_child[c]))
	return (B_FALSE);
	}
	return (B_TRUE);
	}

	boolean_t
	vdev_is_concrete(vdev_t *vd)
	{
	vdev_ops_t *ops = vd->vdev_ops;
	if (ops == &vdev_indirect_ops \|\| ops == &vdev_hole_ops \|\|
	ops == &vdev_missing_ops \|\| ops == &vdev_root_ops) {
	return (B_FALSE);
	} else {
	return (B_TRUE);
	}
	}

	/*
	* Determine if a log device has valid content. If the vdev was
	* removed or faulted in the MOS config then we know that
	* the content on the log device has already been written to the pool.
	*/
	boolean_t
	vdev_log_state_valid(vdev_t *vd)
	{
	if (vd->vdev_ops->vdev_op_leaf && !vd->vdev_faulted &&
	!vd->vdev_removed)
	return (B_TRUE);

	for (int c = 0; c < vd->vdev_children; c++)
	if (vdev_log_state_valid(vd->vdev_child[c]))
	return (B_TRUE);

	return (B_FALSE);
	}

	/*
	* Expand a vdev if possible.
	*/
	void
	vdev_expand(vdev_t *vd, uint64_t txg)
	{
	ASSERT(vd->vdev_top == vd);
	ASSERT(spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER) == SCL_ALL);
	ASSERT(vdev_is_concrete(vd));

	vdev_set_deflate_ratio(vd);

	if ((vd->vdev_asize >> vd->vdev_ms_shift) > vd->vdev_ms_count &&
	vdev_is_concrete(vd)) {
	vdev_metaslab_group_create(vd);
	VERIFY(vdev_metaslab_init(vd, txg) == 0);
	vdev_config_dirty(vd);
	}
	}

	/*
	* Split a vdev.
	*/
	void
	vdev_split(vdev_t *vd)
	{
	vdev_t cvd, pvd = vd->vdev_parent;

	VERIFY3U(pvd->vdev_children, >, 1);

	vdev_remove_child(pvd, vd);
	vdev_compact_children(pvd);

	ASSERT3P(pvd->vdev_child, !=, NULL);

	cvd = pvd->vdev_child[0];
	if (pvd->vdev_children == 1) {
	vdev_remove_parent(cvd);
	cvd->vdev_splitting = B_TRUE;
	}
	vdev_propagate_state(cvd);
	}

	void
	vdev_deadman(vdev_t vd, const char tag)
	{
	for (int c = 0; c < vd->vdev_children; c++) {
	vdev_t *cvd = vd->vdev_child[c];

	vdev_deadman(cvd, tag);
	}

	if (vd->vdev_ops->vdev_op_leaf) {
	vdev_queue_t *vq = &vd->vdev_queue;

	mutex_enter(&vq->vq_lock);
	if (vq->vq_active > 0) {
	spa_t *spa = vd->vdev_spa;
	zio_t *fio;
	uint64_t delta;

	zfs_dbgmsg("slow vdev: %s has %u active IOs",
	vd->vdev_path, vq->vq_active);

	/*
	* Look at the head of all the pending queues,
	* if any I/O has been outstanding for longer than
	* the spa_deadman_synctime invoke the deadman logic.
	*/
	fio = list_head(&vq->vq_active_list);
	delta = gethrtime() - fio->io_timestamp;
	if (delta > spa_deadman_synctime(spa))
	zio_deadman(fio, tag);
	}
	mutex_exit(&vq->vq_lock);
	}
	}

	void
	vdev_defer_resilver(vdev_t *vd)
	{
	ASSERT(vd->vdev_ops->vdev_op_leaf);

	vd->vdev_resilver_deferred = B_TRUE;
	vd->vdev_spa->spa_resilver_deferred = B_TRUE;
	}

	/*
	* Clears the resilver deferred flag on all leaf devs under vd. Returns
	* B_TRUE if we have devices that need to be resilvered and are available to
	* accept resilver I/Os.
	*/
	boolean_t
	vdev_clear_resilver_deferred(vdev_t vd, dmu_tx_t tx)
	{
	boolean_t resilver_needed = B_FALSE;
	spa_t *spa = vd->vdev_spa;

	for (int c = 0; c < vd->vdev_children; c++) {
	vdev_t *cvd = vd->vdev_child[c];
	resilver_needed \|= vdev_clear_resilver_deferred(cvd, tx);
	}

	if (vd == spa->spa_root_vdev &&
	spa_feature_is_active(spa, SPA_FEATURE_RESILVER_DEFER)) {
	spa_feature_decr(spa, SPA_FEATURE_RESILVER_DEFER, tx);
	vdev_config_dirty(vd);
	spa->spa_resilver_deferred = B_FALSE;
	return (resilver_needed);
	}

	if (!vdev_is_concrete(vd) \|\| vd->vdev_aux \|\|
	!vd->vdev_ops->vdev_op_leaf)
	return (resilver_needed);

	vd->vdev_resilver_deferred = B_FALSE;

	return (!vdev_is_dead(vd) && !vd->vdev_offline &&
	vdev_resilver_needed(vd, NULL, NULL));
	}

	boolean_t
	vdev_xlate_is_empty(range_seg64_t *rs)
	{
	return (rs->rs_start == rs->rs_end);
	}

	/*
	* Translate a logical range to the first contiguous physical range for the
	* specified vdev_t. This function is initially called with a leaf vdev and
	* will walk each parent vdev until it reaches a top-level vdev. Once the
	* top-level is reached the physical range is initialized and the recursive
	* function begins to unwind. As it unwinds it calls the parent's vdev
	* specific translation function to do the real conversion.
	*/
	void
	vdev_xlate(vdev_t vd, const range_seg64_t logical_rs,
	range_seg64_t physical_rs, range_seg64_t remain_rs)
	{
	/*
	* Walk up the vdev tree
	*/
	if (vd != vd->vdev_top) {
	vdev_xlate(vd->vdev_parent, logical_rs, physical_rs,
	remain_rs);
	} else {
	/*
	* We've reached the top-level vdev, initialize the physical
	* range to the logical range and set an empty remaining
	* range then start to unwind.
	*/
	physical_rs->rs_start = logical_rs->rs_start;
	physical_rs->rs_end = logical_rs->rs_end;

	remain_rs->rs_start = logical_rs->rs_start;
	remain_rs->rs_end = logical_rs->rs_start;

	return;
	}

	vdev_t *pvd = vd->vdev_parent;
	ASSERT3P(pvd, !=, NULL);
	ASSERT3P(pvd->vdev_ops->vdev_op_xlate, !=, NULL);

	/*
	* As this recursive function unwinds, translate the logical
	* range into its physical and any remaining components by calling
	* the vdev specific translate function.
	*/
	range_seg64_t intermediate = { 0 };
	pvd->vdev_ops->vdev_op_xlate(vd, physical_rs, &intermediate, remain_rs);

	physical_rs->rs_start = intermediate.rs_start;
	physical_rs->rs_end = intermediate.rs_end;
	}

	void
	vdev_xlate_walk(vdev_t vd, const range_seg64_t logical_rs,
	vdev_xlate_func_t func, void arg)
	{
	range_seg64_t iter_rs = *logical_rs;
	range_seg64_t physical_rs;
	range_seg64_t remain_rs;

	while (!vdev_xlate_is_empty(&iter_rs)) {

	vdev_xlate(vd, &iter_rs, &physical_rs, &remain_rs);

	/*
	* With raidz and dRAID, it's possible that the logical range
	* does not live on this leaf vdev. Only when there is a non-
	* zero physical size call the provided function.
	*/
	if (!vdev_xlate_is_empty(&physical_rs))
	func(arg, &physical_rs);

	iter_rs = remain_rs;
	}
	}

	static char *
	vdev_name(vdev_t vd, char buf, int buflen)
	{
	if (vd->vdev_path == NULL) {
	if (strcmp(vd->vdev_ops->vdev_op_type, "root") == 0) {
	strlcpy(buf, vd->vdev_spa->spa_name, buflen);
	} else if (!vd->vdev_ops->vdev_op_leaf) {
	snprintf(buf, buflen, "%s-%llu",
	vd->vdev_ops->vdev_op_type,
	(u_longlong_t)vd->vdev_id);
	}
	} else {
	strlcpy(buf, vd->vdev_path, buflen);
	}
	return (buf);
	}

	/*
	* Look at the vdev tree and determine whether any devices are currently being
	* replaced.
	*/
	boolean_t
	vdev_replace_in_progress(vdev_t *vdev)
	{
	ASSERT(spa_config_held(vdev->vdev_spa, SCL_ALL, RW_READER) != 0);

	if (vdev->vdev_ops == &vdev_replacing_ops)
	return (B_TRUE);

	/*
	* A 'spare' vdev indicates that we have a replace in progress, unless
	* it has exactly two children, and the second, the hot spare, has
	* finished being resilvered.
	*/
	if (vdev->vdev_ops == &vdev_spare_ops && (vdev->vdev_children > 2 \|\|
	!vdev_dtl_empty(vdev->vdev_child[1], DTL_MISSING)))
	return (B_TRUE);

	for (int i = 0; i < vdev->vdev_children; i++) {
	if (vdev_replace_in_progress(vdev->vdev_child[i]))
	return (B_TRUE);
	}

	return (B_FALSE);
	}

	/*
	* Add a (source=src, propname=propval) list to an nvlist.
	*/
	static void
	vdev_prop_add_list(nvlist_t nvl, const char propname, const char *strval,
	uint64_t intval, zprop_source_t src)
	{
	nvlist_t *propval;

	propval = fnvlist_alloc();
	fnvlist_add_uint64(propval, ZPROP_SOURCE, src);

	if (strval != NULL)
	fnvlist_add_string(propval, ZPROP_VALUE, strval);
	else
	fnvlist_add_uint64(propval, ZPROP_VALUE, intval);

	fnvlist_add_nvlist(nvl, propname, propval);
	nvlist_free(propval);
	}

	static void
	vdev_props_set_sync(void arg, dmu_tx_t tx)
	{
	vdev_t *vd;
	nvlist_t *nvp = arg;
	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
	objset_t *mos = spa->spa_meta_objset;
	nvpair_t *elem = NULL;
	uint64_t vdev_guid;
	uint64_t objid;
	nvlist_t *nvprops;

	vdev_guid = fnvlist_lookup_uint64(nvp, ZPOOL_VDEV_PROPS_SET_VDEV);
	nvprops = fnvlist_lookup_nvlist(nvp, ZPOOL_VDEV_PROPS_SET_PROPS);
	vd = spa_lookup_by_guid(spa, vdev_guid, B_TRUE);

	/* this vdev could get removed while waiting for this sync task */
	if (vd == NULL)
	return;

	/*
	* Set vdev property values in the vdev props mos object.
	*/
	if (vd->vdev_root_zap != 0) {
	objid = vd->vdev_root_zap;
	} else if (vd->vdev_top_zap != 0) {
	objid = vd->vdev_top_zap;
	} else if (vd->vdev_leaf_zap != 0) {
	objid = vd->vdev_leaf_zap;
	} else {
	panic("unexpected vdev type");
	}

	mutex_enter(&spa->spa_props_lock);

	while ((elem = nvlist_next_nvpair(nvprops, elem)) != NULL) {
	uint64_t intval;
	const char *strval;
	vdev_prop_t prop;
	const char *propname = nvpair_name(elem);
	zprop_type_t proptype;

	switch (prop = vdev_name_to_prop(propname)) {
	case VDEV_PROP_USERPROP:
	if (vdev_prop_user(propname)) {
	strval = fnvpair_value_string(elem);
	if (strlen(strval) == 0) {
	/* remove the property if value == "" */
	(void) zap_remove(mos, objid, propname,
	tx);
	} else {
	VERIFY0(zap_update(mos, objid, propname,
	1, strlen(strval) + 1, strval, tx));
	}
	spa_history_log_internal(spa, "vdev set", tx,
	"vdev_guid=%llu: %s=%s",
	(u_longlong_t)vdev_guid, nvpair_name(elem),
	strval);
	}
	break;
	default:
	/* normalize the property name */
	propname = vdev_prop_to_name(prop);
	proptype = vdev_prop_get_type(prop);

	if (nvpair_type(elem) == DATA_TYPE_STRING) {
	ASSERT(proptype == PROP_TYPE_STRING);
	strval = fnvpair_value_string(elem);
	VERIFY0(zap_update(mos, objid, propname,
	1, strlen(strval) + 1, strval, tx));
	spa_history_log_internal(spa, "vdev set", tx,
	"vdev_guid=%llu: %s=%s",
	(u_longlong_t)vdev_guid, nvpair_name(elem),
	strval);
	} else if (nvpair_type(elem) == DATA_TYPE_UINT64) {
	intval = fnvpair_value_uint64(elem);

	if (proptype == PROP_TYPE_INDEX) {
	const char *unused;
	VERIFY0(vdev_prop_index_to_string(
	prop, intval, &unused));
	}
	VERIFY0(zap_update(mos, objid, propname,
	sizeof (uint64_t), 1, &intval, tx));
	spa_history_log_internal(spa, "vdev set", tx,
	"vdev_guid=%llu: %s=%lld",
	(u_longlong_t)vdev_guid,
	nvpair_name(elem), (longlong_t)intval);
	} else {
	panic("invalid vdev property type %u",
	nvpair_type(elem));
	}
	}

	}

	mutex_exit(&spa->spa_props_lock);
	}

	int
	vdev_prop_set(vdev_t vd, nvlist_t innvl, nvlist_t *outnvl)
	{
	spa_t *spa = vd->vdev_spa;
	nvpair_t *elem = NULL;
	uint64_t vdev_guid;
	nvlist_t *nvprops;
	int error = 0;

	ASSERT(vd != NULL);

	/* Check that vdev has a zap we can use */
	if (vd->vdev_root_zap == 0 &&
	vd->vdev_top_zap == 0 &&
	vd->vdev_leaf_zap == 0)
	return (SET_ERROR(EINVAL));

	if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_PROPS_SET_VDEV,
	&vdev_guid) != 0)
	return (SET_ERROR(EINVAL));

	if (nvlist_lookup_nvlist(innvl, ZPOOL_VDEV_PROPS_SET_PROPS,
	&nvprops) != 0)
	return (SET_ERROR(EINVAL));

	if ((vd = spa_lookup_by_guid(spa, vdev_guid, B_TRUE)) == NULL)
	return (SET_ERROR(EINVAL));

	while ((elem = nvlist_next_nvpair(nvprops, elem)) != NULL) {
	const char *propname = nvpair_name(elem);
	vdev_prop_t prop = vdev_name_to_prop(propname);
	uint64_t intval = 0;
	const char *strval = NULL;

	if (prop == VDEV_PROP_USERPROP && !vdev_prop_user(propname)) {
	error = EINVAL;
	goto end;
	}

	if (vdev_prop_readonly(prop)) {
	error = EROFS;
	goto end;
	}

	/* Special Processing */
	switch (prop) {
	case VDEV_PROP_PATH:
	if (vd->vdev_path == NULL) {
	error = EROFS;
	break;
	}
	if (nvpair_value_string(elem, &strval) != 0) {
	error = EINVAL;
	break;
	}
	/* New path must start with /dev/ */
	if (strncmp(strval, "/dev/", 5)) {
	error = EINVAL;
	break;
	}
	error = spa_vdev_setpath(spa, vdev_guid, strval);
	break;
	case VDEV_PROP_ALLOCATING:
	if (nvpair_value_uint64(elem, &intval) != 0) {
	error = EINVAL;
	break;
	}
	if (intval != vd->vdev_noalloc)
	break;
	if (intval == 0)
	error = spa_vdev_noalloc(spa, vdev_guid);
	else
	error = spa_vdev_alloc(spa, vdev_guid);
	break;
	case VDEV_PROP_FAILFAST:
	if (nvpair_value_uint64(elem, &intval) != 0) {
	error = EINVAL;
	break;
	}
	vd->vdev_failfast = intval & 1;
	break;
	case VDEV_PROP_CHECKSUM_N:
	if (nvpair_value_uint64(elem, &intval) != 0) {
	error = EINVAL;
	break;
	}
	vd->vdev_checksum_n = intval;
	break;
	case VDEV_PROP_CHECKSUM_T:
	if (nvpair_value_uint64(elem, &intval) != 0) {
	error = EINVAL;
	break;
	}
	vd->vdev_checksum_t = intval;
	break;
	case VDEV_PROP_IO_N:
	if (nvpair_value_uint64(elem, &intval) != 0) {
	error = EINVAL;
	break;
	}
	vd->vdev_io_n = intval;
	break;
	case VDEV_PROP_IO_T:
	if (nvpair_value_uint64(elem, &intval) != 0) {
	error = EINVAL;
	break;
	}
	vd->vdev_io_t = intval;
	break;
	+ case VDEV_PROP_SLOW_IO_N:
	+ if (nvpair_value_uint64(elem, &intval) != 0) {
	+ error = EINVAL;
	+ break;
	+ }
	+ vd->vdev_slow_io_n = intval;
	+ break;
	+ case VDEV_PROP_SLOW_IO_T:
	+ if (nvpair_value_uint64(elem, &intval) != 0) {
	+ error = EINVAL;
	+ break;
	+ }
	+ vd->vdev_slow_io_t = intval;
	+ break;
	default:
	/* Most processing is done in vdev_props_set_sync */
	break;
	}
	end:
	if (error != 0) {
	intval = error;
	vdev_prop_add_list(outnvl, propname, strval, intval, 0);
	return (error);
	}
	}

	return (dsl_sync_task(spa->spa_name, NULL, vdev_props_set_sync,
	innvl, 6, ZFS_SPACE_CHECK_EXTRA_RESERVED));
	}

	int
	vdev_prop_get(vdev_t vd, nvlist_t innvl, nvlist_t *outnvl)
	{
	spa_t *spa = vd->vdev_spa;
	objset_t *mos = spa->spa_meta_objset;
	int err = 0;
	uint64_t objid;
	uint64_t vdev_guid;
	nvpair_t *elem = NULL;
	nvlist_t *nvprops = NULL;
	uint64_t intval = 0;
	char *strval = NULL;
	const char *propname = NULL;
	vdev_prop_t prop;

	ASSERT(vd != NULL);
	ASSERT(mos != NULL);

	if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_PROPS_GET_VDEV,
	&vdev_guid) != 0)
	return (SET_ERROR(EINVAL));

	nvlist_lookup_nvlist(innvl, ZPOOL_VDEV_PROPS_GET_PROPS, &nvprops);

	if (vd->vdev_root_zap != 0) {
	objid = vd->vdev_root_zap;
	} else if (vd->vdev_top_zap != 0) {
	objid = vd->vdev_top_zap;
	} else if (vd->vdev_leaf_zap != 0) {
	objid = vd->vdev_leaf_zap;
	} else {
	return (SET_ERROR(EINVAL));
	}
	ASSERT(objid != 0);

	mutex_enter(&spa->spa_props_lock);

	if (nvprops != NULL) {
	char namebuf[64] = { 0 };

	while ((elem = nvlist_next_nvpair(nvprops, elem)) != NULL) {
	intval = 0;
	strval = NULL;
	propname = nvpair_name(elem);
	prop = vdev_name_to_prop(propname);
	zprop_source_t src = ZPROP_SRC_DEFAULT;
	uint64_t integer_size, num_integers;

	switch (prop) {
	/* Special Read-only Properties */
	case VDEV_PROP_NAME:
	strval = vdev_name(vd, namebuf,
	sizeof (namebuf));
	if (strval == NULL)
	continue;
	vdev_prop_add_list(outnvl, propname, strval, 0,
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_CAPACITY:
	/* percent used */
	intval = (vd->vdev_stat.vs_dspace == 0) ? 0 :
	(vd->vdev_stat.vs_alloc * 100 /
	vd->vdev_stat.vs_dspace);
	vdev_prop_add_list(outnvl, propname, NULL,
	intval, ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_STATE:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_state, ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_GUID:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_guid, ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_ASIZE:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_asize, ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_PSIZE:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_psize, ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_ASHIFT:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_ashift, ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_SIZE:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_dspace, ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_FREE:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_dspace -
	vd->vdev_stat.vs_alloc, ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_ALLOCATED:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_alloc, ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_EXPANDSZ:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_esize, ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_FRAGMENTATION:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_fragmentation,
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_PARITY:
	vdev_prop_add_list(outnvl, propname, NULL,
	vdev_get_nparity(vd), ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_PATH:
	if (vd->vdev_path == NULL)
	continue;
	vdev_prop_add_list(outnvl, propname,
	vd->vdev_path, 0, ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_DEVID:
	if (vd->vdev_devid == NULL)
	continue;
	vdev_prop_add_list(outnvl, propname,
	vd->vdev_devid, 0, ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_PHYS_PATH:
	if (vd->vdev_physpath == NULL)
	continue;
	vdev_prop_add_list(outnvl, propname,
	vd->vdev_physpath, 0, ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_ENC_PATH:
	if (vd->vdev_enc_sysfs_path == NULL)
	continue;
	vdev_prop_add_list(outnvl, propname,
	vd->vdev_enc_sysfs_path, 0, ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_FRU:
	if (vd->vdev_fru == NULL)
	continue;
	vdev_prop_add_list(outnvl, propname,
	vd->vdev_fru, 0, ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_PARENT:
	if (vd->vdev_parent != NULL) {
	strval = vdev_name(vd->vdev_parent,
	namebuf, sizeof (namebuf));
	vdev_prop_add_list(outnvl, propname,
	strval, 0, ZPROP_SRC_NONE);
	}
	continue;
	case VDEV_PROP_CHILDREN:
	if (vd->vdev_children > 0)
	strval = kmem_zalloc(ZAP_MAXVALUELEN,
	KM_SLEEP);
	for (uint64_t i = 0; i < vd->vdev_children;
	i++) {
	const char *vname;

	vname = vdev_name(vd->vdev_child[i],
	namebuf, sizeof (namebuf));
	if (vname == NULL)
	vname = "(unknown)";
	if (strlen(strval) > 0)
	strlcat(strval, ",",
	ZAP_MAXVALUELEN);
	strlcat(strval, vname, ZAP_MAXVALUELEN);
	}
	if (strval != NULL) {
	vdev_prop_add_list(outnvl, propname,
	strval, 0, ZPROP_SRC_NONE);
	kmem_free(strval, ZAP_MAXVALUELEN);
	}
	continue;
	case VDEV_PROP_NUMCHILDREN:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_children, ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_READ_ERRORS:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_read_errors,
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_WRITE_ERRORS:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_write_errors,
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_CHECKSUM_ERRORS:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_checksum_errors,
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_INITIALIZE_ERRORS:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_initialize_errors,
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_OPS_NULL:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_ops[ZIO_TYPE_NULL],
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_OPS_READ:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_ops[ZIO_TYPE_READ],
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_OPS_WRITE:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_ops[ZIO_TYPE_WRITE],
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_OPS_FREE:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_ops[ZIO_TYPE_FREE],
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_OPS_CLAIM:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_ops[ZIO_TYPE_CLAIM],
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_OPS_TRIM:
	/*
	* TRIM ops and bytes are reported to user
	* space as ZIO_TYPE_IOCTL. This is done to
	* preserve the vdev_stat_t structure layout
	* for user space.
	*/
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_ops[ZIO_TYPE_IOCTL],
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_BYTES_NULL:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_bytes[ZIO_TYPE_NULL],
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_BYTES_READ:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_bytes[ZIO_TYPE_READ],
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_BYTES_WRITE:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_bytes[ZIO_TYPE_WRITE],
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_BYTES_FREE:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_bytes[ZIO_TYPE_FREE],
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_BYTES_CLAIM:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_bytes[ZIO_TYPE_CLAIM],
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_BYTES_TRIM:
	/*
	* TRIM ops and bytes are reported to user
	* space as ZIO_TYPE_IOCTL. This is done to
	* preserve the vdev_stat_t structure layout
	* for user space.
	*/
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_stat.vs_bytes[ZIO_TYPE_IOCTL],
	ZPROP_SRC_NONE);
	continue;
	case VDEV_PROP_REMOVING:
	vdev_prop_add_list(outnvl, propname, NULL,
	vd->vdev_removing, ZPROP_SRC_NONE);
	continue;
	/* Numeric Properites */
	case VDEV_PROP_ALLOCATING:
	/* Leaf vdevs cannot have this property */
	if (vd->vdev_mg == NULL &&
	vd->vdev_top != NULL) {
	src = ZPROP_SRC_NONE;
	intval = ZPROP_BOOLEAN_NA;
	} else {
	err = vdev_prop_get_int(vd, prop,
	&intval);
	if (err && err != ENOENT)
	break;

	if (intval ==
	vdev_prop_default_numeric(prop))
	src = ZPROP_SRC_DEFAULT;
	else
	src = ZPROP_SRC_LOCAL;
	}

	vdev_prop_add_list(outnvl, propname, NULL,
	intval, src);
	break;
	case VDEV_PROP_FAILFAST:
	src = ZPROP_SRC_LOCAL;
	strval = NULL;

	err = zap_lookup(mos, objid, nvpair_name(elem),
	sizeof (uint64_t), 1, &intval);
	if (err == ENOENT) {
	intval = vdev_prop_default_numeric(
	prop);
	err = 0;
	} else if (err) {
	break;
	}
	if (intval == vdev_prop_default_numeric(prop))
	src = ZPROP_SRC_DEFAULT;

	vdev_prop_add_list(outnvl, propname, strval,
	intval, src);
	break;
	case VDEV_PROP_CHECKSUM_N:
	case VDEV_PROP_CHECKSUM_T:
	case VDEV_PROP_IO_N:
	case VDEV_PROP_IO_T:
	+ case VDEV_PROP_SLOW_IO_N:
	+ case VDEV_PROP_SLOW_IO_T:
	err = vdev_prop_get_int(vd, prop, &intval);
	if (err && err != ENOENT)
	break;

	if (intval == vdev_prop_default_numeric(prop))
	src = ZPROP_SRC_DEFAULT;
	else
	src = ZPROP_SRC_LOCAL;

	vdev_prop_add_list(outnvl, propname, NULL,
	intval, src);
	break;
	/* Text Properties */
	case VDEV_PROP_COMMENT:
	/* Exists in the ZAP below */
	/* FALLTHRU */
	case VDEV_PROP_USERPROP:
	/* User Properites */
	src = ZPROP_SRC_LOCAL;

	err = zap_length(mos, objid, nvpair_name(elem),
	&integer_size, &num_integers);
	if (err)
	break;

	switch (integer_size) {
	case 8:
	/* User properties cannot be integers */
	err = EINVAL;
	break;
	case 1:
	/* string property */
	strval = kmem_alloc(num_integers,
	KM_SLEEP);
	err = zap_lookup(mos, objid,
	nvpair_name(elem), 1,
	num_integers, strval);
	if (err) {
	kmem_free(strval,
	num_integers);
	break;
	}
	vdev_prop_add_list(outnvl, propname,
	strval, 0, src);
	kmem_free(strval, num_integers);
	break;
	}
	break;
	default:
	err = ENOENT;
	break;
	}
	if (err)
	break;
	}
	} else {
	/*
	* Get all properties from the MOS vdev property object.
	*/
	zap_cursor_t zc;
	zap_attribute_t za;
	for (zap_cursor_init(&zc, mos, objid);
	(err = zap_cursor_retrieve(&zc, &za)) == 0;
	zap_cursor_advance(&zc)) {
	intval = 0;
	strval = NULL;
	zprop_source_t src = ZPROP_SRC_DEFAULT;
	propname = za.za_name;

	switch (za.za_integer_length) {
	case 8:
	/* We do not allow integer user properties */
	/* This is likely an internal value */
	break;
	case 1:
	/* string property */
	strval = kmem_alloc(za.za_num_integers,
	KM_SLEEP);
	err = zap_lookup(mos, objid, za.za_name, 1,
	za.za_num_integers, strval);
	if (err) {
	kmem_free(strval, za.za_num_integers);
	break;
	}
	vdev_prop_add_list(outnvl, propname, strval, 0,
	src);
	kmem_free(strval, za.za_num_integers);
	break;

	default:
	break;
	}
	}
	zap_cursor_fini(&zc);
	}

	mutex_exit(&spa->spa_props_lock);
	if (err && err != ENOENT) {
	return (err);
	}

	return (0);
	}

	EXPORT_SYMBOL(vdev_fault);
	EXPORT_SYMBOL(vdev_degrade);
	EXPORT_SYMBOL(vdev_online);
	EXPORT_SYMBOL(vdev_offline);
	EXPORT_SYMBOL(vdev_clear);

	ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, default_ms_count, UINT, ZMOD_RW,
	"Target number of metaslabs per top-level vdev");

	ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, default_ms_shift, UINT, ZMOD_RW,
	"Default lower limit for metaslab size");

	ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, max_ms_shift, UINT, ZMOD_RW,
	"Default upper limit for metaslab size");

	ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, min_ms_count, UINT, ZMOD_RW,
	"Minimum number of metaslabs per top-level vdev");

	ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, ms_count_limit, UINT, ZMOD_RW,
	"Practical upper limit of total metaslabs per top-level vdev");

	ZFS_MODULE_PARAM(zfs, zfs_, slow_io_events_per_second, UINT, ZMOD_RW,
	"Rate limit slow IO (delay) events to this many per second");

	/* BEGIN CSTYLED */
	ZFS_MODULE_PARAM(zfs, zfs_, checksum_events_per_second, UINT, ZMOD_RW,
	"Rate limit checksum events to this many checksum errors per second "
	"(do not set below ZED threshold).");
	/* END CSTYLED */

	ZFS_MODULE_PARAM(zfs, zfs_, scan_ignore_errors, INT, ZMOD_RW,
	"Ignore errors during resilver/scrub");

	ZFS_MODULE_PARAM(zfs_vdev, vdev_, validate_skip, INT, ZMOD_RW,
	"Bypass vdev_validate()");

	ZFS_MODULE_PARAM(zfs, zfs_, nocacheflush, INT, ZMOD_RW,
	"Disable cache flushes");

	ZFS_MODULE_PARAM(zfs, zfs_, embedded_slog_min_ms, UINT, ZMOD_RW,
	"Minimum number of metaslabs required to dedicate one for log blocks");

	/* BEGIN CSTYLED */
	ZFS_MODULE_PARAM_CALL(zfs_vdev, zfs_vdev_, min_auto_ashift,
	param_set_min_auto_ashift, param_get_uint, ZMOD_RW,
	"Minimum ashift used when creating new top-level vdevs");

	ZFS_MODULE_PARAM_CALL(zfs_vdev, zfs_vdev_, max_auto_ashift,
	param_set_max_auto_ashift, param_get_uint, ZMOD_RW,
	"Maximum ashift used when optimizing for logical -> physical sector "
	"size on new top-level vdevs");
	/* END CSTYLED */
	diff --git a/sys/contrib/openzfs/module/zfs/vdev_label.c b/sys/contrib/openzfs/module/zfs/vdev_label.c
	index 737d8b33e188..5c0e750c4614 100644
	--- a/sys/contrib/openzfs/module/zfs/vdev_label.c
	+++ b/sys/contrib/openzfs/module/zfs/vdev_label.c
	@@ -1,2080 +1,2082 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2012, 2020 by Delphix. All rights reserved.
	* Copyright (c) 2017, Intel Corporation.
	*/

	/*
	* Virtual Device Labels
	* ---------------------
	*
	* The vdev label serves several distinct purposes:
	*
	* 1. Uniquely identify this device as part of a ZFS pool and confirm its
	* identity within the pool.
	*
	* 2. Verify that all the devices given in a configuration are present
	* within the pool.
	*
	* 3. Determine the uberblock for the pool.
	*
	* 4. In case of an import operation, determine the configuration of the
	* toplevel vdev of which it is a part.
	*
	* 5. If an import operation cannot find all the devices in the pool,
	* provide enough information to the administrator to determine which
	* devices are missing.
	*
	* It is important to note that while the kernel is responsible for writing the
	* label, it only consumes the information in the first three cases. The
	* latter information is only consumed in userland when determining the
	* configuration to import a pool.
	*
	*
	* Label Organization
	* ------------------
	*
	* Before describing the contents of the label, it's important to understand how
	* the labels are written and updated with respect to the uberblock.
	*
	* When the pool configuration is altered, either because it was newly created
	* or a device was added, we want to update all the labels such that we can deal
	* with fatal failure at any point. To this end, each disk has two labels which
	* are updated before and after the uberblock is synced. Assuming we have
	* labels and an uberblock with the following transaction groups:
	*
	* L1 UB L2
	* +------+ +------+ +------+
	* \| \| \| \| \| \|
	* \| t10 \| \| t10 \| \| t10 \|
	* \| \| \| \| \| \|
	* +------+ +------+ +------+
	*
	* In this stable state, the labels and the uberblock were all updated within
	* the same transaction group (10). Each label is mirrored and checksummed, so
	* that we can detect when we fail partway through writing the label.
	*
	* In order to identify which labels are valid, the labels are written in the
	* following manner:
	*
	* 1. For each vdev, update 'L1' to the new label
	* 2. Update the uberblock
	* 3. For each vdev, update 'L2' to the new label
	*
	* Given arbitrary failure, we can determine the correct label to use based on
	* the transaction group. If we fail after updating L1 but before updating the
	* UB, we will notice that L1's transaction group is greater than the uberblock,
	* so L2 must be valid. If we fail after writing the uberblock but before
	* writing L2, we will notice that L2's transaction group is less than L1, and
	* therefore L1 is valid.
	*
	* Another added complexity is that not every label is updated when the config
	* is synced. If we add a single device, we do not want to have to re-write
	* every label for every device in the pool. This means that both L1 and L2 may
	* be older than the pool uberblock, because the necessary information is stored
	* on another vdev.
	*
	*
	* On-disk Format
	* --------------
	*
	* The vdev label consists of two distinct parts, and is wrapped within the
	* vdev_label_t structure. The label includes 8k of padding to permit legacy
	* VTOC disk labels, but is otherwise ignored.
	*
	* The first half of the label is a packed nvlist which contains pool wide
	* properties, per-vdev properties, and configuration information. It is
	* described in more detail below.
	*
	* The latter half of the label consists of a redundant array of uberblocks.
	* These uberblocks are updated whenever a transaction group is committed,
	* or when the configuration is updated. When a pool is loaded, we scan each
	* vdev for the 'best' uberblock.
	*
	*
	* Configuration Information
	* -------------------------
	*
	* The nvlist describing the pool and vdev contains the following elements:
	*
	* version ZFS on-disk version
	* name Pool name
	* state Pool state
	* txg Transaction group in which this label was written
	* pool_guid Unique identifier for this pool
	* vdev_tree An nvlist describing vdev tree.
	* features_for_read
	* An nvlist of the features necessary for reading the MOS.
	*
	* Each leaf device label also contains the following:
	*
	* top_guid Unique ID for top-level vdev in which this is contained
	* guid Unique ID for the leaf vdev
	*
	* The 'vs' configuration follows the format described in 'spa_config.c'.
	*/

	#include <sys/zfs_context.h>
	#include <sys/spa.h>
	#include <sys/spa_impl.h>
	#include <sys/dmu.h>
	#include <sys/zap.h>
	#include <sys/vdev.h>
	#include <sys/vdev_impl.h>
	#include <sys/vdev_draid.h>
	#include <sys/uberblock_impl.h>
	#include <sys/metaslab.h>
	#include <sys/metaslab_impl.h>
	#include <sys/zio.h>
	#include <sys/dsl_scan.h>
	#include <sys/abd.h>
	#include <sys/fs/zfs.h>
	#include <sys/byteorder.h>
	#include <sys/zfs_bootenv.h>

	/*
	* Basic routines to read and write from a vdev label.
	* Used throughout the rest of this file.
	*/
	uint64_t
	vdev_label_offset(uint64_t psize, int l, uint64_t offset)
	{
	ASSERT(offset < sizeof (vdev_label_t));
	ASSERT(P2PHASE_TYPED(psize, sizeof (vdev_label_t), uint64_t) == 0);

	return (offset + l * sizeof (vdev_label_t) + (l < VDEV_LABELS / 2 ?
	0 : psize - VDEV_LABELS * sizeof (vdev_label_t)));
	}

	/*
	* Returns back the vdev label associated with the passed in offset.
	*/
	int
	vdev_label_number(uint64_t psize, uint64_t offset)
	{
	int l;

	if (offset >= psize - VDEV_LABEL_END_SIZE) {
	offset -= psize - VDEV_LABEL_END_SIZE;
	offset += (VDEV_LABELS / 2) * sizeof (vdev_label_t);
	}
	l = offset / sizeof (vdev_label_t);
	return (l < VDEV_LABELS ? l : -1);
	}

	static void
	vdev_label_read(zio_t zio, vdev_t vd, int l, abd_t *buf, uint64_t offset,
	uint64_t size, zio_done_func_t done, void private, int flags)
	{
	ASSERT(
	spa_config_held(zio->io_spa, SCL_STATE, RW_READER) == SCL_STATE \|\|
	spa_config_held(zio->io_spa, SCL_STATE, RW_WRITER) == SCL_STATE);
	ASSERT(flags & ZIO_FLAG_CONFIG_WRITER);

	zio_nowait(zio_read_phys(zio, vd,
	vdev_label_offset(vd->vdev_psize, l, offset),
	size, buf, ZIO_CHECKSUM_LABEL, done, private,
	ZIO_PRIORITY_SYNC_READ, flags, B_TRUE));
	}

	void
	vdev_label_write(zio_t zio, vdev_t vd, int l, abd_t *buf, uint64_t offset,
	uint64_t size, zio_done_func_t done, void private, int flags)
	{
	ASSERT(
	spa_config_held(zio->io_spa, SCL_STATE, RW_READER) == SCL_STATE \|\|
	spa_config_held(zio->io_spa, SCL_STATE, RW_WRITER) == SCL_STATE);
	ASSERT(flags & ZIO_FLAG_CONFIG_WRITER);

	zio_nowait(zio_write_phys(zio, vd,
	vdev_label_offset(vd->vdev_psize, l, offset),
	size, buf, ZIO_CHECKSUM_LABEL, done, private,
	ZIO_PRIORITY_SYNC_WRITE, flags, B_TRUE));
	}

	/*
	* Generate the nvlist representing this vdev's stats
	*/
	void
	vdev_config_generate_stats(vdev_t vd, nvlist_t nv)
	{
	nvlist_t *nvx;
	vdev_stat_t *vs;
	vdev_stat_ex_t *vsx;

	vs = kmem_alloc(sizeof (*vs), KM_SLEEP);
	vsx = kmem_alloc(sizeof (*vsx), KM_SLEEP);

	vdev_get_stats_ex(vd, vs, vsx);
	fnvlist_add_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
	(uint64_t )vs, sizeof (vs) / sizeof (uint64_t));

	/*
	* Add extended stats into a special extended stats nvlist. This keeps
	* all the extended stats nicely grouped together. The extended stats
	* nvlist is then added to the main nvlist.
	*/
	nvx = fnvlist_alloc();

	/* ZIOs in flight to disk */
	fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
	vsx->vsx_active_queue[ZIO_PRIORITY_SYNC_READ]);

	fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
	vsx->vsx_active_queue[ZIO_PRIORITY_SYNC_WRITE]);

	fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
	vsx->vsx_active_queue[ZIO_PRIORITY_ASYNC_READ]);

	fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
	vsx->vsx_active_queue[ZIO_PRIORITY_ASYNC_WRITE]);

	fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
	vsx->vsx_active_queue[ZIO_PRIORITY_SCRUB]);

	fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE,
	vsx->vsx_active_queue[ZIO_PRIORITY_TRIM]);

	fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_REBUILD_ACTIVE_QUEUE,
	vsx->vsx_active_queue[ZIO_PRIORITY_REBUILD]);

	/* ZIOs pending */
	fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_SYNC_R_PEND_QUEUE,
	vsx->vsx_pend_queue[ZIO_PRIORITY_SYNC_READ]);

	fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_SYNC_W_PEND_QUEUE,
	vsx->vsx_pend_queue[ZIO_PRIORITY_SYNC_WRITE]);

	fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_ASYNC_R_PEND_QUEUE,
	vsx->vsx_pend_queue[ZIO_PRIORITY_ASYNC_READ]);

	fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_ASYNC_W_PEND_QUEUE,
	vsx->vsx_pend_queue[ZIO_PRIORITY_ASYNC_WRITE]);

	fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_SCRUB_PEND_QUEUE,
	vsx->vsx_pend_queue[ZIO_PRIORITY_SCRUB]);

	fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_TRIM_PEND_QUEUE,
	vsx->vsx_pend_queue[ZIO_PRIORITY_TRIM]);

	fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_REBUILD_PEND_QUEUE,
	vsx->vsx_pend_queue[ZIO_PRIORITY_REBUILD]);

	/* Histograms */
	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
	vsx->vsx_total_histo[ZIO_TYPE_READ],
	ARRAY_SIZE(vsx->vsx_total_histo[ZIO_TYPE_READ]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
	vsx->vsx_total_histo[ZIO_TYPE_WRITE],
	ARRAY_SIZE(vsx->vsx_total_histo[ZIO_TYPE_WRITE]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
	vsx->vsx_disk_histo[ZIO_TYPE_READ],
	ARRAY_SIZE(vsx->vsx_disk_histo[ZIO_TYPE_READ]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
	vsx->vsx_disk_histo[ZIO_TYPE_WRITE],
	ARRAY_SIZE(vsx->vsx_disk_histo[ZIO_TYPE_WRITE]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
	vsx->vsx_queue_histo[ZIO_PRIORITY_SYNC_READ],
	ARRAY_SIZE(vsx->vsx_queue_histo[ZIO_PRIORITY_SYNC_READ]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
	vsx->vsx_queue_histo[ZIO_PRIORITY_SYNC_WRITE],
	ARRAY_SIZE(vsx->vsx_queue_histo[ZIO_PRIORITY_SYNC_WRITE]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
	vsx->vsx_queue_histo[ZIO_PRIORITY_ASYNC_READ],
	ARRAY_SIZE(vsx->vsx_queue_histo[ZIO_PRIORITY_ASYNC_READ]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
	vsx->vsx_queue_histo[ZIO_PRIORITY_ASYNC_WRITE],
	ARRAY_SIZE(vsx->vsx_queue_histo[ZIO_PRIORITY_ASYNC_WRITE]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
	vsx->vsx_queue_histo[ZIO_PRIORITY_SCRUB],
	ARRAY_SIZE(vsx->vsx_queue_histo[ZIO_PRIORITY_SCRUB]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
	vsx->vsx_queue_histo[ZIO_PRIORITY_TRIM],
	ARRAY_SIZE(vsx->vsx_queue_histo[ZIO_PRIORITY_TRIM]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_REBUILD_LAT_HISTO,
	vsx->vsx_queue_histo[ZIO_PRIORITY_REBUILD],
	ARRAY_SIZE(vsx->vsx_queue_histo[ZIO_PRIORITY_REBUILD]));

	/* Request sizes */
	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SYNC_IND_R_HISTO,
	vsx->vsx_ind_histo[ZIO_PRIORITY_SYNC_READ],
	ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_SYNC_READ]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SYNC_IND_W_HISTO,
	vsx->vsx_ind_histo[ZIO_PRIORITY_SYNC_WRITE],
	ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_SYNC_WRITE]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_ASYNC_IND_R_HISTO,
	vsx->vsx_ind_histo[ZIO_PRIORITY_ASYNC_READ],
	ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_ASYNC_READ]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_ASYNC_IND_W_HISTO,
	vsx->vsx_ind_histo[ZIO_PRIORITY_ASYNC_WRITE],
	ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_ASYNC_WRITE]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_IND_SCRUB_HISTO,
	vsx->vsx_ind_histo[ZIO_PRIORITY_SCRUB],
	ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_SCRUB]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_IND_TRIM_HISTO,
	vsx->vsx_ind_histo[ZIO_PRIORITY_TRIM],
	ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_TRIM]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_IND_REBUILD_HISTO,
	vsx->vsx_ind_histo[ZIO_PRIORITY_REBUILD],
	ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_REBUILD]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SYNC_AGG_R_HISTO,
	vsx->vsx_agg_histo[ZIO_PRIORITY_SYNC_READ],
	ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_SYNC_READ]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SYNC_AGG_W_HISTO,
	vsx->vsx_agg_histo[ZIO_PRIORITY_SYNC_WRITE],
	ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_SYNC_WRITE]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_ASYNC_AGG_R_HISTO,
	vsx->vsx_agg_histo[ZIO_PRIORITY_ASYNC_READ],
	ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_ASYNC_READ]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_ASYNC_AGG_W_HISTO,
	vsx->vsx_agg_histo[ZIO_PRIORITY_ASYNC_WRITE],
	ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_ASYNC_WRITE]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_AGG_SCRUB_HISTO,
	vsx->vsx_agg_histo[ZIO_PRIORITY_SCRUB],
	ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_SCRUB]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_AGG_TRIM_HISTO,
	vsx->vsx_agg_histo[ZIO_PRIORITY_TRIM],
	ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_TRIM]));

	fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_AGG_REBUILD_HISTO,
	vsx->vsx_agg_histo[ZIO_PRIORITY_REBUILD],
	ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_REBUILD]));

	/* IO delays */
	fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_SLOW_IOS, vs->vs_slow_ios);

	/* Add extended stats nvlist to main nvlist */
	fnvlist_add_nvlist(nv, ZPOOL_CONFIG_VDEV_STATS_EX, nvx);

	fnvlist_free(nvx);
	kmem_free(vs, sizeof (*vs));
	kmem_free(vsx, sizeof (*vsx));
	}

	static void
	root_vdev_actions_getprogress(vdev_t vd, nvlist_t nvl)
	{
	spa_t *spa = vd->vdev_spa;

	if (vd != spa->spa_root_vdev)
	return;

	/* provide either current or previous scan information */
	pool_scan_stat_t ps;
	if (spa_scan_get_stats(spa, &ps) == 0) {
	fnvlist_add_uint64_array(nvl,
	ZPOOL_CONFIG_SCAN_STATS, (uint64_t *)&ps,
	sizeof (pool_scan_stat_t) / sizeof (uint64_t));
	}

	pool_removal_stat_t prs;
	if (spa_removal_get_stats(spa, &prs) == 0) {
	fnvlist_add_uint64_array(nvl,
	ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t *)&prs,
	sizeof (prs) / sizeof (uint64_t));
	}

	pool_checkpoint_stat_t pcs;
	if (spa_checkpoint_get_stats(spa, &pcs) == 0) {
	fnvlist_add_uint64_array(nvl,
	ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t *)&pcs,
	sizeof (pcs) / sizeof (uint64_t));
	}
	}

	static void
	top_vdev_actions_getprogress(vdev_t vd, nvlist_t nvl)
	{
	if (vd == vd->vdev_top) {
	vdev_rebuild_stat_t vrs;
	if (vdev_rebuild_get_stats(vd, &vrs) == 0) {
	fnvlist_add_uint64_array(nvl,
	ZPOOL_CONFIG_REBUILD_STATS, (uint64_t *)&vrs,
	sizeof (vrs) / sizeof (uint64_t));
	}
	}
	}

	/*
	* Generate the nvlist representing this vdev's config.
	*/
	nvlist_t *
	vdev_config_generate(spa_t spa, vdev_t vd, boolean_t getstats,
	vdev_config_flag_t flags)
	{
	nvlist_t *nv = NULL;
	vdev_indirect_config_t *vic = &vd->vdev_indirect_config;

	nv = fnvlist_alloc();

	fnvlist_add_string(nv, ZPOOL_CONFIG_TYPE, vd->vdev_ops->vdev_op_type);
	if (!(flags & (VDEV_CONFIG_SPARE \| VDEV_CONFIG_L2CACHE)))
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_ID, vd->vdev_id);
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_GUID, vd->vdev_guid);

	if (vd->vdev_path != NULL)
	fnvlist_add_string(nv, ZPOOL_CONFIG_PATH, vd->vdev_path);

	if (vd->vdev_devid != NULL)
	fnvlist_add_string(nv, ZPOOL_CONFIG_DEVID, vd->vdev_devid);

	if (vd->vdev_physpath != NULL)
	fnvlist_add_string(nv, ZPOOL_CONFIG_PHYS_PATH,
	vd->vdev_physpath);

	if (vd->vdev_enc_sysfs_path != NULL)
	fnvlist_add_string(nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH,
	vd->vdev_enc_sysfs_path);

	if (vd->vdev_fru != NULL)
	fnvlist_add_string(nv, ZPOOL_CONFIG_FRU, vd->vdev_fru);

	if (vd->vdev_ops->vdev_op_config_generate != NULL)
	vd->vdev_ops->vdev_op_config_generate(vd, nv);

	if (vd->vdev_wholedisk != -1ULL) {
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_WHOLE_DISK,
	vd->vdev_wholedisk);
	}

	if (vd->vdev_not_present && !(flags & VDEV_CONFIG_MISSING))
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT, 1);

	if (vd->vdev_isspare)
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_IS_SPARE, 1);

	if (flags & VDEV_CONFIG_L2CACHE)
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_ASHIFT, vd->vdev_ashift);

	if (!(flags & (VDEV_CONFIG_SPARE \| VDEV_CONFIG_L2CACHE)) &&
	vd == vd->vdev_top) {
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_METASLAB_ARRAY,
	vd->vdev_ms_array);
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_METASLAB_SHIFT,
	vd->vdev_ms_shift);
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_ASHIFT, vd->vdev_ashift);
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_ASIZE,
	vd->vdev_asize);
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_IS_LOG, vd->vdev_islog);
	if (vd->vdev_noalloc) {
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_NONALLOCATING,
	vd->vdev_noalloc);
	}

	/*
	* Slog devices are removed synchronously so don't
	* persist the vdev_removing flag to the label.
	*/
	if (vd->vdev_removing && !vd->vdev_islog) {
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_REMOVING,
	vd->vdev_removing);
	}

	/* zpool command expects alloc class data */
	if (getstats && vd->vdev_alloc_bias != VDEV_BIAS_NONE) {
	const char *bias = NULL;

	switch (vd->vdev_alloc_bias) {
	case VDEV_BIAS_LOG:
	bias = VDEV_ALLOC_BIAS_LOG;
	break;
	case VDEV_BIAS_SPECIAL:
	bias = VDEV_ALLOC_BIAS_SPECIAL;
	break;
	case VDEV_BIAS_DEDUP:
	bias = VDEV_ALLOC_BIAS_DEDUP;
	break;
	default:
	ASSERT3U(vd->vdev_alloc_bias, ==,
	VDEV_BIAS_NONE);
	}
	fnvlist_add_string(nv, ZPOOL_CONFIG_ALLOCATION_BIAS,
	bias);
	}
	}

	if (vd->vdev_dtl_sm != NULL) {
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_DTL,
	space_map_object(vd->vdev_dtl_sm));
	}

	if (vic->vic_mapping_object != 0) {
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_INDIRECT_OBJECT,
	vic->vic_mapping_object);
	}

	if (vic->vic_births_object != 0) {
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_INDIRECT_BIRTHS,
	vic->vic_births_object);
	}

	if (vic->vic_prev_indirect_vdev != UINT64_MAX) {
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_PREV_INDIRECT_VDEV,
	vic->vic_prev_indirect_vdev);
	}

	if (vd->vdev_crtxg)
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_CREATE_TXG, vd->vdev_crtxg);

	if (vd->vdev_expansion_time)
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_EXPANSION_TIME,
	vd->vdev_expansion_time);

	if (flags & VDEV_CONFIG_MOS) {
	if (vd->vdev_leaf_zap != 0) {
	ASSERT(vd->vdev_ops->vdev_op_leaf);
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_VDEV_LEAF_ZAP,
	vd->vdev_leaf_zap);
	}

	if (vd->vdev_top_zap != 0) {
	ASSERT(vd == vd->vdev_top);
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_VDEV_TOP_ZAP,
	vd->vdev_top_zap);
	}

	if (vd->vdev_ops == &vdev_root_ops && vd->vdev_root_zap != 0 &&
	spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_AVZ_V2)) {
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_VDEV_ROOT_ZAP,
	vd->vdev_root_zap);
	}

	if (vd->vdev_resilver_deferred) {
	ASSERT(vd->vdev_ops->vdev_op_leaf);
	ASSERT(spa->spa_resilver_deferred);
	fnvlist_add_boolean(nv, ZPOOL_CONFIG_RESILVER_DEFER);
	}
	}

	if (getstats) {
	vdev_config_generate_stats(vd, nv);

	root_vdev_actions_getprogress(vd, nv);
	top_vdev_actions_getprogress(vd, nv);

	/*
	* Note: this can be called from open context
	* (spa_get_stats()), so we need the rwlock to prevent
	* the mapping from being changed by condensing.
	*/
	rw_enter(&vd->vdev_indirect_rwlock, RW_READER);
	if (vd->vdev_indirect_mapping != NULL) {
	ASSERT(vd->vdev_indirect_births != NULL);
	vdev_indirect_mapping_t *vim =
	vd->vdev_indirect_mapping;
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_INDIRECT_SIZE,
	vdev_indirect_mapping_size(vim));
	}
	rw_exit(&vd->vdev_indirect_rwlock);
	if (vd->vdev_mg != NULL &&
	vd->vdev_mg->mg_fragmentation != ZFS_FRAG_INVALID) {
	/*
	* Compute approximately how much memory would be used
	* for the indirect mapping if this device were to
	* be removed.
	*
	* Note: If the frag metric is invalid, then not
	* enough metaslabs have been converted to have
	* histograms.
	*/
	uint64_t seg_count = 0;
	uint64_t to_alloc = vd->vdev_stat.vs_alloc;

	/*
	* There are the same number of allocated segments
	* as free segments, so we will have at least one
	* entry per free segment. However, small free
	* segments (smaller than vdev_removal_max_span)
	* will be combined with adjacent allocated segments
	* as a single mapping.
	*/
	for (int i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++) {
	if (i + 1 < highbit64(vdev_removal_max_span)
	- 1) {
	to_alloc +=
	vd->vdev_mg->mg_histogram[i] <<
	(i + 1);
	} else {
	seg_count +=
	vd->vdev_mg->mg_histogram[i];
	}
	}

	/*
	* The maximum length of a mapping is
	* zfs_remove_max_segment, so we need at least one entry
	* per zfs_remove_max_segment of allocated data.
	*/
	seg_count += to_alloc / spa_remove_max_segment(spa);

	fnvlist_add_uint64(nv, ZPOOL_CONFIG_INDIRECT_SIZE,
	seg_count *
	sizeof (vdev_indirect_mapping_entry_phys_t));
	}
	}

	if (!vd->vdev_ops->vdev_op_leaf) {
	nvlist_t **child;
	uint64_t c;

	ASSERT(!vd->vdev_ishole);

	child = kmem_alloc(vd->vdev_children * sizeof (nvlist_t *),
	KM_SLEEP);

	for (c = 0; c < vd->vdev_children; c++) {
	child[c] = vdev_config_generate(spa, vd->vdev_child[c],
	getstats, flags);
	}

	fnvlist_add_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
	(const nvlist_t * const *)child, vd->vdev_children);

	for (c = 0; c < vd->vdev_children; c++)
	nvlist_free(child[c]);

	kmem_free(child, vd->vdev_children * sizeof (nvlist_t *));

	} else {
	const char *aux = NULL;

	if (vd->vdev_offline && !vd->vdev_tmpoffline)
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_OFFLINE, B_TRUE);
	if (vd->vdev_resilver_txg != 0)
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_RESILVER_TXG,
	vd->vdev_resilver_txg);
	if (vd->vdev_rebuild_txg != 0)
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_REBUILD_TXG,
	vd->vdev_rebuild_txg);
	if (vd->vdev_faulted)
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_FAULTED, B_TRUE);
	if (vd->vdev_degraded)
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_DEGRADED, B_TRUE);
	if (vd->vdev_removed)
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_REMOVED, B_TRUE);
	if (vd->vdev_unspare)
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_UNSPARE, B_TRUE);
	if (vd->vdev_ishole)
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_IS_HOLE, B_TRUE);

	/* Set the reason why we're FAULTED/DEGRADED. */
	switch (vd->vdev_stat.vs_aux) {
	case VDEV_AUX_ERR_EXCEEDED:
	aux = "err_exceeded";
	break;

	case VDEV_AUX_EXTERNAL:
	aux = "external";
	break;
	}

	if (aux != NULL && !vd->vdev_tmpoffline) {
	fnvlist_add_string(nv, ZPOOL_CONFIG_AUX_STATE, aux);
	} else {
	/*
	* We're healthy - clear any previous AUX_STATE values.
	*/
	if (nvlist_exists(nv, ZPOOL_CONFIG_AUX_STATE))
	nvlist_remove_all(nv, ZPOOL_CONFIG_AUX_STATE);
	}

	if (vd->vdev_splitting && vd->vdev_orig_guid != 0LL) {
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_ORIG_GUID,
	vd->vdev_orig_guid);
	}
	}

	return (nv);
	}

	/*
	* Generate a view of the top-level vdevs. If we currently have holes
	* in the namespace, then generate an array which contains a list of holey
	* vdevs. Additionally, add the number of top-level children that currently
	* exist.
	*/
	void
	vdev_top_config_generate(spa_t spa, nvlist_t config)
	{
	vdev_t *rvd = spa->spa_root_vdev;
	uint64_t *array;
	uint_t c, idx;

	array = kmem_alloc(rvd->vdev_children * sizeof (uint64_t), KM_SLEEP);

	for (c = 0, idx = 0; c < rvd->vdev_children; c++) {
	vdev_t *tvd = rvd->vdev_child[c];

	if (tvd->vdev_ishole) {
	array[idx++] = c;
	}
	}

	if (idx) {
	VERIFY(nvlist_add_uint64_array(config, ZPOOL_CONFIG_HOLE_ARRAY,
	array, idx) == 0);
	}

	VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_VDEV_CHILDREN,
	rvd->vdev_children) == 0);

	kmem_free(array, rvd->vdev_children * sizeof (uint64_t));
	}

	/*
	* Returns the configuration from the label of the given vdev. For vdevs
	* which don't have a txg value stored on their label (i.e. spares/cache)
	* or have not been completely initialized (txg = 0) just return
	* the configuration from the first valid label we find. Otherwise,
	* find the most up-to-date label that does not exceed the specified
	* 'txg' value.
	*/
	nvlist_t *
	vdev_label_read_config(vdev_t *vd, uint64_t txg)
	{
	spa_t *spa = vd->vdev_spa;
	nvlist_t *config = NULL;
	vdev_phys_t *vp[VDEV_LABELS];
	abd_t *vp_abd[VDEV_LABELS];
	zio_t *zio[VDEV_LABELS];
	uint64_t best_txg = 0;
	uint64_t label_txg = 0;
	int error = 0;
	int flags = ZIO_FLAG_CONFIG_WRITER \| ZIO_FLAG_CANFAIL \|
	ZIO_FLAG_SPECULATIVE;

	ASSERT(vd->vdev_validate_thread == curthread \|\|
	spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);

	if (!vdev_readable(vd))
	return (NULL);

	/*
	* The label for a dRAID distributed spare is not stored on disk.
	* Instead it is generated when needed which allows us to bypass
	* the pipeline when reading the config from the label.
	*/
	if (vd->vdev_ops == &vdev_draid_spare_ops)
	return (vdev_draid_read_config_spare(vd));

	for (int l = 0; l < VDEV_LABELS; l++) {
	vp_abd[l] = abd_alloc_linear(sizeof (vdev_phys_t), B_TRUE);
	vp[l] = abd_to_buf(vp_abd[l]);
	}

	retry:
	for (int l = 0; l < VDEV_LABELS; l++) {
	zio[l] = zio_root(spa, NULL, NULL, flags);

	vdev_label_read(zio[l], vd, l, vp_abd[l],
	offsetof(vdev_label_t, vl_vdev_phys), sizeof (vdev_phys_t),
	NULL, NULL, flags);
	}
	for (int l = 0; l < VDEV_LABELS; l++) {
	nvlist_t *label = NULL;

	if (zio_wait(zio[l]) == 0 &&
	nvlist_unpack(vp[l]->vp_nvlist, sizeof (vp[l]->vp_nvlist),
	&label, 0) == 0) {
	/*
	* Auxiliary vdevs won't have txg values in their
	* labels and newly added vdevs may not have been
	* completely initialized so just return the
	* configuration from the first valid label we
	* encounter.
	*/
	error = nvlist_lookup_uint64(label,
	ZPOOL_CONFIG_POOL_TXG, &label_txg);
	if ((error \|\| label_txg == 0) && !config) {
	config = label;
	for (l++; l < VDEV_LABELS; l++)
	zio_wait(zio[l]);
	break;
	} else if (label_txg <= txg && label_txg > best_txg) {
	best_txg = label_txg;
	nvlist_free(config);
	config = fnvlist_dup(label);
	}
	}

	if (label != NULL) {
	nvlist_free(label);
	label = NULL;
	}
	}

	if (config == NULL && !(flags & ZIO_FLAG_TRYHARD)) {
	flags \|= ZIO_FLAG_TRYHARD;
	goto retry;
	}

	/*
	* We found a valid label but it didn't pass txg restrictions.
	*/
	if (config == NULL && label_txg != 0) {
	vdev_dbgmsg(vd, "label discarded as txg is too large "
	"(%llu > %llu)", (u_longlong_t)label_txg,
	(u_longlong_t)txg);
	}

	for (int l = 0; l < VDEV_LABELS; l++) {
	abd_free(vp_abd[l]);
	}

	return (config);
	}

	/*
	* Determine if a device is in use. The 'spare_guid' parameter will be filled
	* in with the device guid if this spare is active elsewhere on the system.
	*/
	static boolean_t
	vdev_inuse(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason,
	uint64_t spare_guid, uint64_t l2cache_guid)
	{
	spa_t *spa = vd->vdev_spa;
	uint64_t state, pool_guid, device_guid, txg, spare_pool;
	uint64_t vdtxg = 0;
	nvlist_t *label;

	if (spare_guid)
	*spare_guid = 0ULL;
	if (l2cache_guid)
	*l2cache_guid = 0ULL;

	/*
	* Read the label, if any, and perform some basic sanity checks.
	*/
	if ((label = vdev_label_read_config(vd, -1ULL)) == NULL)
	return (B_FALSE);

	(void) nvlist_lookup_uint64(label, ZPOOL_CONFIG_CREATE_TXG,
	&vdtxg);

	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_STATE,
	&state) != 0 \|\|
	nvlist_lookup_uint64(label, ZPOOL_CONFIG_GUID,
	&device_guid) != 0) {
	nvlist_free(label);
	return (B_FALSE);
	}

	if (state != POOL_STATE_SPARE && state != POOL_STATE_L2CACHE &&
	(nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_GUID,
	&pool_guid) != 0 \|\|
	nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_TXG,
	&txg) != 0)) {
	nvlist_free(label);
	return (B_FALSE);
	}

	nvlist_free(label);

	/*
	* Check to see if this device indeed belongs to the pool it claims to
	* be a part of. The only way this is allowed is if the device is a hot
	* spare (which we check for later on).
	*/
	if (state != POOL_STATE_SPARE && state != POOL_STATE_L2CACHE &&
	!spa_guid_exists(pool_guid, device_guid) &&
	!spa_spare_exists(device_guid, NULL, NULL) &&
	!spa_l2cache_exists(device_guid, NULL))
	return (B_FALSE);

	/*
	* If the transaction group is zero, then this an initialized (but
	* unused) label. This is only an error if the create transaction
	* on-disk is the same as the one we're using now, in which case the
	* user has attempted to add the same vdev multiple times in the same
	* transaction.
	*/
	if (state != POOL_STATE_SPARE && state != POOL_STATE_L2CACHE &&
	txg == 0 && vdtxg == crtxg)
	return (B_TRUE);

	/*
	* Check to see if this is a spare device. We do an explicit check for
	* spa_has_spare() here because it may be on our pending list of spares
	* to add.
	*/
	if (spa_spare_exists(device_guid, &spare_pool, NULL) \|\|
	spa_has_spare(spa, device_guid)) {
	if (spare_guid)
	*spare_guid = device_guid;

	switch (reason) {
	case VDEV_LABEL_CREATE:
	return (B_TRUE);

	case VDEV_LABEL_REPLACE:
	return (!spa_has_spare(spa, device_guid) \|\|
	spare_pool != 0ULL);

	case VDEV_LABEL_SPARE:
	return (spa_has_spare(spa, device_guid));
	default:
	break;
	}
	}

	/*
	* Check to see if this is an l2cache device.
	*/
	if (spa_l2cache_exists(device_guid, NULL) \|\|
	spa_has_l2cache(spa, device_guid)) {
	if (l2cache_guid)
	*l2cache_guid = device_guid;

	switch (reason) {
	case VDEV_LABEL_CREATE:
	return (B_TRUE);

	case VDEV_LABEL_REPLACE:
	return (!spa_has_l2cache(spa, device_guid));

	case VDEV_LABEL_L2CACHE:
	return (spa_has_l2cache(spa, device_guid));
	default:
	break;
	}
	}

	/*
	* We can't rely on a pool's state if it's been imported
	* read-only. Instead we look to see if the pools is marked
	* read-only in the namespace and set the state to active.
	*/
	if (state != POOL_STATE_SPARE && state != POOL_STATE_L2CACHE &&
	(spa = spa_by_guid(pool_guid, device_guid)) != NULL &&
	spa_mode(spa) == SPA_MODE_READ)
	state = POOL_STATE_ACTIVE;

	/*
	* If the device is marked ACTIVE, then this device is in use by another
	* pool on the system.
	*/
	return (state == POOL_STATE_ACTIVE);
	}

	/*
	* Initialize a vdev label. We check to make sure each leaf device is not in
	* use, and writable. We put down an initial label which we will later
	* overwrite with a complete label. Note that it's important to do this
	* sequentially, not in parallel, so that we catch cases of multiple use of the
	* same leaf vdev in the vdev we're creating -- e.g. mirroring a disk with
	* itself.
	*/
	int
	vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason)
	{
	spa_t *spa = vd->vdev_spa;
	nvlist_t *label;
	vdev_phys_t *vp;
	abd_t *vp_abd;
	abd_t *bootenv;
	uberblock_t *ub;
	abd_t *ub_abd;
	zio_t *zio;
	char *buf;
	size_t buflen;
	int error;
	uint64_t spare_guid = 0, l2cache_guid = 0;
	int flags = ZIO_FLAG_CONFIG_WRITER \| ZIO_FLAG_CANFAIL;
	boolean_t reason_spare = (reason == VDEV_LABEL_SPARE \|\| (reason ==
	VDEV_LABEL_REMOVE && vd->vdev_isspare));
	boolean_t reason_l2cache = (reason == VDEV_LABEL_L2CACHE \|\| (reason ==
	VDEV_LABEL_REMOVE && vd->vdev_isl2cache));

	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);

	for (int c = 0; c < vd->vdev_children; c++)
	if ((error = vdev_label_init(vd->vdev_child[c],
	crtxg, reason)) != 0)
	return (error);

	/* Track the creation time for this vdev */
	vd->vdev_crtxg = crtxg;

	if (!vd->vdev_ops->vdev_op_leaf \|\| !spa_writeable(spa))
	return (0);

	/*
	* Dead vdevs cannot be initialized.
	*/
	if (vdev_is_dead(vd))
	return (SET_ERROR(EIO));

	/*
	* Determine if the vdev is in use.
	*/
	if (reason != VDEV_LABEL_REMOVE && reason != VDEV_LABEL_SPLIT &&
	vdev_inuse(vd, crtxg, reason, &spare_guid, &l2cache_guid))
	return (SET_ERROR(EBUSY));

	/*
	* If this is a request to add or replace a spare or l2cache device
	* that is in use elsewhere on the system, then we must update the
	* guid (which was initialized to a random value) to reflect the
	* actual GUID (which is shared between multiple pools).
	*/
	if (reason != VDEV_LABEL_REMOVE && reason != VDEV_LABEL_L2CACHE &&
	spare_guid != 0ULL) {
	uint64_t guid_delta = spare_guid - vd->vdev_guid;

	vd->vdev_guid += guid_delta;

	for (vdev_t *pvd = vd; pvd != NULL; pvd = pvd->vdev_parent)
	pvd->vdev_guid_sum += guid_delta;

	/*
	* If this is a replacement, then we want to fallthrough to the
	* rest of the code. If we're adding a spare, then it's already
	* labeled appropriately and we can just return.
	*/
	if (reason == VDEV_LABEL_SPARE)
	return (0);
	ASSERT(reason == VDEV_LABEL_REPLACE \|\|
	reason == VDEV_LABEL_SPLIT);
	}

	if (reason != VDEV_LABEL_REMOVE && reason != VDEV_LABEL_SPARE &&
	l2cache_guid != 0ULL) {
	uint64_t guid_delta = l2cache_guid - vd->vdev_guid;

	vd->vdev_guid += guid_delta;

	for (vdev_t *pvd = vd; pvd != NULL; pvd = pvd->vdev_parent)
	pvd->vdev_guid_sum += guid_delta;

	/*
	* If this is a replacement, then we want to fallthrough to the
	* rest of the code. If we're adding an l2cache, then it's
	* already labeled appropriately and we can just return.
	*/
	if (reason == VDEV_LABEL_L2CACHE)
	return (0);
	ASSERT(reason == VDEV_LABEL_REPLACE);
	}

	/*
	* Initialize its label.
	*/
	vp_abd = abd_alloc_linear(sizeof (vdev_phys_t), B_TRUE);
	abd_zero(vp_abd, sizeof (vdev_phys_t));
	vp = abd_to_buf(vp_abd);

	/*
	* Generate a label describing the pool and our top-level vdev.
	* We mark it as being from txg 0 to indicate that it's not
	* really part of an active pool just yet. The labels will
	* be written again with a meaningful txg by spa_sync().
	*/
	if (reason_spare \|\| reason_l2cache) {
	/*
	* For inactive hot spares and level 2 ARC devices, we generate
	* a special label that identifies as a mutually shared hot
	* spare or l2cache device. We write the label in case of
	* addition or removal of hot spare or l2cache vdev (in which
	* case we want to revert the labels).
	*/
	VERIFY(nvlist_alloc(&label, NV_UNIQUE_NAME, KM_SLEEP) == 0);

	VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_VERSION,
	spa_version(spa)) == 0);
	VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_POOL_STATE,
	reason_spare ? POOL_STATE_SPARE : POOL_STATE_L2CACHE) == 0);
	VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_GUID,
	vd->vdev_guid) == 0);

	/*
	* This is merely to facilitate reporting the ashift of the
	* cache device through zdb. The actual retrieval of the
	* ashift (in vdev_alloc()) uses the nvlist
	* spa->spa_l2cache->sav_config (populated in
	* spa_ld_open_aux_vdevs()).
	*/
	if (reason_l2cache) {
	VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_ASHIFT,
	vd->vdev_ashift) == 0);
	}

	/*
	* Add path information to help find it during pool import
	*/
	if (vd->vdev_path != NULL) {
	VERIFY(nvlist_add_string(label, ZPOOL_CONFIG_PATH,
	vd->vdev_path) == 0);
	}
	if (vd->vdev_devid != NULL) {
	VERIFY(nvlist_add_string(label, ZPOOL_CONFIG_DEVID,
	vd->vdev_devid) == 0);
	}
	if (vd->vdev_physpath != NULL) {
	VERIFY(nvlist_add_string(label, ZPOOL_CONFIG_PHYS_PATH,
	vd->vdev_physpath) == 0);
	}

	/*
	* When spare or l2cache (aux) vdev is added during pool
	* creation, spa->spa_uberblock is not written until this
	* point. Write it on next config sync.
	*/
	if (uberblock_verify(&spa->spa_uberblock))
	spa->spa_aux_sync_uber = B_TRUE;
	} else {
	uint64_t txg = 0ULL;

	if (reason == VDEV_LABEL_SPLIT)
	txg = spa->spa_uberblock.ub_txg;
	label = spa_config_generate(spa, vd, txg, B_FALSE);

	/*
	* Add our creation time. This allows us to detect multiple
	* vdev uses as described above, and automatically expires if we
	* fail.
	*/
	VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_CREATE_TXG,
	crtxg) == 0);
	}

	buf = vp->vp_nvlist;
	buflen = sizeof (vp->vp_nvlist);

	error = nvlist_pack(label, &buf, &buflen, NV_ENCODE_XDR, KM_SLEEP);
	if (error != 0) {
	nvlist_free(label);
	abd_free(vp_abd);
	/* EFAULT means nvlist_pack ran out of room */
	return (SET_ERROR(error == EFAULT ? ENAMETOOLONG : EINVAL));
	}

	/*
	* Initialize uberblock template.
	*/
	ub_abd = abd_alloc_linear(VDEV_UBERBLOCK_RING, B_TRUE);
	abd_zero(ub_abd, VDEV_UBERBLOCK_RING);
	abd_copy_from_buf(ub_abd, &spa->spa_uberblock, sizeof (uberblock_t));
	ub = abd_to_buf(ub_abd);
	ub->ub_txg = 0;

	/* Initialize the 2nd padding area. */
	bootenv = abd_alloc_for_io(VDEV_PAD_SIZE, B_TRUE);
	abd_zero(bootenv, VDEV_PAD_SIZE);

	/*
	* Write everything in parallel.
	*/
	retry:
	zio = zio_root(spa, NULL, NULL, flags);

	for (int l = 0; l < VDEV_LABELS; l++) {

	vdev_label_write(zio, vd, l, vp_abd,
	offsetof(vdev_label_t, vl_vdev_phys),
	sizeof (vdev_phys_t), NULL, NULL, flags);

	/*
	* Skip the 1st padding area.
	* Zero out the 2nd padding area where it might have
	* left over data from previous filesystem format.
	*/
	vdev_label_write(zio, vd, l, bootenv,
	offsetof(vdev_label_t, vl_be),
	VDEV_PAD_SIZE, NULL, NULL, flags);

	vdev_label_write(zio, vd, l, ub_abd,
	offsetof(vdev_label_t, vl_uberblock),
	VDEV_UBERBLOCK_RING, NULL, NULL, flags);
	}

	error = zio_wait(zio);

	if (error != 0 && !(flags & ZIO_FLAG_TRYHARD)) {
	flags \|= ZIO_FLAG_TRYHARD;
	goto retry;
	}

	nvlist_free(label);
	abd_free(bootenv);
	abd_free(ub_abd);
	abd_free(vp_abd);

	/*
	* If this vdev hasn't been previously identified as a spare, then we
	* mark it as such only if a) we are labeling it as a spare, or b) it
	* exists as a spare elsewhere in the system. Do the same for
	* level 2 ARC devices.
	*/
	if (error == 0 && !vd->vdev_isspare &&
	(reason == VDEV_LABEL_SPARE \|\|
	spa_spare_exists(vd->vdev_guid, NULL, NULL)))
	spa_spare_add(vd);

	if (error == 0 && !vd->vdev_isl2cache &&
	(reason == VDEV_LABEL_L2CACHE \|\|
	spa_l2cache_exists(vd->vdev_guid, NULL)))
	spa_l2cache_add(vd);

	return (error);
	}

	/*
	* Done callback for vdev_label_read_bootenv_impl. If this is the first
	* callback to finish, store our abd in the callback pointer. Otherwise, we
	* just free our abd and return.
	*/
	static void
	vdev_label_read_bootenv_done(zio_t *zio)
	{
	zio_t *rio = zio->io_private;
	abd_t **cbp = rio->io_private;

	ASSERT3U(zio->io_size, ==, VDEV_PAD_SIZE);

	if (zio->io_error == 0) {
	mutex_enter(&rio->io_lock);
	if (*cbp == NULL) {
	/* Will free this buffer in vdev_label_read_bootenv. */
	*cbp = zio->io_abd;
	} else {
	abd_free(zio->io_abd);
	}
	mutex_exit(&rio->io_lock);
	} else {
	abd_free(zio->io_abd);
	}
	}

	static void
	vdev_label_read_bootenv_impl(zio_t zio, vdev_t vd, int flags)
	{
	for (int c = 0; c < vd->vdev_children; c++)
	vdev_label_read_bootenv_impl(zio, vd->vdev_child[c], flags);

	/*
	* We just use the first label that has a correct checksum; the
	* bootloader should have rewritten them all to be the same on boot,
	* and any changes we made since boot have been the same across all
	* labels.
	*/
	if (vd->vdev_ops->vdev_op_leaf && vdev_readable(vd)) {
	for (int l = 0; l < VDEV_LABELS; l++) {
	vdev_label_read(zio, vd, l,
	abd_alloc_linear(VDEV_PAD_SIZE, B_FALSE),
	offsetof(vdev_label_t, vl_be), VDEV_PAD_SIZE,
	vdev_label_read_bootenv_done, zio, flags);
	}
	}
	}

	int
	vdev_label_read_bootenv(vdev_t rvd, nvlist_t bootenv)
	{
	nvlist_t *config;
	spa_t *spa = rvd->vdev_spa;
	abd_t *abd = NULL;
	int flags = ZIO_FLAG_CONFIG_WRITER \| ZIO_FLAG_CANFAIL \|
	ZIO_FLAG_SPECULATIVE \| ZIO_FLAG_TRYHARD;

	ASSERT(bootenv);
	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);

	zio_t *zio = zio_root(spa, NULL, &abd, flags);
	vdev_label_read_bootenv_impl(zio, rvd, flags);
	int err = zio_wait(zio);

	if (abd != NULL) {
	char *buf;
	vdev_boot_envblock_t *vbe = abd_to_buf(abd);

	vbe->vbe_version = ntohll(vbe->vbe_version);
	switch (vbe->vbe_version) {
	case VB_RAW:
	/*
	* if we have textual data in vbe_bootenv, create nvlist
	* with key "envmap".
	*/
	fnvlist_add_uint64(bootenv, BOOTENV_VERSION, VB_RAW);
	vbe->vbe_bootenv[sizeof (vbe->vbe_bootenv) - 1] = '\0';
	fnvlist_add_string(bootenv, GRUB_ENVMAP,
	vbe->vbe_bootenv);
	break;

	case VB_NVLIST:
	err = nvlist_unpack(vbe->vbe_bootenv,
	sizeof (vbe->vbe_bootenv), &config, 0);
	if (err == 0) {
	fnvlist_merge(bootenv, config);
	nvlist_free(config);
	break;
	}
	zfs_fallthrough;
	default:
	/* Check for FreeBSD zfs bootonce command string */
	buf = abd_to_buf(abd);
	if (*buf == '\0') {
	fnvlist_add_uint64(bootenv, BOOTENV_VERSION,
	VB_NVLIST);
	break;
	}
	fnvlist_add_string(bootenv, FREEBSD_BOOTONCE, buf);
	}

	/*
	* abd was allocated in vdev_label_read_bootenv_impl()
	*/
	abd_free(abd);
	/*
	* If we managed to read any successfully,
	* return success.
	*/
	return (0);
	}
	return (err);
	}

	int
	vdev_label_write_bootenv(vdev_t vd, nvlist_t env)
	{
	zio_t *zio;
	spa_t *spa = vd->vdev_spa;
	vdev_boot_envblock_t *bootenv;
	int flags = ZIO_FLAG_CONFIG_WRITER \| ZIO_FLAG_CANFAIL;
	int error;
	size_t nvsize;
	char *nvbuf;
	const char *tmp;

	error = nvlist_size(env, &nvsize, NV_ENCODE_XDR);
	if (error != 0)
	return (SET_ERROR(error));

	if (nvsize >= sizeof (bootenv->vbe_bootenv)) {
	return (SET_ERROR(E2BIG));
	}

	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);

	error = ENXIO;
	for (int c = 0; c < vd->vdev_children; c++) {
	int child_err;

	child_err = vdev_label_write_bootenv(vd->vdev_child[c], env);
	/*
	* As long as any of the disks managed to write all of their
	* labels successfully, return success.
	*/
	if (child_err == 0)
	error = child_err;
	}

	if (!vd->vdev_ops->vdev_op_leaf \|\| vdev_is_dead(vd) \|\|
	!vdev_writeable(vd)) {
	return (error);
	}
	ASSERT3U(sizeof (*bootenv), ==, VDEV_PAD_SIZE);
	abd_t *abd = abd_alloc_for_io(VDEV_PAD_SIZE, B_TRUE);
	abd_zero(abd, VDEV_PAD_SIZE);

	bootenv = abd_borrow_buf_copy(abd, VDEV_PAD_SIZE);
	nvbuf = bootenv->vbe_bootenv;
	nvsize = sizeof (bootenv->vbe_bootenv);

	bootenv->vbe_version = fnvlist_lookup_uint64(env, BOOTENV_VERSION);
	switch (bootenv->vbe_version) {
	case VB_RAW:
	if (nvlist_lookup_string(env, GRUB_ENVMAP, &tmp) == 0) {
	(void) strlcpy(bootenv->vbe_bootenv, tmp, nvsize);
	}
	error = 0;
	break;

	case VB_NVLIST:
	error = nvlist_pack(env, &nvbuf, &nvsize, NV_ENCODE_XDR,
	KM_SLEEP);
	break;

	default:
	error = EINVAL;
	break;
	}

	if (error == 0) {
	bootenv->vbe_version = htonll(bootenv->vbe_version);
	abd_return_buf_copy(abd, bootenv, VDEV_PAD_SIZE);
	} else {
	abd_free(abd);
	return (SET_ERROR(error));
	}

	retry:
	zio = zio_root(spa, NULL, NULL, flags);
	for (int l = 0; l < VDEV_LABELS; l++) {
	vdev_label_write(zio, vd, l, abd,
	offsetof(vdev_label_t, vl_be),
	VDEV_PAD_SIZE, NULL, NULL, flags);
	}

	error = zio_wait(zio);
	if (error != 0 && !(flags & ZIO_FLAG_TRYHARD)) {
	flags \|= ZIO_FLAG_TRYHARD;
	goto retry;
	}

	abd_free(abd);
	return (error);
	}

	/*
	* ==========================================================================
	* uberblock load/sync
	* ==========================================================================
	*/

	/*
	* Consider the following situation: txg is safely synced to disk. We've
	* written the first uberblock for txg + 1, and then we lose power. When we
	* come back up, we fail to see the uberblock for txg + 1 because, say,
	* it was on a mirrored device and the replica to which we wrote txg + 1
	* is now offline. If we then make some changes and sync txg + 1, and then
	* the missing replica comes back, then for a few seconds we'll have two
	* conflicting uberblocks on disk with the same txg. The solution is simple:
	* among uberblocks with equal txg, choose the one with the latest timestamp.
	*/
	static int
	vdev_uberblock_compare(const uberblock_t ub1, const uberblock_t ub2)
	{
	int cmp = TREE_CMP(ub1->ub_txg, ub2->ub_txg);

	if (likely(cmp))
	return (cmp);

	cmp = TREE_CMP(ub1->ub_timestamp, ub2->ub_timestamp);
	if (likely(cmp))
	return (cmp);

	/*
	* If MMP_VALID(ub) && MMP_SEQ_VALID(ub) then the host has an MMP-aware
	* ZFS, e.g. OpenZFS >= 0.7.
	*
	* If one ub has MMP and the other does not, they were written by
	* different hosts, which matters for MMP. So we treat no MMP/no SEQ as
	* a 0 value.
	*
	* Since timestamp and txg are the same if we get this far, either is
	* acceptable for importing the pool.
	*/
	unsigned int seq1 = 0;
	unsigned int seq2 = 0;

	if (MMP_VALID(ub1) && MMP_SEQ_VALID(ub1))
	seq1 = MMP_SEQ(ub1);

	if (MMP_VALID(ub2) && MMP_SEQ_VALID(ub2))
	seq2 = MMP_SEQ(ub2);

	return (TREE_CMP(seq1, seq2));
	}

	struct ubl_cbdata {
	uberblock_t ubl_ubbest; / Best uberblock */
	vdev_t ubl_vd; / vdev associated with the above */
	};

	static void
	vdev_uberblock_load_done(zio_t *zio)
	{
	vdev_t *vd = zio->io_vd;
	spa_t *spa = zio->io_spa;
	zio_t *rio = zio->io_private;
	uberblock_t *ub = abd_to_buf(zio->io_abd);
	struct ubl_cbdata *cbp = rio->io_private;

	ASSERT3U(zio->io_size, ==, VDEV_UBERBLOCK_SIZE(vd));

	if (zio->io_error == 0 && uberblock_verify(ub) == 0) {
	mutex_enter(&rio->io_lock);
	if (ub->ub_txg <= spa->spa_load_max_txg &&
	vdev_uberblock_compare(ub, cbp->ubl_ubbest) > 0) {
	/*
	* Keep track of the vdev in which this uberblock
	* was found. We will use this information later
	* to obtain the config nvlist associated with
	* this uberblock.
	*/
	cbp->ubl_ubbest = ub;
	cbp->ubl_vd = vd;
	}
	mutex_exit(&rio->io_lock);
	}

	abd_free(zio->io_abd);
	}

	static void
	vdev_uberblock_load_impl(zio_t zio, vdev_t vd, int flags,
	struct ubl_cbdata *cbp)
	{
	for (int c = 0; c < vd->vdev_children; c++)
	vdev_uberblock_load_impl(zio, vd->vdev_child[c], flags, cbp);

	if (vd->vdev_ops->vdev_op_leaf && vdev_readable(vd) &&
	vd->vdev_ops != &vdev_draid_spare_ops) {
	for (int l = 0; l < VDEV_LABELS; l++) {
	for (int n = 0; n < VDEV_UBERBLOCK_COUNT(vd); n++) {
	vdev_label_read(zio, vd, l,
	abd_alloc_linear(VDEV_UBERBLOCK_SIZE(vd),
	B_TRUE), VDEV_UBERBLOCK_OFFSET(vd, n),
	VDEV_UBERBLOCK_SIZE(vd),
	vdev_uberblock_load_done, zio, flags);
	}
	}
	}
	}

	/*
	* Reads the 'best' uberblock from disk along with its associated
	* configuration. First, we read the uberblock array of each label of each
	* vdev, keeping track of the uberblock with the highest txg in each array.
	* Then, we read the configuration from the same vdev as the best uberblock.
	*/
	void
	vdev_uberblock_load(vdev_t rvd, uberblock_t ub, nvlist_t **config)
	{
	zio_t *zio;
	spa_t *spa = rvd->vdev_spa;
	struct ubl_cbdata cb;
	int flags = ZIO_FLAG_CONFIG_WRITER \| ZIO_FLAG_CANFAIL \|
	ZIO_FLAG_SPECULATIVE \| ZIO_FLAG_TRYHARD;

	ASSERT(ub);
	ASSERT(config);

	memset(ub, 0, sizeof (uberblock_t));
	*config = NULL;

	cb.ubl_ubbest = ub;
	cb.ubl_vd = NULL;

	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
	zio = zio_root(spa, NULL, &cb, flags);
	vdev_uberblock_load_impl(zio, rvd, flags, &cb);
	(void) zio_wait(zio);

	/*
	* It's possible that the best uberblock was discovered on a label
	* that has a configuration which was written in a future txg.
	* Search all labels on this vdev to find the configuration that
	* matches the txg for our uberblock.
	*/
	if (cb.ubl_vd != NULL) {
	vdev_dbgmsg(cb.ubl_vd, "best uberblock found for spa %s. "
	"txg %llu", spa->spa_name, (u_longlong_t)ub->ub_txg);

	*config = vdev_label_read_config(cb.ubl_vd, ub->ub_txg);
	if (*config == NULL && spa->spa_extreme_rewind) {
	vdev_dbgmsg(cb.ubl_vd, "failed to read label config. "
	"Trying again without txg restrictions.");
	*config = vdev_label_read_config(cb.ubl_vd, UINT64_MAX);
	}
	if (*config == NULL) {
	vdev_dbgmsg(cb.ubl_vd, "failed to read label config");
	}
	}
	spa_config_exit(spa, SCL_ALL, FTAG);
	}

	/*
	* For use when a leaf vdev is expanded.
	* The location of labels 2 and 3 changed, and at the new location the
	* uberblock rings are either empty or contain garbage. The sync will write
	* new configs there because the vdev is dirty, but expansion also needs the
	* uberblock rings copied. Read them from label 0 which did not move.
	*
	* Since the point is to populate labels {2,3} with valid uberblocks,
	* we zero uberblocks we fail to read or which are not valid.
	*/

	static void
	vdev_copy_uberblocks(vdev_t *vd)
	{
	abd_t *ub_abd;
	zio_t *write_zio;
	int locks = (SCL_L2ARC \| SCL_ZIO);
	int flags = ZIO_FLAG_CONFIG_WRITER \| ZIO_FLAG_CANFAIL \|
	ZIO_FLAG_SPECULATIVE;

	ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_READER) ==
	SCL_STATE);
	ASSERT(vd->vdev_ops->vdev_op_leaf);

	/*
	* No uberblocks are stored on distributed spares, they may be
	* safely skipped when expanding a leaf vdev.
	*/
	if (vd->vdev_ops == &vdev_draid_spare_ops)
	return;

	spa_config_enter(vd->vdev_spa, locks, FTAG, RW_READER);

	ub_abd = abd_alloc_linear(VDEV_UBERBLOCK_SIZE(vd), B_TRUE);

	write_zio = zio_root(vd->vdev_spa, NULL, NULL, flags);
	for (int n = 0; n < VDEV_UBERBLOCK_COUNT(vd); n++) {
	const int src_label = 0;
	zio_t *zio;

	zio = zio_root(vd->vdev_spa, NULL, NULL, flags);
	vdev_label_read(zio, vd, src_label, ub_abd,
	VDEV_UBERBLOCK_OFFSET(vd, n), VDEV_UBERBLOCK_SIZE(vd),
	NULL, NULL, flags);

	if (zio_wait(zio) \|\| uberblock_verify(abd_to_buf(ub_abd)))
	abd_zero(ub_abd, VDEV_UBERBLOCK_SIZE(vd));

	for (int l = 2; l < VDEV_LABELS; l++)
	vdev_label_write(write_zio, vd, l, ub_abd,
	VDEV_UBERBLOCK_OFFSET(vd, n),
	VDEV_UBERBLOCK_SIZE(vd), NULL, NULL,
	flags \| ZIO_FLAG_DONT_PROPAGATE);
	}
	(void) zio_wait(write_zio);

	spa_config_exit(vd->vdev_spa, locks, FTAG);

	abd_free(ub_abd);
	}

	/*
	* On success, increment root zio's count of good writes.
	* We only get credit for writes to known-visible vdevs; see spa_vdev_add().
	*/
	static void
	vdev_uberblock_sync_done(zio_t *zio)
	{
	uint64_t *good_writes = zio->io_private;

	if (zio->io_error == 0 && zio->io_vd->vdev_top->vdev_ms_array != 0)
	atomic_inc_64(good_writes);
	}

	/*
	* Write the uberblock to all labels of all leaves of the specified vdev.
	*/
	static void
	vdev_uberblock_sync(zio_t zio, uint64_t good_writes,
	uberblock_t ub, vdev_t vd, int flags)
	{
	for (uint64_t c = 0; c < vd->vdev_children; c++) {
	vdev_uberblock_sync(zio, good_writes,
	ub, vd->vdev_child[c], flags);
	}

	if (!vd->vdev_ops->vdev_op_leaf)
	return;

	if (!vdev_writeable(vd))
	return;

	/*
	* There's no need to write uberblocks to a distributed spare, they
	* are already stored on all the leaves of the parent dRAID. For
	* this same reason vdev_uberblock_load_impl() skips distributed
	* spares when reading uberblocks.
	*/
	if (vd->vdev_ops == &vdev_draid_spare_ops)
	return;

	/* If the vdev was expanded, need to copy uberblock rings. */
	if (vd->vdev_state == VDEV_STATE_HEALTHY &&
	vd->vdev_copy_uberblocks == B_TRUE) {
	vdev_copy_uberblocks(vd);
	vd->vdev_copy_uberblocks = B_FALSE;
	}

	int m = spa_multihost(vd->vdev_spa) ? MMP_BLOCKS_PER_LABEL : 0;
	int n = ub->ub_txg % (VDEV_UBERBLOCK_COUNT(vd) - m);

	/* Copy the uberblock_t into the ABD */
	abd_t *ub_abd = abd_alloc_for_io(VDEV_UBERBLOCK_SIZE(vd), B_TRUE);
	abd_zero(ub_abd, VDEV_UBERBLOCK_SIZE(vd));
	abd_copy_from_buf(ub_abd, ub, sizeof (uberblock_t));

	for (int l = 0; l < VDEV_LABELS; l++)
	vdev_label_write(zio, vd, l, ub_abd,
	VDEV_UBERBLOCK_OFFSET(vd, n), VDEV_UBERBLOCK_SIZE(vd),
	vdev_uberblock_sync_done, good_writes,
	flags \| ZIO_FLAG_DONT_PROPAGATE);

	abd_free(ub_abd);
	}

	/* Sync the uberblocks to all vdevs in svd[] */
	static int
	vdev_uberblock_sync_list(vdev_t *svd, int svdcount, uberblock_t ub, int flags)
	{
	spa_t *spa = svd[0]->vdev_spa;
	zio_t *zio;
	uint64_t good_writes = 0;

	zio = zio_root(spa, NULL, NULL, flags);

	for (int v = 0; v < svdcount; v++)
	vdev_uberblock_sync(zio, &good_writes, ub, svd[v], flags);

	if (spa->spa_aux_sync_uber) {
	for (int v = 0; v < spa->spa_spares.sav_count; v++) {
	vdev_uberblock_sync(zio, &good_writes, ub,
	spa->spa_spares.sav_vdevs[v], flags);
	}
	for (int v = 0; v < spa->spa_l2cache.sav_count; v++) {
	vdev_uberblock_sync(zio, &good_writes, ub,
	spa->spa_l2cache.sav_vdevs[v], flags);
	}
	}
	(void) zio_wait(zio);

	/*
	* Flush the uberblocks to disk. This ensures that the odd labels
	* are no longer needed (because the new uberblocks and the even
	* labels are safely on disk), so it is safe to overwrite them.
	*/
	zio = zio_root(spa, NULL, NULL, flags);

	for (int v = 0; v < svdcount; v++) {
	if (vdev_writeable(svd[v])) {
	zio_flush(zio, svd[v]);
	}
	}
	if (spa->spa_aux_sync_uber) {
	spa->spa_aux_sync_uber = B_FALSE;
	for (int v = 0; v < spa->spa_spares.sav_count; v++) {
	if (vdev_writeable(spa->spa_spares.sav_vdevs[v])) {
	zio_flush(zio, spa->spa_spares.sav_vdevs[v]);
	}
	}
	for (int v = 0; v < spa->spa_l2cache.sav_count; v++) {
	if (vdev_writeable(spa->spa_l2cache.sav_vdevs[v])) {
	zio_flush(zio, spa->spa_l2cache.sav_vdevs[v]);
	}
	}
	}

	(void) zio_wait(zio);

	return (good_writes >= 1 ? 0 : EIO);
	}

	/*
	* On success, increment the count of good writes for our top-level vdev.
	*/
	static void
	vdev_label_sync_done(zio_t *zio)
	{
	uint64_t *good_writes = zio->io_private;

	if (zio->io_error == 0)
	atomic_inc_64(good_writes);
	}

	/*
	* If there weren't enough good writes, indicate failure to the parent.
	*/
	static void
	vdev_label_sync_top_done(zio_t *zio)
	{
	uint64_t *good_writes = zio->io_private;

	if (*good_writes == 0)
	zio->io_error = SET_ERROR(EIO);

	kmem_free(good_writes, sizeof (uint64_t));
	}

	/*
	* We ignore errors for log and cache devices, simply free the private data.
	*/
	static void
	vdev_label_sync_ignore_done(zio_t *zio)
	{
	kmem_free(zio->io_private, sizeof (uint64_t));
	}

	/*
	* Write all even or odd labels to all leaves of the specified vdev.
	*/
	static void
	vdev_label_sync(zio_t zio, uint64_t good_writes,
	vdev_t *vd, int l, uint64_t txg, int flags)
	{
	nvlist_t *label;
	vdev_phys_t *vp;
	abd_t *vp_abd;
	char *buf;
	size_t buflen;

	for (int c = 0; c < vd->vdev_children; c++) {
	vdev_label_sync(zio, good_writes,
	vd->vdev_child[c], l, txg, flags);
	}

	if (!vd->vdev_ops->vdev_op_leaf)
	return;

	if (!vdev_writeable(vd))
	return;

	/*
	* The top-level config never needs to be written to a distributed
	* spare. When read vdev_dspare_label_read_config() will generate
	* the config for the vdev_label_read_config().
	*/
	if (vd->vdev_ops == &vdev_draid_spare_ops)
	return;

	/*
	* Generate a label describing the top-level config to which we belong.
	*/
	label = spa_config_generate(vd->vdev_spa, vd, txg, B_FALSE);

	vp_abd = abd_alloc_linear(sizeof (vdev_phys_t), B_TRUE);
	abd_zero(vp_abd, sizeof (vdev_phys_t));
	vp = abd_to_buf(vp_abd);

	buf = vp->vp_nvlist;
	buflen = sizeof (vp->vp_nvlist);

	if (!nvlist_pack(label, &buf, &buflen, NV_ENCODE_XDR, KM_SLEEP)) {
	for (; l < VDEV_LABELS; l += 2) {
	vdev_label_write(zio, vd, l, vp_abd,
	offsetof(vdev_label_t, vl_vdev_phys),
	sizeof (vdev_phys_t),
	vdev_label_sync_done, good_writes,
	flags \| ZIO_FLAG_DONT_PROPAGATE);
	}
	}

	abd_free(vp_abd);
	nvlist_free(label);
	}

	static int
	vdev_label_sync_list(spa_t *spa, int l, uint64_t txg, int flags)
	{
	list_t *dl = &spa->spa_config_dirty_list;
	vdev_t *vd;
	zio_t *zio;
	int error;

	/*
	* Write the new labels to disk.
	*/
	zio = zio_root(spa, NULL, NULL, flags);

	for (vd = list_head(dl); vd != NULL; vd = list_next(dl, vd)) {
	uint64_t *good_writes;

	ASSERT(!vd->vdev_ishole);

	good_writes = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
	zio_t *vio = zio_null(zio, spa, NULL,
	(vd->vdev_islog \|\| vd->vdev_aux != NULL) ?
	vdev_label_sync_ignore_done : vdev_label_sync_top_done,
	good_writes, flags);
	vdev_label_sync(vio, good_writes, vd, l, txg, flags);
	zio_nowait(vio);
	}

	error = zio_wait(zio);

	/*
	* Flush the new labels to disk.
	*/
	zio = zio_root(spa, NULL, NULL, flags);

	for (vd = list_head(dl); vd != NULL; vd = list_next(dl, vd))
	zio_flush(zio, vd);

	(void) zio_wait(zio);

	return (error);
	}

	/*
	* Sync the uberblock and any changes to the vdev configuration.
	*
	* The order of operations is carefully crafted to ensure that
	* if the system panics or loses power at any time, the state on disk
	* is still transactionally consistent. The in-line comments below
	* describe the failure semantics at each stage.
	*
	* Moreover, vdev_config_sync() is designed to be idempotent: if it fails
	* at any time, you can just call it again, and it will resume its work.
	*/
	int
	vdev_config_sync(vdev_t **svd, int svdcount, uint64_t txg)
	{
	spa_t *spa = svd[0]->vdev_spa;
	uberblock_t *ub = &spa->spa_uberblock;
	int error = 0;
	int flags = ZIO_FLAG_CONFIG_WRITER \| ZIO_FLAG_CANFAIL;

	ASSERT(svdcount != 0);
	retry:
	/*
	* Normally, we don't want to try too hard to write every label and
	* uberblock. If there is a flaky disk, we don't want the rest of the
	* sync process to block while we retry. But if we can't write a
	* single label out, we should retry with ZIO_FLAG_TRYHARD before
	* bailing out and declaring the pool faulted.
	*/
	if (error != 0) {
	if ((flags & ZIO_FLAG_TRYHARD) != 0)
	return (error);
	flags \|= ZIO_FLAG_TRYHARD;
	}

	ASSERT(ub->ub_txg <= txg);

	/*
	* If this isn't a resync due to I/O errors,
	* and nothing changed in this transaction group,
	+ * and multihost protection isn't enabled,
	* and the vdev configuration hasn't changed,
	* then there's nothing to do.
	*/
	if (ub->ub_txg < txg) {
	boolean_t changed = uberblock_update(ub, spa->spa_root_vdev,
	txg, spa->spa_mmp.mmp_delay);

	- if (!changed && list_is_empty(&spa->spa_config_dirty_list))
	+ if (!changed && list_is_empty(&spa->spa_config_dirty_list) &&
	+ !spa_multihost(spa))
	return (0);
	}

	if (txg > spa_freeze_txg(spa))
	return (0);

	ASSERT(txg <= spa->spa_final_txg);

	/*
	* Flush the write cache of every disk that's been written to
	* in this transaction group. This ensures that all blocks
	* written in this txg will be committed to stable storage
	* before any uberblock that references them.
	*/
	zio_t *zio = zio_root(spa, NULL, NULL, flags);

	for (vdev_t *vd =
	txg_list_head(&spa->spa_vdev_txg_list, TXG_CLEAN(txg)); vd != NULL;
	vd = txg_list_next(&spa->spa_vdev_txg_list, vd, TXG_CLEAN(txg)))
	zio_flush(zio, vd);

	(void) zio_wait(zio);

	/*
	* Sync out the even labels (L0, L2) for every dirty vdev. If the
	* system dies in the middle of this process, that's OK: all of the
	* even labels that made it to disk will be newer than any uberblock,
	* and will therefore be considered invalid. The odd labels (L1, L3),
	* which have not yet been touched, will still be valid. We flush
	* the new labels to disk to ensure that all even-label updates
	* are committed to stable storage before the uberblock update.
	*/
	if ((error = vdev_label_sync_list(spa, 0, txg, flags)) != 0) {
	if ((flags & ZIO_FLAG_TRYHARD) != 0) {
	zfs_dbgmsg("vdev_label_sync_list() returned error %d "
	"for pool '%s' when syncing out the even labels "
	"of dirty vdevs", error, spa_name(spa));
	}
	goto retry;
	}

	/*
	* Sync the uberblocks to all vdevs in svd[].
	* If the system dies in the middle of this step, there are two cases
	* to consider, and the on-disk state is consistent either way:
	*
	* (1) If none of the new uberblocks made it to disk, then the
	* previous uberblock will be the newest, and the odd labels
	* (which had not yet been touched) will be valid with respect
	* to that uberblock.
	*
	* (2) If one or more new uberblocks made it to disk, then they
	* will be the newest, and the even labels (which had all
	* been successfully committed) will be valid with respect
	* to the new uberblocks.
	*/
	if ((error = vdev_uberblock_sync_list(svd, svdcount, ub, flags)) != 0) {
	if ((flags & ZIO_FLAG_TRYHARD) != 0) {
	zfs_dbgmsg("vdev_uberblock_sync_list() returned error "
	"%d for pool '%s'", error, spa_name(spa));
	}
	goto retry;
	}

	if (spa_multihost(spa))
	mmp_update_uberblock(spa, ub);

	/*
	* Sync out odd labels for every dirty vdev. If the system dies
	* in the middle of this process, the even labels and the new
	* uberblocks will suffice to open the pool. The next time
	* the pool is opened, the first thing we'll do -- before any
	* user data is modified -- is mark every vdev dirty so that
	* all labels will be brought up to date. We flush the new labels
	* to disk to ensure that all odd-label updates are committed to
	* stable storage before the next transaction group begins.
	*/
	if ((error = vdev_label_sync_list(spa, 1, txg, flags)) != 0) {
	if ((flags & ZIO_FLAG_TRYHARD) != 0) {
	zfs_dbgmsg("vdev_label_sync_list() returned error %d "
	"for pool '%s' when syncing out the odd labels of "
	"dirty vdevs", error, spa_name(spa));
	}
	goto retry;
	}

	return (0);
	}
	diff --git a/sys/contrib/openzfs/module/zfs/vdev_raidz.c b/sys/contrib/openzfs/module/zfs/vdev_raidz.c
	index 3445fa9d35d5..34ef57a3548d 100644
	--- a/sys/contrib/openzfs/module/zfs/vdev_raidz.c
	+++ b/sys/contrib/openzfs/module/zfs/vdev_raidz.c
	@@ -1,2673 +1,2674 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2012, 2020 by Delphix. All rights reserved.
	* Copyright (c) 2016 Gvozden Nešković. All rights reserved.
	*/

	#include <sys/zfs_context.h>
	#include <sys/spa.h>
	#include <sys/vdev_impl.h>
	#include <sys/zio.h>
	#include <sys/zio_checksum.h>
	#include <sys/abd.h>
	#include <sys/fs/zfs.h>
	#include <sys/fm/fs/zfs.h>
	#include <sys/vdev_raidz.h>
	#include <sys/vdev_raidz_impl.h>
	#include <sys/vdev_draid.h>

	#ifdef ZFS_DEBUG
	#include <sys/vdev.h> /* For vdev_xlate() in vdev_raidz_io_verify() */
	#endif

	/*
	* Virtual device vector for RAID-Z.
	*
	* This vdev supports single, double, and triple parity. For single parity,
	* we use a simple XOR of all the data columns. For double or triple parity,
	* we use a special case of Reed-Solomon coding. This extends the
	* technique described in "The mathematics of RAID-6" by H. Peter Anvin by
	* drawing on the system described in "A Tutorial on Reed-Solomon Coding for
	* Fault-Tolerance in RAID-like Systems" by James S. Plank on which the
	* former is also based. The latter is designed to provide higher performance
	* for writes.
	*
	* Note that the Plank paper claimed to support arbitrary N+M, but was then
	* amended six years later identifying a critical flaw that invalidates its
	* claims. Nevertheless, the technique can be adapted to work for up to
	* triple parity. For additional parity, the amendment "Note: Correction to
	* the 1997 Tutorial on Reed-Solomon Coding" by James S. Plank and Ying Ding
	* is viable, but the additional complexity means that write performance will
	* suffer.
	*
	* All of the methods above operate on a Galois field, defined over the
	* integers mod 2^N. In our case we choose N=8 for GF(8) so that all elements
	* can be expressed with a single byte. Briefly, the operations on the
	* field are defined as follows:
	*
	* o addition (+) is represented by a bitwise XOR
	* o subtraction (-) is therefore identical to addition: A + B = A - B
	* o multiplication of A by 2 is defined by the following bitwise expression:
	*
	* (A * 2)_7 = A_6
	* (A * 2)_6 = A_5
	* (A * 2)_5 = A_4
	* (A * 2)_4 = A_3 + A_7
	* (A * 2)_3 = A_2 + A_7
	* (A * 2)_2 = A_1 + A_7
	* (A * 2)_1 = A_0
	* (A * 2)_0 = A_7
	*
	* In C, multiplying by 2 is therefore ((a << 1) ^ ((a & 0x80) ? 0x1d : 0)).
	* As an aside, this multiplication is derived from the error correcting
	* primitive polynomial x^8 + x^4 + x^3 + x^2 + 1.
	*
	* Observe that any number in the field (except for 0) can be expressed as a
	* power of 2 -- a generator for the field. We store a table of the powers of
	* 2 and logs base 2 for quick look ups, and exploit the fact that A * B can
	* be rewritten as 2^(log_2(A) + log_2(B)) (where '+' is normal addition rather
	* than field addition). The inverse of a field element A (A^-1) is therefore
	* A ^ (255 - 1) = A^254.
	*
	* The up-to-three parity columns, P, Q, R over several data columns,
	* D_0, ... D_n-1, can be expressed by field operations:
	*
	* P = D_0 + D_1 + ... + D_n-2 + D_n-1
	* Q = 2^n-1 * D_0 + 2^n-2 * D_1 + ... + 2^1 * D_n-2 + 2^0 * D_n-1
	* = ((...((D_0) * 2 + D_1) * 2 + ...) * 2 + D_n-2) * 2 + D_n-1
	* R = 4^n-1 * D_0 + 4^n-2 * D_1 + ... + 4^1 * D_n-2 + 4^0 * D_n-1
	* = ((...((D_0) * 4 + D_1) * 4 + ...) * 4 + D_n-2) * 4 + D_n-1
	*
	* We chose 1, 2, and 4 as our generators because 1 corresponds to the trivial
	* XOR operation, and 2 and 4 can be computed quickly and generate linearly-
	* independent coefficients. (There are no additional coefficients that have
	* this property which is why the uncorrected Plank method breaks down.)
	*
	* See the reconstruction code below for how P, Q and R can used individually
	* or in concert to recover missing data columns.
	*/

	#define VDEV_RAIDZ_P 0
	#define VDEV_RAIDZ_Q 1
	#define VDEV_RAIDZ_R 2

	#define VDEV_RAIDZ_MUL_2(x) (((x) << 1) ^ (((x) & 0x80) ? 0x1d : 0))
	#define VDEV_RAIDZ_MUL_4(x) (VDEV_RAIDZ_MUL_2(VDEV_RAIDZ_MUL_2(x)))

	/*
	* We provide a mechanism to perform the field multiplication operation on a
	* 64-bit value all at once rather than a byte at a time. This works by
	* creating a mask from the top bit in each byte and using that to
	* conditionally apply the XOR of 0x1d.
	*/
	#define VDEV_RAIDZ_64MUL_2(x, mask) \
	{ \
	(mask) = (x) & 0x8080808080808080ULL; \
	(mask) = ((mask) << 1) - ((mask) >> 7); \
	(x) = (((x) << 1) & 0xfefefefefefefefeULL) ^ \
	((mask) & 0x1d1d1d1d1d1d1d1dULL); \
	}

	#define VDEV_RAIDZ_64MUL_4(x, mask) \
	{ \
	VDEV_RAIDZ_64MUL_2((x), mask); \
	VDEV_RAIDZ_64MUL_2((x), mask); \
	}

	static void
	vdev_raidz_row_free(raidz_row_t *rr)
	{
	for (int c = 0; c < rr->rr_cols; c++) {
	raidz_col_t *rc = &rr->rr_col[c];

	if (rc->rc_size != 0)
	abd_free(rc->rc_abd);
	if (rc->rc_orig_data != NULL)
	abd_free(rc->rc_orig_data);
	}

	if (rr->rr_abd_empty != NULL)
	abd_free(rr->rr_abd_empty);

	kmem_free(rr, offsetof(raidz_row_t, rr_col[rr->rr_scols]));
	}

	void
	vdev_raidz_map_free(raidz_map_t *rm)
	{
	for (int i = 0; i < rm->rm_nrows; i++)
	vdev_raidz_row_free(rm->rm_row[i]);

	kmem_free(rm, offsetof(raidz_map_t, rm_row[rm->rm_nrows]));
	}

	static void
	vdev_raidz_map_free_vsd(zio_t *zio)
	{
	raidz_map_t *rm = zio->io_vsd;

	vdev_raidz_map_free(rm);
	}

	const zio_vsd_ops_t vdev_raidz_vsd_ops = {
	.vsd_free = vdev_raidz_map_free_vsd,
	};

	static void
	vdev_raidz_map_alloc_write(zio_t zio, raidz_map_t rm, uint64_t ashift)
	{
	int c;
	int nwrapped = 0;
	uint64_t off = 0;
	raidz_row_t *rr = rm->rm_row[0];

	ASSERT3U(zio->io_type, ==, ZIO_TYPE_WRITE);
	ASSERT3U(rm->rm_nrows, ==, 1);

	/*
	* Pad any parity columns with additional space to account for skip
	* sectors.
	*/
	if (rm->rm_skipstart < rr->rr_firstdatacol) {
	ASSERT0(rm->rm_skipstart);
	nwrapped = rm->rm_nskip;
	} else if (rr->rr_scols < (rm->rm_skipstart + rm->rm_nskip)) {
	nwrapped =
	(rm->rm_skipstart + rm->rm_nskip) % rr->rr_scols;
	}

	/*
	* Optional single skip sectors (rc_size == 0) will be handled in
	* vdev_raidz_io_start_write().
	*/
	int skipped = rr->rr_scols - rr->rr_cols;

	/* Allocate buffers for the parity columns */
	for (c = 0; c < rr->rr_firstdatacol; c++) {
	raidz_col_t *rc = &rr->rr_col[c];

	/*
	* Parity columns will pad out a linear ABD to account for
	* the skip sector. A linear ABD is used here because
	* parity calculations use the ABD buffer directly to calculate
	* parity. This avoids doing a memcpy back to the ABD after the
	* parity has been calculated. By issuing the parity column
	* with the skip sector we can reduce contention on the child
	* VDEV queue locks (vq_lock).
	*/
	if (c < nwrapped) {
	rc->rc_abd = abd_alloc_linear(
	rc->rc_size + (1ULL << ashift), B_FALSE);
	abd_zero_off(rc->rc_abd, rc->rc_size, 1ULL << ashift);
	skipped++;
	} else {
	rc->rc_abd = abd_alloc_linear(rc->rc_size, B_FALSE);
	}
	}

	for (off = 0; c < rr->rr_cols; c++) {
	raidz_col_t *rc = &rr->rr_col[c];
	abd_t *abd = abd_get_offset_struct(&rc->rc_abdstruct,
	zio->io_abd, off, rc->rc_size);

	/*
	* Generate I/O for skip sectors to improve aggregation
	* continuity. We will use gang ABD's to reduce contention
	* on the child VDEV queue locks (vq_lock) by issuing
	* a single I/O that contains the data and skip sector.
	*
	* It is important to make sure that rc_size is not updated
	* even though we are adding a skip sector to the ABD. When
	* calculating the parity in vdev_raidz_generate_parity_row()
	* the rc_size is used to iterate through the ABD's. We can
	* not have zero'd out skip sectors used for calculating
	* parity for raidz, because those same sectors are not used
	* during reconstruction.
	*/
	if (c >= rm->rm_skipstart && skipped < rm->rm_nskip) {
	rc->rc_abd = abd_alloc_gang();
	abd_gang_add(rc->rc_abd, abd, B_TRUE);
	abd_gang_add(rc->rc_abd,
	abd_get_zeros(1ULL << ashift), B_TRUE);
	skipped++;
	} else {
	rc->rc_abd = abd;
	}
	off += rc->rc_size;
	}

	ASSERT3U(off, ==, zio->io_size);
	ASSERT3S(skipped, ==, rm->rm_nskip);
	}

	static void
	vdev_raidz_map_alloc_read(zio_t zio, raidz_map_t rm)
	{
	int c;
	raidz_row_t *rr = rm->rm_row[0];

	ASSERT3U(rm->rm_nrows, ==, 1);

	/* Allocate buffers for the parity columns */
	for (c = 0; c < rr->rr_firstdatacol; c++)
	rr->rr_col[c].rc_abd =
	abd_alloc_linear(rr->rr_col[c].rc_size, B_FALSE);

	for (uint64_t off = 0; c < rr->rr_cols; c++) {
	raidz_col_t *rc = &rr->rr_col[c];
	rc->rc_abd = abd_get_offset_struct(&rc->rc_abdstruct,
	zio->io_abd, off, rc->rc_size);
	off += rc->rc_size;
	}
	}

	/*
	* Divides the IO evenly across all child vdevs; usually, dcols is
	* the number of children in the target vdev.
	*
	* Avoid inlining the function to keep vdev_raidz_io_start(), which
	* is this functions only caller, as small as possible on the stack.
	*/
	noinline raidz_map_t *
	vdev_raidz_map_alloc(zio_t *zio, uint64_t ashift, uint64_t dcols,
	uint64_t nparity)
	{
	raidz_row_t *rr;
	/* The starting RAIDZ (parent) vdev sector of the block. */
	uint64_t b = zio->io_offset >> ashift;
	/* The zio's size in units of the vdev's minimum sector size. */
	uint64_t s = zio->io_size >> ashift;
	/* The first column for this stripe. */
	uint64_t f = b % dcols;
	/* The starting byte offset on each child vdev. */
	uint64_t o = (b / dcols) << ashift;
	uint64_t q, r, c, bc, col, acols, scols, coff, devidx, asize, tot;

	raidz_map_t *rm =
	kmem_zalloc(offsetof(raidz_map_t, rm_row[1]), KM_SLEEP);
	rm->rm_nrows = 1;

	/*
	* "Quotient": The number of data sectors for this stripe on all but
	* the "big column" child vdevs that also contain "remainder" data.
	*/
	q = s / (dcols - nparity);

	/*
	* "Remainder": The number of partial stripe data sectors in this I/O.
	* This will add a sector to some, but not all, child vdevs.
	*/
	r = s - q * (dcols - nparity);

	/* The number of "big columns" - those which contain remainder data. */
	bc = (r == 0 ? 0 : r + nparity);

	/*
	* The total number of data and parity sectors associated with
	* this I/O.
	*/
	tot = s + nparity * (q + (r == 0 ? 0 : 1));

	/*
	* acols: The columns that will be accessed.
	* scols: The columns that will be accessed or skipped.
	*/
	if (q == 0) {
	/* Our I/O request doesn't span all child vdevs. */
	acols = bc;
	scols = MIN(dcols, roundup(bc, nparity + 1));
	} else {
	acols = dcols;
	scols = dcols;
	}

	ASSERT3U(acols, <=, scols);

	rr = kmem_alloc(offsetof(raidz_row_t, rr_col[scols]), KM_SLEEP);
	rm->rm_row[0] = rr;

	rr->rr_cols = acols;
	rr->rr_scols = scols;
	rr->rr_bigcols = bc;
	rr->rr_missingdata = 0;
	rr->rr_missingparity = 0;
	rr->rr_firstdatacol = nparity;
	rr->rr_abd_empty = NULL;
	rr->rr_nempty = 0;
	#ifdef ZFS_DEBUG
	rr->rr_offset = zio->io_offset;
	rr->rr_size = zio->io_size;
	#endif

	asize = 0;

	for (c = 0; c < scols; c++) {
	raidz_col_t *rc = &rr->rr_col[c];
	col = f + c;
	coff = o;
	if (col >= dcols) {
	col -= dcols;
	coff += 1ULL << ashift;
	}
	rc->rc_devidx = col;
	rc->rc_offset = coff;
	rc->rc_abd = NULL;
	rc->rc_orig_data = NULL;
	rc->rc_error = 0;
	rc->rc_tried = 0;
	rc->rc_skipped = 0;
	rc->rc_force_repair = 0;
	rc->rc_allow_repair = 1;
	rc->rc_need_orig_restore = B_FALSE;

	if (c >= acols)
	rc->rc_size = 0;
	else if (c < bc)
	rc->rc_size = (q + 1) << ashift;
	else
	rc->rc_size = q << ashift;

	asize += rc->rc_size;
	}

	ASSERT3U(asize, ==, tot << ashift);
	rm->rm_nskip = roundup(tot, nparity + 1) - tot;
	rm->rm_skipstart = bc;

	/*
	* If all data stored spans all columns, there's a danger that parity
	* will always be on the same device and, since parity isn't read
	* during normal operation, that device's I/O bandwidth won't be
	* used effectively. We therefore switch the parity every 1MB.
	*
	* ... at least that was, ostensibly, the theory. As a practical
	* matter unless we juggle the parity between all devices evenly, we
	* won't see any benefit. Further, occasional writes that aren't a
	* multiple of the LCM of the number of children and the minimum
	* stripe width are sufficient to avoid pessimal behavior.
	* Unfortunately, this decision created an implicit on-disk format
	* requirement that we need to support for all eternity, but only
	* for single-parity RAID-Z.
	*
	* If we intend to skip a sector in the zeroth column for padding
	* we must make sure to note this swap. We will never intend to
	* skip the first column since at least one data and one parity
	* column must appear in each row.
	*/
	ASSERT(rr->rr_cols >= 2);
	ASSERT(rr->rr_col[0].rc_size == rr->rr_col[1].rc_size);

	if (rr->rr_firstdatacol == 1 && (zio->io_offset & (1ULL << 20))) {
	devidx = rr->rr_col[0].rc_devidx;
	o = rr->rr_col[0].rc_offset;
	rr->rr_col[0].rc_devidx = rr->rr_col[1].rc_devidx;
	rr->rr_col[0].rc_offset = rr->rr_col[1].rc_offset;
	rr->rr_col[1].rc_devidx = devidx;
	rr->rr_col[1].rc_offset = o;

	if (rm->rm_skipstart == 0)
	rm->rm_skipstart = 1;
	}

	if (zio->io_type == ZIO_TYPE_WRITE) {
	vdev_raidz_map_alloc_write(zio, rm, ashift);
	} else {
	vdev_raidz_map_alloc_read(zio, rm);
	}

	/* init RAIDZ parity ops */
	rm->rm_ops = vdev_raidz_math_get_ops();

	return (rm);
	}

	struct pqr_struct {
	uint64_t *p;
	uint64_t *q;
	uint64_t *r;
	};

	static int
	vdev_raidz_p_func(void buf, size_t size, void private)
	{
	struct pqr_struct *pqr = private;
	const uint64_t *src = buf;
	int i, cnt = size / sizeof (src[0]);

	ASSERT(pqr->p && !pqr->q && !pqr->r);

	for (i = 0; i < cnt; i++, src++, pqr->p++)
	pqr->p ^= src;

	return (0);
	}

	static int
	vdev_raidz_pq_func(void buf, size_t size, void private)
	{
	struct pqr_struct *pqr = private;
	const uint64_t *src = buf;
	uint64_t mask;
	int i, cnt = size / sizeof (src[0]);

	ASSERT(pqr->p && pqr->q && !pqr->r);

	for (i = 0; i < cnt; i++, src++, pqr->p++, pqr->q++) {
	pqr->p ^= src;
	VDEV_RAIDZ_64MUL_2(*pqr->q, mask);
	pqr->q ^= src;
	}

	return (0);
	}

	static int
	vdev_raidz_pqr_func(void buf, size_t size, void private)
	{
	struct pqr_struct *pqr = private;
	const uint64_t *src = buf;
	uint64_t mask;
	int i, cnt = size / sizeof (src[0]);

	ASSERT(pqr->p && pqr->q && pqr->r);

	for (i = 0; i < cnt; i++, src++, pqr->p++, pqr->q++, pqr->r++) {
	pqr->p ^= src;
	VDEV_RAIDZ_64MUL_2(*pqr->q, mask);
	pqr->q ^= src;
	VDEV_RAIDZ_64MUL_4(*pqr->r, mask);
	pqr->r ^= src;
	}

	return (0);
	}

	static void
	vdev_raidz_generate_parity_p(raidz_row_t *rr)
	{
	uint64_t *p = abd_to_buf(rr->rr_col[VDEV_RAIDZ_P].rc_abd);

	for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
	abd_t *src = rr->rr_col[c].rc_abd;

	if (c == rr->rr_firstdatacol) {
	abd_copy_to_buf(p, src, rr->rr_col[c].rc_size);
	} else {
	struct pqr_struct pqr = { p, NULL, NULL };
	(void) abd_iterate_func(src, 0, rr->rr_col[c].rc_size,
	vdev_raidz_p_func, &pqr);
	}
	}
	}

	static void
	vdev_raidz_generate_parity_pq(raidz_row_t *rr)
	{
	uint64_t *p = abd_to_buf(rr->rr_col[VDEV_RAIDZ_P].rc_abd);
	uint64_t *q = abd_to_buf(rr->rr_col[VDEV_RAIDZ_Q].rc_abd);
	uint64_t pcnt = rr->rr_col[VDEV_RAIDZ_P].rc_size / sizeof (p[0]);
	ASSERT(rr->rr_col[VDEV_RAIDZ_P].rc_size ==
	rr->rr_col[VDEV_RAIDZ_Q].rc_size);

	for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
	abd_t *src = rr->rr_col[c].rc_abd;

	uint64_t ccnt = rr->rr_col[c].rc_size / sizeof (p[0]);

	if (c == rr->rr_firstdatacol) {
	ASSERT(ccnt == pcnt \|\| ccnt == 0);
	abd_copy_to_buf(p, src, rr->rr_col[c].rc_size);
	(void) memcpy(q, p, rr->rr_col[c].rc_size);

	for (uint64_t i = ccnt; i < pcnt; i++) {
	p[i] = 0;
	q[i] = 0;
	}
	} else {
	struct pqr_struct pqr = { p, q, NULL };

	ASSERT(ccnt <= pcnt);
	(void) abd_iterate_func(src, 0, rr->rr_col[c].rc_size,
	vdev_raidz_pq_func, &pqr);

	/*
	* Treat short columns as though they are full of 0s.
	* Note that there's therefore nothing needed for P.
	*/
	uint64_t mask;
	for (uint64_t i = ccnt; i < pcnt; i++) {
	VDEV_RAIDZ_64MUL_2(q[i], mask);
	}
	}
	}
	}

	static void
	vdev_raidz_generate_parity_pqr(raidz_row_t *rr)
	{
	uint64_t *p = abd_to_buf(rr->rr_col[VDEV_RAIDZ_P].rc_abd);
	uint64_t *q = abd_to_buf(rr->rr_col[VDEV_RAIDZ_Q].rc_abd);
	uint64_t *r = abd_to_buf(rr->rr_col[VDEV_RAIDZ_R].rc_abd);
	uint64_t pcnt = rr->rr_col[VDEV_RAIDZ_P].rc_size / sizeof (p[0]);
	ASSERT(rr->rr_col[VDEV_RAIDZ_P].rc_size ==
	rr->rr_col[VDEV_RAIDZ_Q].rc_size);
	ASSERT(rr->rr_col[VDEV_RAIDZ_P].rc_size ==
	rr->rr_col[VDEV_RAIDZ_R].rc_size);

	for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
	abd_t *src = rr->rr_col[c].rc_abd;

	uint64_t ccnt = rr->rr_col[c].rc_size / sizeof (p[0]);

	if (c == rr->rr_firstdatacol) {
	ASSERT(ccnt == pcnt \|\| ccnt == 0);
	abd_copy_to_buf(p, src, rr->rr_col[c].rc_size);
	(void) memcpy(q, p, rr->rr_col[c].rc_size);
	(void) memcpy(r, p, rr->rr_col[c].rc_size);

	for (uint64_t i = ccnt; i < pcnt; i++) {
	p[i] = 0;
	q[i] = 0;
	r[i] = 0;
	}
	} else {
	struct pqr_struct pqr = { p, q, r };

	ASSERT(ccnt <= pcnt);
	(void) abd_iterate_func(src, 0, rr->rr_col[c].rc_size,
	vdev_raidz_pqr_func, &pqr);

	/*
	* Treat short columns as though they are full of 0s.
	* Note that there's therefore nothing needed for P.
	*/
	uint64_t mask;
	for (uint64_t i = ccnt; i < pcnt; i++) {
	VDEV_RAIDZ_64MUL_2(q[i], mask);
	VDEV_RAIDZ_64MUL_4(r[i], mask);
	}
	}
	}
	}

	/*
	* Generate RAID parity in the first virtual columns according to the number of
	* parity columns available.
	*/
	void
	vdev_raidz_generate_parity_row(raidz_map_t rm, raidz_row_t rr)
	{
	ASSERT3U(rr->rr_cols, !=, 0);

	/* Generate using the new math implementation */
	if (vdev_raidz_math_generate(rm, rr) != RAIDZ_ORIGINAL_IMPL)
	return;

	switch (rr->rr_firstdatacol) {
	case 1:
	vdev_raidz_generate_parity_p(rr);
	break;
	case 2:
	vdev_raidz_generate_parity_pq(rr);
	break;
	case 3:
	vdev_raidz_generate_parity_pqr(rr);
	break;
	default:
	cmn_err(CE_PANIC, "invalid RAID-Z configuration");
	}
	}

	void
	vdev_raidz_generate_parity(raidz_map_t *rm)
	{
	for (int i = 0; i < rm->rm_nrows; i++) {
	raidz_row_t *rr = rm->rm_row[i];
	vdev_raidz_generate_parity_row(rm, rr);
	}
	}

	static int
	vdev_raidz_reconst_p_func(void dbuf, void sbuf, size_t size, void *private)
	{
	(void) private;
	uint64_t *dst = dbuf;
	uint64_t *src = sbuf;
	int cnt = size / sizeof (src[0]);

	for (int i = 0; i < cnt; i++) {
	dst[i] ^= src[i];
	}

	return (0);
	}

	static int
	vdev_raidz_reconst_q_pre_func(void dbuf, void sbuf, size_t size,
	void *private)
	{
	(void) private;
	uint64_t *dst = dbuf;
	uint64_t *src = sbuf;
	uint64_t mask;
	int cnt = size / sizeof (dst[0]);

	for (int i = 0; i < cnt; i++, dst++, src++) {
	VDEV_RAIDZ_64MUL_2(*dst, mask);
	dst ^= src;
	}

	return (0);
	}

	static int
	vdev_raidz_reconst_q_pre_tail_func(void buf, size_t size, void private)
	{
	(void) private;
	uint64_t *dst = buf;
	uint64_t mask;
	int cnt = size / sizeof (dst[0]);

	for (int i = 0; i < cnt; i++, dst++) {
	/* same operation as vdev_raidz_reconst_q_pre_func() on dst */
	VDEV_RAIDZ_64MUL_2(*dst, mask);
	}

	return (0);
	}

	struct reconst_q_struct {
	uint64_t *q;
	int exp;
	};

	static int
	vdev_raidz_reconst_q_post_func(void buf, size_t size, void private)
	{
	struct reconst_q_struct *rq = private;
	uint64_t *dst = buf;
	int cnt = size / sizeof (dst[0]);

	for (int i = 0; i < cnt; i++, dst++, rq->q++) {
	int j;
	uint8_t *b;

	dst ^= rq->q;
	for (j = 0, b = (uint8_t *)dst; j < 8; j++, b++) {
	b = vdev_raidz_exp2(b, rq->exp);
	}
	}

	return (0);
	}

	struct reconst_pq_struct {
	uint8_t *p;
	uint8_t *q;
	uint8_t *pxy;
	uint8_t *qxy;
	int aexp;
	int bexp;
	};

	static int
	vdev_raidz_reconst_pq_func(void xbuf, void ybuf, size_t size, void *private)
	{
	struct reconst_pq_struct *rpq = private;
	uint8_t *xd = xbuf;
	uint8_t *yd = ybuf;

	for (int i = 0; i < size;
	i++, rpq->p++, rpq->q++, rpq->pxy++, rpq->qxy++, xd++, yd++) {
	xd = vdev_raidz_exp2(rpq->p ^ *rpq->pxy, rpq->aexp) ^
	vdev_raidz_exp2(rpq->q ^ rpq->qxy, rpq->bexp);
	yd = rpq->p ^ rpq->pxy ^ xd;
	}

	return (0);
	}

	static int
	vdev_raidz_reconst_pq_tail_func(void xbuf, size_t size, void private)
	{
	struct reconst_pq_struct *rpq = private;
	uint8_t *xd = xbuf;

	for (int i = 0; i < size;
	i++, rpq->p++, rpq->q++, rpq->pxy++, rpq->qxy++, xd++) {
	/* same operation as vdev_raidz_reconst_pq_func() on xd */
	xd = vdev_raidz_exp2(rpq->p ^ *rpq->pxy, rpq->aexp) ^
	vdev_raidz_exp2(rpq->q ^ rpq->qxy, rpq->bexp);
	}

	return (0);
	}

	static void
	vdev_raidz_reconstruct_p(raidz_row_t rr, int tgts, int ntgts)
	{
	int x = tgts[0];
	abd_t dst, src;

	ASSERT3U(ntgts, ==, 1);
	ASSERT3U(x, >=, rr->rr_firstdatacol);
	ASSERT3U(x, <, rr->rr_cols);

	ASSERT3U(rr->rr_col[x].rc_size, <=, rr->rr_col[VDEV_RAIDZ_P].rc_size);

	src = rr->rr_col[VDEV_RAIDZ_P].rc_abd;
	dst = rr->rr_col[x].rc_abd;

	abd_copy_from_buf(dst, abd_to_buf(src), rr->rr_col[x].rc_size);

	for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
	uint64_t size = MIN(rr->rr_col[x].rc_size,
	rr->rr_col[c].rc_size);

	src = rr->rr_col[c].rc_abd;

	if (c == x)
	continue;

	(void) abd_iterate_func2(dst, src, 0, 0, size,
	vdev_raidz_reconst_p_func, NULL);
	}
	}

	static void
	vdev_raidz_reconstruct_q(raidz_row_t rr, int tgts, int ntgts)
	{
	int x = tgts[0];
	int c, exp;
	abd_t dst, src;

	ASSERT(ntgts == 1);

	ASSERT(rr->rr_col[x].rc_size <= rr->rr_col[VDEV_RAIDZ_Q].rc_size);

	for (c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
	uint64_t size = (c == x) ? 0 : MIN(rr->rr_col[x].rc_size,
	rr->rr_col[c].rc_size);

	src = rr->rr_col[c].rc_abd;
	dst = rr->rr_col[x].rc_abd;

	if (c == rr->rr_firstdatacol) {
	abd_copy(dst, src, size);
	if (rr->rr_col[x].rc_size > size) {
	abd_zero_off(dst, size,
	rr->rr_col[x].rc_size - size);
	}
	} else {
	ASSERT3U(size, <=, rr->rr_col[x].rc_size);
	(void) abd_iterate_func2(dst, src, 0, 0, size,
	vdev_raidz_reconst_q_pre_func, NULL);
	(void) abd_iterate_func(dst,
	size, rr->rr_col[x].rc_size - size,
	vdev_raidz_reconst_q_pre_tail_func, NULL);
	}
	}

	src = rr->rr_col[VDEV_RAIDZ_Q].rc_abd;
	dst = rr->rr_col[x].rc_abd;
	exp = 255 - (rr->rr_cols - 1 - x);

	struct reconst_q_struct rq = { abd_to_buf(src), exp };
	(void) abd_iterate_func(dst, 0, rr->rr_col[x].rc_size,
	vdev_raidz_reconst_q_post_func, &rq);
	}

	static void
	vdev_raidz_reconstruct_pq(raidz_row_t rr, int tgts, int ntgts)
	{
	uint8_t p, q, pxy, qxy, tmp, a, b, aexp, bexp;
	abd_t pdata, qdata;
	uint64_t xsize, ysize;
	int x = tgts[0];
	int y = tgts[1];
	abd_t xd, yd;

	ASSERT(ntgts == 2);
	ASSERT(x < y);
	ASSERT(x >= rr->rr_firstdatacol);
	ASSERT(y < rr->rr_cols);

	ASSERT(rr->rr_col[x].rc_size >= rr->rr_col[y].rc_size);

	/*
	* Move the parity data aside -- we're going to compute parity as
	* though columns x and y were full of zeros -- Pxy and Qxy. We want to
	* reuse the parity generation mechanism without trashing the actual
	* parity so we make those columns appear to be full of zeros by
	* setting their lengths to zero.
	*/
	pdata = rr->rr_col[VDEV_RAIDZ_P].rc_abd;
	qdata = rr->rr_col[VDEV_RAIDZ_Q].rc_abd;
	xsize = rr->rr_col[x].rc_size;
	ysize = rr->rr_col[y].rc_size;

	rr->rr_col[VDEV_RAIDZ_P].rc_abd =
	abd_alloc_linear(rr->rr_col[VDEV_RAIDZ_P].rc_size, B_TRUE);
	rr->rr_col[VDEV_RAIDZ_Q].rc_abd =
	abd_alloc_linear(rr->rr_col[VDEV_RAIDZ_Q].rc_size, B_TRUE);
	rr->rr_col[x].rc_size = 0;
	rr->rr_col[y].rc_size = 0;

	vdev_raidz_generate_parity_pq(rr);

	rr->rr_col[x].rc_size = xsize;
	rr->rr_col[y].rc_size = ysize;

	p = abd_to_buf(pdata);
	q = abd_to_buf(qdata);
	pxy = abd_to_buf(rr->rr_col[VDEV_RAIDZ_P].rc_abd);
	qxy = abd_to_buf(rr->rr_col[VDEV_RAIDZ_Q].rc_abd);
	xd = rr->rr_col[x].rc_abd;
	yd = rr->rr_col[y].rc_abd;

	/*
	* We now have:
	* Pxy = P + D_x + D_y
	* Qxy = Q + 2^(ndevs - 1 - x) * D_x + 2^(ndevs - 1 - y) * D_y
	*
	* We can then solve for D_x:
	* D_x = A * (P + Pxy) + B * (Q + Qxy)
	* where
	* A = 2^(x - y) * (2^(x - y) + 1)^-1
	* B = 2^(ndevs - 1 - x) * (2^(x - y) + 1)^-1
	*
	* With D_x in hand, we can easily solve for D_y:
	* D_y = P + Pxy + D_x
	*/

	a = vdev_raidz_pow2[255 + x - y];
	b = vdev_raidz_pow2[255 - (rr->rr_cols - 1 - x)];
	tmp = 255 - vdev_raidz_log2[a ^ 1];

	aexp = vdev_raidz_log2[vdev_raidz_exp2(a, tmp)];
	bexp = vdev_raidz_log2[vdev_raidz_exp2(b, tmp)];

	ASSERT3U(xsize, >=, ysize);
	struct reconst_pq_struct rpq = { p, q, pxy, qxy, aexp, bexp };

	(void) abd_iterate_func2(xd, yd, 0, 0, ysize,
	vdev_raidz_reconst_pq_func, &rpq);
	(void) abd_iterate_func(xd, ysize, xsize - ysize,
	vdev_raidz_reconst_pq_tail_func, &rpq);

	abd_free(rr->rr_col[VDEV_RAIDZ_P].rc_abd);
	abd_free(rr->rr_col[VDEV_RAIDZ_Q].rc_abd);

	/*
	* Restore the saved parity data.
	*/
	rr->rr_col[VDEV_RAIDZ_P].rc_abd = pdata;
	rr->rr_col[VDEV_RAIDZ_Q].rc_abd = qdata;
	}

	/*
	* In the general case of reconstruction, we must solve the system of linear
	* equations defined by the coefficients used to generate parity as well as
	* the contents of the data and parity disks. This can be expressed with
	* vectors for the original data (D) and the actual data (d) and parity (p)
	* and a matrix composed of the identity matrix (I) and a dispersal matrix (V):
	*
	* __ __ __ __
	* \| \| __ __ \| p_0 \|
	* \| V \| \| D_0 \| \| p_m-1 \|
	* \| \| x \| : \| = \| d_0 \|
	* \| I \| \| D_n-1 \| \| : \|
	* \| \| ~~ ~~ \| d_n-1 \|
	* ~~ ~~ ~~ ~~
	*
	* I is simply a square identity matrix of size n, and V is a vandermonde
	* matrix defined by the coefficients we chose for the various parity columns
	* (1, 2, 4). Note that these values were chosen both for simplicity, speedy
	* computation as well as linear separability.
	*
	* __ __ __ __
	* \| 1 .. 1 1 1 \| \| p_0 \|
	* \| 2^n-1 .. 4 2 1 \| __ __ \| : \|
	* \| 4^n-1 .. 16 4 1 \| \| D_0 \| \| p_m-1 \|
	* \| 1 .. 0 0 0 \| \| D_1 \| \| d_0 \|
	* \| 0 .. 0 0 0 \| x \| D_2 \| = \| d_1 \|
	* \| : : : : \| \| : \| \| d_2 \|
	* \| 0 .. 1 0 0 \| \| D_n-1 \| \| : \|
	* \| 0 .. 0 1 0 \| ~~ ~~ \| : \|
	* \| 0 .. 0 0 1 \| \| d_n-1 \|
	* ~~ ~~ ~~ ~~
	*
	* Note that I, V, d, and p are known. To compute D, we must invert the
	* matrix and use the known data and parity values to reconstruct the unknown
	* data values. We begin by removing the rows in V\|I and d\|p that correspond
	* to failed or missing columns; we then make V\|I square (n x n) and d\|p
	* sized n by removing rows corresponding to unused parity from the bottom up
	* to generate (V\|I)' and (d\|p)'. We can then generate the inverse of (V\|I)'
	* using Gauss-Jordan elimination. In the example below we use m=3 parity
	* columns, n=8 data columns, with errors in d_1, d_2, and p_1:
	* __ __
	* \| 1 1 1 1 1 1 1 1 \|
	* \| 128 64 32 16 8 4 2 1 \| <-----+-+-- missing disks
	* \| 19 205 116 29 64 16 4 1 \| / /
	* \| 1 0 0 0 0 0 0 0 \| / /
	* \| 0 1 0 0 0 0 0 0 \| <--' /
	* (V\|I) = \| 0 0 1 0 0 0 0 0 \| <---'
	* \| 0 0 0 1 0 0 0 0 \|
	* \| 0 0 0 0 1 0 0 0 \|
	* \| 0 0 0 0 0 1 0 0 \|
	* \| 0 0 0 0 0 0 1 0 \|
	* \| 0 0 0 0 0 0 0 1 \|
	* ~~ ~~
	* __ __
	* \| 1 1 1 1 1 1 1 1 \|
	* \| 128 64 32 16 8 4 2 1 \|
	* \| 19 205 116 29 64 16 4 1 \|
	* \| 1 0 0 0 0 0 0 0 \|
	* \| 0 1 0 0 0 0 0 0 \|
	* (V\|I)' = \| 0 0 1 0 0 0 0 0 \|
	* \| 0 0 0 1 0 0 0 0 \|
	* \| 0 0 0 0 1 0 0 0 \|
	* \| 0 0 0 0 0 1 0 0 \|
	* \| 0 0 0 0 0 0 1 0 \|
	* \| 0 0 0 0 0 0 0 1 \|
	* ~~ ~~
	*
	* Here we employ Gauss-Jordan elimination to find the inverse of (V\|I)'. We
	* have carefully chosen the seed values 1, 2, and 4 to ensure that this
	* matrix is not singular.
	* __ __
	* \| 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 \|
	* \| 19 205 116 29 64 16 4 1 0 1 0 0 0 0 0 0 \|
	* \| 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 \|
	* \| 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 \|
	* \| 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 \|
	* \| 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 \|
	* \| 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 \|
	* \| 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 \|
	* ~~ ~~
	* __ __
	* \| 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 \|
	* \| 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 \|
	* \| 19 205 116 29 64 16 4 1 0 1 0 0 0 0 0 0 \|
	* \| 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 \|
	* \| 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 \|
	* \| 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 \|
	* \| 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 \|
	* \| 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 \|
	* ~~ ~~
	* __ __
	* \| 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 \|
	* \| 0 1 1 0 0 0 0 0 1 0 1 1 1 1 1 1 \|
	* \| 0 205 116 0 0 0 0 0 0 1 19 29 64 16 4 1 \|
	* \| 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 \|
	* \| 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 \|
	* \| 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 \|
	* \| 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 \|
	* \| 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 \|
	* ~~ ~~
	* __ __
	* \| 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 \|
	* \| 0 1 1 0 0 0 0 0 1 0 1 1 1 1 1 1 \|
	* \| 0 0 185 0 0 0 0 0 205 1 222 208 141 221 201 204 \|
	* \| 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 \|
	* \| 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 \|
	* \| 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 \|
	* \| 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 \|
	* \| 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 \|
	* ~~ ~~
	* __ __
	* \| 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 \|
	* \| 0 1 1 0 0 0 0 0 1 0 1 1 1 1 1 1 \|
	* \| 0 0 1 0 0 0 0 0 166 100 4 40 158 168 216 209 \|
	* \| 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 \|
	* \| 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 \|
	* \| 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 \|
	* \| 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 \|
	* \| 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 \|
	* ~~ ~~
	* __ __
	* \| 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 \|
	* \| 0 1 0 0 0 0 0 0 167 100 5 41 159 169 217 208 \|
	* \| 0 0 1 0 0 0 0 0 166 100 4 40 158 168 216 209 \|
	* \| 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 \|
	* \| 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 \|
	* \| 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 \|
	* \| 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 \|
	* \| 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 \|
	* ~~ ~~
	* __ __
	* \| 0 0 1 0 0 0 0 0 \|
	* \| 167 100 5 41 159 169 217 208 \|
	* \| 166 100 4 40 158 168 216 209 \|
	* (V\|I)'^-1 = \| 0 0 0 1 0 0 0 0 \|
	* \| 0 0 0 0 1 0 0 0 \|
	* \| 0 0 0 0 0 1 0 0 \|
	* \| 0 0 0 0 0 0 1 0 \|
	* \| 0 0 0 0 0 0 0 1 \|
	* ~~ ~~
	*
	* We can then simply compute D = (V\|I)'^-1 x (d\|p)' to discover the values
	* of the missing data.
	*
	* As is apparent from the example above, the only non-trivial rows in the
	* inverse matrix correspond to the data disks that we're trying to
	* reconstruct. Indeed, those are the only rows we need as the others would
	* only be useful for reconstructing data known or assumed to be valid. For
	* that reason, we only build the coefficients in the rows that correspond to
	* targeted columns.
	*/

	static void
	vdev_raidz_matrix_init(raidz_row_t rr, int n, int nmap, int map,
	uint8_t **rows)
	{
	int i, j;
	int pow;

	ASSERT(n == rr->rr_cols - rr->rr_firstdatacol);

	/*
	* Fill in the missing rows of interest.
	*/
	for (i = 0; i < nmap; i++) {
	ASSERT3S(0, <=, map[i]);
	ASSERT3S(map[i], <=, 2);

	pow = map[i] * n;
	if (pow > 255)
	pow -= 255;
	ASSERT(pow <= 255);

	for (j = 0; j < n; j++) {
	pow -= map[i];
	if (pow < 0)
	pow += 255;
	rows[i][j] = vdev_raidz_pow2[pow];
	}
	}
	}

	static void
	vdev_raidz_matrix_invert(raidz_row_t rr, int n, int nmissing, int missing,
	uint8_t rows, uint8_t invrows, const uint8_t *used)
	{
	int i, j, ii, jj;
	uint8_t log;

	/*
	* Assert that the first nmissing entries from the array of used
	* columns correspond to parity columns and that subsequent entries
	* correspond to data columns.
	*/
	for (i = 0; i < nmissing; i++) {
	ASSERT3S(used[i], <, rr->rr_firstdatacol);
	}
	for (; i < n; i++) {
	ASSERT3S(used[i], >=, rr->rr_firstdatacol);
	}

	/*
	* First initialize the storage where we'll compute the inverse rows.
	*/
	for (i = 0; i < nmissing; i++) {
	for (j = 0; j < n; j++) {
	invrows[i][j] = (i == j) ? 1 : 0;
	}
	}

	/*
	* Subtract all trivial rows from the rows of consequence.
	*/
	for (i = 0; i < nmissing; i++) {
	for (j = nmissing; j < n; j++) {
	ASSERT3U(used[j], >=, rr->rr_firstdatacol);
	jj = used[j] - rr->rr_firstdatacol;
	ASSERT3S(jj, <, n);
	invrows[i][j] = rows[i][jj];
	rows[i][jj] = 0;
	}
	}

	/*
	* For each of the rows of interest, we must normalize it and subtract
	* a multiple of it from the other rows.
	*/
	for (i = 0; i < nmissing; i++) {
	for (j = 0; j < missing[i]; j++) {
	ASSERT0(rows[i][j]);
	}
	ASSERT3U(rows[i][missing[i]], !=, 0);

	/*
	* Compute the inverse of the first element and multiply each
	* element in the row by that value.
	*/
	log = 255 - vdev_raidz_log2[rows[i][missing[i]]];

	for (j = 0; j < n; j++) {
	rows[i][j] = vdev_raidz_exp2(rows[i][j], log);
	invrows[i][j] = vdev_raidz_exp2(invrows[i][j], log);
	}

	for (ii = 0; ii < nmissing; ii++) {
	if (i == ii)
	continue;

	ASSERT3U(rows[ii][missing[i]], !=, 0);

	log = vdev_raidz_log2[rows[ii][missing[i]]];

	for (j = 0; j < n; j++) {
	rows[ii][j] ^=
	vdev_raidz_exp2(rows[i][j], log);
	invrows[ii][j] ^=
	vdev_raidz_exp2(invrows[i][j], log);
	}
	}
	}

	/*
	* Verify that the data that is left in the rows are properly part of
	* an identity matrix.
	*/
	for (i = 0; i < nmissing; i++) {
	for (j = 0; j < n; j++) {
	if (j == missing[i]) {
	ASSERT3U(rows[i][j], ==, 1);
	} else {
	ASSERT0(rows[i][j]);
	}
	}
	}
	}

	static void
	vdev_raidz_matrix_reconstruct(raidz_row_t *rr, int n, int nmissing,
	int missing, uint8_t invrows, const uint8_t used)
	{
	int i, j, x, cc, c;
	uint8_t *src;
	uint64_t ccount;
	uint8_t *dst[VDEV_RAIDZ_MAXPARITY] = { NULL };
	uint64_t dcount[VDEV_RAIDZ_MAXPARITY] = { 0 };
	uint8_t log = 0;
	uint8_t val;
	int ll;
	uint8_t *invlog[VDEV_RAIDZ_MAXPARITY];
	uint8_t p, pp;
	size_t psize;

	psize = sizeof (invlog[0][0]) * n * nmissing;
	p = kmem_alloc(psize, KM_SLEEP);

	for (pp = p, i = 0; i < nmissing; i++) {
	invlog[i] = pp;
	pp += n;
	}

	for (i = 0; i < nmissing; i++) {
	for (j = 0; j < n; j++) {
	ASSERT3U(invrows[i][j], !=, 0);
	invlog[i][j] = vdev_raidz_log2[invrows[i][j]];
	}
	}

	for (i = 0; i < n; i++) {
	c = used[i];
	ASSERT3U(c, <, rr->rr_cols);

	ccount = rr->rr_col[c].rc_size;
	ASSERT(ccount >= rr->rr_col[missing[0]].rc_size \|\| i > 0);
	if (ccount == 0)
	continue;
	src = abd_to_buf(rr->rr_col[c].rc_abd);
	for (j = 0; j < nmissing; j++) {
	cc = missing[j] + rr->rr_firstdatacol;
	ASSERT3U(cc, >=, rr->rr_firstdatacol);
	ASSERT3U(cc, <, rr->rr_cols);
	ASSERT3U(cc, !=, c);

	dcount[j] = rr->rr_col[cc].rc_size;
	if (dcount[j] != 0)
	dst[j] = abd_to_buf(rr->rr_col[cc].rc_abd);
	}

	for (x = 0; x < ccount; x++, src++) {
	if (*src != 0)
	log = vdev_raidz_log2[*src];

	for (cc = 0; cc < nmissing; cc++) {
	if (x >= dcount[cc])
	continue;

	if (*src == 0) {
	val = 0;
	} else {
	if ((ll = log + invlog[cc][i]) >= 255)
	ll -= 255;
	val = vdev_raidz_pow2[ll];
	}

	if (i == 0)
	dst[cc][x] = val;
	else
	dst[cc][x] ^= val;
	}
	}
	}

	kmem_free(p, psize);
	}

	static void
	vdev_raidz_reconstruct_general(raidz_row_t rr, int tgts, int ntgts)
	{
	- int n, i, c, t, tt;
	- int nmissing_rows;
	+ int i, c, t, tt;
	+ unsigned int n;
	+ unsigned int nmissing_rows;
	int missing_rows[VDEV_RAIDZ_MAXPARITY];
	int parity_map[VDEV_RAIDZ_MAXPARITY];
	uint8_t p, pp;
	size_t psize;
	uint8_t *rows[VDEV_RAIDZ_MAXPARITY];
	uint8_t *invrows[VDEV_RAIDZ_MAXPARITY];
	uint8_t *used;

	abd_t **bufs = NULL;

	/*
	* Matrix reconstruction can't use scatter ABDs yet, so we allocate
	* temporary linear ABDs if any non-linear ABDs are found.
	*/
	for (i = rr->rr_firstdatacol; i < rr->rr_cols; i++) {
	if (!abd_is_linear(rr->rr_col[i].rc_abd)) {
	bufs = kmem_alloc(rr->rr_cols * sizeof (abd_t *),
	KM_PUSHPAGE);

	for (c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
	raidz_col_t *col = &rr->rr_col[c];

	bufs[c] = col->rc_abd;
	if (bufs[c] != NULL) {
	col->rc_abd = abd_alloc_linear(
	col->rc_size, B_TRUE);
	abd_copy(col->rc_abd, bufs[c],
	col->rc_size);
	}
	}

	break;
	}
	}

	n = rr->rr_cols - rr->rr_firstdatacol;

	/*
	* Figure out which data columns are missing.
	*/
	nmissing_rows = 0;
	for (t = 0; t < ntgts; t++) {
	if (tgts[t] >= rr->rr_firstdatacol) {
	missing_rows[nmissing_rows++] =
	tgts[t] - rr->rr_firstdatacol;
	}
	}

	/*
	* Figure out which parity columns to use to help generate the missing
	* data columns.
	*/
	for (tt = 0, c = 0, i = 0; i < nmissing_rows; c++) {
	ASSERT(tt < ntgts);
	ASSERT(c < rr->rr_firstdatacol);

	/*
	* Skip any targeted parity columns.
	*/
	if (c == tgts[tt]) {
	tt++;
	continue;
	}

	parity_map[i] = c;
	i++;
	}

	psize = (sizeof (rows[0][0]) + sizeof (invrows[0][0])) *
	nmissing_rows * n + sizeof (used[0]) * n;
	p = kmem_alloc(psize, KM_SLEEP);

	for (pp = p, i = 0; i < nmissing_rows; i++) {
	rows[i] = pp;
	pp += n;
	invrows[i] = pp;
	pp += n;
	}
	used = pp;

	for (i = 0; i < nmissing_rows; i++) {
	used[i] = parity_map[i];
	}

	for (tt = 0, c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
	if (tt < nmissing_rows &&
	c == missing_rows[tt] + rr->rr_firstdatacol) {
	tt++;
	continue;
	}

	ASSERT3S(i, <, n);
	used[i] = c;
	i++;
	}

	/*
	* Initialize the interesting rows of the matrix.
	*/
	vdev_raidz_matrix_init(rr, n, nmissing_rows, parity_map, rows);

	/*
	* Invert the matrix.
	*/
	vdev_raidz_matrix_invert(rr, n, nmissing_rows, missing_rows, rows,
	invrows, used);

	/*
	* Reconstruct the missing data using the generated matrix.
	*/
	vdev_raidz_matrix_reconstruct(rr, n, nmissing_rows, missing_rows,
	invrows, used);

	kmem_free(p, psize);

	/*
	* copy back from temporary linear abds and free them
	*/
	if (bufs) {
	for (c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
	raidz_col_t *col = &rr->rr_col[c];

	if (bufs[c] != NULL) {
	abd_copy(bufs[c], col->rc_abd, col->rc_size);
	abd_free(col->rc_abd);
	}
	col->rc_abd = bufs[c];
	}
	kmem_free(bufs, rr->rr_cols * sizeof (abd_t *));
	}
	}

	static void
	vdev_raidz_reconstruct_row(raidz_map_t rm, raidz_row_t rr,
	const int *t, int nt)
	{
	int tgts[VDEV_RAIDZ_MAXPARITY], *dt;
	int ntgts;
	int i, c, ret;
	int nbadparity, nbaddata;
	int parity_valid[VDEV_RAIDZ_MAXPARITY];

	nbadparity = rr->rr_firstdatacol;
	nbaddata = rr->rr_cols - nbadparity;
	ntgts = 0;
	for (i = 0, c = 0; c < rr->rr_cols; c++) {
	if (c < rr->rr_firstdatacol)
	parity_valid[c] = B_FALSE;

	if (i < nt && c == t[i]) {
	tgts[ntgts++] = c;
	i++;
	} else if (rr->rr_col[c].rc_error != 0) {
	tgts[ntgts++] = c;
	} else if (c >= rr->rr_firstdatacol) {
	nbaddata--;
	} else {
	parity_valid[c] = B_TRUE;
	nbadparity--;
	}
	}

	ASSERT(ntgts >= nt);
	ASSERT(nbaddata >= 0);
	ASSERT(nbaddata + nbadparity == ntgts);

	dt = &tgts[nbadparity];

	/* Reconstruct using the new math implementation */
	ret = vdev_raidz_math_reconstruct(rm, rr, parity_valid, dt, nbaddata);
	if (ret != RAIDZ_ORIGINAL_IMPL)
	return;

	/*
	* See if we can use any of our optimized reconstruction routines.
	*/
	switch (nbaddata) {
	case 1:
	if (parity_valid[VDEV_RAIDZ_P]) {
	vdev_raidz_reconstruct_p(rr, dt, 1);
	return;
	}

	ASSERT(rr->rr_firstdatacol > 1);

	if (parity_valid[VDEV_RAIDZ_Q]) {
	vdev_raidz_reconstruct_q(rr, dt, 1);
	return;
	}

	ASSERT(rr->rr_firstdatacol > 2);
	break;

	case 2:
	ASSERT(rr->rr_firstdatacol > 1);

	if (parity_valid[VDEV_RAIDZ_P] &&
	parity_valid[VDEV_RAIDZ_Q]) {
	vdev_raidz_reconstruct_pq(rr, dt, 2);
	return;
	}

	ASSERT(rr->rr_firstdatacol > 2);

	break;
	}

	vdev_raidz_reconstruct_general(rr, tgts, ntgts);
	}

	static int
	vdev_raidz_open(vdev_t vd, uint64_t asize, uint64_t *max_asize,
	uint64_t logical_ashift, uint64_t physical_ashift)
	{
	vdev_raidz_t *vdrz = vd->vdev_tsd;
	uint64_t nparity = vdrz->vd_nparity;
	int c;
	int lasterror = 0;
	int numerrors = 0;

	ASSERT(nparity > 0);

	if (nparity > VDEV_RAIDZ_MAXPARITY \|\|
	vd->vdev_children < nparity + 1) {
	vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
	return (SET_ERROR(EINVAL));
	}

	vdev_open_children(vd);

	for (c = 0; c < vd->vdev_children; c++) {
	vdev_t *cvd = vd->vdev_child[c];

	if (cvd->vdev_open_error != 0) {
	lasterror = cvd->vdev_open_error;
	numerrors++;
	continue;
	}

	asize = MIN(asize - 1, cvd->vdev_asize - 1) + 1;
	max_asize = MIN(max_asize - 1, cvd->vdev_max_asize - 1) + 1;
	logical_ashift = MAX(logical_ashift, cvd->vdev_ashift);
	}
	for (c = 0; c < vd->vdev_children; c++) {
	vdev_t *cvd = vd->vdev_child[c];

	if (cvd->vdev_open_error != 0)
	continue;
	physical_ashift = vdev_best_ashift(logical_ashift,
	*physical_ashift, cvd->vdev_physical_ashift);
	}

	asize = vd->vdev_children;
	max_asize = vd->vdev_children;

	if (numerrors > nparity) {
	vd->vdev_stat.vs_aux = VDEV_AUX_NO_REPLICAS;
	return (lasterror);
	}

	return (0);
	}

	static void
	vdev_raidz_close(vdev_t *vd)
	{
	for (int c = 0; c < vd->vdev_children; c++) {
	if (vd->vdev_child[c] != NULL)
	vdev_close(vd->vdev_child[c]);
	}
	}

	static uint64_t
	vdev_raidz_asize(vdev_t *vd, uint64_t psize)
	{
	vdev_raidz_t *vdrz = vd->vdev_tsd;
	uint64_t asize;
	uint64_t ashift = vd->vdev_top->vdev_ashift;
	uint64_t cols = vdrz->vd_logical_width;
	uint64_t nparity = vdrz->vd_nparity;

	asize = ((psize - 1) >> ashift) + 1;
	asize += nparity * ((asize + cols - nparity - 1) / (cols - nparity));
	asize = roundup(asize, nparity + 1) << ashift;

	return (asize);
	}

	/*
	* The allocatable space for a raidz vdev is N * sizeof(smallest child)
	* so each child must provide at least 1/Nth of its asize.
	*/
	static uint64_t
	vdev_raidz_min_asize(vdev_t *vd)
	{
	return ((vd->vdev_min_asize + vd->vdev_children - 1) /
	vd->vdev_children);
	}

	void
	vdev_raidz_child_done(zio_t *zio)
	{
	raidz_col_t *rc = zio->io_private;

	ASSERT3P(rc->rc_abd, !=, NULL);
	rc->rc_error = zio->io_error;
	rc->rc_tried = 1;
	rc->rc_skipped = 0;
	}

	static void
	vdev_raidz_io_verify(vdev_t vd, raidz_row_t rr, int col)
	{
	#ifdef ZFS_DEBUG
	vdev_t *tvd = vd->vdev_top;

	range_seg64_t logical_rs, physical_rs, remain_rs;
	logical_rs.rs_start = rr->rr_offset;
	logical_rs.rs_end = logical_rs.rs_start +
	vdev_raidz_asize(vd, rr->rr_size);

	raidz_col_t *rc = &rr->rr_col[col];
	vdev_t *cvd = vd->vdev_child[rc->rc_devidx];

	vdev_xlate(cvd, &logical_rs, &physical_rs, &remain_rs);
	ASSERT(vdev_xlate_is_empty(&remain_rs));
	ASSERT3U(rc->rc_offset, ==, physical_rs.rs_start);
	ASSERT3U(rc->rc_offset, <, physical_rs.rs_end);
	/*
	* It would be nice to assert that rs_end is equal
	* to rc_offset + rc_size but there might be an
	* optional I/O at the end that is not accounted in
	* rc_size.
	*/
	if (physical_rs.rs_end > rc->rc_offset + rc->rc_size) {
	ASSERT3U(physical_rs.rs_end, ==, rc->rc_offset +
	rc->rc_size + (1 << tvd->vdev_ashift));
	} else {
	ASSERT3U(physical_rs.rs_end, ==, rc->rc_offset + rc->rc_size);
	}
	#endif
	}

	static void
	vdev_raidz_io_start_write(zio_t zio, raidz_row_t rr, uint64_t ashift)
	{
	vdev_t *vd = zio->io_vd;
	raidz_map_t *rm = zio->io_vsd;

	vdev_raidz_generate_parity_row(rm, rr);

	for (int c = 0; c < rr->rr_scols; c++) {
	raidz_col_t *rc = &rr->rr_col[c];
	vdev_t *cvd = vd->vdev_child[rc->rc_devidx];

	/* Verify physical to logical translation */
	vdev_raidz_io_verify(vd, rr, c);

	if (rc->rc_size > 0) {
	ASSERT3P(rc->rc_abd, !=, NULL);
	zio_nowait(zio_vdev_child_io(zio, NULL, cvd,
	rc->rc_offset, rc->rc_abd,
	abd_get_size(rc->rc_abd), zio->io_type,
	zio->io_priority, 0, vdev_raidz_child_done, rc));
	} else {
	/*
	* Generate optional write for skip sector to improve
	* aggregation contiguity.
	*/
	ASSERT3P(rc->rc_abd, ==, NULL);
	zio_nowait(zio_vdev_child_io(zio, NULL, cvd,
	rc->rc_offset, NULL, 1ULL << ashift,
	zio->io_type, zio->io_priority,
	ZIO_FLAG_NODATA \| ZIO_FLAG_OPTIONAL, NULL,
	NULL));
	}
	}
	}

	static void
	vdev_raidz_io_start_read(zio_t zio, raidz_row_t rr)
	{
	vdev_t *vd = zio->io_vd;

	/*
	* Iterate over the columns in reverse order so that we hit the parity
	* last -- any errors along the way will force us to read the parity.
	*/
	for (int c = rr->rr_cols - 1; c >= 0; c--) {
	raidz_col_t *rc = &rr->rr_col[c];
	if (rc->rc_size == 0)
	continue;
	vdev_t *cvd = vd->vdev_child[rc->rc_devidx];
	if (!vdev_readable(cvd)) {
	if (c >= rr->rr_firstdatacol)
	rr->rr_missingdata++;
	else
	rr->rr_missingparity++;
	rc->rc_error = SET_ERROR(ENXIO);
	rc->rc_tried = 1; /* don't even try */
	rc->rc_skipped = 1;
	continue;
	}
	if (vdev_dtl_contains(cvd, DTL_MISSING, zio->io_txg, 1)) {
	if (c >= rr->rr_firstdatacol)
	rr->rr_missingdata++;
	else
	rr->rr_missingparity++;
	rc->rc_error = SET_ERROR(ESTALE);
	rc->rc_skipped = 1;
	continue;
	}
	if (c >= rr->rr_firstdatacol \|\| rr->rr_missingdata > 0 \|\|
	(zio->io_flags & (ZIO_FLAG_SCRUB \| ZIO_FLAG_RESILVER))) {
	zio_nowait(zio_vdev_child_io(zio, NULL, cvd,
	rc->rc_offset, rc->rc_abd, rc->rc_size,
	zio->io_type, zio->io_priority, 0,
	vdev_raidz_child_done, rc));
	}
	}
	}

	/*
	* Start an IO operation on a RAIDZ VDev
	*
	* Outline:
	* - For write operations:
	* 1. Generate the parity data
	* 2. Create child zio write operations to each column's vdev, for both
	* data and parity.
	* 3. If the column skips any sectors for padding, create optional dummy
	* write zio children for those areas to improve aggregation continuity.
	* - For read operations:
	* 1. Create child zio read operations to each data column's vdev to read
	* the range of data required for zio.
	* 2. If this is a scrub or resilver operation, or if any of the data
	* vdevs have had errors, then create zio read operations to the parity
	* columns' VDevs as well.
	*/
	static void
	vdev_raidz_io_start(zio_t *zio)
	{
	vdev_t *vd = zio->io_vd;
	vdev_t *tvd = vd->vdev_top;
	vdev_raidz_t *vdrz = vd->vdev_tsd;

	raidz_map_t *rm = vdev_raidz_map_alloc(zio, tvd->vdev_ashift,
	vdrz->vd_logical_width, vdrz->vd_nparity);
	zio->io_vsd = rm;
	zio->io_vsd_ops = &vdev_raidz_vsd_ops;

	/*
	* Until raidz expansion is implemented all maps for a raidz vdev
	* contain a single row.
	*/
	ASSERT3U(rm->rm_nrows, ==, 1);
	raidz_row_t *rr = rm->rm_row[0];

	if (zio->io_type == ZIO_TYPE_WRITE) {
	vdev_raidz_io_start_write(zio, rr, tvd->vdev_ashift);
	} else {
	ASSERT(zio->io_type == ZIO_TYPE_READ);
	vdev_raidz_io_start_read(zio, rr);
	}

	zio_execute(zio);
	}

	/*
	* Report a checksum error for a child of a RAID-Z device.
	*/
	void
	vdev_raidz_checksum_error(zio_t zio, raidz_col_t rc, abd_t *bad_data)
	{
	vdev_t *vd = zio->io_vd->vdev_child[rc->rc_devidx];

	if (!(zio->io_flags & ZIO_FLAG_SPECULATIVE) &&
	zio->io_priority != ZIO_PRIORITY_REBUILD) {
	zio_bad_cksum_t zbc;
	raidz_map_t *rm = zio->io_vsd;

	zbc.zbc_has_cksum = 0;
	zbc.zbc_injected = rm->rm_ecksuminjected;

	mutex_enter(&vd->vdev_stat_lock);
	vd->vdev_stat.vs_checksum_errors++;
	mutex_exit(&vd->vdev_stat_lock);
	(void) zfs_ereport_post_checksum(zio->io_spa, vd,
	&zio->io_bookmark, zio, rc->rc_offset, rc->rc_size,
	rc->rc_abd, bad_data, &zbc);
	}
	}

	/*
	* We keep track of whether or not there were any injected errors, so that
	* any ereports we generate can note it.
	*/
	static int
	raidz_checksum_verify(zio_t *zio)
	{
	zio_bad_cksum_t zbc = {0};
	raidz_map_t *rm = zio->io_vsd;

	int ret = zio_checksum_error(zio, &zbc);
	if (ret != 0 && zbc.zbc_injected != 0)
	rm->rm_ecksuminjected = 1;

	return (ret);
	}

	/*
	* Generate the parity from the data columns. If we tried and were able to
	* read the parity without error, verify that the generated parity matches the
	* data we read. If it doesn't, we fire off a checksum error. Return the
	* number of such failures.
	*/
	static int
	raidz_parity_verify(zio_t zio, raidz_row_t rr)
	{
	abd_t *orig[VDEV_RAIDZ_MAXPARITY];
	int c, ret = 0;
	raidz_map_t *rm = zio->io_vsd;
	raidz_col_t *rc;

	blkptr_t *bp = zio->io_bp;
	enum zio_checksum checksum = (bp == NULL ? zio->io_prop.zp_checksum :
	(BP_IS_GANG(bp) ? ZIO_CHECKSUM_GANG_HEADER : BP_GET_CHECKSUM(bp)));

	if (checksum == ZIO_CHECKSUM_NOPARITY)
	return (ret);

	for (c = 0; c < rr->rr_firstdatacol; c++) {
	rc = &rr->rr_col[c];
	if (!rc->rc_tried \|\| rc->rc_error != 0)
	continue;

	orig[c] = rc->rc_abd;
	ASSERT3U(abd_get_size(rc->rc_abd), ==, rc->rc_size);
	rc->rc_abd = abd_alloc_linear(rc->rc_size, B_FALSE);
	}

	/*
	* Verify any empty sectors are zero filled to ensure the parity
	* is calculated correctly even if these non-data sectors are damaged.
	*/
	if (rr->rr_nempty && rr->rr_abd_empty != NULL)
	ret += vdev_draid_map_verify_empty(zio, rr);

	/*
	* Regenerates parity even for !tried\|\|rc_error!=0 columns. This
	* isn't harmful but it does have the side effect of fixing stuff
	* we didn't realize was necessary (i.e. even if we return 0).
	*/
	vdev_raidz_generate_parity_row(rm, rr);

	for (c = 0; c < rr->rr_firstdatacol; c++) {
	rc = &rr->rr_col[c];

	if (!rc->rc_tried \|\| rc->rc_error != 0)
	continue;

	if (abd_cmp(orig[c], rc->rc_abd) != 0) {
	vdev_raidz_checksum_error(zio, rc, orig[c]);
	rc->rc_error = SET_ERROR(ECKSUM);
	ret++;
	}
	abd_free(orig[c]);
	}

	return (ret);
	}

	static int
	vdev_raidz_worst_error(raidz_row_t *rr)
	{
	int error = 0;

	for (int c = 0; c < rr->rr_cols; c++)
	error = zio_worst_error(error, rr->rr_col[c].rc_error);

	return (error);
	}

	static void
	vdev_raidz_io_done_verified(zio_t zio, raidz_row_t rr)
	{
	int unexpected_errors = 0;
	int parity_errors = 0;
	int parity_untried = 0;
	int data_errors = 0;

	ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ);

	for (int c = 0; c < rr->rr_cols; c++) {
	raidz_col_t *rc = &rr->rr_col[c];

	if (rc->rc_error) {
	if (c < rr->rr_firstdatacol)
	parity_errors++;
	else
	data_errors++;

	if (!rc->rc_skipped)
	unexpected_errors++;
	} else if (c < rr->rr_firstdatacol && !rc->rc_tried) {
	parity_untried++;
	}

	if (rc->rc_force_repair)
	unexpected_errors++;
	}

	/*
	* If we read more parity disks than were used for
	* reconstruction, confirm that the other parity disks produced
	* correct data.
	*
	* Note that we also regenerate parity when resilvering so we
	* can write it out to failed devices later.
	*/
	if (parity_errors + parity_untried <
	rr->rr_firstdatacol - data_errors \|\|
	(zio->io_flags & ZIO_FLAG_RESILVER)) {
	int n = raidz_parity_verify(zio, rr);
	unexpected_errors += n;
	}

	if (zio->io_error == 0 && spa_writeable(zio->io_spa) &&
	(unexpected_errors > 0 \|\| (zio->io_flags & ZIO_FLAG_RESILVER))) {
	/*
	* Use the good data we have in hand to repair damaged children.
	*/
	for (int c = 0; c < rr->rr_cols; c++) {
	raidz_col_t *rc = &rr->rr_col[c];
	vdev_t *vd = zio->io_vd;
	vdev_t *cvd = vd->vdev_child[rc->rc_devidx];

	if (!rc->rc_allow_repair) {
	continue;
	} else if (!rc->rc_force_repair &&
	(rc->rc_error == 0 \|\| rc->rc_size == 0)) {
	continue;
	}

	zio_nowait(zio_vdev_child_io(zio, NULL, cvd,
	rc->rc_offset, rc->rc_abd, rc->rc_size,
	ZIO_TYPE_WRITE,
	zio->io_priority == ZIO_PRIORITY_REBUILD ?
	ZIO_PRIORITY_REBUILD : ZIO_PRIORITY_ASYNC_WRITE,
	ZIO_FLAG_IO_REPAIR \| (unexpected_errors ?
	ZIO_FLAG_SELF_HEAL : 0), NULL, NULL));
	}
	}
	}

	static void
	raidz_restore_orig_data(raidz_map_t *rm)
	{
	for (int i = 0; i < rm->rm_nrows; i++) {
	raidz_row_t *rr = rm->rm_row[i];
	for (int c = 0; c < rr->rr_cols; c++) {
	raidz_col_t *rc = &rr->rr_col[c];
	if (rc->rc_need_orig_restore) {
	abd_copy(rc->rc_abd,
	rc->rc_orig_data, rc->rc_size);
	rc->rc_need_orig_restore = B_FALSE;
	}
	}
	}
	}

	/*
	* returns EINVAL if reconstruction of the block will not be possible
	* returns ECKSUM if this specific reconstruction failed
	* returns 0 on successful reconstruction
	*/
	static int
	raidz_reconstruct(zio_t zio, int ltgts, int ntgts, int nparity)
	{
	raidz_map_t *rm = zio->io_vsd;

	/* Reconstruct each row */
	for (int r = 0; r < rm->rm_nrows; r++) {
	raidz_row_t *rr = rm->rm_row[r];
	int my_tgts[VDEV_RAIDZ_MAXPARITY]; /* value is child id */
	int t = 0;
	int dead = 0;
	int dead_data = 0;

	for (int c = 0; c < rr->rr_cols; c++) {
	raidz_col_t *rc = &rr->rr_col[c];
	ASSERT0(rc->rc_need_orig_restore);
	if (rc->rc_error != 0) {
	dead++;
	if (c >= nparity)
	dead_data++;
	continue;
	}
	if (rc->rc_size == 0)
	continue;
	for (int lt = 0; lt < ntgts; lt++) {
	if (rc->rc_devidx == ltgts[lt]) {
	if (rc->rc_orig_data == NULL) {
	rc->rc_orig_data =
	abd_alloc_linear(
	rc->rc_size, B_TRUE);
	abd_copy(rc->rc_orig_data,
	rc->rc_abd, rc->rc_size);
	}
	rc->rc_need_orig_restore = B_TRUE;

	dead++;
	if (c >= nparity)
	dead_data++;
	my_tgts[t++] = c;
	break;
	}
	}
	}
	if (dead > nparity) {
	/* reconstruction not possible */
	raidz_restore_orig_data(rm);
	return (EINVAL);
	}
	if (dead_data > 0)
	vdev_raidz_reconstruct_row(rm, rr, my_tgts, t);
	}

	/* Check for success */
	if (raidz_checksum_verify(zio) == 0) {

	/* Reconstruction succeeded - report errors */
	for (int i = 0; i < rm->rm_nrows; i++) {
	raidz_row_t *rr = rm->rm_row[i];

	for (int c = 0; c < rr->rr_cols; c++) {
	raidz_col_t *rc = &rr->rr_col[c];
	if (rc->rc_need_orig_restore) {
	/*
	* Note: if this is a parity column,
	* we don't really know if it's wrong.
	* We need to let
	* vdev_raidz_io_done_verified() check
	* it, and if we set rc_error, it will
	* think that it is a "known" error
	* that doesn't need to be checked
	* or corrected.
	*/
	if (rc->rc_error == 0 &&
	c >= rr->rr_firstdatacol) {
	vdev_raidz_checksum_error(zio,
	rc, rc->rc_orig_data);
	rc->rc_error =
	SET_ERROR(ECKSUM);
	}
	rc->rc_need_orig_restore = B_FALSE;
	}
	}

	vdev_raidz_io_done_verified(zio, rr);
	}

	zio_checksum_verified(zio);

	return (0);
	}

	/* Reconstruction failed - restore original data */
	raidz_restore_orig_data(rm);
	return (ECKSUM);
	}

	/*
	* Iterate over all combinations of N bad vdevs and attempt a reconstruction.
	* Note that the algorithm below is non-optimal because it doesn't take into
	* account how reconstruction is actually performed. For example, with
	* triple-parity RAID-Z the reconstruction procedure is the same if column 4
	* is targeted as invalid as if columns 1 and 4 are targeted since in both
	* cases we'd only use parity information in column 0.
	*
	* The order that we find the various possible combinations of failed
	* disks is dictated by these rules:
	* - Examine each "slot" (the "i" in tgts[i])
	* - Try to increment this slot (tgts[i] = tgts[i] + 1)
	* - if we can't increment because it runs into the next slot,
	* reset our slot to the minimum, and examine the next slot
	*
	* For example, with a 6-wide RAIDZ3, and no known errors (so we have to choose
	* 3 columns to reconstruct), we will generate the following sequence:
	*
	* STATE ACTION
	* 0 1 2 special case: skip since these are all parity
	* 0 1 3 first slot: reset to 0; middle slot: increment to 2
	* 0 2 3 first slot: increment to 1
	* 1 2 3 first: reset to 0; middle: reset to 1; last: increment to 4
	* 0 1 4 first: reset to 0; middle: increment to 2
	* 0 2 4 first: increment to 1
	* 1 2 4 first: reset to 0; middle: increment to 3
	* 0 3 4 first: increment to 1
	* 1 3 4 first: increment to 2
	* 2 3 4 first: reset to 0; middle: reset to 1; last: increment to 5
	* 0 1 5 first: reset to 0; middle: increment to 2
	* 0 2 5 first: increment to 1
	* 1 2 5 first: reset to 0; middle: increment to 3
	* 0 3 5 first: increment to 1
	* 1 3 5 first: increment to 2
	* 2 3 5 first: reset to 0; middle: increment to 4
	* 0 4 5 first: increment to 1
	* 1 4 5 first: increment to 2
	* 2 4 5 first: increment to 3
	* 3 4 5 done
	*
	* This strategy works for dRAID but is less efficient when there are a large
	* number of child vdevs and therefore permutations to check. Furthermore,
	* since the raidz_map_t rows likely do not overlap reconstruction would be
	* possible as long as there are no more than nparity data errors per row.
	* These additional permutations are not currently checked but could be as
	* a future improvement.
	*/
	static int
	vdev_raidz_combrec(zio_t *zio)
	{
	int nparity = vdev_get_nparity(zio->io_vd);
	raidz_map_t *rm = zio->io_vsd;

	/* Check if there's enough data to attempt reconstrution. */
	for (int i = 0; i < rm->rm_nrows; i++) {
	raidz_row_t *rr = rm->rm_row[i];
	int total_errors = 0;

	for (int c = 0; c < rr->rr_cols; c++) {
	if (rr->rr_col[c].rc_error)
	total_errors++;
	}

	if (total_errors > nparity)
	return (vdev_raidz_worst_error(rr));
	}

	for (int num_failures = 1; num_failures <= nparity; num_failures++) {
	int tstore[VDEV_RAIDZ_MAXPARITY + 2];
	int ltgts = &tstore[1]; / value is logical child ID */

	/* Determine number of logical children, n */
	int n = zio->io_vd->vdev_children;

	ASSERT3U(num_failures, <=, nparity);
	ASSERT3U(num_failures, <=, VDEV_RAIDZ_MAXPARITY);

	/* Handle corner cases in combrec logic */
	ltgts[-1] = -1;
	for (int i = 0; i < num_failures; i++) {
	ltgts[i] = i;
	}
	ltgts[num_failures] = n;

	for (;;) {
	int err = raidz_reconstruct(zio, ltgts, num_failures,
	nparity);
	if (err == EINVAL) {
	/*
	* Reconstruction not possible with this #
	* failures; try more failures.
	*/
	break;
	} else if (err == 0)
	return (0);

	/* Compute next targets to try */
	for (int t = 0; ; t++) {
	ASSERT3U(t, <, num_failures);
	ltgts[t]++;
	if (ltgts[t] == n) {
	/* try more failures */
	ASSERT3U(t, ==, num_failures - 1);
	break;
	}

	ASSERT3U(ltgts[t], <, n);
	ASSERT3U(ltgts[t], <=, ltgts[t + 1]);

	/*
	* If that spot is available, we're done here.
	* Try the next combination.
	*/
	if (ltgts[t] != ltgts[t + 1])
	break;

	/*
	* Otherwise, reset this tgt to the minimum,
	* and move on to the next tgt.
	*/
	ltgts[t] = ltgts[t - 1] + 1;
	ASSERT3U(ltgts[t], ==, t);
	}

	/* Increase the number of failures and keep trying. */
	if (ltgts[num_failures - 1] == n)
	break;
	}
	}

	return (ECKSUM);
	}

	void
	vdev_raidz_reconstruct(raidz_map_t rm, const int t, int nt)
	{
	for (uint64_t row = 0; row < rm->rm_nrows; row++) {
	raidz_row_t *rr = rm->rm_row[row];
	vdev_raidz_reconstruct_row(rm, rr, t, nt);
	}
	}

	/*
	* Complete a write IO operation on a RAIDZ VDev
	*
	* Outline:
	* 1. Check for errors on the child IOs.
	* 2. Return, setting an error code if too few child VDevs were written
	* to reconstruct the data later. Note that partial writes are
	* considered successful if they can be reconstructed at all.
	*/
	static void
	vdev_raidz_io_done_write_impl(zio_t zio, raidz_row_t rr)
	{
	int total_errors = 0;

	ASSERT3U(rr->rr_missingparity, <=, rr->rr_firstdatacol);
	ASSERT3U(rr->rr_missingdata, <=, rr->rr_cols - rr->rr_firstdatacol);
	ASSERT3U(zio->io_type, ==, ZIO_TYPE_WRITE);

	for (int c = 0; c < rr->rr_cols; c++) {
	raidz_col_t *rc = &rr->rr_col[c];

	if (rc->rc_error) {
	ASSERT(rc->rc_error != ECKSUM); /* child has no bp */

	total_errors++;
	}
	}

	/*
	* Treat partial writes as a success. If we couldn't write enough
	* columns to reconstruct the data, the I/O failed. Otherwise,
	* good enough.
	*
	* Now that we support write reallocation, it would be better
	* to treat partial failure as real failure unless there are
	* no non-degraded top-level vdevs left, and not update DTLs
	* if we intend to reallocate.
	*/
	if (total_errors > rr->rr_firstdatacol) {
	zio->io_error = zio_worst_error(zio->io_error,
	vdev_raidz_worst_error(rr));
	}
	}

	static void
	vdev_raidz_io_done_reconstruct_known_missing(zio_t zio, raidz_map_t rm,
	raidz_row_t *rr)
	{
	int parity_errors = 0;
	int parity_untried = 0;
	int data_errors = 0;
	int total_errors = 0;

	ASSERT3U(rr->rr_missingparity, <=, rr->rr_firstdatacol);
	ASSERT3U(rr->rr_missingdata, <=, rr->rr_cols - rr->rr_firstdatacol);
	ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ);

	for (int c = 0; c < rr->rr_cols; c++) {
	raidz_col_t *rc = &rr->rr_col[c];

	/*
	* If scrubbing and a replacing/sparing child vdev determined
	* that not all of its children have an identical copy of the
	* data, then clear the error so the column is treated like
	* any other read and force a repair to correct the damage.
	*/
	if (rc->rc_error == ECKSUM) {
	ASSERT(zio->io_flags & ZIO_FLAG_SCRUB);
	vdev_raidz_checksum_error(zio, rc, rc->rc_abd);
	rc->rc_force_repair = 1;
	rc->rc_error = 0;
	}

	if (rc->rc_error) {
	if (c < rr->rr_firstdatacol)
	parity_errors++;
	else
	data_errors++;

	total_errors++;
	} else if (c < rr->rr_firstdatacol && !rc->rc_tried) {
	parity_untried++;
	}
	}

	/*
	* If there were data errors and the number of errors we saw was
	* correctable -- less than or equal to the number of parity disks read
	* -- reconstruct based on the missing data.
	*/
	if (data_errors != 0 &&
	total_errors <= rr->rr_firstdatacol - parity_untried) {
	/*
	* We either attempt to read all the parity columns or
	* none of them. If we didn't try to read parity, we
	* wouldn't be here in the correctable case. There must
	* also have been fewer parity errors than parity
	* columns or, again, we wouldn't be in this code path.
	*/
	ASSERT(parity_untried == 0);
	ASSERT(parity_errors < rr->rr_firstdatacol);

	/*
	* Identify the data columns that reported an error.
	*/
	int n = 0;
	int tgts[VDEV_RAIDZ_MAXPARITY];
	for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
	raidz_col_t *rc = &rr->rr_col[c];
	if (rc->rc_error != 0) {
	ASSERT(n < VDEV_RAIDZ_MAXPARITY);
	tgts[n++] = c;
	}
	}

	ASSERT(rr->rr_firstdatacol >= n);

	vdev_raidz_reconstruct_row(rm, rr, tgts, n);
	}
	}

	/*
	* Return the number of reads issued.
	*/
	static int
	vdev_raidz_read_all(zio_t zio, raidz_row_t rr)
	{
	vdev_t *vd = zio->io_vd;
	int nread = 0;

	rr->rr_missingdata = 0;
	rr->rr_missingparity = 0;

	/*
	* If this rows contains empty sectors which are not required
	* for a normal read then allocate an ABD for them now so they
	* may be read, verified, and any needed repairs performed.
	*/
	if (rr->rr_nempty && rr->rr_abd_empty == NULL)
	vdev_draid_map_alloc_empty(zio, rr);

	for (int c = 0; c < rr->rr_cols; c++) {
	raidz_col_t *rc = &rr->rr_col[c];
	if (rc->rc_tried \|\| rc->rc_size == 0)
	continue;

	zio_nowait(zio_vdev_child_io(zio, NULL,
	vd->vdev_child[rc->rc_devidx],
	rc->rc_offset, rc->rc_abd, rc->rc_size,
	zio->io_type, zio->io_priority, 0,
	vdev_raidz_child_done, rc));
	nread++;
	}
	return (nread);
	}

	/*
	* We're here because either there were too many errors to even attempt
	* reconstruction (total_errors == rm_first_datacol), or vdev_*_combrec()
	* failed. In either case, there is enough bad data to prevent reconstruction.
	* Start checksum ereports for all children which haven't failed.
	*/
	static void
	vdev_raidz_io_done_unrecoverable(zio_t *zio)
	{
	raidz_map_t *rm = zio->io_vsd;

	for (int i = 0; i < rm->rm_nrows; i++) {
	raidz_row_t *rr = rm->rm_row[i];

	for (int c = 0; c < rr->rr_cols; c++) {
	raidz_col_t *rc = &rr->rr_col[c];
	vdev_t *cvd = zio->io_vd->vdev_child[rc->rc_devidx];

	if (rc->rc_error != 0)
	continue;

	zio_bad_cksum_t zbc;
	zbc.zbc_has_cksum = 0;
	zbc.zbc_injected = rm->rm_ecksuminjected;

	mutex_enter(&cvd->vdev_stat_lock);
	cvd->vdev_stat.vs_checksum_errors++;
	mutex_exit(&cvd->vdev_stat_lock);
	(void) zfs_ereport_start_checksum(zio->io_spa,
	cvd, &zio->io_bookmark, zio, rc->rc_offset,
	rc->rc_size, &zbc);
	}
	}
	}

	void
	vdev_raidz_io_done(zio_t *zio)
	{
	raidz_map_t *rm = zio->io_vsd;

	if (zio->io_type == ZIO_TYPE_WRITE) {
	for (int i = 0; i < rm->rm_nrows; i++) {
	vdev_raidz_io_done_write_impl(zio, rm->rm_row[i]);
	}
	} else {
	for (int i = 0; i < rm->rm_nrows; i++) {
	raidz_row_t *rr = rm->rm_row[i];
	vdev_raidz_io_done_reconstruct_known_missing(zio,
	rm, rr);
	}

	if (raidz_checksum_verify(zio) == 0) {
	for (int i = 0; i < rm->rm_nrows; i++) {
	raidz_row_t *rr = rm->rm_row[i];
	vdev_raidz_io_done_verified(zio, rr);
	}
	zio_checksum_verified(zio);
	} else {
	/*
	* A sequential resilver has no checksum which makes
	* combinatoral reconstruction impossible. This code
	* path is unreachable since raidz_checksum_verify()
	* has no checksum to verify and must succeed.
	*/
	ASSERT3U(zio->io_priority, !=, ZIO_PRIORITY_REBUILD);

	/*
	* This isn't a typical situation -- either we got a
	* read error or a child silently returned bad data.
	* Read every block so we can try again with as much
	* data and parity as we can track down. If we've
	* already been through once before, all children will
	* be marked as tried so we'll proceed to combinatorial
	* reconstruction.
	*/
	int nread = 0;
	for (int i = 0; i < rm->rm_nrows; i++) {
	nread += vdev_raidz_read_all(zio,
	rm->rm_row[i]);
	}
	if (nread != 0) {
	/*
	* Normally our stage is VDEV_IO_DONE, but if
	* we've already called redone(), it will have
	* changed to VDEV_IO_START, in which case we
	* don't want to call redone() again.
	*/
	if (zio->io_stage != ZIO_STAGE_VDEV_IO_START)
	zio_vdev_io_redone(zio);
	return;
	}

	zio->io_error = vdev_raidz_combrec(zio);
	if (zio->io_error == ECKSUM &&
	!(zio->io_flags & ZIO_FLAG_SPECULATIVE)) {
	vdev_raidz_io_done_unrecoverable(zio);
	}
	}
	}
	}

	static void
	vdev_raidz_state_change(vdev_t *vd, int faulted, int degraded)
	{
	vdev_raidz_t *vdrz = vd->vdev_tsd;
	if (faulted > vdrz->vd_nparity)
	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	VDEV_AUX_NO_REPLICAS);
	else if (degraded + faulted != 0)
	vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED, VDEV_AUX_NONE);
	else
	vdev_set_state(vd, B_FALSE, VDEV_STATE_HEALTHY, VDEV_AUX_NONE);
	}

	/*
	* Determine if any portion of the provided block resides on a child vdev
	* with a dirty DTL and therefore needs to be resilvered. The function
	* assumes that at least one DTL is dirty which implies that full stripe
	* width blocks must be resilvered.
	*/
	static boolean_t
	vdev_raidz_need_resilver(vdev_t vd, const dva_t dva, size_t psize,
	uint64_t phys_birth)
	{
	vdev_raidz_t *vdrz = vd->vdev_tsd;
	uint64_t dcols = vd->vdev_children;
	uint64_t nparity = vdrz->vd_nparity;
	uint64_t ashift = vd->vdev_top->vdev_ashift;
	/* The starting RAIDZ (parent) vdev sector of the block. */
	uint64_t b = DVA_GET_OFFSET(dva) >> ashift;
	/* The zio's size in units of the vdev's minimum sector size. */
	uint64_t s = ((psize - 1) >> ashift) + 1;
	/* The first column for this stripe. */
	uint64_t f = b % dcols;

	/* Unreachable by sequential resilver. */
	ASSERT3U(phys_birth, !=, TXG_UNKNOWN);

	if (!vdev_dtl_contains(vd, DTL_PARTIAL, phys_birth, 1))
	return (B_FALSE);

	if (s + nparity >= dcols)
	return (B_TRUE);

	for (uint64_t c = 0; c < s + nparity; c++) {
	uint64_t devidx = (f + c) % dcols;
	vdev_t *cvd = vd->vdev_child[devidx];

	/*
	* dsl_scan_need_resilver() already checked vd with
	* vdev_dtl_contains(). So here just check cvd with
	* vdev_dtl_empty(), cheaper and a good approximation.
	*/
	if (!vdev_dtl_empty(cvd, DTL_PARTIAL))
	return (B_TRUE);
	}

	return (B_FALSE);
	}

	static void
	vdev_raidz_xlate(vdev_t cvd, const range_seg64_t logical_rs,
	range_seg64_t physical_rs, range_seg64_t remain_rs)
	{
	(void) remain_rs;

	vdev_t *raidvd = cvd->vdev_parent;
	ASSERT(raidvd->vdev_ops == &vdev_raidz_ops);

	uint64_t width = raidvd->vdev_children;
	uint64_t tgt_col = cvd->vdev_id;
	uint64_t ashift = raidvd->vdev_top->vdev_ashift;

	/* make sure the offsets are block-aligned */
	ASSERT0(logical_rs->rs_start % (1 << ashift));
	ASSERT0(logical_rs->rs_end % (1 << ashift));
	uint64_t b_start = logical_rs->rs_start >> ashift;
	uint64_t b_end = logical_rs->rs_end >> ashift;

	uint64_t start_row = 0;
	if (b_start > tgt_col) /* avoid underflow */
	start_row = ((b_start - tgt_col - 1) / width) + 1;

	uint64_t end_row = 0;
	if (b_end > tgt_col)
	end_row = ((b_end - tgt_col - 1) / width) + 1;

	physical_rs->rs_start = start_row << ashift;
	physical_rs->rs_end = end_row << ashift;

	ASSERT3U(physical_rs->rs_start, <=, logical_rs->rs_start);
	ASSERT3U(physical_rs->rs_end - physical_rs->rs_start, <=,
	logical_rs->rs_end - logical_rs->rs_start);
	}

	/*
	* Initialize private RAIDZ specific fields from the nvlist.
	*/
	static int
	vdev_raidz_init(spa_t spa, nvlist_t nv, void **tsd)
	{
	vdev_raidz_t *vdrz;
	uint64_t nparity;

	uint_t children;
	nvlist_t **child;
	int error = nvlist_lookup_nvlist_array(nv,
	ZPOOL_CONFIG_CHILDREN, &child, &children);
	if (error != 0)
	return (SET_ERROR(EINVAL));

	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NPARITY, &nparity) == 0) {
	if (nparity == 0 \|\| nparity > VDEV_RAIDZ_MAXPARITY)
	return (SET_ERROR(EINVAL));

	/*
	* Previous versions could only support 1 or 2 parity
	* device.
	*/
	if (nparity > 1 && spa_version(spa) < SPA_VERSION_RAIDZ2)
	return (SET_ERROR(EINVAL));
	else if (nparity > 2 && spa_version(spa) < SPA_VERSION_RAIDZ3)
	return (SET_ERROR(EINVAL));
	} else {
	/*
	* We require the parity to be specified for SPAs that
	* support multiple parity levels.
	*/
	if (spa_version(spa) >= SPA_VERSION_RAIDZ2)
	return (SET_ERROR(EINVAL));

	/*
	* Otherwise, we default to 1 parity device for RAID-Z.
	*/
	nparity = 1;
	}

	vdrz = kmem_zalloc(sizeof (*vdrz), KM_SLEEP);
	vdrz->vd_logical_width = children;
	vdrz->vd_nparity = nparity;

	*tsd = vdrz;

	return (0);
	}

	static void
	vdev_raidz_fini(vdev_t *vd)
	{
	kmem_free(vd->vdev_tsd, sizeof (vdev_raidz_t));
	}

	/*
	* Add RAIDZ specific fields to the config nvlist.
	*/
	static void
	vdev_raidz_config_generate(vdev_t vd, nvlist_t nv)
	{
	ASSERT3P(vd->vdev_ops, ==, &vdev_raidz_ops);
	vdev_raidz_t *vdrz = vd->vdev_tsd;

	/*
	* Make sure someone hasn't managed to sneak a fancy new vdev
	* into a crufty old storage pool.
	*/
	ASSERT(vdrz->vd_nparity == 1 \|\|
	(vdrz->vd_nparity <= 2 &&
	spa_version(vd->vdev_spa) >= SPA_VERSION_RAIDZ2) \|\|
	(vdrz->vd_nparity <= 3 &&
	spa_version(vd->vdev_spa) >= SPA_VERSION_RAIDZ3));

	/*
	* Note that we'll add these even on storage pools where they
	* aren't strictly required -- older software will just ignore
	* it.
	*/
	fnvlist_add_uint64(nv, ZPOOL_CONFIG_NPARITY, vdrz->vd_nparity);
	}

	static uint64_t
	vdev_raidz_nparity(vdev_t *vd)
	{
	vdev_raidz_t *vdrz = vd->vdev_tsd;
	return (vdrz->vd_nparity);
	}

	static uint64_t
	vdev_raidz_ndisks(vdev_t *vd)
	{
	return (vd->vdev_children);
	}

	vdev_ops_t vdev_raidz_ops = {
	.vdev_op_init = vdev_raidz_init,
	.vdev_op_fini = vdev_raidz_fini,
	.vdev_op_open = vdev_raidz_open,
	.vdev_op_close = vdev_raidz_close,
	.vdev_op_asize = vdev_raidz_asize,
	.vdev_op_min_asize = vdev_raidz_min_asize,
	.vdev_op_min_alloc = NULL,
	.vdev_op_io_start = vdev_raidz_io_start,
	.vdev_op_io_done = vdev_raidz_io_done,
	.vdev_op_state_change = vdev_raidz_state_change,
	.vdev_op_need_resilver = vdev_raidz_need_resilver,
	.vdev_op_hold = NULL,
	.vdev_op_rele = NULL,
	.vdev_op_remap = NULL,
	.vdev_op_xlate = vdev_raidz_xlate,
	.vdev_op_rebuild_asize = NULL,
	.vdev_op_metaslab_init = NULL,
	.vdev_op_config_generate = vdev_raidz_config_generate,
	.vdev_op_nparity = vdev_raidz_nparity,
	.vdev_op_ndisks = vdev_raidz_ndisks,
	.vdev_op_type = VDEV_TYPE_RAIDZ, /* name of this vdev type */
	.vdev_op_leaf = B_FALSE /* not a leaf vdev */
	};
	diff --git a/sys/contrib/openzfs/module/zfs/zap.c b/sys/contrib/openzfs/module/zfs/zap.c
	index dde05d7005c2..da86defb445c 100644
	--- a/sys/contrib/openzfs/module/zfs/zap.c
	+++ b/sys/contrib/openzfs/module/zfs/zap.c
	@@ -1,1383 +1,1372 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2012, 2018 by Delphix. All rights reserved.
	* Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
	*/

	/*
	* This file contains the top half of the zfs directory structure
	* implementation. The bottom half is in zap_leaf.c.
	*
	* The zdir is an extendable hash data structure. There is a table of
	* pointers to buckets (zap_t->zd_data->zd_leafs). The buckets are
	* each a constant size and hold a variable number of directory entries.
	* The buckets (aka "leaf nodes") are implemented in zap_leaf.c.
	*
	* The pointer table holds a power of 2 number of pointers.
	* (1<<zap_t->zd_data->zd_phys->zd_prefix_len). The bucket pointed to
	* by the pointer at index i in the table holds entries whose hash value
	* has a zd_prefix_len - bit prefix
	*/

	#include <sys/spa.h>
	#include <sys/dmu.h>
	#include <sys/zfs_context.h>
	#include <sys/zfs_znode.h>
	#include <sys/fs/zfs.h>
	#include <sys/zap.h>
	#include <sys/zap_impl.h>
	#include <sys/zap_leaf.h>

	/*
	* If zap_iterate_prefetch is set, we will prefetch the entire ZAP object
	* (all leaf blocks) when we start iterating over it.
	*
	* For zap_cursor_init(), the callers all intend to iterate through all the
	* entries. There are a few cases where an error (typically i/o error) could
	* cause it to bail out early.
	*
	* For zap_cursor_init_serialized(), there are callers that do the iteration
	* outside of ZFS. Typically they would iterate over everything, but we
	* don't have control of that. E.g. zfs_ioc_snapshot_list_next(),
	* zcp_snapshots_iter(), and other iterators over things in the MOS - these
	* are called by /sbin/zfs and channel programs. The other example is
	* zfs_readdir() which iterates over directory entries for the getdents()
	* syscall. /sbin/ls iterates to the end (unless it receives a signal), but
	* userland doesn't have to.
	*
	* Given that the ZAP entries aren't returned in a specific order, the only
	* legitimate use cases for partial iteration would be:
	*
	* 1. Pagination: e.g. you only want to display 100 entries at a time, so you
	* get the first 100 and then wait for the user to hit "next page", which
	* they may never do).
	*
	* 2. You want to know if there are more than X entries, without relying on
	* the zfs-specific implementation of the directory's st_size (which is
	* the number of entries).
	*/
	static int zap_iterate_prefetch = B_TRUE;

	int fzap_default_block_shift = 14; /* 16k blocksize */

	static uint64_t zap_allocate_blocks(zap_t *zap, int nblocks);

	void
	fzap_byteswap(void *vbuf, size_t size)
	{
	uint64_t block_type = (uint64_t )vbuf;

	if (block_type == ZBT_LEAF \|\| block_type == BSWAP_64(ZBT_LEAF))
	zap_leaf_byteswap(vbuf, size);
	else {
	/* it's a ptrtbl block */
	byteswap_uint64_array(vbuf, size);
	}
	}

	void
	fzap_upgrade(zap_t zap, dmu_tx_t tx, zap_flags_t flags)
	{
	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));
	zap->zap_ismicro = FALSE;

	zap->zap_dbu.dbu_evict_func_sync = zap_evict_sync;
	zap->zap_dbu.dbu_evict_func_async = NULL;

	mutex_init(&zap->zap_f.zap_num_entries_mtx, 0, MUTEX_DEFAULT, 0);
	zap->zap_f.zap_block_shift = highbit64(zap->zap_dbuf->db_size) - 1;

	zap_phys_t *zp = zap_f_phys(zap);
	/*
	* explicitly zero it since it might be coming from an
	* initialized microzap
	*/
	memset(zap->zap_dbuf->db_data, 0, zap->zap_dbuf->db_size);
	zp->zap_block_type = ZBT_HEADER;
	zp->zap_magic = ZAP_MAGIC;

	zp->zap_ptrtbl.zt_shift = ZAP_EMBEDDED_PTRTBL_SHIFT(zap);

	zp->zap_freeblk = 2; /* block 1 will be the first leaf */
	zp->zap_num_leafs = 1;
	zp->zap_num_entries = 0;
	zp->zap_salt = zap->zap_salt;
	zp->zap_normflags = zap->zap_normflags;
	zp->zap_flags = flags;

	/* block 1 will be the first leaf */
	for (int i = 0; i < (1<<zp->zap_ptrtbl.zt_shift); i++)
	ZAP_EMBEDDED_PTRTBL_ENT(zap, i) = 1;

	/*
	* set up block 1 - the first leaf
	*/
	dmu_buf_t *db;
	- VERIFY0(dmu_buf_hold(zap->zap_objset, zap->zap_object,
	+ VERIFY0(dmu_buf_hold_by_dnode(zap->zap_dnode,
	1<<FZAP_BLOCK_SHIFT(zap), FTAG, &db, DMU_READ_NO_PREFETCH));
	dmu_buf_will_dirty(db, tx);

	zap_leaf_t *l = kmem_zalloc(sizeof (zap_leaf_t), KM_SLEEP);
	l->l_dbuf = db;

	zap_leaf_init(l, zp->zap_normflags != 0);

	kmem_free(l, sizeof (zap_leaf_t));
	dmu_buf_rele(db, FTAG);
	}

	static int
	zap_tryupgradedir(zap_t zap, dmu_tx_t tx)
	{
	if (RW_WRITE_HELD(&zap->zap_rwlock))
	return (1);
	if (rw_tryupgrade(&zap->zap_rwlock)) {
	dmu_buf_will_dirty(zap->zap_dbuf, tx);
	return (1);
	}
	return (0);
	}

	/*
	* Generic routines for dealing with the pointer & cookie tables.
	*/

	static int
	zap_table_grow(zap_t zap, zap_table_phys_t tbl,
	void (transfer_func)(const uint64_t src, uint64_t *dst, int n),
	dmu_tx_t *tx)
	{
	uint64_t newblk;
	int bs = FZAP_BLOCK_SHIFT(zap);
	int hepb = 1<<(bs-4);
	/* hepb = half the number of entries in a block */

	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));
	ASSERT(tbl->zt_blk != 0);
	ASSERT(tbl->zt_numblks > 0);

	if (tbl->zt_nextblk != 0) {
	newblk = tbl->zt_nextblk;
	} else {
	newblk = zap_allocate_blocks(zap, tbl->zt_numblks * 2);
	tbl->zt_nextblk = newblk;
	ASSERT0(tbl->zt_blks_copied);
	- dmu_prefetch(zap->zap_objset, zap->zap_object, 0,
	+ dmu_prefetch_by_dnode(zap->zap_dnode, 0,
	tbl->zt_blk << bs, tbl->zt_numblks << bs,
	ZIO_PRIORITY_SYNC_READ);
	}

	/*
	* Copy the ptrtbl from the old to new location.
	*/

	uint64_t b = tbl->zt_blks_copied;
	dmu_buf_t *db_old;
	- int err = dmu_buf_hold(zap->zap_objset, zap->zap_object,
	+ int err = dmu_buf_hold_by_dnode(zap->zap_dnode,
	(tbl->zt_blk + b) << bs, FTAG, &db_old, DMU_READ_NO_PREFETCH);
	if (err != 0)
	return (err);

	/* first half of entries in old[b] go to new[2b+0] /
	dmu_buf_t *db_new;
	- VERIFY0(dmu_buf_hold(zap->zap_objset, zap->zap_object,
	+ VERIFY0(dmu_buf_hold_by_dnode(zap->zap_dnode,
	(newblk + 2*b+0) << bs, FTAG, &db_new, DMU_READ_NO_PREFETCH));
	dmu_buf_will_dirty(db_new, tx);
	transfer_func(db_old->db_data, db_new->db_data, hepb);
	dmu_buf_rele(db_new, FTAG);

	/* second half of entries in old[b] go to new[2b+1] /
	- VERIFY0(dmu_buf_hold(zap->zap_objset, zap->zap_object,
	+ VERIFY0(dmu_buf_hold_by_dnode(zap->zap_dnode,
	(newblk + 2*b+1) << bs, FTAG, &db_new, DMU_READ_NO_PREFETCH));
	dmu_buf_will_dirty(db_new, tx);
	transfer_func((uint64_t *)db_old->db_data + hepb,
	db_new->db_data, hepb);
	dmu_buf_rele(db_new, FTAG);

	dmu_buf_rele(db_old, FTAG);

	tbl->zt_blks_copied++;

	dprintf("copied block %llu of %llu\n",
	(u_longlong_t)tbl->zt_blks_copied,
	(u_longlong_t)tbl->zt_numblks);

	if (tbl->zt_blks_copied == tbl->zt_numblks) {
	(void) dmu_free_range(zap->zap_objset, zap->zap_object,
	tbl->zt_blk << bs, tbl->zt_numblks << bs, tx);

	tbl->zt_blk = newblk;
	tbl->zt_numblks *= 2;
	tbl->zt_shift++;
	tbl->zt_nextblk = 0;
	tbl->zt_blks_copied = 0;

	dprintf("finished; numblocks now %llu (%uk entries)\n",
	(u_longlong_t)tbl->zt_numblks, 1<<(tbl->zt_shift-10));
	}

	return (0);
	}

	static int
	zap_table_store(zap_t zap, zap_table_phys_t tbl, uint64_t idx, uint64_t val,
	dmu_tx_t *tx)
	{
	int bs = FZAP_BLOCK_SHIFT(zap);

	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));
	ASSERT(tbl->zt_blk != 0);

	dprintf("storing %llx at index %llx\n", (u_longlong_t)val,
	(u_longlong_t)idx);

	uint64_t blk = idx >> (bs-3);
	uint64_t off = idx & ((1<<(bs-3))-1);

	dmu_buf_t *db;
	- int err = dmu_buf_hold(zap->zap_objset, zap->zap_object,
	+ int err = dmu_buf_hold_by_dnode(zap->zap_dnode,
	(tbl->zt_blk + blk) << bs, FTAG, &db, DMU_READ_NO_PREFETCH);
	if (err != 0)
	return (err);
	dmu_buf_will_dirty(db, tx);

	if (tbl->zt_nextblk != 0) {
	uint64_t idx2 = idx * 2;
	uint64_t blk2 = idx2 >> (bs-3);
	uint64_t off2 = idx2 & ((1<<(bs-3))-1);
	dmu_buf_t *db2;

	- err = dmu_buf_hold(zap->zap_objset, zap->zap_object,
	+ err = dmu_buf_hold_by_dnode(zap->zap_dnode,
	(tbl->zt_nextblk + blk2) << bs, FTAG, &db2,
	DMU_READ_NO_PREFETCH);
	if (err != 0) {
	dmu_buf_rele(db, FTAG);
	return (err);
	}
	dmu_buf_will_dirty(db2, tx);
	((uint64_t *)db2->db_data)[off2] = val;
	((uint64_t *)db2->db_data)[off2+1] = val;
	dmu_buf_rele(db2, FTAG);
	}

	((uint64_t *)db->db_data)[off] = val;
	dmu_buf_rele(db, FTAG);

	return (0);
	}

	static int
	zap_table_load(zap_t zap, zap_table_phys_t tbl, uint64_t idx, uint64_t *valp)
	{
	int bs = FZAP_BLOCK_SHIFT(zap);

	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));

	uint64_t blk = idx >> (bs-3);
	uint64_t off = idx & ((1<<(bs-3))-1);

	- /*
	- * Note: this is equivalent to dmu_buf_hold(), but we use
	- * _dnode_enter / _by_dnode because it's faster because we don't
	- * have to hold the dnode.
	- */
	- dnode_t *dn = dmu_buf_dnode_enter(zap->zap_dbuf);
	dmu_buf_t *db;
	- int err = dmu_buf_hold_by_dnode(dn,
	+ int err = dmu_buf_hold_by_dnode(zap->zap_dnode,
	(tbl->zt_blk + blk) << bs, FTAG, &db, DMU_READ_NO_PREFETCH);
	- dmu_buf_dnode_exit(zap->zap_dbuf);
	if (err != 0)
	return (err);
	valp = ((uint64_t )db->db_data)[off];
	dmu_buf_rele(db, FTAG);

	if (tbl->zt_nextblk != 0) {
	/*
	* read the nextblk for the sake of i/o error checking,
	* so that zap_table_load() will catch errors for
	* zap_table_store.
	*/
	blk = (idx*2) >> (bs-3);

	- dn = dmu_buf_dnode_enter(zap->zap_dbuf);
	- err = dmu_buf_hold_by_dnode(dn,
	+ err = dmu_buf_hold_by_dnode(zap->zap_dnode,
	(tbl->zt_nextblk + blk) << bs, FTAG, &db,
	DMU_READ_NO_PREFETCH);
	- dmu_buf_dnode_exit(zap->zap_dbuf);
	if (err == 0)
	dmu_buf_rele(db, FTAG);
	}
	return (err);
	}

	/*
	* Routines for growing the ptrtbl.
	*/

	static void
	zap_ptrtbl_transfer(const uint64_t src, uint64_t dst, int n)
	{
	for (int i = 0; i < n; i++) {
	uint64_t lb = src[i];
	dst[2 * i + 0] = lb;
	dst[2 * i + 1] = lb;
	}
	}

	static int
	zap_grow_ptrtbl(zap_t zap, dmu_tx_t tx)
	{
	/*
	* The pointer table should never use more hash bits than we
	* have (otherwise we'd be using useless zero bits to index it).
	* If we are within 2 bits of running out, stop growing, since
	* this is already an aberrant condition.
	*/
	if (zap_f_phys(zap)->zap_ptrtbl.zt_shift >= zap_hashbits(zap) - 2)
	return (SET_ERROR(ENOSPC));

	if (zap_f_phys(zap)->zap_ptrtbl.zt_numblks == 0) {
	/*
	* We are outgrowing the "embedded" ptrtbl (the one
	* stored in the header block). Give it its own entire
	* block, which will double the size of the ptrtbl.
	*/
	ASSERT3U(zap_f_phys(zap)->zap_ptrtbl.zt_shift, ==,
	ZAP_EMBEDDED_PTRTBL_SHIFT(zap));
	ASSERT0(zap_f_phys(zap)->zap_ptrtbl.zt_blk);

	uint64_t newblk = zap_allocate_blocks(zap, 1);
	dmu_buf_t *db_new;
	- int err = dmu_buf_hold(zap->zap_objset, zap->zap_object,
	+ int err = dmu_buf_hold_by_dnode(zap->zap_dnode,
	newblk << FZAP_BLOCK_SHIFT(zap), FTAG, &db_new,
	DMU_READ_NO_PREFETCH);
	if (err != 0)
	return (err);
	dmu_buf_will_dirty(db_new, tx);
	zap_ptrtbl_transfer(&ZAP_EMBEDDED_PTRTBL_ENT(zap, 0),
	db_new->db_data, 1 << ZAP_EMBEDDED_PTRTBL_SHIFT(zap));
	dmu_buf_rele(db_new, FTAG);

	zap_f_phys(zap)->zap_ptrtbl.zt_blk = newblk;
	zap_f_phys(zap)->zap_ptrtbl.zt_numblks = 1;
	zap_f_phys(zap)->zap_ptrtbl.zt_shift++;

	ASSERT3U(1ULL << zap_f_phys(zap)->zap_ptrtbl.zt_shift, ==,
	zap_f_phys(zap)->zap_ptrtbl.zt_numblks <<
	(FZAP_BLOCK_SHIFT(zap)-3));

	return (0);
	} else {
	return (zap_table_grow(zap, &zap_f_phys(zap)->zap_ptrtbl,
	zap_ptrtbl_transfer, tx));
	}
	}

	static void
	zap_increment_num_entries(zap_t zap, int delta, dmu_tx_t tx)
	{
	dmu_buf_will_dirty(zap->zap_dbuf, tx);
	mutex_enter(&zap->zap_f.zap_num_entries_mtx);
	ASSERT(delta > 0 \|\| zap_f_phys(zap)->zap_num_entries >= -delta);
	zap_f_phys(zap)->zap_num_entries += delta;
	mutex_exit(&zap->zap_f.zap_num_entries_mtx);
	}

	static uint64_t
	zap_allocate_blocks(zap_t *zap, int nblocks)
	{
	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));
	uint64_t newblk = zap_f_phys(zap)->zap_freeblk;
	zap_f_phys(zap)->zap_freeblk += nblocks;
	return (newblk);
	}

	static void
	zap_leaf_evict_sync(void *dbu)
	{
	zap_leaf_t *l = dbu;

	rw_destroy(&l->l_rwlock);
	kmem_free(l, sizeof (zap_leaf_t));
	}

	static zap_leaf_t *
	zap_create_leaf(zap_t zap, dmu_tx_t tx)
	{
	zap_leaf_t *l = kmem_zalloc(sizeof (zap_leaf_t), KM_SLEEP);

	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));

	rw_init(&l->l_rwlock, NULL, RW_NOLOCKDEP, NULL);
	rw_enter(&l->l_rwlock, RW_WRITER);
	l->l_blkid = zap_allocate_blocks(zap, 1);
	l->l_dbuf = NULL;

	- VERIFY0(dmu_buf_hold(zap->zap_objset, zap->zap_object,
	+ VERIFY0(dmu_buf_hold_by_dnode(zap->zap_dnode,
	l->l_blkid << FZAP_BLOCK_SHIFT(zap), NULL, &l->l_dbuf,
	DMU_READ_NO_PREFETCH));
	dmu_buf_init_user(&l->l_dbu, zap_leaf_evict_sync, NULL, &l->l_dbuf);
	VERIFY3P(NULL, ==, dmu_buf_set_user(l->l_dbuf, &l->l_dbu));
	dmu_buf_will_dirty(l->l_dbuf, tx);

	zap_leaf_init(l, zap->zap_normflags != 0);

	zap_f_phys(zap)->zap_num_leafs++;

	return (l);
	}

	int
	fzap_count(zap_t zap, uint64_t count)
	{
	ASSERT(!zap->zap_ismicro);
	mutex_enter(&zap->zap_f.zap_num_entries_mtx); /* unnecessary */
	*count = zap_f_phys(zap)->zap_num_entries;
	mutex_exit(&zap->zap_f.zap_num_entries_mtx);
	return (0);
	}

	/*
	* Routines for obtaining zap_leaf_t's
	*/

	void
	zap_put_leaf(zap_leaf_t *l)
	{
	rw_exit(&l->l_rwlock);
	dmu_buf_rele(l->l_dbuf, NULL);
	}

	static zap_leaf_t *
	zap_open_leaf(uint64_t blkid, dmu_buf_t *db)
	{
	ASSERT(blkid != 0);

	zap_leaf_t *l = kmem_zalloc(sizeof (zap_leaf_t), KM_SLEEP);
	rw_init(&l->l_rwlock, NULL, RW_DEFAULT, NULL);
	rw_enter(&l->l_rwlock, RW_WRITER);
	l->l_blkid = blkid;
	l->l_bs = highbit64(db->db_size) - 1;
	l->l_dbuf = db;

	dmu_buf_init_user(&l->l_dbu, zap_leaf_evict_sync, NULL, &l->l_dbuf);
	zap_leaf_t *winner = dmu_buf_set_user(db, &l->l_dbu);

	rw_exit(&l->l_rwlock);
	if (winner != NULL) {
	/* someone else set it first */
	zap_leaf_evict_sync(&l->l_dbu);
	l = winner;
	}

	/*
	* lhr_pad was previously used for the next leaf in the leaf
	* chain. There should be no chained leafs (as we have removed
	* support for them).
	*/
	ASSERT0(zap_leaf_phys(l)->l_hdr.lh_pad1);

	/*
	* There should be more hash entries than there can be
	* chunks to put in the hash table
	*/
	ASSERT3U(ZAP_LEAF_HASH_NUMENTRIES(l), >, ZAP_LEAF_NUMCHUNKS(l) / 3);

	/* The chunks should begin at the end of the hash table */
	ASSERT3P(&ZAP_LEAF_CHUNK(l, 0), ==, (zap_leaf_chunk_t *)
	&zap_leaf_phys(l)->l_hash[ZAP_LEAF_HASH_NUMENTRIES(l)]);

	/* The chunks should end at the end of the block */
	ASSERT3U((uintptr_t)&ZAP_LEAF_CHUNK(l, ZAP_LEAF_NUMCHUNKS(l)) -
	(uintptr_t)zap_leaf_phys(l), ==, l->l_dbuf->db_size);

	return (l);
	}

	static int
	zap_get_leaf_byblk(zap_t zap, uint64_t blkid, dmu_tx_t tx, krw_t lt,
	zap_leaf_t **lp)
	{
	dmu_buf_t *db;

	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));

	/*
	* If system crashed just after dmu_free_long_range in zfs_rmnode, we
	* would be left with an empty xattr dir in delete queue. blkid=0
	* would be passed in when doing zfs_purgedir. If that's the case we
	* should just return immediately. The underlying objects should
	* already be freed, so this should be perfectly fine.
	*/
	if (blkid == 0)
	return (SET_ERROR(ENOENT));

	int bs = FZAP_BLOCK_SHIFT(zap);
	- dnode_t *dn = dmu_buf_dnode_enter(zap->zap_dbuf);
	- int err = dmu_buf_hold_by_dnode(dn,
	+ int err = dmu_buf_hold_by_dnode(zap->zap_dnode,
	blkid << bs, NULL, &db, DMU_READ_NO_PREFETCH);
	- dmu_buf_dnode_exit(zap->zap_dbuf);
	if (err != 0)
	return (err);

	ASSERT3U(db->db_object, ==, zap->zap_object);
	ASSERT3U(db->db_offset, ==, blkid << bs);
	ASSERT3U(db->db_size, ==, 1 << bs);
	ASSERT(blkid != 0);

	zap_leaf_t *l = dmu_buf_get_user(db);

	if (l == NULL)
	l = zap_open_leaf(blkid, db);

	rw_enter(&l->l_rwlock, lt);
	/*
	* Must lock before dirtying, otherwise zap_leaf_phys(l) could change,
	* causing ASSERT below to fail.
	*/
	if (lt == RW_WRITER)
	dmu_buf_will_dirty(db, tx);
	ASSERT3U(l->l_blkid, ==, blkid);
	ASSERT3P(l->l_dbuf, ==, db);
	ASSERT3U(zap_leaf_phys(l)->l_hdr.lh_block_type, ==, ZBT_LEAF);
	ASSERT3U(zap_leaf_phys(l)->l_hdr.lh_magic, ==, ZAP_LEAF_MAGIC);

	*lp = l;
	return (0);
	}

	static int
	zap_idx_to_blk(zap_t zap, uint64_t idx, uint64_t valp)
	{
	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));

	if (zap_f_phys(zap)->zap_ptrtbl.zt_numblks == 0) {
	ASSERT3U(idx, <,
	(1ULL << zap_f_phys(zap)->zap_ptrtbl.zt_shift));
	*valp = ZAP_EMBEDDED_PTRTBL_ENT(zap, idx);
	return (0);
	} else {
	return (zap_table_load(zap, &zap_f_phys(zap)->zap_ptrtbl,
	idx, valp));
	}
	}

	static int
	zap_set_idx_to_blk(zap_t zap, uint64_t idx, uint64_t blk, dmu_tx_t tx)
	{
	ASSERT(tx != NULL);
	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));

	if (zap_f_phys(zap)->zap_ptrtbl.zt_blk == 0) {
	ZAP_EMBEDDED_PTRTBL_ENT(zap, idx) = blk;
	return (0);
	} else {
	return (zap_table_store(zap, &zap_f_phys(zap)->zap_ptrtbl,
	idx, blk, tx));
	}
	}

	static int
	zap_deref_leaf(zap_t zap, uint64_t h, dmu_tx_t tx, krw_t lt, zap_leaf_t **lp)
	{
	uint64_t blk;

	ASSERT(zap->zap_dbuf == NULL \|\|
	zap_f_phys(zap) == zap->zap_dbuf->db_data);

	/* Reality check for corrupt zap objects (leaf or header). */
	if ((zap_f_phys(zap)->zap_block_type != ZBT_LEAF &&
	zap_f_phys(zap)->zap_block_type != ZBT_HEADER) \|\|
	zap_f_phys(zap)->zap_magic != ZAP_MAGIC) {
	return (SET_ERROR(EIO));
	}

	uint64_t idx = ZAP_HASH_IDX(h, zap_f_phys(zap)->zap_ptrtbl.zt_shift);
	int err = zap_idx_to_blk(zap, idx, &blk);
	if (err != 0)
	return (err);
	err = zap_get_leaf_byblk(zap, blk, tx, lt, lp);

	ASSERT(err \|\|
	ZAP_HASH_IDX(h, zap_leaf_phys(*lp)->l_hdr.lh_prefix_len) ==
	zap_leaf_phys(*lp)->l_hdr.lh_prefix);
	return (err);
	}

	static int
	zap_expand_leaf(zap_name_t zn, zap_leaf_t l,
	const void tag, dmu_tx_t tx, zap_leaf_t **lp)
	{
	zap_t *zap = zn->zn_zap;
	uint64_t hash = zn->zn_hash;
	int err;
	int old_prefix_len = zap_leaf_phys(l)->l_hdr.lh_prefix_len;

	ASSERT3U(old_prefix_len, <=, zap_f_phys(zap)->zap_ptrtbl.zt_shift);
	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));

	ASSERT3U(ZAP_HASH_IDX(hash, old_prefix_len), ==,
	zap_leaf_phys(l)->l_hdr.lh_prefix);

	if (zap_tryupgradedir(zap, tx) == 0 \|\|
	old_prefix_len == zap_f_phys(zap)->zap_ptrtbl.zt_shift) {
	/* We failed to upgrade, or need to grow the pointer table */
	objset_t *os = zap->zap_objset;
	uint64_t object = zap->zap_object;

	zap_put_leaf(l);
	zap_unlockdir(zap, tag);
	err = zap_lockdir(os, object, tx, RW_WRITER,
	FALSE, FALSE, tag, &zn->zn_zap);
	zap = zn->zn_zap;
	if (err != 0)
	return (err);
	ASSERT(!zap->zap_ismicro);

	while (old_prefix_len ==
	zap_f_phys(zap)->zap_ptrtbl.zt_shift) {
	err = zap_grow_ptrtbl(zap, tx);
	if (err != 0)
	return (err);
	}

	err = zap_deref_leaf(zap, hash, tx, RW_WRITER, &l);
	if (err != 0)
	return (err);

	if (zap_leaf_phys(l)->l_hdr.lh_prefix_len != old_prefix_len) {
	/* it split while our locks were down */
	*lp = l;
	return (0);
	}
	}
	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));
	ASSERT3U(old_prefix_len, <, zap_f_phys(zap)->zap_ptrtbl.zt_shift);
	ASSERT3U(ZAP_HASH_IDX(hash, old_prefix_len), ==,
	zap_leaf_phys(l)->l_hdr.lh_prefix);

	int prefix_diff = zap_f_phys(zap)->zap_ptrtbl.zt_shift -
	(old_prefix_len + 1);
	uint64_t sibling =
	(ZAP_HASH_IDX(hash, old_prefix_len + 1) \| 1) << prefix_diff;

	/* check for i/o errors before doing zap_leaf_split */
	for (int i = 0; i < (1ULL << prefix_diff); i++) {
	uint64_t blk;
	err = zap_idx_to_blk(zap, sibling + i, &blk);
	if (err != 0)
	return (err);
	ASSERT3U(blk, ==, l->l_blkid);
	}

	zap_leaf_t *nl = zap_create_leaf(zap, tx);
	zap_leaf_split(l, nl, zap->zap_normflags != 0);

	/* set sibling pointers */
	for (int i = 0; i < (1ULL << prefix_diff); i++) {
	err = zap_set_idx_to_blk(zap, sibling + i, nl->l_blkid, tx);
	ASSERT0(err); /* we checked for i/o errors above */
	}

	ASSERT3U(zap_leaf_phys(l)->l_hdr.lh_prefix_len, >, 0);

	if (hash & (1ULL << (64 - zap_leaf_phys(l)->l_hdr.lh_prefix_len))) {
	/* we want the sibling */
	zap_put_leaf(l);
	*lp = nl;
	} else {
	zap_put_leaf(nl);
	*lp = l;
	}

	return (0);
	}

	static void
	zap_put_leaf_maybe_grow_ptrtbl(zap_name_t zn, zap_leaf_t l,
	const void tag, dmu_tx_t tx)
	{
	zap_t *zap = zn->zn_zap;
	int shift = zap_f_phys(zap)->zap_ptrtbl.zt_shift;
	int leaffull = (zap_leaf_phys(l)->l_hdr.lh_prefix_len == shift &&
	zap_leaf_phys(l)->l_hdr.lh_nfree < ZAP_LEAF_LOW_WATER);

	zap_put_leaf(l);

	if (leaffull \|\| zap_f_phys(zap)->zap_ptrtbl.zt_nextblk) {
	/*
	* We are in the middle of growing the pointer table, or
	* this leaf will soon make us grow it.
	*/
	if (zap_tryupgradedir(zap, tx) == 0) {
	objset_t *os = zap->zap_objset;
	uint64_t zapobj = zap->zap_object;

	zap_unlockdir(zap, tag);
	int err = zap_lockdir(os, zapobj, tx,
	RW_WRITER, FALSE, FALSE, tag, &zn->zn_zap);
	zap = zn->zn_zap;
	if (err != 0)
	return;
	}

	/* could have finished growing while our locks were down */
	if (zap_f_phys(zap)->zap_ptrtbl.zt_shift == shift)
	(void) zap_grow_ptrtbl(zap, tx);
	}
	}

	static int
	fzap_checkname(zap_name_t *zn)
	{
	if (zn->zn_key_orig_numints * zn->zn_key_intlen > ZAP_MAXNAMELEN)
	return (SET_ERROR(ENAMETOOLONG));
	return (0);
	}

	static int
	fzap_checksize(uint64_t integer_size, uint64_t num_integers)
	{
	/* Only integer sizes supported by C */
	switch (integer_size) {
	case 1:
	case 2:
	case 4:
	case 8:
	break;
	default:
	return (SET_ERROR(EINVAL));
	}

	if (integer_size * num_integers > ZAP_MAXVALUELEN)
	return (SET_ERROR(E2BIG));

	return (0);
	}

	static int
	fzap_check(zap_name_t *zn, uint64_t integer_size, uint64_t num_integers)
	{
	int err = fzap_checkname(zn);
	if (err != 0)
	return (err);
	return (fzap_checksize(integer_size, num_integers));
	}

	/*
	* Routines for manipulating attributes.
	*/
	int
	fzap_lookup(zap_name_t *zn,
	uint64_t integer_size, uint64_t num_integers, void *buf,
	char realname, int rn_len, boolean_t ncp)
	{
	zap_leaf_t *l;
	zap_entry_handle_t zeh;

	int err = fzap_checkname(zn);
	if (err != 0)
	return (err);

	err = zap_deref_leaf(zn->zn_zap, zn->zn_hash, NULL, RW_READER, &l);
	if (err != 0)
	return (err);
	err = zap_leaf_lookup(l, zn, &zeh);
	if (err == 0) {
	if ((err = fzap_checksize(integer_size, num_integers)) != 0) {
	zap_put_leaf(l);
	return (err);
	}

	err = zap_entry_read(&zeh, integer_size, num_integers, buf);
	(void) zap_entry_read_name(zn->zn_zap, &zeh, rn_len, realname);
	if (ncp) {
	*ncp = zap_entry_normalization_conflict(&zeh,
	zn, NULL, zn->zn_zap);
	}
	}

	zap_put_leaf(l);
	return (err);
	}

	int
	fzap_add_cd(zap_name_t *zn,
	uint64_t integer_size, uint64_t num_integers,
	const void val, uint32_t cd, const void tag, dmu_tx_t *tx)
	{
	zap_leaf_t *l;
	int err;
	zap_entry_handle_t zeh;
	zap_t *zap = zn->zn_zap;

	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));
	ASSERT(!zap->zap_ismicro);
	ASSERT(fzap_check(zn, integer_size, num_integers) == 0);

	err = zap_deref_leaf(zap, zn->zn_hash, tx, RW_WRITER, &l);
	if (err != 0)
	return (err);
	retry:
	err = zap_leaf_lookup(l, zn, &zeh);
	if (err == 0) {
	err = SET_ERROR(EEXIST);
	goto out;
	}
	if (err != ENOENT)
	goto out;

	err = zap_entry_create(l, zn, cd,
	integer_size, num_integers, val, &zeh);

	if (err == 0) {
	zap_increment_num_entries(zap, 1, tx);
	} else if (err == EAGAIN) {
	err = zap_expand_leaf(zn, l, tag, tx, &l);
	zap = zn->zn_zap; /* zap_expand_leaf() may change zap */
	if (err == 0) {
	goto retry;
	} else if (err == ENOSPC) {
	/*
	* If we failed to expand the leaf, then bailout
	* as there is no point trying
	* zap_put_leaf_maybe_grow_ptrtbl().
	*/
	return (err);
	}
	}

	out:
	if (zap != NULL)
	zap_put_leaf_maybe_grow_ptrtbl(zn, l, tag, tx);
	return (err);
	}

	int
	fzap_add(zap_name_t *zn,
	uint64_t integer_size, uint64_t num_integers,
	const void val, const void tag, dmu_tx_t *tx)
	{
	int err = fzap_check(zn, integer_size, num_integers);
	if (err != 0)
	return (err);

	return (fzap_add_cd(zn, integer_size, num_integers,
	val, ZAP_NEED_CD, tag, tx));
	}

	int
	fzap_update(zap_name_t *zn,
	int integer_size, uint64_t num_integers, const void *val,
	const void tag, dmu_tx_t tx)
	{
	zap_leaf_t *l;
	int err;
	boolean_t create;
	zap_entry_handle_t zeh;
	zap_t *zap = zn->zn_zap;

	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));
	err = fzap_check(zn, integer_size, num_integers);
	if (err != 0)
	return (err);

	err = zap_deref_leaf(zap, zn->zn_hash, tx, RW_WRITER, &l);
	if (err != 0)
	return (err);
	retry:
	err = zap_leaf_lookup(l, zn, &zeh);
	create = (err == ENOENT);
	ASSERT(err == 0 \|\| err == ENOENT);

	if (create) {
	err = zap_entry_create(l, zn, ZAP_NEED_CD,
	integer_size, num_integers, val, &zeh);
	if (err == 0)
	zap_increment_num_entries(zap, 1, tx);
	} else {
	err = zap_entry_update(&zeh, integer_size, num_integers, val);
	}

	if (err == EAGAIN) {
	err = zap_expand_leaf(zn, l, tag, tx, &l);
	zap = zn->zn_zap; /* zap_expand_leaf() may change zap */
	if (err == 0)
	goto retry;
	}

	if (zap != NULL)
	zap_put_leaf_maybe_grow_ptrtbl(zn, l, tag, tx);
	return (err);
	}

	int
	fzap_length(zap_name_t *zn,
	uint64_t integer_size, uint64_t num_integers)
	{
	zap_leaf_t *l;
	int err;
	zap_entry_handle_t zeh;

	err = zap_deref_leaf(zn->zn_zap, zn->zn_hash, NULL, RW_READER, &l);
	if (err != 0)
	return (err);
	err = zap_leaf_lookup(l, zn, &zeh);
	if (err != 0)
	goto out;

	if (integer_size != NULL)
	*integer_size = zeh.zeh_integer_size;
	if (num_integers != NULL)
	*num_integers = zeh.zeh_num_integers;
	out:
	zap_put_leaf(l);
	return (err);
	}

	int
	fzap_remove(zap_name_t zn, dmu_tx_t tx)
	{
	zap_leaf_t *l;
	int err;
	zap_entry_handle_t zeh;

	err = zap_deref_leaf(zn->zn_zap, zn->zn_hash, tx, RW_WRITER, &l);
	if (err != 0)
	return (err);
	err = zap_leaf_lookup(l, zn, &zeh);
	if (err == 0) {
	zap_entry_remove(&zeh);
	zap_increment_num_entries(zn->zn_zap, -1, tx);
	}
	zap_put_leaf(l);
	return (err);
	}

	void
	fzap_prefetch(zap_name_t *zn)
	{
	uint64_t blk;
	zap_t *zap = zn->zn_zap;

	uint64_t idx = ZAP_HASH_IDX(zn->zn_hash,
	zap_f_phys(zap)->zap_ptrtbl.zt_shift);
	if (zap_idx_to_blk(zap, idx, &blk) != 0)
	return;
	int bs = FZAP_BLOCK_SHIFT(zap);
	- dmu_prefetch(zap->zap_objset, zap->zap_object, 0, blk << bs, 1 << bs,
	+ dmu_prefetch_by_dnode(zap->zap_dnode, 0, blk << bs, 1 << bs,
	ZIO_PRIORITY_SYNC_READ);
	}

	/*
	* Helper functions for consumers.
	*/

	uint64_t
	zap_create_link(objset_t *os, dmu_object_type_t ot, uint64_t parent_obj,
	const char name, dmu_tx_t tx)
	{
	return (zap_create_link_dnsize(os, ot, parent_obj, name, 0, tx));
	}

	uint64_t
	zap_create_link_dnsize(objset_t *os, dmu_object_type_t ot, uint64_t parent_obj,
	const char name, int dnodesize, dmu_tx_t tx)
	{
	uint64_t new_obj;

	new_obj = zap_create_dnsize(os, ot, DMU_OT_NONE, 0, dnodesize, tx);
	VERIFY(new_obj != 0);
	VERIFY0(zap_add(os, parent_obj, name, sizeof (uint64_t), 1, &new_obj,
	tx));

	return (new_obj);
	}

	int
	zap_value_search(objset_t *os, uint64_t zapobj, uint64_t value, uint64_t mask,
	char *name)
	{
	zap_cursor_t zc;
	int err;

	if (mask == 0)
	mask = -1ULL;

	zap_attribute_t za = kmem_alloc(sizeof (za), KM_SLEEP);
	for (zap_cursor_init(&zc, os, zapobj);
	(err = zap_cursor_retrieve(&zc, za)) == 0;
	zap_cursor_advance(&zc)) {
	if ((za->za_first_integer & mask) == (value & mask)) {
	(void) strlcpy(name, za->za_name, MAXNAMELEN);
	break;
	}
	}
	zap_cursor_fini(&zc);
	kmem_free(za, sizeof (*za));
	return (err);
	}

	int
	zap_join(objset_t os, uint64_t fromobj, uint64_t intoobj, dmu_tx_t tx)
	{
	zap_cursor_t zc;
	int err = 0;

	zap_attribute_t za = kmem_alloc(sizeof (za), KM_SLEEP);
	for (zap_cursor_init(&zc, os, fromobj);
	zap_cursor_retrieve(&zc, za) == 0;
	(void) zap_cursor_advance(&zc)) {
	if (za->za_integer_length != 8 \|\| za->za_num_integers != 1) {
	err = SET_ERROR(EINVAL);
	break;
	}
	err = zap_add(os, intoobj, za->za_name,
	8, 1, &za->za_first_integer, tx);
	if (err != 0)
	break;
	}
	zap_cursor_fini(&zc);
	kmem_free(za, sizeof (*za));
	return (err);
	}

	int
	zap_join_key(objset_t *os, uint64_t fromobj, uint64_t intoobj,
	uint64_t value, dmu_tx_t *tx)
	{
	zap_cursor_t zc;
	int err = 0;

	zap_attribute_t za = kmem_alloc(sizeof (za), KM_SLEEP);
	for (zap_cursor_init(&zc, os, fromobj);
	zap_cursor_retrieve(&zc, za) == 0;
	(void) zap_cursor_advance(&zc)) {
	if (za->za_integer_length != 8 \|\| za->za_num_integers != 1) {
	err = SET_ERROR(EINVAL);
	break;
	}
	err = zap_add(os, intoobj, za->za_name,
	8, 1, &value, tx);
	if (err != 0)
	break;
	}
	zap_cursor_fini(&zc);
	kmem_free(za, sizeof (*za));
	return (err);
	}

	int
	zap_join_increment(objset_t *os, uint64_t fromobj, uint64_t intoobj,
	dmu_tx_t *tx)
	{
	zap_cursor_t zc;
	int err = 0;

	zap_attribute_t za = kmem_alloc(sizeof (za), KM_SLEEP);
	for (zap_cursor_init(&zc, os, fromobj);
	zap_cursor_retrieve(&zc, za) == 0;
	(void) zap_cursor_advance(&zc)) {
	uint64_t delta = 0;

	if (za->za_integer_length != 8 \|\| za->za_num_integers != 1) {
	err = SET_ERROR(EINVAL);
	break;
	}

	err = zap_lookup(os, intoobj, za->za_name, 8, 1, &delta);
	if (err != 0 && err != ENOENT)
	break;
	delta += za->za_first_integer;
	err = zap_update(os, intoobj, za->za_name, 8, 1, &delta, tx);
	if (err != 0)
	break;
	}
	zap_cursor_fini(&zc);
	kmem_free(za, sizeof (*za));
	return (err);
	}

	int
	zap_add_int(objset_t os, uint64_t obj, uint64_t value, dmu_tx_t tx)
	{
	char name[20];

	(void) snprintf(name, sizeof (name), "%llx", (longlong_t)value);
	return (zap_add(os, obj, name, 8, 1, &value, tx));
	}

	int
	zap_remove_int(objset_t os, uint64_t obj, uint64_t value, dmu_tx_t tx)
	{
	char name[20];

	(void) snprintf(name, sizeof (name), "%llx", (longlong_t)value);
	return (zap_remove(os, obj, name, tx));
	}

	int
	zap_lookup_int(objset_t *os, uint64_t obj, uint64_t value)
	{
	char name[20];

	(void) snprintf(name, sizeof (name), "%llx", (longlong_t)value);
	return (zap_lookup(os, obj, name, 8, 1, &value));
	}

	int
	zap_add_int_key(objset_t *os, uint64_t obj,
	uint64_t key, uint64_t value, dmu_tx_t *tx)
	{
	char name[20];

	(void) snprintf(name, sizeof (name), "%llx", (longlong_t)key);
	return (zap_add(os, obj, name, 8, 1, &value, tx));
	}

	int
	zap_update_int_key(objset_t *os, uint64_t obj,
	uint64_t key, uint64_t value, dmu_tx_t *tx)
	{
	char name[20];

	(void) snprintf(name, sizeof (name), "%llx", (longlong_t)key);
	return (zap_update(os, obj, name, 8, 1, &value, tx));
	}

	int
	zap_lookup_int_key(objset_t os, uint64_t obj, uint64_t key, uint64_t valuep)
	{
	char name[20];

	(void) snprintf(name, sizeof (name), "%llx", (longlong_t)key);
	return (zap_lookup(os, obj, name, 8, 1, valuep));
	}

	int
	zap_increment(objset_t os, uint64_t obj, const char name, int64_t delta,
	dmu_tx_t *tx)
	{
	uint64_t value = 0;

	if (delta == 0)
	return (0);

	int err = zap_lookup(os, obj, name, 8, 1, &value);
	if (err != 0 && err != ENOENT)
	return (err);
	value += delta;
	if (value == 0)
	err = zap_remove(os, obj, name, tx);
	else
	err = zap_update(os, obj, name, 8, 1, &value, tx);
	return (err);
	}

	int
	zap_increment_int(objset_t *os, uint64_t obj, uint64_t key, int64_t delta,
	dmu_tx_t *tx)
	{
	char name[20];

	(void) snprintf(name, sizeof (name), "%llx", (longlong_t)key);
	return (zap_increment(os, obj, name, delta, tx));
	}

	/*
	* Routines for iterating over the attributes.
	*/

	int
	fzap_cursor_retrieve(zap_t zap, zap_cursor_t zc, zap_attribute_t *za)
	{
	int err = ENOENT;
	zap_entry_handle_t zeh;
	zap_leaf_t *l;

	/* retrieve the next entry at or after zc_hash/zc_cd */
	/* if no entry, return ENOENT */

	/*
	* If we are reading from the beginning, we're almost certain to
	* iterate over the entire ZAP object. If there are multiple leaf
	* blocks (freeblk > 2), prefetch the whole object (up to
	* dmu_prefetch_max bytes), so that we read the leaf blocks
	* concurrently. (Unless noprefetch was requested via
	* zap_cursor_init_noprefetch()).
	*/
	if (zc->zc_hash == 0 && zap_iterate_prefetch &&
	zc->zc_prefetch && zap_f_phys(zap)->zap_freeblk > 2) {
	- dmu_prefetch(zc->zc_objset, zc->zc_zapobj, 0, 0,
	+ dmu_prefetch_by_dnode(zap->zap_dnode, 0, 0,
	zap_f_phys(zap)->zap_freeblk << FZAP_BLOCK_SHIFT(zap),
	ZIO_PRIORITY_ASYNC_READ);
	}

	if (zc->zc_leaf &&
	(ZAP_HASH_IDX(zc->zc_hash,
	zap_leaf_phys(zc->zc_leaf)->l_hdr.lh_prefix_len) !=
	zap_leaf_phys(zc->zc_leaf)->l_hdr.lh_prefix)) {
	rw_enter(&zc->zc_leaf->l_rwlock, RW_READER);
	zap_put_leaf(zc->zc_leaf);
	zc->zc_leaf = NULL;
	}

	again:
	if (zc->zc_leaf == NULL) {
	err = zap_deref_leaf(zap, zc->zc_hash, NULL, RW_READER,
	&zc->zc_leaf);
	if (err != 0)
	return (err);
	} else {
	rw_enter(&zc->zc_leaf->l_rwlock, RW_READER);
	}
	l = zc->zc_leaf;

	err = zap_leaf_lookup_closest(l, zc->zc_hash, zc->zc_cd, &zeh);

	if (err == ENOENT) {
	if (zap_leaf_phys(l)->l_hdr.lh_prefix_len == 0) {
	zc->zc_hash = -1ULL;
	zc->zc_cd = 0;
	} else {
	uint64_t nocare = (1ULL <<
	(64 - zap_leaf_phys(l)->l_hdr.lh_prefix_len)) - 1;

	zc->zc_hash = (zc->zc_hash & ~nocare) + nocare + 1;
	zc->zc_cd = 0;

	if (zc->zc_hash == 0) {
	zc->zc_hash = -1ULL;
	} else {
	zap_put_leaf(zc->zc_leaf);
	zc->zc_leaf = NULL;
	goto again;
	}
	}
	}

	if (err == 0) {
	zc->zc_hash = zeh.zeh_hash;
	zc->zc_cd = zeh.zeh_cd;
	za->za_integer_length = zeh.zeh_integer_size;
	za->za_num_integers = zeh.zeh_num_integers;
	if (zeh.zeh_num_integers == 0) {
	za->za_first_integer = 0;
	} else {
	err = zap_entry_read(&zeh, 8, 1, &za->za_first_integer);
	ASSERT(err == 0 \|\| err == EOVERFLOW);
	}
	err = zap_entry_read_name(zap, &zeh,
	sizeof (za->za_name), za->za_name);
	ASSERT(err == 0);

	za->za_normalization_conflict =
	zap_entry_normalization_conflict(&zeh,
	NULL, za->za_name, zap);
	}
	rw_exit(&zc->zc_leaf->l_rwlock);
	return (err);
	}

	static void
	zap_stats_ptrtbl(zap_t zap, uint64_t tbl, int len, zap_stats_t *zs)
	{
	uint64_t lastblk = 0;

	/*
	* NB: if a leaf has more pointers than an entire ptrtbl block
	* can hold, then it'll be accounted for more than once, since
	* we won't have lastblk.
	*/
	for (int i = 0; i < len; i++) {
	zap_leaf_t *l;

	if (tbl[i] == lastblk)
	continue;
	lastblk = tbl[i];

	int err = zap_get_leaf_byblk(zap, tbl[i], NULL, RW_READER, &l);
	if (err == 0) {
	zap_leaf_stats(zap, l, zs);
	zap_put_leaf(l);
	}
	}
	}

	void
	fzap_get_stats(zap_t zap, zap_stats_t zs)
	{
	int bs = FZAP_BLOCK_SHIFT(zap);
	zs->zs_blocksize = 1ULL << bs;

	/*
	* Set zap_phys_t fields
	*/
	zs->zs_num_leafs = zap_f_phys(zap)->zap_num_leafs;
	zs->zs_num_entries = zap_f_phys(zap)->zap_num_entries;
	zs->zs_num_blocks = zap_f_phys(zap)->zap_freeblk;
	zs->zs_block_type = zap_f_phys(zap)->zap_block_type;
	zs->zs_magic = zap_f_phys(zap)->zap_magic;
	zs->zs_salt = zap_f_phys(zap)->zap_salt;

	/*
	* Set zap_ptrtbl fields
	*/
	zs->zs_ptrtbl_len = 1ULL << zap_f_phys(zap)->zap_ptrtbl.zt_shift;
	zs->zs_ptrtbl_nextblk = zap_f_phys(zap)->zap_ptrtbl.zt_nextblk;
	zs->zs_ptrtbl_blks_copied =
	zap_f_phys(zap)->zap_ptrtbl.zt_blks_copied;
	zs->zs_ptrtbl_zt_blk = zap_f_phys(zap)->zap_ptrtbl.zt_blk;
	zs->zs_ptrtbl_zt_numblks = zap_f_phys(zap)->zap_ptrtbl.zt_numblks;
	zs->zs_ptrtbl_zt_shift = zap_f_phys(zap)->zap_ptrtbl.zt_shift;

	if (zap_f_phys(zap)->zap_ptrtbl.zt_numblks == 0) {
	/* the ptrtbl is entirely in the header block. */
	zap_stats_ptrtbl(zap, &ZAP_EMBEDDED_PTRTBL_ENT(zap, 0),
	1 << ZAP_EMBEDDED_PTRTBL_SHIFT(zap), zs);
	} else {
	- dmu_prefetch(zap->zap_objset, zap->zap_object, 0,
	+ dmu_prefetch_by_dnode(zap->zap_dnode, 0,
	zap_f_phys(zap)->zap_ptrtbl.zt_blk << bs,
	zap_f_phys(zap)->zap_ptrtbl.zt_numblks << bs,
	ZIO_PRIORITY_SYNC_READ);

	for (int b = 0; b < zap_f_phys(zap)->zap_ptrtbl.zt_numblks;
	b++) {
	dmu_buf_t *db;
	int err;

	- err = dmu_buf_hold(zap->zap_objset, zap->zap_object,
	+ err = dmu_buf_hold_by_dnode(zap->zap_dnode,
	(zap_f_phys(zap)->zap_ptrtbl.zt_blk + b) << bs,
	FTAG, &db, DMU_READ_NO_PREFETCH);
	if (err == 0) {
	zap_stats_ptrtbl(zap, db->db_data,
	1<<(bs-3), zs);
	dmu_buf_rele(db, FTAG);
	}
	}
	}
	}

	/* CSTYLED */
	ZFS_MODULE_PARAM(zfs, , zap_iterate_prefetch, INT, ZMOD_RW,
	"When iterating ZAP object, prefetch it");
	diff --git a/sys/contrib/openzfs/module/zfs/zap_leaf.c b/sys/contrib/openzfs/module/zfs/zap_leaf.c
	index e6afb1c58c95..032aca92695e 100644
	--- a/sys/contrib/openzfs/module/zfs/zap_leaf.c
	+++ b/sys/contrib/openzfs/module/zfs/zap_leaf.c
	@@ -1,848 +1,839 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2013, 2016 by Delphix. All rights reserved.
	* Copyright 2017 Nexenta Systems, Inc.
	*/

	/*
	* The 512-byte leaf is broken into 32 16-byte chunks.
	* chunk number n means l_chunk[n], even though the header precedes it.
	* the names are stored null-terminated.
	*/

	#include <sys/zio.h>
	#include <sys/spa.h>
	#include <sys/dmu.h>
	#include <sys/zfs_context.h>
	#include <sys/fs/zfs.h>
	#include <sys/zap.h>
	#include <sys/zap_impl.h>
	#include <sys/zap_leaf.h>
	#include <sys/arc.h>

	-static uint16_t zap_leaf_rehash_entry(zap_leaf_t l, uint16_t entry);
	+static uint16_t zap_leaf_rehash_entry(zap_leaf_t l, struct zap_leaf_entry *le,
	+ uint16_t entry);

	#define CHAIN_END 0xffff /* end of the chunk chain */

	#define LEAF_HASH(l, h) \
	((ZAP_LEAF_HASH_NUMENTRIES(l)-1) & \
	((h) >> \
	(64 - ZAP_LEAF_HASH_SHIFT(l) - zap_leaf_phys(l)->l_hdr.lh_prefix_len)))

	#define LEAF_HASH_ENTPTR(l, h) (&zap_leaf_phys(l)->l_hash[LEAF_HASH(l, h)])

	-static void
	-zap_memset(void *a, int c, size_t n)
	-{
	- char *cp = a;
	- char *cpend = cp + n;
	-
	- while (cp < cpend)
	- *cp++ = c;
	-}
	-
	static void
	stv(int len, void *addr, uint64_t value)
	{
	switch (len) {
	case 1:
	(uint8_t )addr = value;
	return;
	case 2:
	(uint16_t )addr = value;
	return;
	case 4:
	(uint32_t )addr = value;
	return;
	case 8:
	(uint64_t )addr = value;
	return;
	default:
	- cmn_err(CE_PANIC, "bad int len %d", len);
	+ PANIC("bad int len %d", len);
	}
	}

	static uint64_t
	ldv(int len, const void *addr)
	{
	switch (len) {
	case 1:
	return ((uint8_t )addr);
	case 2:
	return ((uint16_t )addr);
	case 4:
	return ((uint32_t )addr);
	case 8:
	return ((uint64_t )addr);
	default:
	- cmn_err(CE_PANIC, "bad int len %d", len);
	+ PANIC("bad int len %d", len);
	}
	return (0xFEEDFACEDEADBEEFULL);
	}

	void
	-zap_leaf_byteswap(zap_leaf_phys_t *buf, int size)
	+zap_leaf_byteswap(zap_leaf_phys_t *buf, size_t size)
	{
	zap_leaf_t l;
	dmu_buf_t l_dbuf;

	l_dbuf.db_data = buf;
	l.l_bs = highbit64(size) - 1;
	l.l_dbuf = &l_dbuf;

	buf->l_hdr.lh_block_type = BSWAP_64(buf->l_hdr.lh_block_type);
	buf->l_hdr.lh_prefix = BSWAP_64(buf->l_hdr.lh_prefix);
	buf->l_hdr.lh_magic = BSWAP_32(buf->l_hdr.lh_magic);
	buf->l_hdr.lh_nfree = BSWAP_16(buf->l_hdr.lh_nfree);
	buf->l_hdr.lh_nentries = BSWAP_16(buf->l_hdr.lh_nentries);
	buf->l_hdr.lh_prefix_len = BSWAP_16(buf->l_hdr.lh_prefix_len);
	buf->l_hdr.lh_freelist = BSWAP_16(buf->l_hdr.lh_freelist);

	- for (int i = 0; i < ZAP_LEAF_HASH_NUMENTRIES(&l); i++)
	+ for (uint_t i = 0; i < ZAP_LEAF_HASH_NUMENTRIES(&l); i++)
	buf->l_hash[i] = BSWAP_16(buf->l_hash[i]);

	- for (int i = 0; i < ZAP_LEAF_NUMCHUNKS(&l); i++) {
	+ for (uint_t i = 0; i < ZAP_LEAF_NUMCHUNKS(&l); i++) {
	zap_leaf_chunk_t *lc = &ZAP_LEAF_CHUNK(&l, i);
	struct zap_leaf_entry *le;

	switch (lc->l_free.lf_type) {
	case ZAP_CHUNK_ENTRY:
	le = &lc->l_entry;

	le->le_type = BSWAP_8(le->le_type);
	le->le_value_intlen = BSWAP_8(le->le_value_intlen);
	le->le_next = BSWAP_16(le->le_next);
	le->le_name_chunk = BSWAP_16(le->le_name_chunk);
	le->le_name_numints = BSWAP_16(le->le_name_numints);
	le->le_value_chunk = BSWAP_16(le->le_value_chunk);
	le->le_value_numints = BSWAP_16(le->le_value_numints);
	le->le_cd = BSWAP_32(le->le_cd);
	le->le_hash = BSWAP_64(le->le_hash);
	break;
	case ZAP_CHUNK_FREE:
	lc->l_free.lf_type = BSWAP_8(lc->l_free.lf_type);
	lc->l_free.lf_next = BSWAP_16(lc->l_free.lf_next);
	break;
	case ZAP_CHUNK_ARRAY:
	lc->l_array.la_type = BSWAP_8(lc->l_array.la_type);
	lc->l_array.la_next = BSWAP_16(lc->l_array.la_next);
	/* la_array doesn't need swapping */
	break;
	default:
	cmn_err(CE_PANIC, "bad leaf type %d",
	lc->l_free.lf_type);
	}
	}
	}

	void
	zap_leaf_init(zap_leaf_t *l, boolean_t sort)
	{
	l->l_bs = highbit64(l->l_dbuf->db_size) - 1;
	- zap_memset(&zap_leaf_phys(l)->l_hdr, 0,
	+ memset(&zap_leaf_phys(l)->l_hdr, 0,
	sizeof (struct zap_leaf_header));
	- zap_memset(zap_leaf_phys(l)->l_hash, CHAIN_END,
	+ memset(zap_leaf_phys(l)->l_hash, CHAIN_END,
	2*ZAP_LEAF_HASH_NUMENTRIES(l));
	- for (int i = 0; i < ZAP_LEAF_NUMCHUNKS(l); i++) {
	+ for (uint_t i = 0; i < ZAP_LEAF_NUMCHUNKS(l); i++) {
	ZAP_LEAF_CHUNK(l, i).l_free.lf_type = ZAP_CHUNK_FREE;
	ZAP_LEAF_CHUNK(l, i).l_free.lf_next = i+1;
	}
	ZAP_LEAF_CHUNK(l, ZAP_LEAF_NUMCHUNKS(l)-1).l_free.lf_next = CHAIN_END;
	zap_leaf_phys(l)->l_hdr.lh_block_type = ZBT_LEAF;
	zap_leaf_phys(l)->l_hdr.lh_magic = ZAP_LEAF_MAGIC;
	zap_leaf_phys(l)->l_hdr.lh_nfree = ZAP_LEAF_NUMCHUNKS(l);
	if (sort)
	zap_leaf_phys(l)->l_hdr.lh_flags \|= ZLF_ENTRIES_CDSORTED;
	}

	/*
	* Routines which manipulate leaf chunks (l_chunk[]).
	*/

	static uint16_t
	zap_leaf_chunk_alloc(zap_leaf_t *l)
	{
	ASSERT(zap_leaf_phys(l)->l_hdr.lh_nfree > 0);

	- int chunk = zap_leaf_phys(l)->l_hdr.lh_freelist;
	+ uint_t chunk = zap_leaf_phys(l)->l_hdr.lh_freelist;
	ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(l));
	ASSERT3U(ZAP_LEAF_CHUNK(l, chunk).l_free.lf_type, ==, ZAP_CHUNK_FREE);

	zap_leaf_phys(l)->l_hdr.lh_freelist =
	ZAP_LEAF_CHUNK(l, chunk).l_free.lf_next;

	zap_leaf_phys(l)->l_hdr.lh_nfree--;

	return (chunk);
	}

	static void
	zap_leaf_chunk_free(zap_leaf_t *l, uint16_t chunk)
	{
	struct zap_leaf_free *zlf = &ZAP_LEAF_CHUNK(l, chunk).l_free;
	ASSERT3U(zap_leaf_phys(l)->l_hdr.lh_nfree, <, ZAP_LEAF_NUMCHUNKS(l));
	ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(l));
	ASSERT(zlf->lf_type != ZAP_CHUNK_FREE);

	zlf->lf_type = ZAP_CHUNK_FREE;
	zlf->lf_next = zap_leaf_phys(l)->l_hdr.lh_freelist;
	memset(zlf->lf_pad, 0, sizeof (zlf->lf_pad)); /* help it to compress */
	zap_leaf_phys(l)->l_hdr.lh_freelist = chunk;

	zap_leaf_phys(l)->l_hdr.lh_nfree++;
	}

	/*
	* Routines which manipulate leaf arrays (zap_leaf_array type chunks).
	*/

	static uint16_t
	zap_leaf_array_create(zap_leaf_t l, const char buf,
	int integer_size, int num_integers)
	{
	uint16_t chunk_head;
	uint16_t *chunkp = &chunk_head;
	- int byten = 0;
	+ int byten = integer_size;
	uint64_t value = 0;
	int shift = (integer_size - 1) * 8;
	int len = num_integers;

	ASSERT3U(num_integers * integer_size, <=, ZAP_MAXVALUELEN);

	+ if (len > 0)
	+ value = ldv(integer_size, buf);
	while (len > 0) {
	uint16_t chunk = zap_leaf_chunk_alloc(l);
	struct zap_leaf_array *la = &ZAP_LEAF_CHUNK(l, chunk).l_array;

	la->la_type = ZAP_CHUNK_ARRAY;
	for (int i = 0; i < ZAP_LEAF_ARRAY_BYTES; i++) {
	- if (byten == 0)
	- value = ldv(integer_size, buf);
	la->la_array[i] = value >> shift;
	value <<= 8;
	- if (++byten == integer_size) {
	- byten = 0;
	- buf += integer_size;
	+ if (--byten == 0) {
	if (--len == 0)
	break;
	+ byten = integer_size;
	+ buf += integer_size;
	+ value = ldv(integer_size, buf);
	}
	}

	*chunkp = chunk;
	chunkp = &la->la_next;
	}
	*chunkp = CHAIN_END;

	return (chunk_head);
	}

	static void
	zap_leaf_array_free(zap_leaf_t l, uint16_t chunkp)
	{
	uint16_t chunk = *chunkp;

	*chunkp = CHAIN_END;

	while (chunk != CHAIN_END) {
	- int nextchunk = ZAP_LEAF_CHUNK(l, chunk).l_array.la_next;
	+ uint_t nextchunk = ZAP_LEAF_CHUNK(l, chunk).l_array.la_next;
	ASSERT3U(ZAP_LEAF_CHUNK(l, chunk).l_array.la_type, ==,
	ZAP_CHUNK_ARRAY);
	zap_leaf_chunk_free(l, chunk);
	chunk = nextchunk;
	}
	}

	/* array_len and buf_len are in integers, not bytes */
	static void
	zap_leaf_array_read(zap_leaf_t *l, uint16_t chunk,
	int array_int_len, int array_len, int buf_int_len, uint64_t buf_len,
	void *buf)
	{
	int len = MIN(array_len, buf_len);
	int byten = 0;
	uint64_t value = 0;
	char *p = buf;

	ASSERT3U(array_int_len, <=, buf_int_len);

	/* Fast path for one 8-byte integer */
	if (array_int_len == 8 && buf_int_len == 8 && len == 1) {
	struct zap_leaf_array *la = &ZAP_LEAF_CHUNK(l, chunk).l_array;
	uint8_t *ip = la->la_array;
	uint64_t *buf64 = buf;

	*buf64 = (uint64_t)ip[0] << 56 \| (uint64_t)ip[1] << 48 \|
	(uint64_t)ip[2] << 40 \| (uint64_t)ip[3] << 32 \|
	(uint64_t)ip[4] << 24 \| (uint64_t)ip[5] << 16 \|
	(uint64_t)ip[6] << 8 \| (uint64_t)ip[7];
	return;
	}

	/* Fast path for an array of 1-byte integers (eg. the entry name) */
	if (array_int_len == 1 && buf_int_len == 1 &&
	buf_len > array_len + ZAP_LEAF_ARRAY_BYTES) {
	while (chunk != CHAIN_END) {
	struct zap_leaf_array *la =
	&ZAP_LEAF_CHUNK(l, chunk).l_array;
	memcpy(p, la->la_array, ZAP_LEAF_ARRAY_BYTES);
	p += ZAP_LEAF_ARRAY_BYTES;
	chunk = la->la_next;
	}
	return;
	}

	while (len > 0) {
	struct zap_leaf_array *la = &ZAP_LEAF_CHUNK(l, chunk).l_array;

	ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(l));
	for (int i = 0; i < ZAP_LEAF_ARRAY_BYTES; i++) {
	value = (value << 8) \| la->la_array[i];
	byten++;
	if (byten == array_int_len) {
	stv(buf_int_len, p, value);
	byten = 0;
	len--;
	if (len == 0)
	return;
	p += buf_int_len;
	}
	}
	chunk = la->la_next;
	}
	}

	static boolean_t
	zap_leaf_array_match(zap_leaf_t l, zap_name_t zn,
	- int chunk, int array_numints)
	+ uint_t chunk, int array_numints)
	{
	int bseen = 0;

	if (zap_getflags(zn->zn_zap) & ZAP_FLAG_UINT64_KEY) {
	uint64_t *thiskey =
	kmem_alloc(array_numints * sizeof (*thiskey), KM_SLEEP);
	ASSERT(zn->zn_key_intlen == sizeof (*thiskey));

	zap_leaf_array_read(l, chunk, sizeof (*thiskey), array_numints,
	sizeof (*thiskey), array_numints, thiskey);
	boolean_t match = memcmp(thiskey, zn->zn_key_orig,
	array_numints * sizeof (*thiskey)) == 0;
	kmem_free(thiskey, array_numints * sizeof (*thiskey));
	return (match);
	}

	ASSERT(zn->zn_key_intlen == 1);
	if (zn->zn_matchtype & MT_NORMALIZE) {
	char *thisname = kmem_alloc(array_numints, KM_SLEEP);

	zap_leaf_array_read(l, chunk, sizeof (char), array_numints,
	sizeof (char), array_numints, thisname);
	boolean_t match = zap_match(zn, thisname);
	kmem_free(thisname, array_numints);
	return (match);
	}

	/*
	* Fast path for exact matching.
	* First check that the lengths match, so that we don't read
	* past the end of the zn_key_orig array.
	*/
	if (array_numints != zn->zn_key_orig_numints)
	return (B_FALSE);
	while (bseen < array_numints) {
	struct zap_leaf_array *la = &ZAP_LEAF_CHUNK(l, chunk).l_array;
	int toread = MIN(array_numints - bseen, ZAP_LEAF_ARRAY_BYTES);
	ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(l));
	if (memcmp(la->la_array, (char *)zn->zn_key_orig + bseen,
	toread))
	break;
	chunk = la->la_next;
	bseen += toread;
	}
	return (bseen == array_numints);
	}

	/*
	* Routines which manipulate leaf entries.
	*/

	int
	zap_leaf_lookup(zap_leaf_t l, zap_name_t zn, zap_entry_handle_t *zeh)
	{
	struct zap_leaf_entry *le;

	ASSERT3U(zap_leaf_phys(l)->l_hdr.lh_magic, ==, ZAP_LEAF_MAGIC);

	for (uint16_t *chunkp = LEAF_HASH_ENTPTR(l, zn->zn_hash);
	*chunkp != CHAIN_END; chunkp = &le->le_next) {
	uint16_t chunk = *chunkp;
	le = ZAP_LEAF_ENTRY(l, chunk);

	ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(l));
	ASSERT3U(le->le_type, ==, ZAP_CHUNK_ENTRY);

	if (le->le_hash != zn->zn_hash)
	continue;

	/*
	* NB: the entry chain is always sorted by cd on
	* normalized zap objects, so this will find the
	* lowest-cd match for MT_NORMALIZE.
	*/
	ASSERT((zn->zn_matchtype == 0) \|\|
	(zap_leaf_phys(l)->l_hdr.lh_flags & ZLF_ENTRIES_CDSORTED));
	if (zap_leaf_array_match(l, zn, le->le_name_chunk,
	le->le_name_numints)) {
	zeh->zeh_num_integers = le->le_value_numints;
	zeh->zeh_integer_size = le->le_value_intlen;
	zeh->zeh_cd = le->le_cd;
	zeh->zeh_hash = le->le_hash;
	zeh->zeh_chunkp = chunkp;
	zeh->zeh_leaf = l;
	return (0);
	}
	}

	return (SET_ERROR(ENOENT));
	}

	/* Return (h1,cd1 >= h2,cd2) */
	#define HCD_GTEQ(h1, cd1, h2, cd2) \
	((h1 > h2) ? TRUE : ((h1 == h2 && cd1 >= cd2) ? TRUE : FALSE))

	int
	zap_leaf_lookup_closest(zap_leaf_t *l,
	uint64_t h, uint32_t cd, zap_entry_handle_t *zeh)
	{
	uint64_t besth = -1ULL;
	uint32_t bestcd = -1U;
	uint16_t bestlh = ZAP_LEAF_HASH_NUMENTRIES(l)-1;
	struct zap_leaf_entry *le;

	ASSERT3U(zap_leaf_phys(l)->l_hdr.lh_magic, ==, ZAP_LEAF_MAGIC);

	for (uint16_t lh = LEAF_HASH(l, h); lh <= bestlh; lh++) {
	for (uint16_t chunk = zap_leaf_phys(l)->l_hash[lh];
	chunk != CHAIN_END; chunk = le->le_next) {
	le = ZAP_LEAF_ENTRY(l, chunk);

	ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(l));
	ASSERT3U(le->le_type, ==, ZAP_CHUNK_ENTRY);

	if (HCD_GTEQ(le->le_hash, le->le_cd, h, cd) &&
	HCD_GTEQ(besth, bestcd, le->le_hash, le->le_cd)) {
	ASSERT3U(bestlh, >=, lh);
	bestlh = lh;
	besth = le->le_hash;
	bestcd = le->le_cd;

	zeh->zeh_num_integers = le->le_value_numints;
	zeh->zeh_integer_size = le->le_value_intlen;
	zeh->zeh_cd = le->le_cd;
	zeh->zeh_hash = le->le_hash;
	zeh->zeh_fakechunk = chunk;
	zeh->zeh_chunkp = &zeh->zeh_fakechunk;
	zeh->zeh_leaf = l;
	}
	}
	}

	return (bestcd == -1U ? SET_ERROR(ENOENT) : 0);
	}

	int
	zap_entry_read(const zap_entry_handle_t *zeh,
	uint8_t integer_size, uint64_t num_integers, void *buf)
	{
	struct zap_leaf_entry *le =
	ZAP_LEAF_ENTRY(zeh->zeh_leaf, *zeh->zeh_chunkp);
	ASSERT3U(le->le_type, ==, ZAP_CHUNK_ENTRY);

	if (le->le_value_intlen > integer_size)
	return (SET_ERROR(EINVAL));

	zap_leaf_array_read(zeh->zeh_leaf, le->le_value_chunk,
	le->le_value_intlen, le->le_value_numints,
	integer_size, num_integers, buf);

	if (zeh->zeh_num_integers > num_integers)
	return (SET_ERROR(EOVERFLOW));
	return (0);

	}

	int
	zap_entry_read_name(zap_t zap, const zap_entry_handle_t zeh, uint16_t buflen,
	char *buf)
	{
	struct zap_leaf_entry *le =
	ZAP_LEAF_ENTRY(zeh->zeh_leaf, *zeh->zeh_chunkp);
	ASSERT3U(le->le_type, ==, ZAP_CHUNK_ENTRY);

	if (zap_getflags(zap) & ZAP_FLAG_UINT64_KEY) {
	zap_leaf_array_read(zeh->zeh_leaf, le->le_name_chunk, 8,
	le->le_name_numints, 8, buflen / 8, buf);
	} else {
	zap_leaf_array_read(zeh->zeh_leaf, le->le_name_chunk, 1,
	le->le_name_numints, 1, buflen, buf);
	}
	if (le->le_name_numints > buflen)
	return (SET_ERROR(EOVERFLOW));
	return (0);
	}

	int
	zap_entry_update(zap_entry_handle_t *zeh,
	uint8_t integer_size, uint64_t num_integers, const void *buf)
	{
	zap_leaf_t *l = zeh->zeh_leaf;
	struct zap_leaf_entry le = ZAP_LEAF_ENTRY(l, zeh->zeh_chunkp);

	int delta_chunks = ZAP_LEAF_ARRAY_NCHUNKS(num_integers * integer_size) -
	ZAP_LEAF_ARRAY_NCHUNKS(le->le_value_numints * le->le_value_intlen);

	if ((int)zap_leaf_phys(l)->l_hdr.lh_nfree < delta_chunks)
	return (SET_ERROR(EAGAIN));

	zap_leaf_array_free(l, &le->le_value_chunk);
	le->le_value_chunk =
	zap_leaf_array_create(l, buf, integer_size, num_integers);
	le->le_value_numints = num_integers;
	le->le_value_intlen = integer_size;
	return (0);
	}

	void
	zap_entry_remove(zap_entry_handle_t *zeh)
	{
	zap_leaf_t *l = zeh->zeh_leaf;

	ASSERT3P(zeh->zeh_chunkp, !=, &zeh->zeh_fakechunk);

	uint16_t entry_chunk = *zeh->zeh_chunkp;
	struct zap_leaf_entry *le = ZAP_LEAF_ENTRY(l, entry_chunk);
	ASSERT3U(le->le_type, ==, ZAP_CHUNK_ENTRY);

	zap_leaf_array_free(l, &le->le_name_chunk);
	zap_leaf_array_free(l, &le->le_value_chunk);

	*zeh->zeh_chunkp = le->le_next;
	zap_leaf_chunk_free(l, entry_chunk);

	zap_leaf_phys(l)->l_hdr.lh_nentries--;
	}

	int
	zap_entry_create(zap_leaf_t l, zap_name_t zn, uint32_t cd,
	uint8_t integer_size, uint64_t num_integers, const void *buf,
	zap_entry_handle_t *zeh)
	{
	uint16_t chunk;
	struct zap_leaf_entry *le;
	uint64_t h = zn->zn_hash;

	uint64_t valuelen = integer_size * num_integers;

	- int numchunks = 1 + ZAP_LEAF_ARRAY_NCHUNKS(zn->zn_key_orig_numints *
	+ uint_t numchunks = 1 + ZAP_LEAF_ARRAY_NCHUNKS(zn->zn_key_orig_numints *
	zn->zn_key_intlen) + ZAP_LEAF_ARRAY_NCHUNKS(valuelen);
	if (numchunks > ZAP_LEAF_NUMCHUNKS(l))
	return (SET_ERROR(E2BIG));

	if (cd == ZAP_NEED_CD) {
	/* find the lowest unused cd */
	if (zap_leaf_phys(l)->l_hdr.lh_flags & ZLF_ENTRIES_CDSORTED) {
	cd = 0;

	for (chunk = *LEAF_HASH_ENTPTR(l, h);
	chunk != CHAIN_END; chunk = le->le_next) {
	le = ZAP_LEAF_ENTRY(l, chunk);
	if (le->le_cd > cd)
	break;
	if (le->le_hash == h) {
	ASSERT3U(cd, ==, le->le_cd);
	cd++;
	}
	}
	} else {
	/* old unsorted format; do it the O(n^2) way */
	for (cd = 0; ; cd++) {
	for (chunk = *LEAF_HASH_ENTPTR(l, h);
	chunk != CHAIN_END; chunk = le->le_next) {
	le = ZAP_LEAF_ENTRY(l, chunk);
	if (le->le_hash == h &&
	le->le_cd == cd) {
	break;
	}
	}
	/* If this cd is not in use, we are good. */
	if (chunk == CHAIN_END)
	break;
	}
	}
	/*
	* We would run out of space in a block before we could
	* store enough entries to run out of CD values.
	*/
	ASSERT3U(cd, <, zap_maxcd(zn->zn_zap));
	}

	if (zap_leaf_phys(l)->l_hdr.lh_nfree < numchunks)
	return (SET_ERROR(EAGAIN));

	/* make the entry */
	chunk = zap_leaf_chunk_alloc(l);
	le = ZAP_LEAF_ENTRY(l, chunk);
	le->le_type = ZAP_CHUNK_ENTRY;
	le->le_name_chunk = zap_leaf_array_create(l, zn->zn_key_orig,
	zn->zn_key_intlen, zn->zn_key_orig_numints);
	le->le_name_numints = zn->zn_key_orig_numints;
	le->le_value_chunk =
	zap_leaf_array_create(l, buf, integer_size, num_integers);
	le->le_value_numints = num_integers;
	le->le_value_intlen = integer_size;
	le->le_hash = h;
	le->le_cd = cd;

	/* link it into the hash chain */
	/* XXX if we did the search above, we could just use that */
	- uint16_t *chunkp = zap_leaf_rehash_entry(l, chunk);
	+ uint16_t *chunkp = zap_leaf_rehash_entry(l, le, chunk);

	zap_leaf_phys(l)->l_hdr.lh_nentries++;

	zeh->zeh_leaf = l;
	zeh->zeh_num_integers = num_integers;
	zeh->zeh_integer_size = le->le_value_intlen;
	zeh->zeh_cd = le->le_cd;
	zeh->zeh_hash = le->le_hash;
	zeh->zeh_chunkp = chunkp;

	return (0);
	}

	/*
	* Determine if there is another entry with the same normalized form.
	* For performance purposes, either zn or name must be provided (the
	* other can be NULL). Note, there usually won't be any hash
	* conflicts, in which case we don't need the concatenated/normalized
	* form of the name. But all callers have one of these on hand anyway,
	* so might as well take advantage. A cleaner but slower interface
	* would accept neither argument, and compute the normalized name as
	* needed (using zap_name_alloc_str(zap_entry_read_name(zeh))).
	*/
	boolean_t
	zap_entry_normalization_conflict(zap_entry_handle_t zeh, zap_name_t zn,
	const char name, zap_t zap)
	{
	struct zap_leaf_entry *le;
	boolean_t allocdzn = B_FALSE;

	if (zap->zap_normflags == 0)
	return (B_FALSE);

	for (uint16_t chunk = *LEAF_HASH_ENTPTR(zeh->zeh_leaf, zeh->zeh_hash);
	chunk != CHAIN_END; chunk = le->le_next) {
	le = ZAP_LEAF_ENTRY(zeh->zeh_leaf, chunk);
	if (le->le_hash != zeh->zeh_hash)
	continue;
	if (le->le_cd == zeh->zeh_cd)
	continue;

	if (zn == NULL) {
	zn = zap_name_alloc_str(zap, name, MT_NORMALIZE);
	allocdzn = B_TRUE;
	}
	if (zap_leaf_array_match(zeh->zeh_leaf, zn,
	le->le_name_chunk, le->le_name_numints)) {
	if (allocdzn)
	zap_name_free(zn);
	return (B_TRUE);
	}
	}
	if (allocdzn)
	zap_name_free(zn);
	return (B_FALSE);
	}

	/*
	* Routines for transferring entries between leafs.
	*/

	static uint16_t *
	-zap_leaf_rehash_entry(zap_leaf_t *l, uint16_t entry)
	+zap_leaf_rehash_entry(zap_leaf_t l, struct zap_leaf_entry le, uint16_t entry)
	{
	- struct zap_leaf_entry *le = ZAP_LEAF_ENTRY(l, entry);
	struct zap_leaf_entry *le2;
	uint16_t *chunkp;

	/*
	* keep the entry chain sorted by cd
	* NB: this will not cause problems for unsorted leafs, though
	* it is unnecessary there.
	*/
	for (chunkp = LEAF_HASH_ENTPTR(l, le->le_hash);
	*chunkp != CHAIN_END; chunkp = &le2->le_next) {
	le2 = ZAP_LEAF_ENTRY(l, *chunkp);
	if (le2->le_cd > le->le_cd)
	break;
	}

	le->le_next = *chunkp;
	*chunkp = entry;
	return (chunkp);
	}

	static uint16_t
	zap_leaf_transfer_array(zap_leaf_t l, uint16_t chunk, zap_leaf_t nl)
	{
	uint16_t new_chunk;
	uint16_t *nchunkp = &new_chunk;

	while (chunk != CHAIN_END) {
	uint16_t nchunk = zap_leaf_chunk_alloc(nl);
	struct zap_leaf_array *nla =
	&ZAP_LEAF_CHUNK(nl, nchunk).l_array;
	struct zap_leaf_array *la =
	&ZAP_LEAF_CHUNK(l, chunk).l_array;
	- int nextchunk = la->la_next;
	+ uint_t nextchunk = la->la_next;

	ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(l));
	ASSERT3U(nchunk, <, ZAP_LEAF_NUMCHUNKS(l));

	nla = la; /* structure assignment */

	zap_leaf_chunk_free(l, chunk);
	chunk = nextchunk;
	*nchunkp = nchunk;
	nchunkp = &nla->la_next;
	}
	*nchunkp = CHAIN_END;
	return (new_chunk);
	}

	static void
	-zap_leaf_transfer_entry(zap_leaf_t l, int entry, zap_leaf_t nl)
	+zap_leaf_transfer_entry(zap_leaf_t l, uint_t entry, zap_leaf_t nl)
	{
	struct zap_leaf_entry *le = ZAP_LEAF_ENTRY(l, entry);
	ASSERT3U(le->le_type, ==, ZAP_CHUNK_ENTRY);

	uint16_t chunk = zap_leaf_chunk_alloc(nl);
	struct zap_leaf_entry *nle = ZAP_LEAF_ENTRY(nl, chunk);
	nle = le; /* structure assignment */

	- (void) zap_leaf_rehash_entry(nl, chunk);
	+ (void) zap_leaf_rehash_entry(nl, nle, chunk);

	nle->le_name_chunk = zap_leaf_transfer_array(l, le->le_name_chunk, nl);
	nle->le_value_chunk =
	zap_leaf_transfer_array(l, le->le_value_chunk, nl);

	zap_leaf_chunk_free(l, entry);

	zap_leaf_phys(l)->l_hdr.lh_nentries--;
	zap_leaf_phys(nl)->l_hdr.lh_nentries++;
	}

	/*
	* Transfer the entries whose hash prefix ends in 1 to the new leaf.
	*/
	void
	zap_leaf_split(zap_leaf_t l, zap_leaf_t nl, boolean_t sort)
	{
	- int bit = 64 - 1 - zap_leaf_phys(l)->l_hdr.lh_prefix_len;
	+ uint_t bit = 64 - 1 - zap_leaf_phys(l)->l_hdr.lh_prefix_len;

	/* set new prefix and prefix_len */
	zap_leaf_phys(l)->l_hdr.lh_prefix <<= 1;
	zap_leaf_phys(l)->l_hdr.lh_prefix_len++;
	zap_leaf_phys(nl)->l_hdr.lh_prefix =
	zap_leaf_phys(l)->l_hdr.lh_prefix \| 1;
	zap_leaf_phys(nl)->l_hdr.lh_prefix_len =
	zap_leaf_phys(l)->l_hdr.lh_prefix_len;

	/* break existing hash chains */
	- zap_memset(zap_leaf_phys(l)->l_hash, CHAIN_END,
	+ memset(zap_leaf_phys(l)->l_hash, CHAIN_END,
	2*ZAP_LEAF_HASH_NUMENTRIES(l));

	if (sort)
	zap_leaf_phys(l)->l_hdr.lh_flags \|= ZLF_ENTRIES_CDSORTED;

	/*
	* Transfer entries whose hash bit 'bit' is set to nl; rehash
	* the remaining entries
	*
	* NB: We could find entries via the hashtable instead. That
	* would be O(hashents+numents) rather than O(numblks+numents),
	* but this accesses memory more sequentially, and when we're
	* called, the block is usually pretty full.
	*/
	- for (int i = 0; i < ZAP_LEAF_NUMCHUNKS(l); i++) {
	+ for (uint_t i = 0; i < ZAP_LEAF_NUMCHUNKS(l); i++) {
	struct zap_leaf_entry *le = ZAP_LEAF_ENTRY(l, i);
	if (le->le_type != ZAP_CHUNK_ENTRY)
	continue;

	if (le->le_hash & (1ULL << bit))
	zap_leaf_transfer_entry(l, i, nl);
	else
	- (void) zap_leaf_rehash_entry(l, i);
	+ (void) zap_leaf_rehash_entry(l, le, i);
	}
	}

	void
	zap_leaf_stats(zap_t zap, zap_leaf_t l, zap_stats_t *zs)
	{
	- int n = zap_f_phys(zap)->zap_ptrtbl.zt_shift -
	+ uint_t n = zap_f_phys(zap)->zap_ptrtbl.zt_shift -
	zap_leaf_phys(l)->l_hdr.lh_prefix_len;
	n = MIN(n, ZAP_HISTOGRAM_SIZE-1);
	zs->zs_leafs_with_2n_pointers[n]++;


	n = zap_leaf_phys(l)->l_hdr.lh_nentries/5;
	n = MIN(n, ZAP_HISTOGRAM_SIZE-1);
	zs->zs_blocks_with_n5_entries[n]++;

	n = ((1<<FZAP_BLOCK_SHIFT(zap)) -
	zap_leaf_phys(l)->l_hdr.lh_nfree * (ZAP_LEAF_ARRAY_BYTES+1))*10 /
	(1<<FZAP_BLOCK_SHIFT(zap));
	n = MIN(n, ZAP_HISTOGRAM_SIZE-1);
	zs->zs_blocks_n_tenths_full[n]++;

	- for (int i = 0; i < ZAP_LEAF_HASH_NUMENTRIES(l); i++) {
	- int nentries = 0;
	- int chunk = zap_leaf_phys(l)->l_hash[i];
	+ for (uint_t i = 0; i < ZAP_LEAF_HASH_NUMENTRIES(l); i++) {
	+ uint_t nentries = 0;
	+ uint_t chunk = zap_leaf_phys(l)->l_hash[i];

	while (chunk != CHAIN_END) {
	struct zap_leaf_entry *le =
	ZAP_LEAF_ENTRY(l, chunk);

	n = 1 + ZAP_LEAF_ARRAY_NCHUNKS(le->le_name_numints) +
	ZAP_LEAF_ARRAY_NCHUNKS(le->le_value_numints *
	le->le_value_intlen);
	n = MIN(n, ZAP_HISTOGRAM_SIZE-1);
	zs->zs_entries_using_n_chunks[n]++;

	chunk = le->le_next;
	nentries++;
	}

	n = nentries;
	n = MIN(n, ZAP_HISTOGRAM_SIZE-1);
	zs->zs_buckets_with_n_entries[n]++;
	}
	}
	diff --git a/sys/contrib/openzfs/module/zfs/zap_micro.c b/sys/contrib/openzfs/module/zfs/zap_micro.c
	index 085d9cd8b4b6..d806988af96d 100644
	--- a/sys/contrib/openzfs/module/zfs/zap_micro.c
	+++ b/sys/contrib/openzfs/module/zfs/zap_micro.c
	@@ -1,1737 +1,1823 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2011, 2018 by Delphix. All rights reserved.
	* Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
	* Copyright 2017 Nexenta Systems, Inc.
	*/

	#include <sys/zio.h>
	#include <sys/spa.h>
	#include <sys/dmu.h>
	#include <sys/zfs_context.h>
	#include <sys/zap.h>
	#include <sys/zap_impl.h>
	#include <sys/zap_leaf.h>
	#include <sys/btree.h>
	#include <sys/arc.h>
	#include <sys/dmu_objset.h>

	#ifdef _KERNEL
	#include <sys/sunddi.h>
	#endif

	int zap_micro_max_size = MZAP_MAX_BLKSZ;

	static int mzap_upgrade(zap_t **zapp,
	const void tag, dmu_tx_t tx, zap_flags_t flags);

	uint64_t
	zap_getflags(zap_t *zap)
	{
	if (zap->zap_ismicro)
	return (0);
	return (zap_f_phys(zap)->zap_flags);
	}

	int
	zap_hashbits(zap_t *zap)
	{
	if (zap_getflags(zap) & ZAP_FLAG_HASH64)
	return (48);
	else
	return (28);
	}

	uint32_t
	zap_maxcd(zap_t *zap)
	{
	if (zap_getflags(zap) & ZAP_FLAG_HASH64)
	return ((1<<16)-1);
	else
	return (-1U);
	}

	static uint64_t
	zap_hash(zap_name_t *zn)
	{
	zap_t *zap = zn->zn_zap;
	uint64_t h = 0;

	if (zap_getflags(zap) & ZAP_FLAG_PRE_HASHED_KEY) {
	ASSERT(zap_getflags(zap) & ZAP_FLAG_UINT64_KEY);
	h = (uint64_t )zn->zn_key_orig;
	} else {
	h = zap->zap_salt;
	ASSERT(h != 0);
	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);

	if (zap_getflags(zap) & ZAP_FLAG_UINT64_KEY) {
	const uint64_t *wp = zn->zn_key_norm;

	ASSERT(zn->zn_key_intlen == 8);
	for (int i = 0; i < zn->zn_key_norm_numints;
	wp++, i++) {
	uint64_t word = *wp;

	for (int j = 0; j < 8; j++) {
	h = (h >> 8) ^
	zfs_crc64_table[(h ^ word) & 0xFF];
	word >>= NBBY;
	}
	}
	} else {
	const uint8_t *cp = zn->zn_key_norm;

	/*
	* We previously stored the terminating null on
	* disk, but didn't hash it, so we need to
	* continue to not hash it. (The
	* zn_key_*_numints includes the terminating
	* null for non-binary keys.)
	*/
	int len = zn->zn_key_norm_numints - 1;

	ASSERT(zn->zn_key_intlen == 1);
	for (int i = 0; i < len; cp++, i++) {
	h = (h >> 8) ^
	zfs_crc64_table[(h ^ *cp) & 0xFF];
	}
	}
	}
	/*
	* Don't use all 64 bits, since we need some in the cookie for
	* the collision differentiator. We MUST use the high bits,
	* since those are the ones that we first pay attention to when
	* choosing the bucket.
	*/
	h &= ~((1ULL << (64 - zap_hashbits(zap))) - 1);

	return (h);
	}

	static int
	zap_normalize(zap_t zap, const char name, char *namenorm, int normflags)
	{
	ASSERT(!(zap_getflags(zap) & ZAP_FLAG_UINT64_KEY));

	size_t inlen = strlen(name) + 1;
	size_t outlen = ZAP_MAXNAMELEN;

	int err = 0;
	(void) u8_textprep_str((char *)name, &inlen, namenorm, &outlen,
	normflags \| U8_TEXTPREP_IGNORE_NULL \| U8_TEXTPREP_IGNORE_INVALID,
	U8_UNICODE_LATEST, &err);

	return (err);
	}

	boolean_t
	zap_match(zap_name_t zn, const char matchname)
	{
	ASSERT(!(zap_getflags(zn->zn_zap) & ZAP_FLAG_UINT64_KEY));

	if (zn->zn_matchtype & MT_NORMALIZE) {
	char norm[ZAP_MAXNAMELEN];

	if (zap_normalize(zn->zn_zap, matchname, norm,
	zn->zn_normflags) != 0)
	return (B_FALSE);

	return (strcmp(zn->zn_key_norm, norm) == 0);
	} else {
	return (strcmp(zn->zn_key_orig, matchname) == 0);
	}
	}

	static zap_name_t *
	zap_name_alloc(zap_t *zap)
	{
	zap_name_t *zn = kmem_alloc(sizeof (zap_name_t), KM_SLEEP);
	zn->zn_zap = zap;
	return (zn);
	}

	void
	zap_name_free(zap_name_t *zn)
	{
	kmem_free(zn, sizeof (zap_name_t));
	}

	static int
	zap_name_init_str(zap_name_t zn, const char key, matchtype_t mt)
	{
	zap_t *zap = zn->zn_zap;

	zn->zn_key_intlen = sizeof (*key);
	zn->zn_key_orig = key;
	zn->zn_key_orig_numints = strlen(zn->zn_key_orig) + 1;
	zn->zn_matchtype = mt;
	zn->zn_normflags = zap->zap_normflags;

	/*
	* If we're dealing with a case sensitive lookup on a mixed or
	* insensitive fs, remove U8_TEXTPREP_TOUPPER or the lookup
	* will fold case to all caps overriding the lookup request.
	*/
	if (mt & MT_MATCH_CASE)
	zn->zn_normflags &= ~U8_TEXTPREP_TOUPPER;

	if (zap->zap_normflags) {
	/*
	* We must use zap_normflags because this normalization is
	* what the hash is computed from.
	*/
	if (zap_normalize(zap, key, zn->zn_normbuf,
	zap->zap_normflags) != 0)
	return (SET_ERROR(ENOTSUP));
	zn->zn_key_norm = zn->zn_normbuf;
	zn->zn_key_norm_numints = strlen(zn->zn_key_norm) + 1;
	} else {
	if (mt != 0)
	return (SET_ERROR(ENOTSUP));
	zn->zn_key_norm = zn->zn_key_orig;
	zn->zn_key_norm_numints = zn->zn_key_orig_numints;
	}

	zn->zn_hash = zap_hash(zn);

	if (zap->zap_normflags != zn->zn_normflags) {
	/*
	* We must use zn_normflags because this normalization is
	* what the matching is based on. (Not the hash!)
	*/
	if (zap_normalize(zap, key, zn->zn_normbuf,
	zn->zn_normflags) != 0)
	return (SET_ERROR(ENOTSUP));
	zn->zn_key_norm_numints = strlen(zn->zn_key_norm) + 1;
	}

	return (0);
	}

	zap_name_t *
	zap_name_alloc_str(zap_t zap, const char key, matchtype_t mt)
	{
	zap_name_t *zn = zap_name_alloc(zap);
	if (zap_name_init_str(zn, key, mt) != 0) {
	zap_name_free(zn);
	return (NULL);
	}
	return (zn);
	}

	static zap_name_t *
	zap_name_alloc_uint64(zap_t zap, const uint64_t key, int numints)
	{
	zap_name_t *zn = kmem_alloc(sizeof (zap_name_t), KM_SLEEP);

	ASSERT(zap->zap_normflags == 0);
	zn->zn_zap = zap;
	zn->zn_key_intlen = sizeof (*key);
	zn->zn_key_orig = zn->zn_key_norm = key;
	zn->zn_key_orig_numints = zn->zn_key_norm_numints = numints;
	zn->zn_matchtype = 0;

	zn->zn_hash = zap_hash(zn);
	return (zn);
	}

	static void
	mzap_byteswap(mzap_phys_t *buf, size_t size)
	{
	buf->mz_block_type = BSWAP_64(buf->mz_block_type);
	buf->mz_salt = BSWAP_64(buf->mz_salt);
	buf->mz_normflags = BSWAP_64(buf->mz_normflags);
	int max = (size / MZAP_ENT_LEN) - 1;
	for (int i = 0; i < max; i++) {
	buf->mz_chunk[i].mze_value =
	BSWAP_64(buf->mz_chunk[i].mze_value);
	buf->mz_chunk[i].mze_cd =
	BSWAP_32(buf->mz_chunk[i].mze_cd);
	}
	}

	void
	zap_byteswap(void *buf, size_t size)
	{
	uint64_t block_type = (uint64_t )buf;

	if (block_type == ZBT_MICRO \|\| block_type == BSWAP_64(ZBT_MICRO)) {
	/* ASSERT(magic == ZAP_LEAF_MAGIC); */
	mzap_byteswap(buf, size);
	} else {
	fzap_byteswap(buf, size);
	}
	}

	__attribute__((always_inline)) inline
	static int
	mze_compare(const void arg1, const void arg2)
	{
	const mzap_ent_t *mze1 = arg1;
	const mzap_ent_t *mze2 = arg2;

	return (TREE_CMP((uint64_t)(mze1->mze_hash) << 32 \| mze1->mze_cd,
	(uint64_t)(mze2->mze_hash) << 32 \| mze2->mze_cd));
	}

	ZFS_BTREE_FIND_IN_BUF_FUNC(mze_find_in_buf, mzap_ent_t,
	mze_compare)

	static void
	mze_insert(zap_t *zap, uint16_t chunkid, uint64_t hash)
	{
	mzap_ent_t mze;

	ASSERT(zap->zap_ismicro);
	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));

	mze.mze_chunkid = chunkid;
	ASSERT0(hash & 0xffffffff);
	mze.mze_hash = hash >> 32;
	ASSERT3U(MZE_PHYS(zap, &mze)->mze_cd, <=, 0xffff);
	mze.mze_cd = (uint16_t)MZE_PHYS(zap, &mze)->mze_cd;
	ASSERT(MZE_PHYS(zap, &mze)->mze_name[0] != 0);
	zfs_btree_add(&zap->zap_m.zap_tree, &mze);
	}

	static mzap_ent_t *
	mze_find(zap_name_t zn, zfs_btree_index_t idx)
	{
	mzap_ent_t mze_tofind;
	mzap_ent_t *mze;
	zfs_btree_t *tree = &zn->zn_zap->zap_m.zap_tree;

	ASSERT(zn->zn_zap->zap_ismicro);
	ASSERT(RW_LOCK_HELD(&zn->zn_zap->zap_rwlock));

	ASSERT0(zn->zn_hash & 0xffffffff);
	mze_tofind.mze_hash = zn->zn_hash >> 32;
	mze_tofind.mze_cd = 0;

	mze = zfs_btree_find(tree, &mze_tofind, idx);
	if (mze == NULL)
	mze = zfs_btree_next(tree, idx, idx);
	for (; mze && mze->mze_hash == mze_tofind.mze_hash;
	mze = zfs_btree_next(tree, idx, idx)) {
	ASSERT3U(mze->mze_cd, ==, MZE_PHYS(zn->zn_zap, mze)->mze_cd);
	if (zap_match(zn, MZE_PHYS(zn->zn_zap, mze)->mze_name))
	return (mze);
	}

	return (NULL);
	}

	static uint32_t
	mze_find_unused_cd(zap_t *zap, uint64_t hash)
	{
	mzap_ent_t mze_tofind;
	zfs_btree_index_t idx;
	zfs_btree_t *tree = &zap->zap_m.zap_tree;

	ASSERT(zap->zap_ismicro);
	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));

	ASSERT0(hash & 0xffffffff);
	hash >>= 32;
	mze_tofind.mze_hash = hash;
	mze_tofind.mze_cd = 0;

	uint32_t cd = 0;
	for (mzap_ent_t *mze = zfs_btree_find(tree, &mze_tofind, &idx);
	mze && mze->mze_hash == hash;
	mze = zfs_btree_next(tree, &idx, &idx)) {
	if (mze->mze_cd != cd)
	break;
	cd++;
	}

	return (cd);
	}

	/*
	* Each mzap entry requires at max : 4 chunks
	* 3 chunks for names + 1 chunk for value.
	*/
	#define MZAP_ENT_CHUNKS (1 + ZAP_LEAF_ARRAY_NCHUNKS(MZAP_NAME_LEN) + \
	ZAP_LEAF_ARRAY_NCHUNKS(sizeof (uint64_t)))

	/*
	* Check if the current entry keeps the colliding entries under the fatzap leaf
	* size.
	*/
	static boolean_t
	mze_canfit_fzap_leaf(zap_name_t *zn, uint64_t hash)
	{
	zap_t *zap = zn->zn_zap;
	mzap_ent_t mze_tofind;
	zfs_btree_index_t idx;
	zfs_btree_t *tree = &zap->zap_m.zap_tree;
	uint32_t mzap_ents = 0;

	ASSERT0(hash & 0xffffffff);
	hash >>= 32;
	mze_tofind.mze_hash = hash;
	mze_tofind.mze_cd = 0;

	for (mzap_ent_t *mze = zfs_btree_find(tree, &mze_tofind, &idx);
	mze && mze->mze_hash == hash;
	mze = zfs_btree_next(tree, &idx, &idx)) {
	mzap_ents++;
	}

	/* Include the new entry being added */
	mzap_ents++;

	return (ZAP_LEAF_NUMCHUNKS_DEF > (mzap_ents * MZAP_ENT_CHUNKS));
	}

	static void
	mze_destroy(zap_t *zap)
	{
	zfs_btree_clear(&zap->zap_m.zap_tree);
	zfs_btree_destroy(&zap->zap_m.zap_tree);
	}

	static zap_t *
	-mzap_open(objset_t os, uint64_t obj, dmu_buf_t db)
	+mzap_open(dmu_buf_t *db)
	{
	zap_t *winner;
	uint64_t zap_hdr = (uint64_t )db->db_data;
	uint64_t zap_block_type = zap_hdr[0];
	uint64_t zap_magic = zap_hdr[1];

	ASSERT3U(MZAP_ENT_LEN, ==, sizeof (mzap_ent_phys_t));

	zap_t *zap = kmem_zalloc(sizeof (zap_t), KM_SLEEP);
	rw_init(&zap->zap_rwlock, NULL, RW_DEFAULT, NULL);
	rw_enter(&zap->zap_rwlock, RW_WRITER);
	- zap->zap_objset = os;
	- zap->zap_object = obj;
	+ zap->zap_objset = dmu_buf_get_objset(db);
	+ zap->zap_object = db->db_object;
	zap->zap_dbuf = db;

	if (zap_block_type != ZBT_MICRO) {
	mutex_init(&zap->zap_f.zap_num_entries_mtx, 0, MUTEX_DEFAULT,
	0);
	zap->zap_f.zap_block_shift = highbit64(db->db_size) - 1;
	if (zap_block_type != ZBT_HEADER \|\| zap_magic != ZAP_MAGIC) {
	winner = NULL; /* No actual winner here... */
	goto handle_winner;
	}
	} else {
	zap->zap_ismicro = TRUE;
	}

	/*
	* Make sure that zap_ismicro is set before we let others see
	* it, because zap_lockdir() checks zap_ismicro without the lock
	* held.
	*/
	dmu_buf_init_user(&zap->zap_dbu, zap_evict_sync, NULL, &zap->zap_dbuf);
	winner = dmu_buf_set_user(db, &zap->zap_dbu);

	if (winner != NULL)
	goto handle_winner;

	if (zap->zap_ismicro) {
	zap->zap_salt = zap_m_phys(zap)->mz_salt;
	zap->zap_normflags = zap_m_phys(zap)->mz_normflags;
	zap->zap_m.zap_num_chunks = db->db_size / MZAP_ENT_LEN - 1;

	/*
	* Reduce B-tree leaf from 4KB to 512 bytes to reduce memmove()
	* overhead on massive inserts below. It still allows to store
	* 62 entries before we have to add 2KB B-tree core node.
	*/
	zfs_btree_create_custom(&zap->zap_m.zap_tree, mze_compare,
	mze_find_in_buf, sizeof (mzap_ent_t), 512);

	zap_name_t *zn = zap_name_alloc(zap);
	for (uint16_t i = 0; i < zap->zap_m.zap_num_chunks; i++) {
	mzap_ent_phys_t *mze =
	&zap_m_phys(zap)->mz_chunk[i];
	if (mze->mze_name[0]) {
	zap->zap_m.zap_num_entries++;
	zap_name_init_str(zn, mze->mze_name, 0);
	mze_insert(zap, i, zn->zn_hash);
	}
	}
	zap_name_free(zn);
	} else {
	zap->zap_salt = zap_f_phys(zap)->zap_salt;
	zap->zap_normflags = zap_f_phys(zap)->zap_normflags;

	ASSERT3U(sizeof (struct zap_leaf_header), ==,
	2*ZAP_LEAF_CHUNKSIZE);

	/*
	* The embedded pointer table should not overlap the
	* other members.
	*/
	ASSERT3P(&ZAP_EMBEDDED_PTRTBL_ENT(zap, 0), >,
	&zap_f_phys(zap)->zap_salt);

	/*
	* The embedded pointer table should end at the end of
	* the block
	*/
	ASSERT3U((uintptr_t)&ZAP_EMBEDDED_PTRTBL_ENT(zap,
	1<<ZAP_EMBEDDED_PTRTBL_SHIFT(zap)) -
	(uintptr_t)zap_f_phys(zap), ==,
	zap->zap_dbuf->db_size);
	}
	rw_exit(&zap->zap_rwlock);
	return (zap);

	handle_winner:
	rw_exit(&zap->zap_rwlock);
	rw_destroy(&zap->zap_rwlock);
	if (!zap->zap_ismicro)
	mutex_destroy(&zap->zap_f.zap_num_entries_mtx);
	kmem_free(zap, sizeof (zap_t));
	return (winner);
	}

	/*
	* This routine "consumes" the caller's hold on the dbuf, which must
	* have the specified tag.
	*/
	static int
	-zap_lockdir_impl(dmu_buf_t db, const void tag, dmu_tx_t *tx,
	+zap_lockdir_impl(dnode_t dn, dmu_buf_t db, const void tag, dmu_tx_t tx,
	krw_t lti, boolean_t fatreader, boolean_t adding, zap_t **zapp)
	{
	ASSERT0(db->db_offset);
	objset_t *os = dmu_buf_get_objset(db);
	uint64_t obj = db->db_object;
	dmu_object_info_t doi;

	*zapp = NULL;

	- dmu_object_info_from_db(db, &doi);
	+ dmu_object_info_from_dnode(dn, &doi);
	if (DMU_OT_BYTESWAP(doi.doi_type) != DMU_BSWAP_ZAP)
	return (SET_ERROR(EINVAL));

	zap_t *zap = dmu_buf_get_user(db);
	if (zap == NULL) {
	- zap = mzap_open(os, obj, db);
	+ zap = mzap_open(db);
	if (zap == NULL) {
	/*
	* mzap_open() didn't like what it saw on-disk.
	* Check for corruption!
	*/
	return (SET_ERROR(EIO));
	}
	}

	/*
	* We're checking zap_ismicro without the lock held, in order to
	* tell what type of lock we want. Once we have some sort of
	* lock, see if it really is the right type. In practice this
	* can only be different if it was upgraded from micro to fat,
	* and micro wanted WRITER but fat only needs READER.
	*/
	krw_t lt = (!zap->zap_ismicro && fatreader) ? RW_READER : lti;
	rw_enter(&zap->zap_rwlock, lt);
	if (lt != ((!zap->zap_ismicro && fatreader) ? RW_READER : lti)) {
	/* it was upgraded, now we only need reader */
	ASSERT(lt == RW_WRITER);
	ASSERT(RW_READER ==
	((!zap->zap_ismicro && fatreader) ? RW_READER : lti));
	rw_downgrade(&zap->zap_rwlock);
	lt = RW_READER;
	}

	zap->zap_objset = os;
	+ zap->zap_dnode = dn;

	if (lt == RW_WRITER)
	dmu_buf_will_dirty(db, tx);

	ASSERT3P(zap->zap_dbuf, ==, db);

	ASSERT(!zap->zap_ismicro \|\|
	zap->zap_m.zap_num_entries <= zap->zap_m.zap_num_chunks);
	if (zap->zap_ismicro && tx && adding &&
	zap->zap_m.zap_num_entries == zap->zap_m.zap_num_chunks) {
	uint64_t newsz = db->db_size + SPA_MINBLOCKSIZE;
	if (newsz > zap_micro_max_size) {
	dprintf("upgrading obj %llu: num_entries=%u\n",
	(u_longlong_t)obj, zap->zap_m.zap_num_entries);
	*zapp = zap;
	int err = mzap_upgrade(zapp, tag, tx, 0);
	if (err != 0)
	rw_exit(&zap->zap_rwlock);
	return (err);
	}
	VERIFY0(dmu_object_set_blocksize(os, obj, newsz, 0, tx));
	zap->zap_m.zap_num_chunks =
	db->db_size / MZAP_ENT_LEN - 1;
	}

	*zapp = zap;
	return (0);
	}

	static int
	zap_lockdir_by_dnode(dnode_t dn, dmu_tx_t tx,
	krw_t lti, boolean_t fatreader, boolean_t adding, const void *tag,
	zap_t **zapp)
	{
	dmu_buf_t *db;
	+ int err;

	- int err = dmu_buf_hold_by_dnode(dn, 0, tag, &db, DMU_READ_NO_PREFETCH);
	- if (err != 0) {
	+ err = dmu_buf_hold_by_dnode(dn, 0, tag, &db, DMU_READ_NO_PREFETCH);
	+ if (err != 0)
	return (err);
	- }
	-#ifdef ZFS_DEBUG
	- {
	- dmu_object_info_t doi;
	- dmu_object_info_from_db(db, &doi);
	- ASSERT3U(DMU_OT_BYTESWAP(doi.doi_type), ==, DMU_BSWAP_ZAP);
	- }
	-#endif
	-
	- err = zap_lockdir_impl(db, tag, tx, lti, fatreader, adding, zapp);
	- if (err != 0) {
	+ err = zap_lockdir_impl(dn, db, tag, tx, lti, fatreader, adding, zapp);
	+ if (err != 0)
	dmu_buf_rele(db, tag);
	- }
	+ else
	+ VERIFY(dnode_add_ref(dn, tag));
	return (err);
	}

	int
	zap_lockdir(objset_t os, uint64_t obj, dmu_tx_t tx,
	krw_t lti, boolean_t fatreader, boolean_t adding, const void *tag,
	zap_t **zapp)
	{
	+ dnode_t *dn;
	dmu_buf_t *db;
	+ int err;

	- int err = dmu_buf_hold(os, obj, 0, tag, &db, DMU_READ_NO_PREFETCH);
	+ err = dnode_hold(os, obj, tag, &dn);
	if (err != 0)
	return (err);
	-#ifdef ZFS_DEBUG
	- {
	- dmu_object_info_t doi;
	- dmu_object_info_from_db(db, &doi);
	- ASSERT3U(DMU_OT_BYTESWAP(doi.doi_type), ==, DMU_BSWAP_ZAP);
	+ err = dmu_buf_hold_by_dnode(dn, 0, tag, &db, DMU_READ_NO_PREFETCH);
	+ if (err != 0) {
	+ dnode_rele(dn, tag);
	+ return (err);
	}
	-#endif
	- err = zap_lockdir_impl(db, tag, tx, lti, fatreader, adding, zapp);
	- if (err != 0)
	+ err = zap_lockdir_impl(dn, db, tag, tx, lti, fatreader, adding, zapp);
	+ if (err != 0) {
	dmu_buf_rele(db, tag);
	+ dnode_rele(dn, tag);
	+ }
	return (err);
	}

	void
	zap_unlockdir(zap_t zap, const void tag)
	{
	rw_exit(&zap->zap_rwlock);
	+ dnode_rele(zap->zap_dnode, tag);
	dmu_buf_rele(zap->zap_dbuf, tag);
	}

	static int
	mzap_upgrade(zap_t *zapp, const void tag, dmu_tx_t *tx, zap_flags_t flags)
	{
	int err = 0;
	zap_t zap = zapp;

	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));

	int sz = zap->zap_dbuf->db_size;
	mzap_phys_t *mzp = vmem_alloc(sz, KM_SLEEP);
	memcpy(mzp, zap->zap_dbuf->db_data, sz);
	int nchunks = zap->zap_m.zap_num_chunks;

	if (!flags) {
	err = dmu_object_set_blocksize(zap->zap_objset, zap->zap_object,
	1ULL << fzap_default_block_shift, 0, tx);
	if (err != 0) {
	vmem_free(mzp, sz);
	return (err);
	}
	}

	dprintf("upgrading obj=%llu with %u chunks\n",
	(u_longlong_t)zap->zap_object, nchunks);
	/* XXX destroy the tree later, so we can use the stored hash value */
	mze_destroy(zap);

	fzap_upgrade(zap, tx, flags);

	zap_name_t *zn = zap_name_alloc(zap);
	for (int i = 0; i < nchunks; i++) {
	mzap_ent_phys_t *mze = &mzp->mz_chunk[i];
	if (mze->mze_name[0] == 0)
	continue;
	dprintf("adding %s=%llu\n",
	mze->mze_name, (u_longlong_t)mze->mze_value);
	zap_name_init_str(zn, mze->mze_name, 0);
	/* If we fail here, we would end up losing entries */
	VERIFY0(fzap_add_cd(zn, 8, 1, &mze->mze_value, mze->mze_cd,
	tag, tx));
	zap = zn->zn_zap; /* fzap_add_cd() may change zap */
	}
	zap_name_free(zn);
	vmem_free(mzp, sz);
	*zapp = zap;
	return (0);
	}

	/*
	* The "normflags" determine the behavior of the matchtype_t which is
	* passed to zap_lookup_norm(). Names which have the same normalized
	* version will be stored with the same hash value, and therefore we can
	* perform normalization-insensitive lookups. We can be Unicode form-
	* insensitive and/or case-insensitive. The following flags are valid for
	* "normflags":
	*
	* U8_TEXTPREP_NFC
	* U8_TEXTPREP_NFD
	* U8_TEXTPREP_NFKC
	* U8_TEXTPREP_NFKD
	* U8_TEXTPREP_TOUPPER
	*
	* The _NF (Normalization Form) flags are mutually exclusive; at most one
	* of them may be supplied.
	*/
	void
	mzap_create_impl(dnode_t dn, int normflags, zap_flags_t flags, dmu_tx_t tx)
	{
	dmu_buf_t *db;

	VERIFY0(dmu_buf_hold_by_dnode(dn, 0, FTAG, &db, DMU_READ_NO_PREFETCH));

	dmu_buf_will_dirty(db, tx);
	mzap_phys_t *zp = db->db_data;
	zp->mz_block_type = ZBT_MICRO;
	zp->mz_salt =
	((uintptr_t)db ^ (uintptr_t)tx ^ (dn->dn_object << 1)) \| 1ULL;
	zp->mz_normflags = normflags;

	if (flags != 0) {
	zap_t *zap;
	/* Only fat zap supports flags; upgrade immediately. */
	- VERIFY0(zap_lockdir_impl(db, FTAG, tx, RW_WRITER,
	+ VERIFY(dnode_add_ref(dn, FTAG));
	+ VERIFY0(zap_lockdir_impl(dn, db, FTAG, tx, RW_WRITER,
	B_FALSE, B_FALSE, &zap));
	VERIFY0(mzap_upgrade(&zap, FTAG, tx, flags));
	zap_unlockdir(zap, FTAG);
	} else {
	dmu_buf_rele(db, FTAG);
	}
	}

	static uint64_t
	zap_create_impl(objset_t *os, int normflags, zap_flags_t flags,
	dmu_object_type_t ot, int leaf_blockshift, int indirect_blockshift,
	dmu_object_type_t bonustype, int bonuslen, int dnodesize,
	dnode_t *allocated_dnode, const void tag, dmu_tx_t *tx)
	{
	uint64_t obj;

	ASSERT3U(DMU_OT_BYTESWAP(ot), ==, DMU_BSWAP_ZAP);

	if (allocated_dnode == NULL) {
	dnode_t *dn;
	obj = dmu_object_alloc_hold(os, ot, 1ULL << leaf_blockshift,
	indirect_blockshift, bonustype, bonuslen, dnodesize,
	&dn, FTAG, tx);
	mzap_create_impl(dn, normflags, flags, tx);
	dnode_rele(dn, FTAG);
	} else {
	obj = dmu_object_alloc_hold(os, ot, 1ULL << leaf_blockshift,
	indirect_blockshift, bonustype, bonuslen, dnodesize,
	allocated_dnode, tag, tx);
	mzap_create_impl(*allocated_dnode, normflags, flags, tx);
	}

	return (obj);
	}

	int
	zap_create_claim(objset_t *os, uint64_t obj, dmu_object_type_t ot,
	dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx)
	{
	return (zap_create_claim_dnsize(os, obj, ot, bonustype, bonuslen,
	0, tx));
	}

	int
	zap_create_claim_dnsize(objset_t *os, uint64_t obj, dmu_object_type_t ot,
	dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx)
	{
	return (zap_create_claim_norm_dnsize(os, obj,
	0, ot, bonustype, bonuslen, dnodesize, tx));
	}

	int
	zap_create_claim_norm(objset_t *os, uint64_t obj, int normflags,
	dmu_object_type_t ot,
	dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx)
	{
	return (zap_create_claim_norm_dnsize(os, obj, normflags, ot, bonustype,
	bonuslen, 0, tx));
	}

	int
	zap_create_claim_norm_dnsize(objset_t *os, uint64_t obj, int normflags,
	dmu_object_type_t ot, dmu_object_type_t bonustype, int bonuslen,
	int dnodesize, dmu_tx_t *tx)
	{
	dnode_t *dn;
	int error;

	ASSERT3U(DMU_OT_BYTESWAP(ot), ==, DMU_BSWAP_ZAP);
	error = dmu_object_claim_dnsize(os, obj, ot, 0, bonustype, bonuslen,
	dnodesize, tx);
	if (error != 0)
	return (error);

	error = dnode_hold(os, obj, FTAG, &dn);
	if (error != 0)
	return (error);

	mzap_create_impl(dn, normflags, 0, tx);

	dnode_rele(dn, FTAG);

	return (0);
	}

	uint64_t
	zap_create(objset_t *os, dmu_object_type_t ot,
	dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx)
	{
	return (zap_create_norm(os, 0, ot, bonustype, bonuslen, tx));
	}

	uint64_t
	zap_create_dnsize(objset_t *os, dmu_object_type_t ot,
	dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx)
	{
	return (zap_create_norm_dnsize(os, 0, ot, bonustype, bonuslen,
	dnodesize, tx));
	}

	uint64_t
	zap_create_norm(objset_t *os, int normflags, dmu_object_type_t ot,
	dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx)
	{
	return (zap_create_norm_dnsize(os, normflags, ot, bonustype, bonuslen,
	0, tx));
	}

	uint64_t
	zap_create_norm_dnsize(objset_t *os, int normflags, dmu_object_type_t ot,
	dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx)
	{
	return (zap_create_impl(os, normflags, 0, ot, 0, 0,
	bonustype, bonuslen, dnodesize, NULL, NULL, tx));
	}

	uint64_t
	zap_create_flags(objset_t *os, int normflags, zap_flags_t flags,
	dmu_object_type_t ot, int leaf_blockshift, int indirect_blockshift,
	dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx)
	{
	return (zap_create_flags_dnsize(os, normflags, flags, ot,
	leaf_blockshift, indirect_blockshift, bonustype, bonuslen, 0, tx));
	}

	uint64_t
	zap_create_flags_dnsize(objset_t *os, int normflags, zap_flags_t flags,
	dmu_object_type_t ot, int leaf_blockshift, int indirect_blockshift,
	dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx)
	{
	return (zap_create_impl(os, normflags, flags, ot, leaf_blockshift,
	indirect_blockshift, bonustype, bonuslen, dnodesize, NULL, NULL,
	tx));
	}

	/*
	* Create a zap object and return a pointer to the newly allocated dnode via
	* the allocated_dnode argument. The returned dnode will be held and the
	* caller is responsible for releasing the hold by calling dnode_rele().
	*/
	uint64_t
	zap_create_hold(objset_t *os, int normflags, zap_flags_t flags,
	dmu_object_type_t ot, int leaf_blockshift, int indirect_blockshift,
	dmu_object_type_t bonustype, int bonuslen, int dnodesize,
	dnode_t *allocated_dnode, const void tag, dmu_tx_t *tx)
	{
	return (zap_create_impl(os, normflags, flags, ot, leaf_blockshift,
	indirect_blockshift, bonustype, bonuslen, dnodesize,
	allocated_dnode, tag, tx));
	}

	int
	zap_destroy(objset_t os, uint64_t zapobj, dmu_tx_t tx)
	{
	/*
	* dmu_object_free will free the object number and free the
	* data. Freeing the data will cause our pageout function to be
	* called, which will destroy our data (zap_leaf_t's and zap_t).
	*/

	return (dmu_object_free(os, zapobj, tx));
	}

	void
	zap_evict_sync(void *dbu)
	{
	zap_t *zap = dbu;

	rw_destroy(&zap->zap_rwlock);

	if (zap->zap_ismicro)
	mze_destroy(zap);
	else
	mutex_destroy(&zap->zap_f.zap_num_entries_mtx);

	kmem_free(zap, sizeof (zap_t));
	}

	int
	zap_count(objset_t os, uint64_t zapobj, uint64_t count)
	{
	zap_t *zap;

	int err =
	zap_lockdir(os, zapobj, NULL, RW_READER, TRUE, FALSE, FTAG, &zap);
	if (err != 0)
	return (err);
	if (!zap->zap_ismicro) {
	err = fzap_count(zap, count);
	} else {
	*count = zap->zap_m.zap_num_entries;
	}
	zap_unlockdir(zap, FTAG);
	return (err);
	}

	/*
	* zn may be NULL; if not specified, it will be computed if needed.
	* See also the comment above zap_entry_normalization_conflict().
	*/
	static boolean_t
	mzap_normalization_conflict(zap_t zap, zap_name_t zn, mzap_ent_t *mze,
	zfs_btree_index_t *idx)
	{
	boolean_t allocdzn = B_FALSE;
	mzap_ent_t *other;
	zfs_btree_index_t oidx;

	if (zap->zap_normflags == 0)
	return (B_FALSE);

	for (other = zfs_btree_prev(&zap->zap_m.zap_tree, idx, &oidx);
	other && other->mze_hash == mze->mze_hash;
	other = zfs_btree_prev(&zap->zap_m.zap_tree, &oidx, &oidx)) {

	if (zn == NULL) {
	zn = zap_name_alloc_str(zap,
	MZE_PHYS(zap, mze)->mze_name, MT_NORMALIZE);
	allocdzn = B_TRUE;
	}
	if (zap_match(zn, MZE_PHYS(zap, other)->mze_name)) {
	if (allocdzn)
	zap_name_free(zn);
	return (B_TRUE);
	}
	}

	for (other = zfs_btree_next(&zap->zap_m.zap_tree, idx, &oidx);
	other && other->mze_hash == mze->mze_hash;
	other = zfs_btree_next(&zap->zap_m.zap_tree, &oidx, &oidx)) {

	if (zn == NULL) {
	zn = zap_name_alloc_str(zap,
	MZE_PHYS(zap, mze)->mze_name, MT_NORMALIZE);
	allocdzn = B_TRUE;
	}
	if (zap_match(zn, MZE_PHYS(zap, other)->mze_name)) {
	if (allocdzn)
	zap_name_free(zn);
	return (B_TRUE);
	}
	}

	if (allocdzn)
	zap_name_free(zn);
	return (B_FALSE);
	}

	/*
	* Routines for manipulating attributes.
	*/

	int
	zap_lookup(objset_t os, uint64_t zapobj, const char name,
	uint64_t integer_size, uint64_t num_integers, void *buf)
	{
	return (zap_lookup_norm(os, zapobj, name, integer_size,
	num_integers, buf, 0, NULL, 0, NULL));
	}

	static int
	zap_lookup_impl(zap_t zap, const char name,
	uint64_t integer_size, uint64_t num_integers, void *buf,
	matchtype_t mt, char *realname, int rn_len,
	boolean_t *ncp)
	{
	int err = 0;

	zap_name_t *zn = zap_name_alloc_str(zap, name, mt);
	if (zn == NULL)
	return (SET_ERROR(ENOTSUP));

	if (!zap->zap_ismicro) {
	err = fzap_lookup(zn, integer_size, num_integers, buf,
	realname, rn_len, ncp);
	} else {
	zfs_btree_index_t idx;
	mzap_ent_t *mze = mze_find(zn, &idx);
	if (mze == NULL) {
	err = SET_ERROR(ENOENT);
	} else {
	if (num_integers < 1) {
	err = SET_ERROR(EOVERFLOW);
	} else if (integer_size != 8) {
	err = SET_ERROR(EINVAL);
	} else {
	(uint64_t )buf =
	MZE_PHYS(zap, mze)->mze_value;
	if (realname != NULL)
	(void) strlcpy(realname,
	MZE_PHYS(zap, mze)->mze_name,
	rn_len);
	if (ncp) {
	*ncp = mzap_normalization_conflict(zap,
	zn, mze, &idx);
	}
	}
	}
	}
	zap_name_free(zn);
	return (err);
	}

	int
	zap_lookup_norm(objset_t os, uint64_t zapobj, const char name,
	uint64_t integer_size, uint64_t num_integers, void *buf,
	matchtype_t mt, char *realname, int rn_len,
	boolean_t *ncp)
	{
	zap_t *zap;

	int err =
	zap_lockdir(os, zapobj, NULL, RW_READER, TRUE, FALSE, FTAG, &zap);
	if (err != 0)
	return (err);
	err = zap_lookup_impl(zap, name, integer_size,
	num_integers, buf, mt, realname, rn_len, ncp);
	zap_unlockdir(zap, FTAG);
	return (err);
	}

	int
	zap_prefetch(objset_t os, uint64_t zapobj, const char name)
	{
	zap_t *zap;
	int err;
	zap_name_t *zn;

	err = zap_lockdir(os, zapobj, NULL, RW_READER, TRUE, FALSE, FTAG, &zap);
	if (err)
	return (err);
	zn = zap_name_alloc_str(zap, name, 0);
	if (zn == NULL) {
	zap_unlockdir(zap, FTAG);
	return (SET_ERROR(ENOTSUP));
	}

	fzap_prefetch(zn);
	zap_name_free(zn);
	zap_unlockdir(zap, FTAG);
	return (err);
	}

	int
	zap_lookup_by_dnode(dnode_t dn, const char name,
	uint64_t integer_size, uint64_t num_integers, void *buf)
	{
	return (zap_lookup_norm_by_dnode(dn, name, integer_size,
	num_integers, buf, 0, NULL, 0, NULL));
	}

	int
	zap_lookup_norm_by_dnode(dnode_t dn, const char name,
	uint64_t integer_size, uint64_t num_integers, void *buf,
	matchtype_t mt, char *realname, int rn_len,
	boolean_t *ncp)
	{
	zap_t *zap;

	int err = zap_lockdir_by_dnode(dn, NULL, RW_READER, TRUE, FALSE,
	FTAG, &zap);
	if (err != 0)
	return (err);
	err = zap_lookup_impl(zap, name, integer_size,
	num_integers, buf, mt, realname, rn_len, ncp);
	zap_unlockdir(zap, FTAG);
	return (err);
	}

	int
	zap_prefetch_uint64(objset_t os, uint64_t zapobj, const uint64_t key,
	int key_numints)
	{
	zap_t *zap;

	int err =
	zap_lockdir(os, zapobj, NULL, RW_READER, TRUE, FALSE, FTAG, &zap);
	if (err != 0)
	return (err);
	zap_name_t *zn = zap_name_alloc_uint64(zap, key, key_numints);
	if (zn == NULL) {
	zap_unlockdir(zap, FTAG);
	return (SET_ERROR(ENOTSUP));
	}

	fzap_prefetch(zn);
	zap_name_free(zn);
	zap_unlockdir(zap, FTAG);
	return (err);
	}

	int
	zap_lookup_uint64(objset_t os, uint64_t zapobj, const uint64_t key,
	int key_numints, uint64_t integer_size, uint64_t num_integers, void *buf)
	{
	zap_t *zap;

	int err =
	zap_lockdir(os, zapobj, NULL, RW_READER, TRUE, FALSE, FTAG, &zap);
	if (err != 0)
	return (err);
	zap_name_t *zn = zap_name_alloc_uint64(zap, key, key_numints);
	if (zn == NULL) {
	zap_unlockdir(zap, FTAG);
	return (SET_ERROR(ENOTSUP));
	}

	err = fzap_lookup(zn, integer_size, num_integers, buf,
	NULL, 0, NULL);
	zap_name_free(zn);
	zap_unlockdir(zap, FTAG);
	return (err);
	}

	int
	zap_contains(objset_t os, uint64_t zapobj, const char name)
	{
	int err = zap_lookup_norm(os, zapobj, name, 0,
	0, NULL, 0, NULL, 0, NULL);
	if (err == EOVERFLOW \|\| err == EINVAL)
	err = 0; /* found, but skipped reading the value */
	return (err);
	}

	int
	zap_length(objset_t os, uint64_t zapobj, const char name,
	uint64_t integer_size, uint64_t num_integers)
	{
	zap_t *zap;

	int err =
	zap_lockdir(os, zapobj, NULL, RW_READER, TRUE, FALSE, FTAG, &zap);
	if (err != 0)
	return (err);
	zap_name_t *zn = zap_name_alloc_str(zap, name, 0);
	if (zn == NULL) {
	zap_unlockdir(zap, FTAG);
	return (SET_ERROR(ENOTSUP));
	}
	if (!zap->zap_ismicro) {
	err = fzap_length(zn, integer_size, num_integers);
	} else {
	zfs_btree_index_t idx;
	mzap_ent_t *mze = mze_find(zn, &idx);
	if (mze == NULL) {
	err = SET_ERROR(ENOENT);
	} else {
	if (integer_size)
	*integer_size = 8;
	if (num_integers)
	*num_integers = 1;
	}
	}
	zap_name_free(zn);
	zap_unlockdir(zap, FTAG);
	return (err);
	}

	int
	zap_length_uint64(objset_t os, uint64_t zapobj, const uint64_t key,
	int key_numints, uint64_t integer_size, uint64_t num_integers)
	{
	zap_t *zap;

	int err =
	zap_lockdir(os, zapobj, NULL, RW_READER, TRUE, FALSE, FTAG, &zap);
	if (err != 0)
	return (err);
	zap_name_t *zn = zap_name_alloc_uint64(zap, key, key_numints);
	if (zn == NULL) {
	zap_unlockdir(zap, FTAG);
	return (SET_ERROR(ENOTSUP));
	}
	err = fzap_length(zn, integer_size, num_integers);
	zap_name_free(zn);
	zap_unlockdir(zap, FTAG);
	return (err);
	}

	static void
	mzap_addent(zap_name_t *zn, uint64_t value)
	{
	zap_t *zap = zn->zn_zap;
	uint16_t start = zap->zap_m.zap_alloc_next;

	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));

	#ifdef ZFS_DEBUG
	for (int i = 0; i < zap->zap_m.zap_num_chunks; i++) {
	mzap_ent_phys_t *mze = &zap_m_phys(zap)->mz_chunk[i];
	ASSERT(strcmp(zn->zn_key_orig, mze->mze_name) != 0);
	}
	#endif

	uint32_t cd = mze_find_unused_cd(zap, zn->zn_hash);
	/* given the limited size of the microzap, this can't happen */
	ASSERT(cd < zap_maxcd(zap));

	again:
	for (uint16_t i = start; i < zap->zap_m.zap_num_chunks; i++) {
	mzap_ent_phys_t *mze = &zap_m_phys(zap)->mz_chunk[i];
	if (mze->mze_name[0] == 0) {
	mze->mze_value = value;
	mze->mze_cd = cd;
	(void) strlcpy(mze->mze_name, zn->zn_key_orig,
	sizeof (mze->mze_name));
	zap->zap_m.zap_num_entries++;
	zap->zap_m.zap_alloc_next = i+1;
	if (zap->zap_m.zap_alloc_next ==
	zap->zap_m.zap_num_chunks)
	zap->zap_m.zap_alloc_next = 0;
	mze_insert(zap, i, zn->zn_hash);
	return;
	}
	}
	if (start != 0) {
	start = 0;
	goto again;
	}
	cmn_err(CE_PANIC, "out of entries!");
	}

	static int
	zap_add_impl(zap_t zap, const char key,
	int integer_size, uint64_t num_integers,
	const void val, dmu_tx_t tx, const void *tag)
	{
	const uint64_t *intval = val;
	int err = 0;

	zap_name_t *zn = zap_name_alloc_str(zap, key, 0);
	if (zn == NULL) {
	zap_unlockdir(zap, tag);
	return (SET_ERROR(ENOTSUP));
	}
	if (!zap->zap_ismicro) {
	err = fzap_add(zn, integer_size, num_integers, val, tag, tx);
	zap = zn->zn_zap; /* fzap_add() may change zap */
	} else if (integer_size != 8 \|\| num_integers != 1 \|\|
	strlen(key) >= MZAP_NAME_LEN \|\|
	!mze_canfit_fzap_leaf(zn, zn->zn_hash)) {
	err = mzap_upgrade(&zn->zn_zap, tag, tx, 0);
	if (err == 0) {
	err = fzap_add(zn, integer_size, num_integers, val,
	tag, tx);
	}
	zap = zn->zn_zap; /* fzap_add() may change zap */
	} else {
	zfs_btree_index_t idx;
	if (mze_find(zn, &idx) != NULL) {
	err = SET_ERROR(EEXIST);
	} else {
	mzap_addent(zn, *intval);
	}
	}
	ASSERT(zap == zn->zn_zap);
	zap_name_free(zn);
	if (zap != NULL) /* may be NULL if fzap_add() failed */
	zap_unlockdir(zap, tag);
	return (err);
	}

	int
	zap_add(objset_t os, uint64_t zapobj, const char key,
	int integer_size, uint64_t num_integers,
	const void val, dmu_tx_t tx)
	{
	zap_t *zap;
	int err;

	err = zap_lockdir(os, zapobj, tx, RW_WRITER, TRUE, TRUE, FTAG, &zap);
	if (err != 0)
	return (err);
	err = zap_add_impl(zap, key, integer_size, num_integers, val, tx, FTAG);
	/* zap_add_impl() calls zap_unlockdir() */
	return (err);
	}

	int
	zap_add_by_dnode(dnode_t dn, const char key,
	int integer_size, uint64_t num_integers,
	const void val, dmu_tx_t tx)
	{
	zap_t *zap;
	int err;

	err = zap_lockdir_by_dnode(dn, tx, RW_WRITER, TRUE, TRUE, FTAG, &zap);
	if (err != 0)
	return (err);
	err = zap_add_impl(zap, key, integer_size, num_integers, val, tx, FTAG);
	/* zap_add_impl() calls zap_unlockdir() */
	return (err);
	}

	+static int
	+zap_add_uint64_impl(zap_t zap, const uint64_t key,
	+ int key_numints, int integer_size, uint64_t num_integers,
	+ const void val, dmu_tx_t tx, const void *tag)
	+{
	+ int err;
	+
	+ zap_name_t *zn = zap_name_alloc_uint64(zap, key, key_numints);
	+ if (zn == NULL) {
	+ zap_unlockdir(zap, tag);
	+ return (SET_ERROR(ENOTSUP));
	+ }
	+ err = fzap_add(zn, integer_size, num_integers, val, tag, tx);
	+ zap = zn->zn_zap; /* fzap_add() may change zap */
	+ zap_name_free(zn);
	+ if (zap != NULL) /* may be NULL if fzap_add() failed */
	+ zap_unlockdir(zap, tag);
	+ return (err);
	+}
	+
	int
	zap_add_uint64(objset_t os, uint64_t zapobj, const uint64_t key,
	int key_numints, int integer_size, uint64_t num_integers,
	const void val, dmu_tx_t tx)
	{
	zap_t *zap;

	int err =
	zap_lockdir(os, zapobj, tx, RW_WRITER, TRUE, TRUE, FTAG, &zap);
	if (err != 0)
	return (err);
	- zap_name_t *zn = zap_name_alloc_uint64(zap, key, key_numints);
	- if (zn == NULL) {
	- zap_unlockdir(zap, FTAG);
	- return (SET_ERROR(ENOTSUP));
	- }
	- err = fzap_add(zn, integer_size, num_integers, val, FTAG, tx);
	- zap = zn->zn_zap; /* fzap_add() may change zap */
	- zap_name_free(zn);
	- if (zap != NULL) /* may be NULL if fzap_add() failed */
	- zap_unlockdir(zap, FTAG);
	+ err = zap_add_uint64_impl(zap, key, key_numints,
	+ integer_size, num_integers, val, tx, FTAG);
	+ /* zap_add_uint64_impl() calls zap_unlockdir() */
	+ return (err);
	+}
	+
	+int
	+zap_add_uint64_by_dnode(dnode_t dn, const uint64_t key,
	+ int key_numints, int integer_size, uint64_t num_integers,
	+ const void val, dmu_tx_t tx)
	+{
	+ zap_t *zap;
	+
	+ int err =
	+ zap_lockdir_by_dnode(dn, tx, RW_WRITER, TRUE, TRUE, FTAG, &zap);
	+ if (err != 0)
	+ return (err);
	+ err = zap_add_uint64_impl(zap, key, key_numints,
	+ integer_size, num_integers, val, tx, FTAG);
	+ /* zap_add_uint64_impl() calls zap_unlockdir() */
	return (err);
	}

	int
	zap_update(objset_t os, uint64_t zapobj, const char name,
	int integer_size, uint64_t num_integers, const void val, dmu_tx_t tx)
	{
	zap_t *zap;
	const uint64_t *intval = val;

	int err =
	zap_lockdir(os, zapobj, tx, RW_WRITER, TRUE, TRUE, FTAG, &zap);
	if (err != 0)
	return (err);
	zap_name_t *zn = zap_name_alloc_str(zap, name, 0);
	if (zn == NULL) {
	zap_unlockdir(zap, FTAG);
	return (SET_ERROR(ENOTSUP));
	}
	if (!zap->zap_ismicro) {
	err = fzap_update(zn, integer_size, num_integers, val,
	FTAG, tx);
	zap = zn->zn_zap; /* fzap_update() may change zap */
	} else if (integer_size != 8 \|\| num_integers != 1 \|\|
	strlen(name) >= MZAP_NAME_LEN) {
	dprintf("upgrading obj %llu: intsz=%u numint=%llu name=%s\n",
	(u_longlong_t)zapobj, integer_size,
	(u_longlong_t)num_integers, name);
	err = mzap_upgrade(&zn->zn_zap, FTAG, tx, 0);
	if (err == 0) {
	err = fzap_update(zn, integer_size, num_integers,
	val, FTAG, tx);
	}
	zap = zn->zn_zap; /* fzap_update() may change zap */
	} else {
	zfs_btree_index_t idx;
	mzap_ent_t *mze = mze_find(zn, &idx);
	if (mze != NULL) {
	MZE_PHYS(zap, mze)->mze_value = *intval;
	} else {
	mzap_addent(zn, *intval);
	}
	}
	ASSERT(zap == zn->zn_zap);
	zap_name_free(zn);
	if (zap != NULL) /* may be NULL if fzap_upgrade() failed */
	zap_unlockdir(zap, FTAG);
	return (err);
	}

	-int
	-zap_update_uint64(objset_t os, uint64_t zapobj, const uint64_t key,
	- int key_numints,
	- int integer_size, uint64_t num_integers, const void val, dmu_tx_t tx)
	+static int
	+zap_update_uint64_impl(zap_t zap, const uint64_t key, int key_numints,
	+ int integer_size, uint64_t num_integers, const void val, dmu_tx_t tx,
	+ const void *tag)
	{
	- zap_t *zap;
	+ int err;

	- int err =
	- zap_lockdir(os, zapobj, tx, RW_WRITER, TRUE, TRUE, FTAG, &zap);
	- if (err != 0)
	- return (err);
	zap_name_t *zn = zap_name_alloc_uint64(zap, key, key_numints);
	if (zn == NULL) {
	- zap_unlockdir(zap, FTAG);
	+ zap_unlockdir(zap, tag);
	return (SET_ERROR(ENOTSUP));
	}
	- err = fzap_update(zn, integer_size, num_integers, val, FTAG, tx);
	+ err = fzap_update(zn, integer_size, num_integers, val, tag, tx);
	zap = zn->zn_zap; /* fzap_update() may change zap */
	zap_name_free(zn);
	if (zap != NULL) /* may be NULL if fzap_upgrade() failed */
	- zap_unlockdir(zap, FTAG);
	+ zap_unlockdir(zap, tag);
	+ return (err);
	+}
	+
	+int
	+zap_update_uint64(objset_t os, uint64_t zapobj, const uint64_t key,
	+ int key_numints, int integer_size, uint64_t num_integers, const void *val,
	+ dmu_tx_t *tx)
	+{
	+ zap_t *zap;
	+
	+ int err =
	+ zap_lockdir(os, zapobj, tx, RW_WRITER, TRUE, TRUE, FTAG, &zap);
	+ if (err != 0)
	+ return (err);
	+ err = zap_update_uint64_impl(zap, key, key_numints,
	+ integer_size, num_integers, val, tx, FTAG);
	+ /* zap_update_uint64_impl() calls zap_unlockdir() */
	+ return (err);
	+}
	+
	+int
	+zap_update_uint64_by_dnode(dnode_t dn, const uint64_t key, int key_numints,
	+ int integer_size, uint64_t num_integers, const void val, dmu_tx_t tx)
	+{
	+ zap_t *zap;
	+
	+ int err =
	+ zap_lockdir_by_dnode(dn, tx, RW_WRITER, TRUE, TRUE, FTAG, &zap);
	+ if (err != 0)
	+ return (err);
	+ err = zap_update_uint64_impl(zap, key, key_numints,
	+ integer_size, num_integers, val, tx, FTAG);
	+ /* zap_update_uint64_impl() calls zap_unlockdir() */
	return (err);
	}

	int
	zap_remove(objset_t os, uint64_t zapobj, const char name, dmu_tx_t *tx)
	{
	return (zap_remove_norm(os, zapobj, name, 0, tx));
	}

	static int
	zap_remove_impl(zap_t zap, const char name,
	matchtype_t mt, dmu_tx_t *tx)
	{
	int err = 0;

	zap_name_t *zn = zap_name_alloc_str(zap, name, mt);
	if (zn == NULL)
	return (SET_ERROR(ENOTSUP));
	if (!zap->zap_ismicro) {
	err = fzap_remove(zn, tx);
	} else {
	zfs_btree_index_t idx;
	mzap_ent_t *mze = mze_find(zn, &idx);
	if (mze == NULL) {
	err = SET_ERROR(ENOENT);
	} else {
	zap->zap_m.zap_num_entries--;
	memset(MZE_PHYS(zap, mze), 0, sizeof (mzap_ent_phys_t));
	zfs_btree_remove_idx(&zap->zap_m.zap_tree, &idx);
	}
	}
	zap_name_free(zn);
	return (err);
	}

	int
	zap_remove_norm(objset_t os, uint64_t zapobj, const char name,
	matchtype_t mt, dmu_tx_t *tx)
	{
	zap_t *zap;
	int err;

	err = zap_lockdir(os, zapobj, tx, RW_WRITER, TRUE, FALSE, FTAG, &zap);
	if (err)
	return (err);
	err = zap_remove_impl(zap, name, mt, tx);
	zap_unlockdir(zap, FTAG);
	return (err);
	}

	int
	zap_remove_by_dnode(dnode_t dn, const char name, dmu_tx_t *tx)
	{
	zap_t *zap;
	int err;

	err = zap_lockdir_by_dnode(dn, tx, RW_WRITER, TRUE, FALSE, FTAG, &zap);
	if (err)
	return (err);
	err = zap_remove_impl(zap, name, 0, tx);
	zap_unlockdir(zap, FTAG);
	return (err);
	}

	+static int
	+zap_remove_uint64_impl(zap_t zap, const uint64_t key, int key_numints,
	+ dmu_tx_t tx, const void tag)
	+{
	+ int err;
	+
	+ zap_name_t *zn = zap_name_alloc_uint64(zap, key, key_numints);
	+ if (zn == NULL) {
	+ zap_unlockdir(zap, tag);
	+ return (SET_ERROR(ENOTSUP));
	+ }
	+ err = fzap_remove(zn, tx);
	+ zap_name_free(zn);
	+ zap_unlockdir(zap, tag);
	+ return (err);
	+}
	+
	int
	zap_remove_uint64(objset_t os, uint64_t zapobj, const uint64_t key,
	int key_numints, dmu_tx_t *tx)
	{
	zap_t *zap;

	int err =
	zap_lockdir(os, zapobj, tx, RW_WRITER, TRUE, FALSE, FTAG, &zap);
	if (err != 0)
	return (err);
	- zap_name_t *zn = zap_name_alloc_uint64(zap, key, key_numints);
	- if (zn == NULL) {
	- zap_unlockdir(zap, FTAG);
	- return (SET_ERROR(ENOTSUP));
	- }
	- err = fzap_remove(zn, tx);
	- zap_name_free(zn);
	- zap_unlockdir(zap, FTAG);
	+ err = zap_remove_uint64_impl(zap, key, key_numints, tx, FTAG);
	+ /* zap_remove_uint64_impl() calls zap_unlockdir() */
	+ return (err);
	+}
	+
	+int
	+zap_remove_uint64_by_dnode(dnode_t dn, const uint64_t key, int key_numints,
	+ dmu_tx_t *tx)
	+{
	+ zap_t *zap;
	+
	+ int err =
	+ zap_lockdir_by_dnode(dn, tx, RW_WRITER, TRUE, FALSE, FTAG, &zap);
	+ if (err != 0)
	+ return (err);
	+ err = zap_remove_uint64_impl(zap, key, key_numints, tx, FTAG);
	+ /* zap_remove_uint64_impl() calls zap_unlockdir() */
	return (err);
	}

	/*
	* Routines for iterating over the attributes.
	*/

	static void
	zap_cursor_init_impl(zap_cursor_t zc, objset_t os, uint64_t zapobj,
	uint64_t serialized, boolean_t prefetch)
	{
	zc->zc_objset = os;
	zc->zc_zap = NULL;
	zc->zc_leaf = NULL;
	zc->zc_zapobj = zapobj;
	zc->zc_serialized = serialized;
	zc->zc_hash = 0;
	zc->zc_cd = 0;
	zc->zc_prefetch = prefetch;
	}
	void
	zap_cursor_init_serialized(zap_cursor_t zc, objset_t os, uint64_t zapobj,
	uint64_t serialized)
	{
	zap_cursor_init_impl(zc, os, zapobj, serialized, B_TRUE);
	}

	/*
	* Initialize a cursor at the beginning of the ZAP object. The entire
	* ZAP object will be prefetched.
	*/
	void
	zap_cursor_init(zap_cursor_t zc, objset_t os, uint64_t zapobj)
	{
	zap_cursor_init_impl(zc, os, zapobj, 0, B_TRUE);
	}

	/*
	* Initialize a cursor at the beginning, but request that we not prefetch
	* the entire ZAP object.
	*/
	void
	zap_cursor_init_noprefetch(zap_cursor_t zc, objset_t os, uint64_t zapobj)
	{
	zap_cursor_init_impl(zc, os, zapobj, 0, B_FALSE);
	}

	void
	zap_cursor_fini(zap_cursor_t *zc)
	{
	if (zc->zc_zap) {
	rw_enter(&zc->zc_zap->zap_rwlock, RW_READER);
	zap_unlockdir(zc->zc_zap, NULL);
	zc->zc_zap = NULL;
	}
	if (zc->zc_leaf) {
	rw_enter(&zc->zc_leaf->l_rwlock, RW_READER);
	zap_put_leaf(zc->zc_leaf);
	zc->zc_leaf = NULL;
	}
	zc->zc_objset = NULL;
	}

	uint64_t
	zap_cursor_serialize(zap_cursor_t *zc)
	{
	if (zc->zc_hash == -1ULL)
	return (-1ULL);
	if (zc->zc_zap == NULL)
	return (zc->zc_serialized);
	ASSERT((zc->zc_hash & zap_maxcd(zc->zc_zap)) == 0);
	ASSERT(zc->zc_cd < zap_maxcd(zc->zc_zap));

	/*
	* We want to keep the high 32 bits of the cursor zero if we can, so
	* that 32-bit programs can access this. So usually use a small
	* (28-bit) hash value so we can fit 4 bits of cd into the low 32-bits
	* of the cursor.
	*
	* [ collision differentiator \| zap_hashbits()-bit hash value ]
	*/
	return ((zc->zc_hash >> (64 - zap_hashbits(zc->zc_zap))) \|
	((uint64_t)zc->zc_cd << zap_hashbits(zc->zc_zap)));
	}

	int
	zap_cursor_retrieve(zap_cursor_t zc, zap_attribute_t za)
	{
	int err;

	if (zc->zc_hash == -1ULL)
	return (SET_ERROR(ENOENT));

	if (zc->zc_zap == NULL) {
	int hb;
	err = zap_lockdir(zc->zc_objset, zc->zc_zapobj, NULL,
	RW_READER, TRUE, FALSE, NULL, &zc->zc_zap);
	if (err != 0)
	return (err);

	/*
	* To support zap_cursor_init_serialized, advance, retrieve,
	* we must add to the existing zc_cd, which may already
	* be 1 due to the zap_cursor_advance.
	*/
	ASSERT(zc->zc_hash == 0);
	hb = zap_hashbits(zc->zc_zap);
	zc->zc_hash = zc->zc_serialized << (64 - hb);
	zc->zc_cd += zc->zc_serialized >> hb;
	if (zc->zc_cd >= zap_maxcd(zc->zc_zap)) /* corrupt serialized */
	zc->zc_cd = 0;
	} else {
	rw_enter(&zc->zc_zap->zap_rwlock, RW_READER);
	}
	if (!zc->zc_zap->zap_ismicro) {
	err = fzap_cursor_retrieve(zc->zc_zap, zc, za);
	} else {
	zfs_btree_index_t idx;
	mzap_ent_t mze_tofind;

	mze_tofind.mze_hash = zc->zc_hash >> 32;
	mze_tofind.mze_cd = zc->zc_cd;

	mzap_ent_t *mze = zfs_btree_find(&zc->zc_zap->zap_m.zap_tree,
	&mze_tofind, &idx);
	if (mze == NULL) {
	mze = zfs_btree_next(&zc->zc_zap->zap_m.zap_tree,
	&idx, &idx);
	}
	if (mze) {
	mzap_ent_phys_t *mzep = MZE_PHYS(zc->zc_zap, mze);
	ASSERT3U(mze->mze_cd, ==, mzep->mze_cd);
	za->za_normalization_conflict =
	mzap_normalization_conflict(zc->zc_zap, NULL,
	mze, &idx);
	za->za_integer_length = 8;
	za->za_num_integers = 1;
	za->za_first_integer = mzep->mze_value;
	(void) strlcpy(za->za_name, mzep->mze_name,
	sizeof (za->za_name));
	zc->zc_hash = (uint64_t)mze->mze_hash << 32;
	zc->zc_cd = mze->mze_cd;
	err = 0;
	} else {
	zc->zc_hash = -1ULL;
	err = SET_ERROR(ENOENT);
	}
	}
	rw_exit(&zc->zc_zap->zap_rwlock);
	return (err);
	}

	void
	zap_cursor_advance(zap_cursor_t *zc)
	{
	if (zc->zc_hash == -1ULL)
	return;
	zc->zc_cd++;
	}

	int
	zap_get_stats(objset_t os, uint64_t zapobj, zap_stats_t zs)
	{
	zap_t *zap;

	int err =
	zap_lockdir(os, zapobj, NULL, RW_READER, TRUE, FALSE, FTAG, &zap);
	if (err != 0)
	return (err);

	memset(zs, 0, sizeof (zap_stats_t));

	if (zap->zap_ismicro) {
	zs->zs_blocksize = zap->zap_dbuf->db_size;
	zs->zs_num_entries = zap->zap_m.zap_num_entries;
	zs->zs_num_blocks = 1;
	} else {
	fzap_get_stats(zap, zs);
	}
	zap_unlockdir(zap, FTAG);
	return (0);
	}

	#if defined(_KERNEL)
	EXPORT_SYMBOL(zap_create);
	EXPORT_SYMBOL(zap_create_dnsize);
	EXPORT_SYMBOL(zap_create_norm);
	EXPORT_SYMBOL(zap_create_norm_dnsize);
	EXPORT_SYMBOL(zap_create_flags);
	EXPORT_SYMBOL(zap_create_flags_dnsize);
	EXPORT_SYMBOL(zap_create_claim);
	EXPORT_SYMBOL(zap_create_claim_norm);
	EXPORT_SYMBOL(zap_create_claim_norm_dnsize);
	EXPORT_SYMBOL(zap_create_hold);
	EXPORT_SYMBOL(zap_destroy);
	EXPORT_SYMBOL(zap_lookup);
	EXPORT_SYMBOL(zap_lookup_by_dnode);
	EXPORT_SYMBOL(zap_lookup_norm);
	EXPORT_SYMBOL(zap_lookup_uint64);
	EXPORT_SYMBOL(zap_contains);
	EXPORT_SYMBOL(zap_prefetch);
	EXPORT_SYMBOL(zap_prefetch_uint64);
	EXPORT_SYMBOL(zap_add);
	EXPORT_SYMBOL(zap_add_by_dnode);
	EXPORT_SYMBOL(zap_add_uint64);
	+EXPORT_SYMBOL(zap_add_uint64_by_dnode);
	EXPORT_SYMBOL(zap_update);
	EXPORT_SYMBOL(zap_update_uint64);
	+EXPORT_SYMBOL(zap_update_uint64_by_dnode);
	EXPORT_SYMBOL(zap_length);
	EXPORT_SYMBOL(zap_length_uint64);
	EXPORT_SYMBOL(zap_remove);
	EXPORT_SYMBOL(zap_remove_by_dnode);
	EXPORT_SYMBOL(zap_remove_norm);
	EXPORT_SYMBOL(zap_remove_uint64);
	+EXPORT_SYMBOL(zap_remove_uint64_by_dnode);
	EXPORT_SYMBOL(zap_count);
	EXPORT_SYMBOL(zap_value_search);
	EXPORT_SYMBOL(zap_join);
	EXPORT_SYMBOL(zap_join_increment);
	EXPORT_SYMBOL(zap_add_int);
	EXPORT_SYMBOL(zap_remove_int);
	EXPORT_SYMBOL(zap_lookup_int);
	EXPORT_SYMBOL(zap_increment_int);
	EXPORT_SYMBOL(zap_add_int_key);
	EXPORT_SYMBOL(zap_lookup_int_key);
	EXPORT_SYMBOL(zap_increment);
	EXPORT_SYMBOL(zap_cursor_init);
	EXPORT_SYMBOL(zap_cursor_fini);
	EXPORT_SYMBOL(zap_cursor_retrieve);
	EXPORT_SYMBOL(zap_cursor_advance);
	EXPORT_SYMBOL(zap_cursor_serialize);
	EXPORT_SYMBOL(zap_cursor_init_serialized);
	EXPORT_SYMBOL(zap_get_stats);

	/* CSTYLED */
	ZFS_MODULE_PARAM(zfs, , zap_micro_max_size, INT, ZMOD_RW,
	"Maximum micro ZAP size, before converting to a fat ZAP, in bytes");
	#endif
	diff --git a/sys/contrib/openzfs/module/zfs/zfs_fm.c b/sys/contrib/openzfs/module/zfs/zfs_fm.c
	index c4eb74e873db..481af2ba826b 100644
	--- a/sys/contrib/openzfs/module/zfs/zfs_fm.c
	+++ b/sys/contrib/openzfs/module/zfs/zfs_fm.c
	@@ -1,1572 +1,1598 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
	* Use is subject to license terms.
	*/

	/*
	* Copyright (c) 2012,2021 by Delphix. All rights reserved.
	*/

	#include <sys/spa.h>
	#include <sys/spa_impl.h>
	#include <sys/vdev.h>
	#include <sys/vdev_impl.h>
	#include <sys/zio.h>
	#include <sys/zio_checksum.h>

	#include <sys/fm/fs/zfs.h>
	#include <sys/fm/protocol.h>
	#include <sys/fm/util.h>
	#include <sys/sysevent.h>

	/*
	* This general routine is responsible for generating all the different ZFS
	* ereports. The payload is dependent on the class, and which arguments are
	* supplied to the function:
	*
	* EREPORT POOL VDEV IO
	* block X X X
	* data X X
	* device X X
	* pool X
	*
	* If we are in a loading state, all errors are chained together by the same
	* SPA-wide ENA (Error Numeric Association).
	*
	* For isolated I/O requests, we get the ENA from the zio_t. The propagation
	* gets very complicated due to RAID-Z, gang blocks, and vdev caching. We want
	* to chain together all ereports associated with a logical piece of data. For
	* read I/Os, there are basically three 'types' of I/O, which form a roughly
	* layered diagram:
	*
	* +---------------+
	* \| Aggregate I/O \| No associated logical data or device
	* +---------------+
	* \|
	* V
	* +---------------+ Reads associated with a piece of logical data.
	* \| Read I/O \| This includes reads on behalf of RAID-Z,
	* +---------------+ mirrors, gang blocks, retries, etc.
	* \|
	* V
	* +---------------+ Reads associated with a particular device, but
	* \| Physical I/O \| no logical data. Issued as part of vdev caching
	* +---------------+ and I/O aggregation.
	*
	* Note that 'physical I/O' here is not the same terminology as used in the rest
	* of ZIO. Typically, 'physical I/O' simply means that there is no attached
	* blockpointer. But I/O with no associated block pointer can still be related
	* to a logical piece of data (i.e. RAID-Z requests).
	*
	* Purely physical I/O always have unique ENAs. They are not related to a
	* particular piece of logical data, and therefore cannot be chained together.
	* We still generate an ereport, but the DE doesn't correlate it with any
	* logical piece of data. When such an I/O fails, the delegated I/O requests
	* will issue a retry, which will trigger the 'real' ereport with the correct
	* ENA.
	*
	* We keep track of the ENA for a ZIO chain through the 'io_logical' member.
	* When a new logical I/O is issued, we set this to point to itself. Child I/Os
	* then inherit this pointer, so that when it is first set subsequent failures
	* will use the same ENA. For vdev cache fill and queue aggregation I/O,
	* this pointer is set to NULL, and no ereport will be generated (since it
	* doesn't actually correspond to any particular device or piece of data,
	* and the caller will always retry without caching or queueing anyway).
	*
	* For checksum errors, we want to include more information about the actual
	* error which occurs. Accordingly, we build an ereport when the error is
	* noticed, but instead of sending it in immediately, we hang it off of the
	* io_cksum_report field of the logical IO. When the logical IO completes
	* (successfully or not), zfs_ereport_finish_checksum() is called with the
	* good and bad versions of the buffer (if available), and we annotate the
	* ereport with information about the differences.
	*/

	#ifdef _KERNEL
	/*
	* Duplicate ereport Detection
	*
	* Some ereports are retained momentarily for detecting duplicates. These
	* are kept in a recent_events_node_t in both a time-ordered list and an AVL
	* tree of recent unique ereports.
	*
	* The lifespan of these recent ereports is bounded (15 mins) and a cleaner
	* task is used to purge stale entries.
	*/
	static list_t recent_events_list;
	static avl_tree_t recent_events_tree;
	static kmutex_t recent_events_lock;
	static taskqid_t recent_events_cleaner_tqid;

	/*
	* Each node is about 128 bytes so 2,000 would consume 1/4 MiB.
	*
	* This setting can be changed dynamically and setting it to zero
	* disables duplicate detection.
	*/
	static unsigned int zfs_zevent_retain_max = 2000;

	/*
	* The lifespan for a recent ereport entry. The default of 15 minutes is
	* intended to outlive the zfs diagnosis engine's threshold of 10 errors
	* over a period of 10 minutes.
	*/
	static unsigned int zfs_zevent_retain_expire_secs = 900;

	typedef enum zfs_subclass {
	ZSC_IO,
	ZSC_DATA,
	ZSC_CHECKSUM
	} zfs_subclass_t;

	typedef struct {
	/* common criteria */
	uint64_t re_pool_guid;
	uint64_t re_vdev_guid;
	int re_io_error;
	uint64_t re_io_size;
	uint64_t re_io_offset;
	zfs_subclass_t re_subclass;
	zio_priority_t re_io_priority;

	/* logical zio criteria (optional) */
	zbookmark_phys_t re_io_bookmark;

	/* internal state */
	avl_node_t re_tree_link;
	list_node_t re_list_link;
	uint64_t re_timestamp;
	} recent_events_node_t;

	static int
	recent_events_compare(const void a, const void b)
	{
	const recent_events_node_t *node1 = a;
	const recent_events_node_t *node2 = b;
	int cmp;

	/*
	* The comparison order here is somewhat arbitrary.
	* What's important is that if every criteria matches, then it
	* is a duplicate (i.e. compare returns 0)
	*/
	if ((cmp = TREE_CMP(node1->re_subclass, node2->re_subclass)) != 0)
	return (cmp);
	if ((cmp = TREE_CMP(node1->re_pool_guid, node2->re_pool_guid)) != 0)
	return (cmp);
	if ((cmp = TREE_CMP(node1->re_vdev_guid, node2->re_vdev_guid)) != 0)
	return (cmp);
	if ((cmp = TREE_CMP(node1->re_io_error, node2->re_io_error)) != 0)
	return (cmp);
	if ((cmp = TREE_CMP(node1->re_io_priority, node2->re_io_priority)) != 0)
	return (cmp);
	if ((cmp = TREE_CMP(node1->re_io_size, node2->re_io_size)) != 0)
	return (cmp);
	if ((cmp = TREE_CMP(node1->re_io_offset, node2->re_io_offset)) != 0)
	return (cmp);

	const zbookmark_phys_t *zb1 = &node1->re_io_bookmark;
	const zbookmark_phys_t *zb2 = &node2->re_io_bookmark;

	if ((cmp = TREE_CMP(zb1->zb_objset, zb2->zb_objset)) != 0)
	return (cmp);
	if ((cmp = TREE_CMP(zb1->zb_object, zb2->zb_object)) != 0)
	return (cmp);
	if ((cmp = TREE_CMP(zb1->zb_level, zb2->zb_level)) != 0)
	return (cmp);
	if ((cmp = TREE_CMP(zb1->zb_blkid, zb2->zb_blkid)) != 0)
	return (cmp);

	return (0);
	}

	/*
	* workaround: vdev properties don't have inheritance
	*/
	static uint64_t
	vdev_prop_get_inherited(vdev_t *vd, vdev_prop_t prop)
	{
	uint64_t propdef, propval;

	propdef = vdev_prop_default_numeric(prop);
	switch (prop) {
	case VDEV_PROP_CHECKSUM_N:
	propval = vd->vdev_checksum_n;
	break;
	case VDEV_PROP_CHECKSUM_T:
	propval = vd->vdev_checksum_t;
	break;
	case VDEV_PROP_IO_N:
	propval = vd->vdev_io_n;
	break;
	case VDEV_PROP_IO_T:
	propval = vd->vdev_io_t;
	break;
	+ case VDEV_PROP_SLOW_IO_N:
	+ propval = vd->vdev_slow_io_n;
	+ break;
	+ case VDEV_PROP_SLOW_IO_T:
	+ propval = vd->vdev_slow_io_t;
	+ break;
	default:
	propval = propdef;
	break;
	}

	if (propval != propdef)
	return (propval);

	if (vd->vdev_parent == NULL)
	return (propdef);

	return (vdev_prop_get_inherited(vd->vdev_parent, prop));
	}

	static void zfs_ereport_schedule_cleaner(void);

	/*
	* background task to clean stale recent event nodes.
	*/
	static void
	zfs_ereport_cleaner(void *arg)
	{
	recent_events_node_t *entry;
	uint64_t now = gethrtime();

	/*
	* purge expired entries
	*/
	mutex_enter(&recent_events_lock);
	while ((entry = list_tail(&recent_events_list)) != NULL) {
	uint64_t age = NSEC2SEC(now - entry->re_timestamp);
	if (age <= zfs_zevent_retain_expire_secs)
	break;

	/* remove expired node */
	avl_remove(&recent_events_tree, entry);
	list_remove(&recent_events_list, entry);
	kmem_free(entry, sizeof (*entry));
	}

	/* Restart the cleaner if more entries remain */
	recent_events_cleaner_tqid = 0;
	if (!list_is_empty(&recent_events_list))
	zfs_ereport_schedule_cleaner();

	mutex_exit(&recent_events_lock);
	}

	static void
	zfs_ereport_schedule_cleaner(void)
	{
	ASSERT(MUTEX_HELD(&recent_events_lock));

	uint64_t timeout = SEC2NSEC(zfs_zevent_retain_expire_secs + 1);

	recent_events_cleaner_tqid = taskq_dispatch_delay(
	system_delay_taskq, zfs_ereport_cleaner, NULL, TQ_SLEEP,
	ddi_get_lbolt() + NSEC_TO_TICK(timeout));
	}

	/*
	* Clear entries for a given vdev or all vdevs in a pool when vdev == NULL
	*/
	void
	zfs_ereport_clear(spa_t spa, vdev_t vd)
	{
	uint64_t vdev_guid, pool_guid;

	ASSERT(vd != NULL \|\| spa != NULL);
	if (vd == NULL) {
	vdev_guid = 0;
	pool_guid = spa_guid(spa);
	} else {
	vdev_guid = vd->vdev_guid;
	pool_guid = 0;
	}

	mutex_enter(&recent_events_lock);

	recent_events_node_t *next = list_head(&recent_events_list);
	while (next != NULL) {
	recent_events_node_t *entry = next;

	next = list_next(&recent_events_list, next);

	if (entry->re_vdev_guid == vdev_guid \|\|
	entry->re_pool_guid == pool_guid) {
	avl_remove(&recent_events_tree, entry);
	list_remove(&recent_events_list, entry);
	kmem_free(entry, sizeof (*entry));
	}
	}

	mutex_exit(&recent_events_lock);
	}

	/*
	* Check if an ereport would be a duplicate of one recently posted.
	*
	* An ereport is considered a duplicate if the set of criteria in
	* recent_events_node_t all match.
	*
	* Only FM_EREPORT_ZFS_IO, FM_EREPORT_ZFS_DATA, and FM_EREPORT_ZFS_CHECKSUM
	* are candidates for duplicate checking.
	*/
	static boolean_t
	zfs_ereport_is_duplicate(const char subclass, spa_t spa, vdev_t *vd,
	const zbookmark_phys_t zb, zio_t zio, uint64_t offset, uint64_t size)
	{
	recent_events_node_t search = {0}, *entry;

	if (vd == NULL \|\| zio == NULL)
	return (B_FALSE);

	if (zfs_zevent_retain_max == 0)
	return (B_FALSE);

	if (strcmp(subclass, FM_EREPORT_ZFS_IO) == 0)
	search.re_subclass = ZSC_IO;
	else if (strcmp(subclass, FM_EREPORT_ZFS_DATA) == 0)
	search.re_subclass = ZSC_DATA;
	else if (strcmp(subclass, FM_EREPORT_ZFS_CHECKSUM) == 0)
	search.re_subclass = ZSC_CHECKSUM;
	else
	return (B_FALSE);

	search.re_pool_guid = spa_guid(spa);
	search.re_vdev_guid = vd->vdev_guid;
	search.re_io_error = zio->io_error;
	search.re_io_priority = zio->io_priority;
	/* if size is supplied use it over what's in zio */
	if (size) {
	search.re_io_size = size;
	search.re_io_offset = offset;
	} else {
	search.re_io_size = zio->io_size;
	search.re_io_offset = zio->io_offset;
	}

	/* grab optional logical zio criteria */
	if (zb != NULL) {
	search.re_io_bookmark.zb_objset = zb->zb_objset;
	search.re_io_bookmark.zb_object = zb->zb_object;
	search.re_io_bookmark.zb_level = zb->zb_level;
	search.re_io_bookmark.zb_blkid = zb->zb_blkid;
	}

	uint64_t now = gethrtime();

	mutex_enter(&recent_events_lock);

	/* check if we have seen this one recently */
	entry = avl_find(&recent_events_tree, &search, NULL);
	if (entry != NULL) {
	uint64_t age = NSEC2SEC(now - entry->re_timestamp);

	/*
	* There is still an active cleaner (since we're here).
	* Reset the last seen time for this duplicate entry
	* so that its lifespand gets extended.
	*/
	list_remove(&recent_events_list, entry);
	list_insert_head(&recent_events_list, entry);
	entry->re_timestamp = now;

	zfs_zevent_track_duplicate();
	mutex_exit(&recent_events_lock);

	return (age <= zfs_zevent_retain_expire_secs);
	}

	if (avl_numnodes(&recent_events_tree) >= zfs_zevent_retain_max) {
	/* recycle oldest node */
	entry = list_tail(&recent_events_list);
	ASSERT(entry != NULL);
	list_remove(&recent_events_list, entry);
	avl_remove(&recent_events_tree, entry);
	} else {
	entry = kmem_alloc(sizeof (recent_events_node_t), KM_SLEEP);
	}

	/* record this as a recent ereport */
	*entry = search;
	avl_add(&recent_events_tree, entry);
	list_insert_head(&recent_events_list, entry);
	entry->re_timestamp = now;

	/* Start a cleaner if not already scheduled */
	if (recent_events_cleaner_tqid == 0)
	zfs_ereport_schedule_cleaner();

	mutex_exit(&recent_events_lock);
	return (B_FALSE);
	}

	void
	zfs_zevent_post_cb(nvlist_t nvl, nvlist_t detector)
	{
	if (nvl)
	fm_nvlist_destroy(nvl, FM_NVA_FREE);

	if (detector)
	fm_nvlist_destroy(detector, FM_NVA_FREE);
	}

	/*
	* We want to rate limit ZIO delay, deadman, and checksum events so as to not
	* flood zevent consumers when a disk is acting up.
	*
	* Returns 1 if we're ratelimiting, 0 if not.
	*/
	static int
	zfs_is_ratelimiting_event(const char subclass, vdev_t vd)
	{
	int rc = 0;
	/*
	* zfs_ratelimit() returns 1 if we're not ratelimiting and 0 if we
	* are. Invert it to get our return value.
	*/
	if (strcmp(subclass, FM_EREPORT_ZFS_DELAY) == 0) {
	rc = !zfs_ratelimit(&vd->vdev_delay_rl);
	} else if (strcmp(subclass, FM_EREPORT_ZFS_DEADMAN) == 0) {
	rc = !zfs_ratelimit(&vd->vdev_deadman_rl);
	} else if (strcmp(subclass, FM_EREPORT_ZFS_CHECKSUM) == 0) {
	rc = !zfs_ratelimit(&vd->vdev_checksum_rl);
	}

	if (rc) {
	/* We're rate limiting */
	fm_erpt_dropped_increment();
	}

	return (rc);
	}

	/*
	* Return B_TRUE if the event actually posted, B_FALSE if not.
	*/
	static boolean_t
	zfs_ereport_start(nvlist_t ereport_out, nvlist_t detector_out,
	const char subclass, spa_t spa, vdev_t vd, const zbookmark_phys_t zb,
	zio_t *zio, uint64_t stateoroffset, uint64_t size)
	{
	nvlist_t ereport, detector;

	uint64_t ena;
	char class[64];

	if ((ereport = fm_nvlist_create(NULL)) == NULL)
	return (B_FALSE);

	if ((detector = fm_nvlist_create(NULL)) == NULL) {
	fm_nvlist_destroy(ereport, FM_NVA_FREE);
	return (B_FALSE);
	}

	/*
	* Serialize ereport generation
	*/
	mutex_enter(&spa->spa_errlist_lock);

	/*
	* Determine the ENA to use for this event. If we are in a loading
	* state, use a SPA-wide ENA. Otherwise, if we are in an I/O state, use
	* a root zio-wide ENA. Otherwise, simply use a unique ENA.
	*/
	if (spa_load_state(spa) != SPA_LOAD_NONE) {
	if (spa->spa_ena == 0)
	spa->spa_ena = fm_ena_generate(0, FM_ENA_FMT1);
	ena = spa->spa_ena;
	} else if (zio != NULL && zio->io_logical != NULL) {
	if (zio->io_logical->io_ena == 0)
	zio->io_logical->io_ena =
	fm_ena_generate(0, FM_ENA_FMT1);
	ena = zio->io_logical->io_ena;
	} else {
	ena = fm_ena_generate(0, FM_ENA_FMT1);
	}

	/*
	* Construct the full class, detector, and other standard FMA fields.
	*/
	(void) snprintf(class, sizeof (class), "%s.%s",
	ZFS_ERROR_CLASS, subclass);

	fm_fmri_zfs_set(detector, FM_ZFS_SCHEME_VERSION, spa_guid(spa),
	vd != NULL ? vd->vdev_guid : 0);

	fm_ereport_set(ereport, FM_EREPORT_VERSION, class, ena, detector, NULL);

	/*
	* Construct the per-ereport payload, depending on which parameters are
	* passed in.
	*/

	/*
	* Generic payload members common to all ereports.
	*/
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_POOL, DATA_TYPE_STRING, spa_name(spa),
	FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, DATA_TYPE_UINT64, spa_guid(spa),
	FM_EREPORT_PAYLOAD_ZFS_POOL_STATE, DATA_TYPE_UINT64,
	(uint64_t)spa_state(spa),
	FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, DATA_TYPE_INT32,
	(int32_t)spa_load_state(spa), NULL);

	fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE,
	DATA_TYPE_STRING,
	spa_get_failmode(spa) == ZIO_FAILURE_MODE_WAIT ?
	FM_EREPORT_FAILMODE_WAIT :
	spa_get_failmode(spa) == ZIO_FAILURE_MODE_CONTINUE ?
	FM_EREPORT_FAILMODE_CONTINUE : FM_EREPORT_FAILMODE_PANIC,
	NULL);

	if (vd != NULL) {
	vdev_t *pvd = vd->vdev_parent;
	vdev_queue_t *vq = &vd->vdev_queue;
	vdev_stat_t *vs = &vd->vdev_stat;
	vdev_t *spare_vd;
	uint64_t *spare_guids;
	char **spare_paths;
	int i, spare_count;

	fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
	DATA_TYPE_UINT64, vd->vdev_guid,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
	DATA_TYPE_STRING, vd->vdev_ops->vdev_op_type, NULL);
	if (vd->vdev_path != NULL)
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_PATH,
	DATA_TYPE_STRING, vd->vdev_path, NULL);
	if (vd->vdev_devid != NULL)
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_DEVID,
	DATA_TYPE_STRING, vd->vdev_devid, NULL);
	if (vd->vdev_fru != NULL)
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_FRU,
	DATA_TYPE_STRING, vd->vdev_fru, NULL);
	if (vd->vdev_enc_sysfs_path != NULL)
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_ENC_SYSFS_PATH,
	DATA_TYPE_STRING, vd->vdev_enc_sysfs_path, NULL);
	if (vd->vdev_ashift)
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_ASHIFT,
	DATA_TYPE_UINT64, vd->vdev_ashift, NULL);

	if (vq != NULL) {
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_COMP_TS,
	DATA_TYPE_UINT64, vq->vq_io_complete_ts, NULL);
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_DELTA_TS,
	DATA_TYPE_UINT64, vq->vq_io_delta_ts, NULL);
	}

	if (vs != NULL) {
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_READ_ERRORS,
	DATA_TYPE_UINT64, vs->vs_read_errors,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_WRITE_ERRORS,
	DATA_TYPE_UINT64, vs->vs_write_errors,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_ERRORS,
	DATA_TYPE_UINT64, vs->vs_checksum_errors,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_DELAYS,
	DATA_TYPE_UINT64, vs->vs_slow_ios,
	NULL);
	}

	if (pvd != NULL) {
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_PARENT_GUID,
	DATA_TYPE_UINT64, pvd->vdev_guid,
	FM_EREPORT_PAYLOAD_ZFS_PARENT_TYPE,
	DATA_TYPE_STRING, pvd->vdev_ops->vdev_op_type,
	NULL);
	if (pvd->vdev_path)
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_PARENT_PATH,
	DATA_TYPE_STRING, pvd->vdev_path, NULL);
	if (pvd->vdev_devid)
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_PARENT_DEVID,
	DATA_TYPE_STRING, pvd->vdev_devid, NULL);
	}

	spare_count = spa->spa_spares.sav_count;
	spare_paths = kmem_zalloc(sizeof (char ) spare_count,
	KM_SLEEP);
	spare_guids = kmem_zalloc(sizeof (uint64_t) * spare_count,
	KM_SLEEP);

	for (i = 0; i < spare_count; i++) {
	spare_vd = spa->spa_spares.sav_vdevs[i];
	if (spare_vd) {
	spare_paths[i] = spare_vd->vdev_path;
	spare_guids[i] = spare_vd->vdev_guid;
	}
	}

	fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_VDEV_SPARE_PATHS,
	DATA_TYPE_STRING_ARRAY, spare_count, spare_paths,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_SPARE_GUIDS,
	DATA_TYPE_UINT64_ARRAY, spare_count, spare_guids, NULL);

	kmem_free(spare_guids, sizeof (uint64_t) * spare_count);
	kmem_free(spare_paths, sizeof (char ) spare_count);
	}

	if (zio != NULL) {
	/*
	* Payload common to all I/Os.
	*/
	fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_ZIO_ERR,
	DATA_TYPE_INT32, zio->io_error, NULL);
	fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS,
	DATA_TYPE_INT32, zio->io_flags, NULL);
	fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_ZIO_STAGE,
	DATA_TYPE_UINT32, zio->io_stage, NULL);
	fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_ZIO_PIPELINE,
	DATA_TYPE_UINT32, zio->io_pipeline, NULL);
	fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_ZIO_DELAY,
	DATA_TYPE_UINT64, zio->io_delay, NULL);
	fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_ZIO_TIMESTAMP,
	DATA_TYPE_UINT64, zio->io_timestamp, NULL);
	fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_ZIO_DELTA,
	DATA_TYPE_UINT64, zio->io_delta, NULL);
	fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_ZIO_PRIORITY,
	DATA_TYPE_UINT32, zio->io_priority, NULL);

	/*
	* If the 'size' parameter is non-zero, it indicates this is a
	* RAID-Z or other I/O where the physical offset and length are
	* provided for us, instead of within the zio_t.
	*/
	if (vd != NULL) {
	if (size)
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_ZIO_OFFSET,
	DATA_TYPE_UINT64, stateoroffset,
	FM_EREPORT_PAYLOAD_ZFS_ZIO_SIZE,
	DATA_TYPE_UINT64, size, NULL);
	else
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_ZIO_OFFSET,
	DATA_TYPE_UINT64, zio->io_offset,
	FM_EREPORT_PAYLOAD_ZFS_ZIO_SIZE,
	DATA_TYPE_UINT64, zio->io_size, NULL);
	}
	} else if (vd != NULL) {
	/*
	* If we have a vdev but no zio, this is a device fault, and the
	* 'stateoroffset' parameter indicates the previous state of the
	* vdev.
	*/
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_PREV_STATE,
	DATA_TYPE_UINT64, stateoroffset, NULL);
	}

	/*
	* Payload for I/Os with corresponding logical information.
	*/
	if (zb != NULL && (zio == NULL \|\| zio->io_logical != NULL)) {
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_ZIO_OBJSET,
	DATA_TYPE_UINT64, zb->zb_objset,
	FM_EREPORT_PAYLOAD_ZFS_ZIO_OBJECT,
	DATA_TYPE_UINT64, zb->zb_object,
	FM_EREPORT_PAYLOAD_ZFS_ZIO_LEVEL,
	DATA_TYPE_INT64, zb->zb_level,
	FM_EREPORT_PAYLOAD_ZFS_ZIO_BLKID,
	DATA_TYPE_UINT64, zb->zb_blkid, NULL);
	}

	/*
	* Payload for tuning the zed
	*/
	if (vd != NULL && strcmp(subclass, FM_EREPORT_ZFS_CHECKSUM) == 0) {
	uint64_t cksum_n, cksum_t;

	cksum_n = vdev_prop_get_inherited(vd, VDEV_PROP_CHECKSUM_N);
	if (cksum_n != vdev_prop_default_numeric(VDEV_PROP_CHECKSUM_N))
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_N,
	DATA_TYPE_UINT64,
	cksum_n,
	NULL);

	cksum_t = vdev_prop_get_inherited(vd, VDEV_PROP_CHECKSUM_T);
	if (cksum_t != vdev_prop_default_numeric(VDEV_PROP_CHECKSUM_T))
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_T,
	DATA_TYPE_UINT64,
	cksum_t,
	NULL);
	}

	if (vd != NULL && strcmp(subclass, FM_EREPORT_ZFS_IO) == 0) {
	uint64_t io_n, io_t;

	io_n = vdev_prop_get_inherited(vd, VDEV_PROP_IO_N);
	if (io_n != vdev_prop_default_numeric(VDEV_PROP_IO_N))
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_N,
	DATA_TYPE_UINT64,
	io_n,
	NULL);

	io_t = vdev_prop_get_inherited(vd, VDEV_PROP_IO_T);
	if (io_t != vdev_prop_default_numeric(VDEV_PROP_IO_T))
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_T,
	DATA_TYPE_UINT64,
	io_t,
	NULL);
	}

	+ if (vd != NULL && strcmp(subclass, FM_EREPORT_ZFS_DELAY) == 0) {
	+ uint64_t slow_io_n, slow_io_t;
	+
	+ slow_io_n = vdev_prop_get_inherited(vd, VDEV_PROP_SLOW_IO_N);
	+ if (slow_io_n != vdev_prop_default_numeric(VDEV_PROP_SLOW_IO_N))
	+ fm_payload_set(ereport,
	+ FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_N,
	+ DATA_TYPE_UINT64,
	+ slow_io_n,
	+ NULL);
	+
	+ slow_io_t = vdev_prop_get_inherited(vd, VDEV_PROP_SLOW_IO_T);
	+ if (slow_io_t != vdev_prop_default_numeric(VDEV_PROP_SLOW_IO_T))
	+ fm_payload_set(ereport,
	+ FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_T,
	+ DATA_TYPE_UINT64,
	+ slow_io_t,
	+ NULL);
	+ }
	+
	mutex_exit(&spa->spa_errlist_lock);

	*ereport_out = ereport;
	*detector_out = detector;
	return (B_TRUE);
	}

	/* if it's <= 128 bytes, save the corruption directly */
	#define ZFM_MAX_INLINE (128 / sizeof (uint64_t))

	#define MAX_RANGES 16

	typedef struct zfs_ecksum_info {
	/* inline arrays of bits set and cleared. */
	uint64_t zei_bits_set[ZFM_MAX_INLINE];
	uint64_t zei_bits_cleared[ZFM_MAX_INLINE];

	/*
	* for each range, the number of bits set and cleared. The Hamming
	* distance between the good and bad buffers is the sum of them all.
	*/
	uint32_t zei_range_sets[MAX_RANGES];
	uint32_t zei_range_clears[MAX_RANGES];

	struct zei_ranges {
	uint32_t zr_start;
	uint32_t zr_end;
	} zei_ranges[MAX_RANGES];

	size_t zei_range_count;
	uint32_t zei_mingap;
	uint32_t zei_allowed_mingap;

	} zfs_ecksum_info_t;

	static void
	update_bad_bits(uint64_t value_arg, uint32_t *count)
	{
	size_t i;
	size_t bits = 0;
	uint64_t value = BE_64(value_arg);

	/* We store the bits in big-endian (largest-first) order */
	for (i = 0; i < 64; i++) {
	if (value & (1ull << i))
	++bits;
	}
	/* update the count of bits changed */
	*count += bits;
	}

	/*
	* We've now filled up the range array, and need to increase "mingap" and
	* shrink the range list accordingly. zei_mingap is always the smallest
	* distance between array entries, so we set the new_allowed_gap to be
	* one greater than that. We then go through the list, joining together
	* any ranges which are closer than the new_allowed_gap.
	*
	* By construction, there will be at least one. We also update zei_mingap
	* to the new smallest gap, to prepare for our next invocation.
	*/
	static void
	zei_shrink_ranges(zfs_ecksum_info_t *eip)
	{
	uint32_t mingap = UINT32_MAX;
	uint32_t new_allowed_gap = eip->zei_mingap + 1;

	size_t idx, output;
	size_t max = eip->zei_range_count;

	struct zei_ranges *r = eip->zei_ranges;

	ASSERT3U(eip->zei_range_count, >, 0);
	ASSERT3U(eip->zei_range_count, <=, MAX_RANGES);

	output = idx = 0;
	while (idx < max - 1) {
	uint32_t start = r[idx].zr_start;
	uint32_t end = r[idx].zr_end;

	while (idx < max - 1) {
	idx++;

	uint32_t nstart = r[idx].zr_start;
	uint32_t nend = r[idx].zr_end;

	uint32_t gap = nstart - end;
	if (gap < new_allowed_gap) {
	end = nend;
	continue;
	}
	if (gap < mingap)
	mingap = gap;
	break;
	}
	r[output].zr_start = start;
	r[output].zr_end = end;
	output++;
	}
	ASSERT3U(output, <, eip->zei_range_count);
	eip->zei_range_count = output;
	eip->zei_mingap = mingap;
	eip->zei_allowed_mingap = new_allowed_gap;
	}

	static void
	zei_add_range(zfs_ecksum_info_t *eip, int start, int end)
	{
	struct zei_ranges *r = eip->zei_ranges;
	size_t count = eip->zei_range_count;

	if (count >= MAX_RANGES) {
	zei_shrink_ranges(eip);
	count = eip->zei_range_count;
	}
	if (count == 0) {
	eip->zei_mingap = UINT32_MAX;
	eip->zei_allowed_mingap = 1;
	} else {
	int gap = start - r[count - 1].zr_end;

	if (gap < eip->zei_allowed_mingap) {
	r[count - 1].zr_end = end;
	return;
	}
	if (gap < eip->zei_mingap)
	eip->zei_mingap = gap;
	}
	r[count].zr_start = start;
	r[count].zr_end = end;
	eip->zei_range_count++;
	}

	static size_t
	zei_range_total_size(zfs_ecksum_info_t *eip)
	{
	struct zei_ranges *r = eip->zei_ranges;
	size_t count = eip->zei_range_count;
	size_t result = 0;
	size_t idx;

	for (idx = 0; idx < count; idx++)
	result += (r[idx].zr_end - r[idx].zr_start);

	return (result);
	}

	static zfs_ecksum_info_t *
	annotate_ecksum(nvlist_t ereport, zio_bad_cksum_t info,
	const abd_t goodabd, const abd_t badabd, size_t size,
	boolean_t drop_if_identical)
	{
	const uint64_t *good;
	const uint64_t *bad;

	size_t nui64s = size / sizeof (uint64_t);

	size_t inline_size;
	int no_inline = 0;
	size_t idx;
	size_t range;

	size_t offset = 0;
	ssize_t start = -1;

	zfs_ecksum_info_t eip = kmem_zalloc(sizeof (eip), KM_SLEEP);

	/* don't do any annotation for injected checksum errors */
	if (info != NULL && info->zbc_injected)
	return (eip);

	if (info != NULL && info->zbc_has_cksum) {
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_CKSUM_ALGO,
	DATA_TYPE_STRING,
	info->zbc_checksum_name,
	NULL);

	if (info->zbc_byteswapped) {
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_CKSUM_BYTESWAP,
	DATA_TYPE_BOOLEAN, 1,
	NULL);
	}
	}

	if (badabd == NULL \|\| goodabd == NULL)
	return (eip);

	ASSERT3U(nui64s, <=, UINT32_MAX);
	ASSERT3U(size, ==, nui64s * sizeof (uint64_t));
	ASSERT3U(size, <=, SPA_MAXBLOCKSIZE);
	ASSERT3U(size, <=, UINT32_MAX);

	good = (const uint64_t ) abd_borrow_buf_copy((abd_t )goodabd, size);
	bad = (const uint64_t ) abd_borrow_buf_copy((abd_t )badabd, size);

	/* build up the range list by comparing the two buffers. */
	for (idx = 0; idx < nui64s; idx++) {
	if (good[idx] == bad[idx]) {
	if (start == -1)
	continue;

	zei_add_range(eip, start, idx);
	start = -1;
	} else {
	if (start != -1)
	continue;

	start = idx;
	}
	}
	if (start != -1)
	zei_add_range(eip, start, idx);

	/* See if it will fit in our inline buffers */
	inline_size = zei_range_total_size(eip);
	if (inline_size > ZFM_MAX_INLINE)
	no_inline = 1;

	/*
	* If there is no change and we want to drop if the buffers are
	* identical, do so.
	*/
	if (inline_size == 0 && drop_if_identical) {
	kmem_free(eip, sizeof (*eip));
	abd_return_buf((abd_t )goodabd, (void )good, size);
	abd_return_buf((abd_t )badabd, (void )bad, size);
	return (NULL);
	}

	/*
	* Now walk through the ranges, filling in the details of the
	* differences. Also convert our uint64_t-array offsets to byte
	* offsets.
	*/
	for (range = 0; range < eip->zei_range_count; range++) {
	size_t start = eip->zei_ranges[range].zr_start;
	size_t end = eip->zei_ranges[range].zr_end;

	for (idx = start; idx < end; idx++) {
	uint64_t set, cleared;

	// bits set in bad, but not in good
	set = ((~good[idx]) & bad[idx]);
	// bits set in good, but not in bad
	cleared = (good[idx] & (~bad[idx]));

	if (!no_inline) {
	ASSERT3U(offset, <, inline_size);
	eip->zei_bits_set[offset] = set;
	eip->zei_bits_cleared[offset] = cleared;
	offset++;
	}

	update_bad_bits(set, &eip->zei_range_sets[range]);
	update_bad_bits(cleared, &eip->zei_range_clears[range]);
	}

	/* convert to byte offsets */
	eip->zei_ranges[range].zr_start *= sizeof (uint64_t);
	eip->zei_ranges[range].zr_end *= sizeof (uint64_t);
	}

	abd_return_buf((abd_t )goodabd, (void )good, size);
	abd_return_buf((abd_t )badabd, (void )bad, size);

	eip->zei_allowed_mingap *= sizeof (uint64_t);
	inline_size *= sizeof (uint64_t);

	/* fill in ereport */
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_BAD_OFFSET_RANGES,
	DATA_TYPE_UINT32_ARRAY, 2 * eip->zei_range_count,
	(uint32_t *)eip->zei_ranges,
	FM_EREPORT_PAYLOAD_ZFS_BAD_RANGE_MIN_GAP,
	DATA_TYPE_UINT32, eip->zei_allowed_mingap,
	FM_EREPORT_PAYLOAD_ZFS_BAD_RANGE_SETS,
	DATA_TYPE_UINT32_ARRAY, eip->zei_range_count, eip->zei_range_sets,
	FM_EREPORT_PAYLOAD_ZFS_BAD_RANGE_CLEARS,
	DATA_TYPE_UINT32_ARRAY, eip->zei_range_count, eip->zei_range_clears,
	NULL);

	if (!no_inline) {
	fm_payload_set(ereport,
	FM_EREPORT_PAYLOAD_ZFS_BAD_SET_BITS,
	DATA_TYPE_UINT8_ARRAY,
	inline_size, (uint8_t *)eip->zei_bits_set,
	FM_EREPORT_PAYLOAD_ZFS_BAD_CLEARED_BITS,
	DATA_TYPE_UINT8_ARRAY,
	inline_size, (uint8_t *)eip->zei_bits_cleared,
	NULL);
	}
	return (eip);
	}
	#else
	void
	zfs_ereport_clear(spa_t spa, vdev_t vd)
	{
	(void) spa, (void) vd;
	}
	#endif

	/*
	* Make sure our event is still valid for the given zio/vdev/pool. For example,
	* we don't want to keep logging events for a faulted or missing vdev.
	*/
	boolean_t
	zfs_ereport_is_valid(const char subclass, spa_t spa, vdev_t vd, zio_t zio)
	{
	#ifdef _KERNEL
	/*
	* If we are doing a spa_tryimport() or in recovery mode,
	* ignore errors.
	*/
	if (spa_load_state(spa) == SPA_LOAD_TRYIMPORT \|\|
	spa_load_state(spa) == SPA_LOAD_RECOVER)
	return (B_FALSE);

	/*
	* If we are in the middle of opening a pool, and the previous attempt
	* failed, don't bother logging any new ereports - we're just going to
	* get the same diagnosis anyway.
	*/
	if (spa_load_state(spa) != SPA_LOAD_NONE &&
	spa->spa_last_open_failed)
	return (B_FALSE);

	if (zio != NULL) {
	/*
	* If this is not a read or write zio, ignore the error. This
	* can occur if the DKIOCFLUSHWRITECACHE ioctl fails.
	*/
	if (zio->io_type != ZIO_TYPE_READ &&
	zio->io_type != ZIO_TYPE_WRITE)
	return (B_FALSE);

	if (vd != NULL) {
	/*
	* If the vdev has already been marked as failing due
	* to a failed probe, then ignore any subsequent I/O
	* errors, as the DE will automatically fault the vdev
	* on the first such failure. This also catches cases
	* where vdev_remove_wanted is set and the device has
	* not yet been asynchronously placed into the REMOVED
	* state.
	*/
	if (zio->io_vd == vd && !vdev_accessible(vd, zio))
	return (B_FALSE);

	/*
	* Ignore checksum errors for reads from DTL regions of
	* leaf vdevs.
	*/
	if (zio->io_type == ZIO_TYPE_READ &&
	zio->io_error == ECKSUM &&
	vd->vdev_ops->vdev_op_leaf &&
	vdev_dtl_contains(vd, DTL_MISSING, zio->io_txg, 1))
	return (B_FALSE);
	}
	}

	/*
	* For probe failure, we want to avoid posting ereports if we've
	* already removed the device in the meantime.
	*/
	if (vd != NULL &&
	strcmp(subclass, FM_EREPORT_ZFS_PROBE_FAILURE) == 0 &&
	(vd->vdev_remove_wanted \|\| vd->vdev_state == VDEV_STATE_REMOVED))
	return (B_FALSE);

	/* Ignore bogus delay events (like from ioctls or unqueued IOs) */
	if ((strcmp(subclass, FM_EREPORT_ZFS_DELAY) == 0) &&
	(zio != NULL) && (!zio->io_timestamp)) {
	return (B_FALSE);
	}
	#else
	(void) subclass, (void) spa, (void) vd, (void) zio;
	#endif
	return (B_TRUE);
	}

	/*
	* Post an ereport for the given subclass
	*
	* Returns
	* - 0 if an event was posted
	* - EINVAL if there was a problem posting event
	* - EBUSY if the event was rate limited
	* - EALREADY if the event was already posted (duplicate)
	*/
	int
	zfs_ereport_post(const char subclass, spa_t spa, vdev_t *vd,
	const zbookmark_phys_t zb, zio_t zio, uint64_t state)
	{
	int rc = 0;
	#ifdef _KERNEL
	nvlist_t *ereport = NULL;
	nvlist_t *detector = NULL;

	if (!zfs_ereport_is_valid(subclass, spa, vd, zio))
	return (EINVAL);

	if (zfs_ereport_is_duplicate(subclass, spa, vd, zb, zio, 0, 0))
	return (SET_ERROR(EALREADY));

	if (zfs_is_ratelimiting_event(subclass, vd))
	return (SET_ERROR(EBUSY));

	if (!zfs_ereport_start(&ereport, &detector, subclass, spa, vd,
	zb, zio, state, 0))
	return (SET_ERROR(EINVAL)); /* couldn't post event */

	if (ereport == NULL)
	return (SET_ERROR(EINVAL));

	/* Cleanup is handled by the callback function */
	rc = zfs_zevent_post(ereport, detector, zfs_zevent_post_cb);
	#else
	(void) subclass, (void) spa, (void) vd, (void) zb, (void) zio,
	(void) state;
	#endif
	return (rc);
	}

	/*
	* Prepare a checksum ereport
	*
	* Returns
	* - 0 if an event was posted
	* - EINVAL if there was a problem posting event
	* - EBUSY if the event was rate limited
	* - EALREADY if the event was already posted (duplicate)
	*/
	int
	zfs_ereport_start_checksum(spa_t spa, vdev_t vd, const zbookmark_phys_t *zb,
	struct zio zio, uint64_t offset, uint64_t length, zio_bad_cksum_t info)
	{
	zio_cksum_report_t *report;

	#ifdef _KERNEL
	if (!zfs_ereport_is_valid(FM_EREPORT_ZFS_CHECKSUM, spa, vd, zio))
	return (SET_ERROR(EINVAL));

	if (zfs_ereport_is_duplicate(FM_EREPORT_ZFS_CHECKSUM, spa, vd, zb, zio,
	offset, length))
	return (SET_ERROR(EALREADY));

	if (zfs_is_ratelimiting_event(FM_EREPORT_ZFS_CHECKSUM, vd))
	return (SET_ERROR(EBUSY));
	#else
	(void) zb, (void) offset;
	#endif

	report = kmem_zalloc(sizeof (*report), KM_SLEEP);

	zio_vsd_default_cksum_report(zio, report);

	/* copy the checksum failure information if it was provided */
	if (info != NULL) {
	report->zcr_ckinfo = kmem_zalloc(sizeof (*info), KM_SLEEP);
	memcpy(report->zcr_ckinfo, info, sizeof (*info));
	}

	report->zcr_sector = 1ULL << vd->vdev_top->vdev_ashift;
	report->zcr_align =
	vdev_psize_to_asize(vd->vdev_top, report->zcr_sector);
	report->zcr_length = length;

	#ifdef _KERNEL
	(void) zfs_ereport_start(&report->zcr_ereport, &report->zcr_detector,
	FM_EREPORT_ZFS_CHECKSUM, spa, vd, zb, zio, offset, length);

	if (report->zcr_ereport == NULL) {
	zfs_ereport_free_checksum(report);
	return (0);
	}
	#endif

	mutex_enter(&spa->spa_errlist_lock);
	report->zcr_next = zio->io_logical->io_cksum_report;
	zio->io_logical->io_cksum_report = report;
	mutex_exit(&spa->spa_errlist_lock);
	return (0);
	}

	void
	zfs_ereport_finish_checksum(zio_cksum_report_t report, const abd_t good_data,
	const abd_t *bad_data, boolean_t drop_if_identical)
	{
	#ifdef _KERNEL
	zfs_ecksum_info_t *info;

	info = annotate_ecksum(report->zcr_ereport, report->zcr_ckinfo,
	good_data, bad_data, report->zcr_length, drop_if_identical);
	if (info != NULL)
	zfs_zevent_post(report->zcr_ereport,
	report->zcr_detector, zfs_zevent_post_cb);
	else
	zfs_zevent_post_cb(report->zcr_ereport, report->zcr_detector);

	report->zcr_ereport = report->zcr_detector = NULL;
	if (info != NULL)
	kmem_free(info, sizeof (*info));
	#else
	(void) report, (void) good_data, (void) bad_data,
	(void) drop_if_identical;
	#endif
	}

	void
	zfs_ereport_free_checksum(zio_cksum_report_t *rpt)
	{
	#ifdef _KERNEL
	if (rpt->zcr_ereport != NULL) {
	fm_nvlist_destroy(rpt->zcr_ereport,
	FM_NVA_FREE);
	fm_nvlist_destroy(rpt->zcr_detector,
	FM_NVA_FREE);
	}
	#endif
	rpt->zcr_free(rpt->zcr_cbdata, rpt->zcr_cbinfo);

	if (rpt->zcr_ckinfo != NULL)
	kmem_free(rpt->zcr_ckinfo, sizeof (*rpt->zcr_ckinfo));

	kmem_free(rpt, sizeof (*rpt));
	}

	/*
	* Post a checksum ereport
	*
	* Returns
	* - 0 if an event was posted
	* - EINVAL if there was a problem posting event
	* - EBUSY if the event was rate limited
	* - EALREADY if the event was already posted (duplicate)
	*/
	int
	zfs_ereport_post_checksum(spa_t spa, vdev_t vd, const zbookmark_phys_t *zb,
	struct zio *zio, uint64_t offset, uint64_t length,
	const abd_t good_data, const abd_t bad_data, zio_bad_cksum_t *zbc)
	{
	int rc = 0;
	#ifdef _KERNEL
	nvlist_t *ereport = NULL;
	nvlist_t *detector = NULL;
	zfs_ecksum_info_t *info;

	if (!zfs_ereport_is_valid(FM_EREPORT_ZFS_CHECKSUM, spa, vd, zio))
	return (SET_ERROR(EINVAL));

	if (zfs_ereport_is_duplicate(FM_EREPORT_ZFS_CHECKSUM, spa, vd, zb, zio,
	offset, length))
	return (SET_ERROR(EALREADY));

	if (zfs_is_ratelimiting_event(FM_EREPORT_ZFS_CHECKSUM, vd))
	return (SET_ERROR(EBUSY));

	if (!zfs_ereport_start(&ereport, &detector, FM_EREPORT_ZFS_CHECKSUM,
	spa, vd, zb, zio, offset, length) \|\| (ereport == NULL)) {
	return (SET_ERROR(EINVAL));
	}

	info = annotate_ecksum(ereport, zbc, good_data, bad_data, length,
	B_FALSE);

	if (info != NULL) {
	rc = zfs_zevent_post(ereport, detector, zfs_zevent_post_cb);
	kmem_free(info, sizeof (*info));
	}
	#else
	(void) spa, (void) vd, (void) zb, (void) zio, (void) offset,
	(void) length, (void) good_data, (void) bad_data, (void) zbc;
	#endif
	return (rc);
	}

	/*
	* The 'sysevent.fs.zfs.*' events are signals posted to notify user space of
	* change in the pool. All sysevents are listed in sys/sysevent/eventdefs.h
	* and are designed to be consumed by the ZFS Event Daemon (ZED). For
	* additional details refer to the zed(8) man page.
	*/
	nvlist_t *
	zfs_event_create(spa_t spa, vdev_t vd, const char type, const char name,
	nvlist_t *aux)
	{
	nvlist_t *resource = NULL;
	#ifdef _KERNEL
	char class[64];

	if (spa_load_state(spa) == SPA_LOAD_TRYIMPORT)
	return (NULL);

	if ((resource = fm_nvlist_create(NULL)) == NULL)
	return (NULL);

	(void) snprintf(class, sizeof (class), "%s.%s.%s", type,
	ZFS_ERROR_CLASS, name);
	VERIFY0(nvlist_add_uint8(resource, FM_VERSION, FM_RSRC_VERSION));
	VERIFY0(nvlist_add_string(resource, FM_CLASS, class));
	VERIFY0(nvlist_add_string(resource,
	FM_EREPORT_PAYLOAD_ZFS_POOL, spa_name(spa)));
	VERIFY0(nvlist_add_uint64(resource,
	FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, spa_guid(spa)));
	VERIFY0(nvlist_add_uint64(resource,
	FM_EREPORT_PAYLOAD_ZFS_POOL_STATE, spa_state(spa)));
	VERIFY0(nvlist_add_int32(resource,
	FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, spa_load_state(spa)));

	if (vd) {
	VERIFY0(nvlist_add_uint64(resource,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, vd->vdev_guid));
	VERIFY0(nvlist_add_uint64(resource,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE, vd->vdev_state));
	if (vd->vdev_path != NULL)
	VERIFY0(nvlist_add_string(resource,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_PATH, vd->vdev_path));
	if (vd->vdev_devid != NULL)
	VERIFY0(nvlist_add_string(resource,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_DEVID, vd->vdev_devid));
	if (vd->vdev_fru != NULL)
	VERIFY0(nvlist_add_string(resource,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_FRU, vd->vdev_fru));
	if (vd->vdev_enc_sysfs_path != NULL)
	VERIFY0(nvlist_add_string(resource,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_ENC_SYSFS_PATH,
	vd->vdev_enc_sysfs_path));
	}

	/* also copy any optional payload data */
	if (aux) {
	nvpair_t *elem = NULL;

	while ((elem = nvlist_next_nvpair(aux, elem)) != NULL)
	(void) nvlist_add_nvpair(resource, elem);
	}
	#else
	(void) spa, (void) vd, (void) type, (void) name, (void) aux;
	#endif
	return (resource);
	}

	static void
	zfs_post_common(spa_t spa, vdev_t vd, const char type, const char name,
	nvlist_t *aux)
	{
	#ifdef _KERNEL
	nvlist_t *resource;

	resource = zfs_event_create(spa, vd, type, name, aux);
	if (resource)
	zfs_zevent_post(resource, NULL, zfs_zevent_post_cb);
	#else
	(void) spa, (void) vd, (void) type, (void) name, (void) aux;
	#endif
	}

	/*
	* The 'resource.fs.zfs.removed' event is an internal signal that the given vdev
	* has been removed from the system. This will cause the DE to ignore any
	* recent I/O errors, inferring that they are due to the asynchronous device
	* removal.
	*/
	void
	zfs_post_remove(spa_t spa, vdev_t vd)
	{
	zfs_post_common(spa, vd, FM_RSRC_CLASS, FM_RESOURCE_REMOVED, NULL);
	}

	/*
	* The 'resource.fs.zfs.autoreplace' event is an internal signal that the pool
	* has the 'autoreplace' property set, and therefore any broken vdevs will be
	* handled by higher level logic, and no vdev fault should be generated.
	*/
	void
	zfs_post_autoreplace(spa_t spa, vdev_t vd)
	{
	zfs_post_common(spa, vd, FM_RSRC_CLASS, FM_RESOURCE_AUTOREPLACE, NULL);
	}

	/*
	* The 'resource.fs.zfs.statechange' event is an internal signal that the
	* given vdev has transitioned its state to DEGRADED or HEALTHY. This will
	* cause the retire agent to repair any outstanding fault management cases
	* open because the device was not found (fault.fs.zfs.device).
	*/
	void
	zfs_post_state_change(spa_t spa, vdev_t vd, uint64_t laststate)
	{
	#ifdef _KERNEL
	nvlist_t *aux;

	/*
	* Add optional supplemental keys to payload
	*/
	aux = fm_nvlist_create(NULL);
	if (vd && aux) {
	if (vd->vdev_physpath) {
	fnvlist_add_string(aux,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_PHYSPATH,
	vd->vdev_physpath);
	}
	if (vd->vdev_enc_sysfs_path) {
	fnvlist_add_string(aux,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_ENC_SYSFS_PATH,
	vd->vdev_enc_sysfs_path);
	}

	fnvlist_add_uint64(aux,
	FM_EREPORT_PAYLOAD_ZFS_VDEV_LASTSTATE, laststate);
	}

	zfs_post_common(spa, vd, FM_RSRC_CLASS, FM_RESOURCE_STATECHANGE,
	aux);

	if (aux)
	fm_nvlist_destroy(aux, FM_NVA_FREE);
	#else
	(void) spa, (void) vd, (void) laststate;
	#endif
	}

	#ifdef _KERNEL
	void
	zfs_ereport_init(void)
	{
	mutex_init(&recent_events_lock, NULL, MUTEX_DEFAULT, NULL);
	list_create(&recent_events_list, sizeof (recent_events_node_t),
	offsetof(recent_events_node_t, re_list_link));
	avl_create(&recent_events_tree, recent_events_compare,
	sizeof (recent_events_node_t), offsetof(recent_events_node_t,
	re_tree_link));
	}

	/*
	* This 'early' fini needs to run before zfs_fini() which on Linux waits
	* for the system_delay_taskq to drain.
	*/
	void
	zfs_ereport_taskq_fini(void)
	{
	mutex_enter(&recent_events_lock);
	if (recent_events_cleaner_tqid != 0) {
	taskq_cancel_id(system_delay_taskq, recent_events_cleaner_tqid);
	recent_events_cleaner_tqid = 0;
	}
	mutex_exit(&recent_events_lock);
	}

	void
	zfs_ereport_fini(void)
	{
	recent_events_node_t *entry;

	while ((entry = list_remove_head(&recent_events_list)) != NULL) {
	avl_remove(&recent_events_tree, entry);
	kmem_free(entry, sizeof (*entry));
	}
	avl_destroy(&recent_events_tree);
	list_destroy(&recent_events_list);
	mutex_destroy(&recent_events_lock);
	}

	void
	zfs_ereport_snapshot_post(const char subclass, spa_t spa, const char *name)
	{
	nvlist_t *aux;

	aux = fm_nvlist_create(NULL);
	fnvlist_add_string(aux, FM_EREPORT_PAYLOAD_ZFS_SNAPSHOT_NAME, name);

	zfs_post_common(spa, NULL, FM_RSRC_CLASS, subclass, aux);
	fm_nvlist_destroy(aux, FM_NVA_FREE);
	}

	/*
	* Post when a event when a zvol is created or removed
	*
	* This is currently only used by macOS, since it uses the event to create
	* symlinks between the volume name (mypool/myvol) and the actual /dev
	* device (/dev/disk3). For example:
	*
	* /var/run/zfs/dsk/mypool/myvol -> /dev/disk3
	*
	* name: The full name of the zvol ("mypool/myvol")
	* dev_name: The full /dev name for the zvol ("/dev/disk3")
	* raw_name: The raw /dev name for the zvol ("/dev/rdisk3")
	*/
	void
	zfs_ereport_zvol_post(const char subclass, const char name,
	const char dev_name, const char raw_name)
	{
	nvlist_t *aux;
	char *r;

	boolean_t locked = mutex_owned(&spa_namespace_lock);
	if (!locked) mutex_enter(&spa_namespace_lock);
	spa_t *spa = spa_lookup(name);
	if (!locked) mutex_exit(&spa_namespace_lock);

	if (spa == NULL)
	return;

	aux = fm_nvlist_create(NULL);
	fnvlist_add_string(aux, FM_EREPORT_PAYLOAD_ZFS_DEVICE_NAME, dev_name);
	fnvlist_add_string(aux, FM_EREPORT_PAYLOAD_ZFS_RAW_DEVICE_NAME,
	raw_name);
	r = strchr(name, '/');
	if (r && r[1])
	fnvlist_add_string(aux, FM_EREPORT_PAYLOAD_ZFS_VOLUME, &r[1]);

	zfs_post_common(spa, NULL, FM_RSRC_CLASS, subclass, aux);
	fm_nvlist_destroy(aux, FM_NVA_FREE);
	}

	EXPORT_SYMBOL(zfs_ereport_post);
	EXPORT_SYMBOL(zfs_ereport_is_valid);
	EXPORT_SYMBOL(zfs_ereport_post_checksum);
	EXPORT_SYMBOL(zfs_post_remove);
	EXPORT_SYMBOL(zfs_post_autoreplace);
	EXPORT_SYMBOL(zfs_post_state_change);

	ZFS_MODULE_PARAM(zfs_zevent, zfs_zevent_, retain_max, UINT, ZMOD_RW,
	"Maximum recent zevents records to retain for duplicate checking");
	ZFS_MODULE_PARAM(zfs_zevent, zfs_zevent_, retain_expire_secs, UINT, ZMOD_RW,
	"Expiration time for recent zevents records");
	#endif /* _KERNEL */
	diff --git a/sys/contrib/openzfs/module/zfs/zfs_ioctl.c b/sys/contrib/openzfs/module/zfs/zfs_ioctl.c
	index 2738385e260b..b07837113293 100644
	--- a/sys/contrib/openzfs/module/zfs/zfs_ioctl.c
	+++ b/sys/contrib/openzfs/module/zfs/zfs_ioctl.c
	@@ -1,7931 +1,7934 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Portions Copyright 2011 Martin Matuska
	* Copyright 2015, OmniTI Computer Consulting, Inc. All rights reserved.
	* Copyright (c) 2012 Pawel Jakub Dawidek
	* Copyright (c) 2014, 2016 Joyent, Inc. All rights reserved.
	* Copyright 2016 Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2014, Joyent, Inc. All rights reserved.
	- * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
	+ * Copyright (c) 2011, 2024 by Delphix. All rights reserved.
	* Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
	* Copyright (c) 2013 Steven Hartland. All rights reserved.
	* Copyright (c) 2014 Integros [integros.com]
	* Copyright 2016 Toomas Soome <tsoome@me.com>
	* Copyright (c) 2016 Actifio, Inc. All rights reserved.
	* Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
	* Copyright 2017 RackTop Systems.
	* Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
	* Copyright (c) 2019 Datto Inc.
	* Copyright (c) 2019, 2020 by Christian Schwarz. All rights reserved.
	* Copyright (c) 2019, 2021, Klara Inc.
	* Copyright (c) 2019, Allan Jude
	*/

	/*
	* ZFS ioctls.
	*
	* This file handles the ioctls to /dev/zfs, used for configuring ZFS storage
	* pools and filesystems, e.g. with /sbin/zfs and /sbin/zpool.
	*
	* There are two ways that we handle ioctls: the legacy way where almost
	* all of the logic is in the ioctl callback, and the new way where most
	* of the marshalling is handled in the common entry point, zfsdev_ioctl().
	*
	* Non-legacy ioctls should be registered by calling
	* zfs_ioctl_register() from zfs_ioctl_init(). The ioctl is invoked
	* from userland by lzc_ioctl().
	*
	* The registration arguments are as follows:
	*
	* const char *name
	* The name of the ioctl. This is used for history logging. If the
	* ioctl returns successfully (the callback returns 0), and allow_log
	* is true, then a history log entry will be recorded with the input &
	* output nvlists. The log entry can be printed with "zpool history -i".
	*
	* zfs_ioc_t ioc
	* The ioctl request number, which userland will pass to ioctl(2).
	* We want newer versions of libzfs and libzfs_core to run against
	* existing zfs kernel modules (i.e. a deferred reboot after an update).
	* Therefore the ioctl numbers cannot change from release to release.
	*
	* zfs_secpolicy_func_t *secpolicy
	* This function will be called before the zfs_ioc_func_t, to
	* determine if this operation is permitted. It should return EPERM
	* on failure, and 0 on success. Checks include determining if the
	* dataset is visible in this zone, and if the user has either all
	* zfs privileges in the zone (SYS_MOUNT), or has been granted permission
	* to do this operation on this dataset with "zfs allow".
	*
	* zfs_ioc_namecheck_t namecheck
	* This specifies what to expect in the zfs_cmd_t:zc_name -- a pool
	* name, a dataset name, or nothing. If the name is not well-formed,
	* the ioctl will fail and the callback will not be called.
	* Therefore, the callback can assume that the name is well-formed
	* (e.g. is null-terminated, doesn't have more than one '@' character,
	* doesn't have invalid characters).
	*
	* zfs_ioc_poolcheck_t pool_check
	* This specifies requirements on the pool state. If the pool does
	* not meet them (is suspended or is readonly), the ioctl will fail
	* and the callback will not be called. If any checks are specified
	* (i.e. it is not POOL_CHECK_NONE), namecheck must not be NO_NAME.
	* Multiple checks can be or-ed together (e.g. POOL_CHECK_SUSPENDED \|
	* POOL_CHECK_READONLY).
	*
	* zfs_ioc_key_t *nvl_keys
	* The list of expected/allowable innvl input keys. This list is used
	* to validate the nvlist input to the ioctl.
	*
	* boolean_t smush_outnvlist
	* If smush_outnvlist is true, then the output is presumed to be a
	* list of errors, and it will be "smushed" down to fit into the
	* caller's buffer, by removing some entries and replacing them with a
	* single "N_MORE_ERRORS" entry indicating how many were removed. See
	* nvlist_smush() for details. If smush_outnvlist is false, and the
	* outnvlist does not fit into the userland-provided buffer, then the
	* ioctl will fail with ENOMEM.
	*
	* zfs_ioc_func_t *func
	* The callback function that will perform the operation.
	*
	* The callback should return 0 on success, or an error number on
	* failure. If the function fails, the userland ioctl will return -1,
	* and errno will be set to the callback's return value. The callback
	* will be called with the following arguments:
	*
	* const char *name
	* The name of the pool or dataset to operate on, from
	* zfs_cmd_t:zc_name. The 'namecheck' argument specifies the
	* expected type (pool, dataset, or none).
	*
	* nvlist_t *innvl
	* The input nvlist, deserialized from zfs_cmd_t:zc_nvlist_src. Or
	* NULL if no input nvlist was provided. Changes to this nvlist are
	* ignored. If the input nvlist could not be deserialized, the
	* ioctl will fail and the callback will not be called.
	*
	* nvlist_t *outnvl
	* The output nvlist, initially empty. The callback can fill it in,
	* and it will be returned to userland by serializing it into
	* zfs_cmd_t:zc_nvlist_dst. If it is non-empty, and serialization
	* fails (e.g. because the caller didn't supply a large enough
	* buffer), then the overall ioctl will fail. See the
	* 'smush_nvlist' argument above for additional behaviors.
	*
	* There are two typical uses of the output nvlist:
	* - To return state, e.g. property values. In this case,
	* smush_outnvlist should be false. If the buffer was not large
	* enough, the caller will reallocate a larger buffer and try
	* the ioctl again.
	*
	* - To return multiple errors from an ioctl which makes on-disk
	* changes. In this case, smush_outnvlist should be true.
	* Ioctls which make on-disk modifications should generally not
	* use the outnvl if they succeed, because the caller can not
	* distinguish between the operation failing, and
	* deserialization failing.
	*
	* IOCTL Interface Errors
	*
	* The following ioctl input errors can be returned:
	* ZFS_ERR_IOC_CMD_UNAVAIL the ioctl number is not supported by kernel
	* ZFS_ERR_IOC_ARG_UNAVAIL an input argument is not supported by kernel
	* ZFS_ERR_IOC_ARG_REQUIRED a required input argument is missing
	* ZFS_ERR_IOC_ARG_BADTYPE an input argument has an invalid type
	*/

	#include <sys/types.h>
	#include <sys/param.h>
	#include <sys/errno.h>
	#include <sys/uio_impl.h>
	#include <sys/file.h>
	#include <sys/kmem.h>
	#include <sys/cmn_err.h>
	#include <sys/stat.h>
	#include <sys/zfs_ioctl.h>
	#include <sys/zfs_quota.h>
	#include <sys/zfs_vfsops.h>
	#include <sys/zfs_znode.h>
	#include <sys/zap.h>
	#include <sys/spa.h>
	#include <sys/spa_impl.h>
	#include <sys/vdev.h>
	#include <sys/vdev_impl.h>
	#include <sys/dmu.h>
	#include <sys/dsl_dir.h>
	#include <sys/dsl_dataset.h>
	#include <sys/dsl_prop.h>
	#include <sys/dsl_deleg.h>
	#include <sys/dmu_objset.h>
	#include <sys/dmu_impl.h>
	#include <sys/dmu_redact.h>
	#include <sys/dmu_tx.h>
	#include <sys/sunddi.h>
	#include <sys/policy.h>
	#include <sys/zone.h>
	#include <sys/nvpair.h>
	#include <sys/pathname.h>
	#include <sys/fs/zfs.h>
	#include <sys/zfs_ctldir.h>
	#include <sys/zfs_dir.h>
	#include <sys/zfs_onexit.h>
	#include <sys/zvol.h>
	#include <sys/dsl_scan.h>
	#include <sys/fm/util.h>
	#include <sys/dsl_crypt.h>
	#include <sys/rrwlock.h>
	#include <sys/zfs_file.h>

	#include <sys/dmu_recv.h>
	#include <sys/dmu_send.h>
	#include <sys/dmu_recv.h>
	#include <sys/dsl_destroy.h>
	#include <sys/dsl_bookmark.h>
	#include <sys/dsl_userhold.h>
	#include <sys/zfeature.h>
	#include <sys/zcp.h>
	#include <sys/zio_checksum.h>
	#include <sys/vdev_removal.h>
	#include <sys/vdev_impl.h>
	#include <sys/vdev_initialize.h>
	#include <sys/vdev_trim.h>

	#include "zfs_namecheck.h"
	#include "zfs_prop.h"
	#include "zfs_deleg.h"
	#include "zfs_comutil.h"

	#include <sys/lua/lua.h>
	#include <sys/lua/lauxlib.h>
	#include <sys/zfs_ioctl_impl.h>

	kmutex_t zfsdev_state_lock;
	static zfsdev_state_t zfsdev_state_listhead;

	/*
	* Limit maximum nvlist size. We don't want users passing in insane values
	* for zc->zc_nvlist_src_size, since we will need to allocate that much memory.
	* Defaults to 0=auto which is handled by platform code.
	*/
	uint64_t zfs_max_nvlist_src_size = 0;

	/*
	* When logging the output nvlist of an ioctl in the on-disk history, limit
	* the logged size to this many bytes. This must be less than DMU_MAX_ACCESS.
	* This applies primarily to zfs_ioc_channel_program().
	*/
	static uint64_t zfs_history_output_max = 1024 * 1024;

	uint_t zfs_fsyncer_key;
	uint_t zfs_allow_log_key;

	/* DATA_TYPE_ANY is used when zkey_type can vary. */
	#define DATA_TYPE_ANY DATA_TYPE_UNKNOWN

	typedef struct zfs_ioc_vec {
	zfs_ioc_legacy_func_t *zvec_legacy_func;
	zfs_ioc_func_t *zvec_func;
	zfs_secpolicy_func_t *zvec_secpolicy;
	zfs_ioc_namecheck_t zvec_namecheck;
	boolean_t zvec_allow_log;
	zfs_ioc_poolcheck_t zvec_pool_check;
	boolean_t zvec_smush_outnvlist;
	const char *zvec_name;
	const zfs_ioc_key_t *zvec_nvl_keys;
	size_t zvec_nvl_key_count;
	} zfs_ioc_vec_t;

	/* This array is indexed by zfs_userquota_prop_t */
	static const char *userquota_perms[] = {
	ZFS_DELEG_PERM_USERUSED,
	ZFS_DELEG_PERM_USERQUOTA,
	ZFS_DELEG_PERM_GROUPUSED,
	ZFS_DELEG_PERM_GROUPQUOTA,
	ZFS_DELEG_PERM_USEROBJUSED,
	ZFS_DELEG_PERM_USEROBJQUOTA,
	ZFS_DELEG_PERM_GROUPOBJUSED,
	ZFS_DELEG_PERM_GROUPOBJQUOTA,
	ZFS_DELEG_PERM_PROJECTUSED,
	ZFS_DELEG_PERM_PROJECTQUOTA,
	ZFS_DELEG_PERM_PROJECTOBJUSED,
	ZFS_DELEG_PERM_PROJECTOBJQUOTA,
	};

	static int zfs_ioc_userspace_upgrade(zfs_cmd_t *zc);
	static int zfs_ioc_id_quota_upgrade(zfs_cmd_t *zc);
	static int zfs_check_settable(const char name, nvpair_t property,
	cred_t *cr);
	static int zfs_check_clearable(const char dataset, nvlist_t props,
	nvlist_t **errors);
	static int zfs_fill_zplprops_root(uint64_t, nvlist_t , nvlist_t ,
	boolean_t *);
	int zfs_set_prop_nvlist(const char , zprop_source_t, nvlist_t , nvlist_t *);
	static int get_nvlist(uint64_t nvl, uint64_t size, int iflag, nvlist_t **nvp);

	static void
	history_str_free(char *buf)
	{
	kmem_free(buf, HIS_MAX_RECORD_LEN);
	}

	static char *
	history_str_get(zfs_cmd_t *zc)
	{
	char *buf;

	if (zc->zc_history == 0)
	return (NULL);

	buf = kmem_alloc(HIS_MAX_RECORD_LEN, KM_SLEEP);
	if (copyinstr((void *)(uintptr_t)zc->zc_history,
	buf, HIS_MAX_RECORD_LEN, NULL) != 0) {
	history_str_free(buf);
	return (NULL);
	}

	buf[HIS_MAX_RECORD_LEN -1] = '\0';

	return (buf);
	}

	/*
	* Return non-zero if the spa version is less than requested version.
	*/
	static int
	zfs_earlier_version(const char *name, int version)
	{
	spa_t *spa;

	if (spa_open(name, &spa, FTAG) == 0) {
	if (spa_version(spa) < version) {
	spa_close(spa, FTAG);
	return (1);
	}
	spa_close(spa, FTAG);
	}
	return (0);
	}

	/*
	* Return TRUE if the ZPL version is less than requested version.
	*/
	static boolean_t
	zpl_earlier_version(const char *name, int version)
	{
	objset_t *os;
	boolean_t rc = B_TRUE;

	if (dmu_objset_hold(name, FTAG, &os) == 0) {
	uint64_t zplversion;

	if (dmu_objset_type(os) != DMU_OST_ZFS) {
	dmu_objset_rele(os, FTAG);
	return (B_TRUE);
	}
	/* XXX reading from non-owned objset */
	if (zfs_get_zplprop(os, ZFS_PROP_VERSION, &zplversion) == 0)
	rc = zplversion < version;
	dmu_objset_rele(os, FTAG);
	}
	return (rc);
	}

	static void
	zfs_log_history(zfs_cmd_t *zc)
	{
	spa_t *spa;
	char *buf;

	if ((buf = history_str_get(zc)) == NULL)
	return;

	if (spa_open(zc->zc_name, &spa, FTAG) == 0) {
	if (spa_version(spa) >= SPA_VERSION_ZPOOL_HISTORY)
	(void) spa_history_log(spa, buf);
	spa_close(spa, FTAG);
	}
	history_str_free(buf);
	}

	/*
	* Policy for top-level read operations (list pools). Requires no privileges,
	* and can be used in the local zone, as there is no associated dataset.
	*/
	static int
	zfs_secpolicy_none(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) zc, (void) innvl, (void) cr;
	return (0);
	}

	/*
	* Policy for dataset read operations (list children, get statistics). Requires
	* no privileges, but must be visible in the local zone.
	*/
	static int
	zfs_secpolicy_read(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) innvl, (void) cr;
	if (INGLOBALZONE(curproc) \|\|
	zone_dataset_visible(zc->zc_name, NULL))
	return (0);

	return (SET_ERROR(ENOENT));
	}

	static int
	zfs_dozonecheck_impl(const char dataset, uint64_t zoned, cred_t cr)
	{
	int writable = 1;

	/*
	* The dataset must be visible by this zone -- check this first
	* so they don't see EPERM on something they shouldn't know about.
	*/
	if (!INGLOBALZONE(curproc) &&
	!zone_dataset_visible(dataset, &writable))
	return (SET_ERROR(ENOENT));

	if (INGLOBALZONE(curproc)) {
	/*
	* If the fs is zoned, only root can access it from the
	* global zone.
	*/
	if (secpolicy_zfs(cr) && zoned)
	return (SET_ERROR(EPERM));
	} else {
	/*
	* If we are in a local zone, the 'zoned' property must be set.
	*/
	if (!zoned)
	return (SET_ERROR(EPERM));

	/* must be writable by this zone */
	if (!writable)
	return (SET_ERROR(EPERM));
	}
	return (0);
	}

	static int
	zfs_dozonecheck(const char dataset, cred_t cr)
	{
	uint64_t zoned;

	if (dsl_prop_get_integer(dataset, zfs_prop_to_name(ZFS_PROP_ZONED),
	&zoned, NULL))
	return (SET_ERROR(ENOENT));

	return (zfs_dozonecheck_impl(dataset, zoned, cr));
	}

	static int
	zfs_dozonecheck_ds(const char dataset, dsl_dataset_t ds, cred_t *cr)
	{
	uint64_t zoned;

	if (dsl_prop_get_int_ds(ds, zfs_prop_to_name(ZFS_PROP_ZONED), &zoned))
	return (SET_ERROR(ENOENT));

	return (zfs_dozonecheck_impl(dataset, zoned, cr));
	}

	static int
	zfs_secpolicy_write_perms_ds(const char name, dsl_dataset_t ds,
	const char perm, cred_t cr)
	{
	int error;

	error = zfs_dozonecheck_ds(name, ds, cr);
	if (error == 0) {
	error = secpolicy_zfs(cr);
	if (error != 0)
	error = dsl_deleg_access_impl(ds, perm, cr);
	}
	return (error);
	}

	static int
	zfs_secpolicy_write_perms(const char name, const char perm, cred_t *cr)
	{
	int error;
	dsl_dataset_t *ds;
	dsl_pool_t *dp;

	/*
	* First do a quick check for root in the global zone, which
	* is allowed to do all write_perms. This ensures that zfs_ioc_*
	* will get to handle nonexistent datasets.
	*/
	if (INGLOBALZONE(curproc) && secpolicy_zfs(cr) == 0)
	return (0);

	error = dsl_pool_hold(name, FTAG, &dp);
	if (error != 0)
	return (error);

	error = dsl_dataset_hold(dp, name, FTAG, &ds);
	if (error != 0) {
	dsl_pool_rele(dp, FTAG);
	return (error);
	}

	error = zfs_secpolicy_write_perms_ds(name, ds, perm, cr);

	dsl_dataset_rele(ds, FTAG);
	dsl_pool_rele(dp, FTAG);
	return (error);
	}

	/*
	* Policy for setting the security label property.
	*
	* Returns 0 for success, non-zero for access and other errors.
	*/
	static int
	zfs_set_slabel_policy(const char name, const char strval, cred_t *cr)
	{
	#ifdef HAVE_MLSLABEL
	char ds_hexsl[MAXNAMELEN];
	bslabel_t ds_sl, new_sl;
	boolean_t new_default = FALSE;
	uint64_t zoned;
	int needed_priv = -1;
	int error;

	/* First get the existing dataset label. */
	error = dsl_prop_get(name, zfs_prop_to_name(ZFS_PROP_MLSLABEL),
	1, sizeof (ds_hexsl), &ds_hexsl, NULL);
	if (error != 0)
	return (SET_ERROR(EPERM));

	if (strcasecmp(strval, ZFS_MLSLABEL_DEFAULT) == 0)
	new_default = TRUE;

	/* The label must be translatable */
	if (!new_default && (hexstr_to_label(strval, &new_sl) != 0))
	return (SET_ERROR(EINVAL));

	/*
	* In a non-global zone, disallow attempts to set a label that
	* doesn't match that of the zone; otherwise no other checks
	* are needed.
	*/
	if (!INGLOBALZONE(curproc)) {
	if (new_default \|\| !blequal(&new_sl, CR_SL(CRED())))
	return (SET_ERROR(EPERM));
	return (0);
	}

	/*
	* For global-zone datasets (i.e., those whose zoned property is
	* "off", verify that the specified new label is valid for the
	* global zone.
	*/
	if (dsl_prop_get_integer(name,
	zfs_prop_to_name(ZFS_PROP_ZONED), &zoned, NULL))
	return (SET_ERROR(EPERM));
	if (!zoned) {
	if (zfs_check_global_label(name, strval) != 0)
	return (SET_ERROR(EPERM));
	}

	/*
	* If the existing dataset label is nondefault, check if the
	* dataset is mounted (label cannot be changed while mounted).
	* Get the zfsvfs_t; if there isn't one, then the dataset isn't
	* mounted (or isn't a dataset, doesn't exist, ...).
	*/
	if (strcasecmp(ds_hexsl, ZFS_MLSLABEL_DEFAULT) != 0) {
	objset_t *os;
	static const char *setsl_tag = "setsl_tag";

	/*
	* Try to own the dataset; abort if there is any error,
	* (e.g., already mounted, in use, or other error).
	*/
	error = dmu_objset_own(name, DMU_OST_ZFS, B_TRUE, B_TRUE,
	setsl_tag, &os);
	if (error != 0)
	return (SET_ERROR(EPERM));

	dmu_objset_disown(os, B_TRUE, setsl_tag);

	if (new_default) {
	needed_priv = PRIV_FILE_DOWNGRADE_SL;
	goto out_check;
	}

	if (hexstr_to_label(strval, &new_sl) != 0)
	return (SET_ERROR(EPERM));

	if (blstrictdom(&ds_sl, &new_sl))
	needed_priv = PRIV_FILE_DOWNGRADE_SL;
	else if (blstrictdom(&new_sl, &ds_sl))
	needed_priv = PRIV_FILE_UPGRADE_SL;
	} else {
	/* dataset currently has a default label */
	if (!new_default)
	needed_priv = PRIV_FILE_UPGRADE_SL;
	}

	out_check:
	if (needed_priv != -1)
	return (PRIV_POLICY(cr, needed_priv, B_FALSE, EPERM, NULL));
	return (0);
	#else
	return (SET_ERROR(ENOTSUP));
	#endif /* HAVE_MLSLABEL */
	}

	static int
	zfs_secpolicy_setprop(const char dsname, zfs_prop_t prop, nvpair_t propval,
	cred_t *cr)
	{
	const char *strval;

	/*
	* Check permissions for special properties.
	*/
	switch (prop) {
	default:
	break;
	case ZFS_PROP_ZONED:
	/*
	* Disallow setting of 'zoned' from within a local zone.
	*/
	if (!INGLOBALZONE(curproc))
	return (SET_ERROR(EPERM));
	break;

	case ZFS_PROP_QUOTA:
	case ZFS_PROP_FILESYSTEM_LIMIT:
	case ZFS_PROP_SNAPSHOT_LIMIT:
	if (!INGLOBALZONE(curproc)) {
	uint64_t zoned;
	char setpoint[ZFS_MAX_DATASET_NAME_LEN];
	/*
	* Unprivileged users are allowed to modify the
	* limit on things under (ie. contained by)
	* the thing they own.
	*/
	if (dsl_prop_get_integer(dsname,
	zfs_prop_to_name(ZFS_PROP_ZONED), &zoned, setpoint))
	return (SET_ERROR(EPERM));
	if (!zoned \|\| strlen(dsname) <= strlen(setpoint))
	return (SET_ERROR(EPERM));
	}
	break;

	case ZFS_PROP_MLSLABEL:
	if (!is_system_labeled())
	return (SET_ERROR(EPERM));

	if (nvpair_value_string(propval, &strval) == 0) {
	int err;

	err = zfs_set_slabel_policy(dsname, strval, CRED());
	if (err != 0)
	return (err);
	}
	break;
	}

	return (zfs_secpolicy_write_perms(dsname, zfs_prop_to_name(prop), cr));
	}

	static int
	zfs_secpolicy_set_fsacl(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	/*
	* permission to set permissions will be evaluated later in
	* dsl_deleg_can_allow()
	*/
	(void) innvl;
	return (zfs_dozonecheck(zc->zc_name, cr));
	}

	static int
	zfs_secpolicy_rollback(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) innvl;
	return (zfs_secpolicy_write_perms(zc->zc_name,
	ZFS_DELEG_PERM_ROLLBACK, cr));
	}

	static int
	zfs_secpolicy_send(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) innvl;
	dsl_pool_t *dp;
	dsl_dataset_t *ds;
	const char *cp;
	int error;

	/*
	* Generate the current snapshot name from the given objsetid, then
	* use that name for the secpolicy/zone checks.
	*/
	cp = strchr(zc->zc_name, '@');
	if (cp == NULL)
	return (SET_ERROR(EINVAL));
	error = dsl_pool_hold(zc->zc_name, FTAG, &dp);
	if (error != 0)
	return (error);

	error = dsl_dataset_hold_obj(dp, zc->zc_sendobj, FTAG, &ds);
	if (error != 0) {
	dsl_pool_rele(dp, FTAG);
	return (error);
	}

	dsl_dataset_name(ds, zc->zc_name);

	error = zfs_secpolicy_write_perms_ds(zc->zc_name, ds,
	ZFS_DELEG_PERM_SEND, cr);
	dsl_dataset_rele(ds, FTAG);
	dsl_pool_rele(dp, FTAG);

	return (error);
	}

	static int
	zfs_secpolicy_send_new(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) innvl;
	return (zfs_secpolicy_write_perms(zc->zc_name,
	ZFS_DELEG_PERM_SEND, cr));
	}

	static int
	zfs_secpolicy_share(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) zc, (void) innvl, (void) cr;
	return (SET_ERROR(ENOTSUP));
	}

	static int
	zfs_secpolicy_smb_acl(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) zc, (void) innvl, (void) cr;
	return (SET_ERROR(ENOTSUP));
	}

	static int
	zfs_get_parent(const char datasetname, char parent, int parentsize)
	{
	char *cp;

	/*
	* Remove the @bla or /bla from the end of the name to get the parent.
	*/
	(void) strlcpy(parent, datasetname, parentsize);
	cp = strrchr(parent, '@');
	if (cp != NULL) {
	cp[0] = '\0';
	} else {
	cp = strrchr(parent, '/');
	if (cp == NULL)
	return (SET_ERROR(ENOENT));
	cp[0] = '\0';
	}

	return (0);
	}

	int
	zfs_secpolicy_destroy_perms(const char name, cred_t cr)
	{
	int error;

	if ((error = zfs_secpolicy_write_perms(name,
	ZFS_DELEG_PERM_MOUNT, cr)) != 0)
	return (error);

	return (zfs_secpolicy_write_perms(name, ZFS_DELEG_PERM_DESTROY, cr));
	}

	static int
	zfs_secpolicy_destroy(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) innvl;
	return (zfs_secpolicy_destroy_perms(zc->zc_name, cr));
	}

	/*
	* Destroying snapshots with delegated permissions requires
	* descendant mount and destroy permissions.
	*/
	static int
	zfs_secpolicy_destroy_snaps(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) zc;
	nvlist_t *snaps;
	nvpair_t pair, nextpair;
	int error = 0;

	snaps = fnvlist_lookup_nvlist(innvl, "snaps");

	for (pair = nvlist_next_nvpair(snaps, NULL); pair != NULL;
	pair = nextpair) {
	nextpair = nvlist_next_nvpair(snaps, pair);
	error = zfs_secpolicy_destroy_perms(nvpair_name(pair), cr);
	if (error == ENOENT) {
	/*
	* Ignore any snapshots that don't exist (we consider
	* them "already destroyed"). Remove the name from the
	* nvl here in case the snapshot is created between
	* now and when we try to destroy it (in which case
	* we don't want to destroy it since we haven't
	* checked for permission).
	*/
	fnvlist_remove_nvpair(snaps, pair);
	error = 0;
	}
	if (error != 0)
	break;
	}

	return (error);
	}

	int
	zfs_secpolicy_rename_perms(const char from, const char to, cred_t *cr)
	{
	char parentname[ZFS_MAX_DATASET_NAME_LEN];
	int error;

	if ((error = zfs_secpolicy_write_perms(from,
	ZFS_DELEG_PERM_RENAME, cr)) != 0)
	return (error);

	if ((error = zfs_secpolicy_write_perms(from,
	ZFS_DELEG_PERM_MOUNT, cr)) != 0)
	return (error);

	if ((error = zfs_get_parent(to, parentname,
	sizeof (parentname))) != 0)
	return (error);

	if ((error = zfs_secpolicy_write_perms(parentname,
	ZFS_DELEG_PERM_CREATE, cr)) != 0)
	return (error);

	if ((error = zfs_secpolicy_write_perms(parentname,
	ZFS_DELEG_PERM_MOUNT, cr)) != 0)
	return (error);

	return (error);
	}

	static int
	zfs_secpolicy_rename(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) innvl;
	return (zfs_secpolicy_rename_perms(zc->zc_name, zc->zc_value, cr));
	}

	static int
	zfs_secpolicy_promote(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) innvl;
	dsl_pool_t *dp;
	dsl_dataset_t *clone;
	int error;

	error = zfs_secpolicy_write_perms(zc->zc_name,
	ZFS_DELEG_PERM_PROMOTE, cr);
	if (error != 0)
	return (error);

	error = dsl_pool_hold(zc->zc_name, FTAG, &dp);
	if (error != 0)
	return (error);

	error = dsl_dataset_hold(dp, zc->zc_name, FTAG, &clone);

	if (error == 0) {
	char parentname[ZFS_MAX_DATASET_NAME_LEN];
	dsl_dataset_t *origin = NULL;
	dsl_dir_t *dd;
	dd = clone->ds_dir;

	error = dsl_dataset_hold_obj(dd->dd_pool,
	dsl_dir_phys(dd)->dd_origin_obj, FTAG, &origin);
	if (error != 0) {
	dsl_dataset_rele(clone, FTAG);
	dsl_pool_rele(dp, FTAG);
	return (error);
	}

	error = zfs_secpolicy_write_perms_ds(zc->zc_name, clone,
	ZFS_DELEG_PERM_MOUNT, cr);

	dsl_dataset_name(origin, parentname);
	if (error == 0) {
	error = zfs_secpolicy_write_perms_ds(parentname, origin,
	ZFS_DELEG_PERM_PROMOTE, cr);
	}
	dsl_dataset_rele(clone, FTAG);
	dsl_dataset_rele(origin, FTAG);
	}
	dsl_pool_rele(dp, FTAG);
	return (error);
	}

	static int
	zfs_secpolicy_recv(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) innvl;
	int error;

	if ((error = zfs_secpolicy_write_perms(zc->zc_name,
	ZFS_DELEG_PERM_RECEIVE, cr)) != 0)
	return (error);

	if ((error = zfs_secpolicy_write_perms(zc->zc_name,
	ZFS_DELEG_PERM_MOUNT, cr)) != 0)
	return (error);

	return (zfs_secpolicy_write_perms(zc->zc_name,
	ZFS_DELEG_PERM_CREATE, cr));
	}

	int
	zfs_secpolicy_snapshot_perms(const char name, cred_t cr)
	{
	return (zfs_secpolicy_write_perms(name,
	ZFS_DELEG_PERM_SNAPSHOT, cr));
	}

	/*
	* Check for permission to create each snapshot in the nvlist.
	*/
	static int
	zfs_secpolicy_snapshot(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) zc;
	nvlist_t *snaps;
	int error = 0;
	nvpair_t *pair;

	snaps = fnvlist_lookup_nvlist(innvl, "snaps");

	for (pair = nvlist_next_nvpair(snaps, NULL); pair != NULL;
	pair = nvlist_next_nvpair(snaps, pair)) {
	char name = (char )nvpair_name(pair);
	char *atp = strchr(name, '@');

	if (atp == NULL) {
	error = SET_ERROR(EINVAL);
	break;
	}
	*atp = '\0';
	error = zfs_secpolicy_snapshot_perms(name, cr);
	*atp = '@';
	if (error != 0)
	break;
	}
	return (error);
	}

	/*
	* Check for permission to create each bookmark in the nvlist.
	*/
	static int
	zfs_secpolicy_bookmark(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) zc;
	int error = 0;

	for (nvpair_t *pair = nvlist_next_nvpair(innvl, NULL);
	pair != NULL; pair = nvlist_next_nvpair(innvl, pair)) {
	char name = (char )nvpair_name(pair);
	char *hashp = strchr(name, '#');

	if (hashp == NULL) {
	error = SET_ERROR(EINVAL);
	break;
	}
	*hashp = '\0';
	error = zfs_secpolicy_write_perms(name,
	ZFS_DELEG_PERM_BOOKMARK, cr);
	*hashp = '#';
	if (error != 0)
	break;
	}
	return (error);
	}

	static int
	zfs_secpolicy_destroy_bookmarks(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) zc;
	nvpair_t pair, nextpair;
	int error = 0;

	for (pair = nvlist_next_nvpair(innvl, NULL); pair != NULL;
	pair = nextpair) {
	char name = (char )nvpair_name(pair);
	char *hashp = strchr(name, '#');
	nextpair = nvlist_next_nvpair(innvl, pair);

	if (hashp == NULL) {
	error = SET_ERROR(EINVAL);
	break;
	}

	*hashp = '\0';
	error = zfs_secpolicy_write_perms(name,
	ZFS_DELEG_PERM_DESTROY, cr);
	*hashp = '#';
	if (error == ENOENT) {
	/*
	* Ignore any filesystems that don't exist (we consider
	* their bookmarks "already destroyed"). Remove
	* the name from the nvl here in case the filesystem
	* is created between now and when we try to destroy
	* the bookmark (in which case we don't want to
	* destroy it since we haven't checked for permission).
	*/
	fnvlist_remove_nvpair(innvl, pair);
	error = 0;
	}
	if (error != 0)
	break;
	}

	return (error);
	}

	static int
	zfs_secpolicy_log_history(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) zc, (void) innvl, (void) cr;
	/*
	* Even root must have a proper TSD so that we know what pool
	* to log to.
	*/
	if (tsd_get(zfs_allow_log_key) == NULL)
	return (SET_ERROR(EPERM));
	return (0);
	}

	static int
	zfs_secpolicy_create_clone(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	char parentname[ZFS_MAX_DATASET_NAME_LEN];
	int error;
	const char *origin;

	if ((error = zfs_get_parent(zc->zc_name, parentname,
	sizeof (parentname))) != 0)
	return (error);

	if (nvlist_lookup_string(innvl, "origin", &origin) == 0 &&
	(error = zfs_secpolicy_write_perms(origin,
	ZFS_DELEG_PERM_CLONE, cr)) != 0)
	return (error);

	if ((error = zfs_secpolicy_write_perms(parentname,
	ZFS_DELEG_PERM_CREATE, cr)) != 0)
	return (error);

	return (zfs_secpolicy_write_perms(parentname,
	ZFS_DELEG_PERM_MOUNT, cr));
	}

	/*
	* Policy for pool operations - create/destroy pools, add vdevs, etc. Requires
	* SYS_CONFIG privilege, which is not available in a local zone.
	*/
	int
	zfs_secpolicy_config(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) zc, (void) innvl;

	if (secpolicy_sys_config(cr, B_FALSE) != 0)
	return (SET_ERROR(EPERM));

	return (0);
	}

	/*
	* Policy for object to name lookups.
	*/
	static int
	zfs_secpolicy_diff(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) innvl;
	int error;

	if (secpolicy_sys_config(cr, B_FALSE) == 0)
	return (0);

	error = zfs_secpolicy_write_perms(zc->zc_name, ZFS_DELEG_PERM_DIFF, cr);
	return (error);
	}

	/*
	* Policy for fault injection. Requires all privileges.
	*/
	static int
	zfs_secpolicy_inject(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) zc, (void) innvl;
	return (secpolicy_zinject(cr));
	}

	static int
	zfs_secpolicy_inherit_prop(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) innvl;
	zfs_prop_t prop = zfs_name_to_prop(zc->zc_value);

	if (prop == ZPROP_USERPROP) {
	if (!zfs_prop_user(zc->zc_value))
	return (SET_ERROR(EINVAL));
	return (zfs_secpolicy_write_perms(zc->zc_name,
	ZFS_DELEG_PERM_USERPROP, cr));
	} else {
	return (zfs_secpolicy_setprop(zc->zc_name, prop,
	NULL, cr));
	}
	}

	static int
	zfs_secpolicy_userspace_one(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	int err = zfs_secpolicy_read(zc, innvl, cr);
	if (err)
	return (err);

	if (zc->zc_objset_type >= ZFS_NUM_USERQUOTA_PROPS)
	return (SET_ERROR(EINVAL));

	if (zc->zc_value[0] == 0) {
	/*
	* They are asking about a posix uid/gid. If it's
	* themself, allow it.
	*/
	if (zc->zc_objset_type == ZFS_PROP_USERUSED \|\|
	zc->zc_objset_type == ZFS_PROP_USERQUOTA \|\|
	zc->zc_objset_type == ZFS_PROP_USEROBJUSED \|\|
	zc->zc_objset_type == ZFS_PROP_USEROBJQUOTA) {
	if (zc->zc_guid == crgetuid(cr))
	return (0);
	} else if (zc->zc_objset_type == ZFS_PROP_GROUPUSED \|\|
	zc->zc_objset_type == ZFS_PROP_GROUPQUOTA \|\|
	zc->zc_objset_type == ZFS_PROP_GROUPOBJUSED \|\|
	zc->zc_objset_type == ZFS_PROP_GROUPOBJQUOTA) {
	if (groupmember(zc->zc_guid, cr))
	return (0);
	}
	/* else is for project quota/used */
	}

	return (zfs_secpolicy_write_perms(zc->zc_name,
	userquota_perms[zc->zc_objset_type], cr));
	}

	static int
	zfs_secpolicy_userspace_many(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	int err = zfs_secpolicy_read(zc, innvl, cr);
	if (err)
	return (err);

	if (zc->zc_objset_type >= ZFS_NUM_USERQUOTA_PROPS)
	return (SET_ERROR(EINVAL));

	return (zfs_secpolicy_write_perms(zc->zc_name,
	userquota_perms[zc->zc_objset_type], cr));
	}

	static int
	zfs_secpolicy_userspace_upgrade(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) innvl;
	return (zfs_secpolicy_setprop(zc->zc_name, ZFS_PROP_VERSION,
	NULL, cr));
	}

	static int
	zfs_secpolicy_hold(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) zc;
	nvpair_t *pair;
	nvlist_t *holds;
	int error;

	holds = fnvlist_lookup_nvlist(innvl, "holds");

	for (pair = nvlist_next_nvpair(holds, NULL); pair != NULL;
	pair = nvlist_next_nvpair(holds, pair)) {
	char fsname[ZFS_MAX_DATASET_NAME_LEN];
	error = dmu_fsname(nvpair_name(pair), fsname);
	if (error != 0)
	return (error);
	error = zfs_secpolicy_write_perms(fsname,
	ZFS_DELEG_PERM_HOLD, cr);
	if (error != 0)
	return (error);
	}
	return (0);
	}

	static int
	zfs_secpolicy_release(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	(void) zc;
	nvpair_t *pair;
	int error;

	for (pair = nvlist_next_nvpair(innvl, NULL); pair != NULL;
	pair = nvlist_next_nvpair(innvl, pair)) {
	char fsname[ZFS_MAX_DATASET_NAME_LEN];
	error = dmu_fsname(nvpair_name(pair), fsname);
	if (error != 0)
	return (error);
	error = zfs_secpolicy_write_perms(fsname,
	ZFS_DELEG_PERM_RELEASE, cr);
	if (error != 0)
	return (error);
	}
	return (0);
	}

	/*
	* Policy for allowing temporary snapshots to be taken or released
	*/
	static int
	zfs_secpolicy_tmp_snapshot(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	/*
	* A temporary snapshot is the same as a snapshot,
	* hold, destroy and release all rolled into one.
	* Delegated diff alone is sufficient that we allow this.
	*/
	int error;

	if (zfs_secpolicy_write_perms(zc->zc_name,
	ZFS_DELEG_PERM_DIFF, cr) == 0)
	return (0);

	error = zfs_secpolicy_snapshot_perms(zc->zc_name, cr);

	if (innvl != NULL) {
	if (error == 0)
	error = zfs_secpolicy_hold(zc, innvl, cr);
	if (error == 0)
	error = zfs_secpolicy_release(zc, innvl, cr);
	if (error == 0)
	error = zfs_secpolicy_destroy(zc, innvl, cr);
	}
	return (error);
	}

	static int
	zfs_secpolicy_load_key(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	return (zfs_secpolicy_write_perms(zc->zc_name,
	ZFS_DELEG_PERM_LOAD_KEY, cr));
	}

	static int
	zfs_secpolicy_change_key(zfs_cmd_t zc, nvlist_t innvl, cred_t *cr)
	{
	return (zfs_secpolicy_write_perms(zc->zc_name,
	ZFS_DELEG_PERM_CHANGE_KEY, cr));
	}

	/*
	* Returns the nvlist as specified by the user in the zfs_cmd_t.
	*/
	static int
	get_nvlist(uint64_t nvl, uint64_t size, int iflag, nvlist_t **nvp)
	{
	char *packed;
	int error;
	nvlist_t *list = NULL;

	/*
	* Read in and unpack the user-supplied nvlist.
	*/
	if (size == 0)
	return (SET_ERROR(EINVAL));

	packed = vmem_alloc(size, KM_SLEEP);

	if (ddi_copyin((void *)(uintptr_t)nvl, packed, size, iflag) != 0) {
	vmem_free(packed, size);
	return (SET_ERROR(EFAULT));
	}

	if ((error = nvlist_unpack(packed, size, &list, 0)) != 0) {
	vmem_free(packed, size);
	return (error);
	}

	vmem_free(packed, size);

	*nvp = list;
	return (0);
	}

	/*
	* Reduce the size of this nvlist until it can be serialized in 'max' bytes.
	* Entries will be removed from the end of the nvlist, and one int32 entry
	* named "N_MORE_ERRORS" will be added indicating how many entries were
	* removed.
	*/
	static int
	nvlist_smush(nvlist_t *errors, size_t max)
	{
	size_t size;

	size = fnvlist_size(errors);

	if (size > max) {
	nvpair_t *more_errors;
	int n = 0;

	if (max < 1024)
	return (SET_ERROR(ENOMEM));

	fnvlist_add_int32(errors, ZPROP_N_MORE_ERRORS, 0);
	more_errors = nvlist_prev_nvpair(errors, NULL);

	do {
	nvpair_t *pair = nvlist_prev_nvpair(errors,
	more_errors);
	fnvlist_remove_nvpair(errors, pair);
	n++;
	size = fnvlist_size(errors);
	} while (size > max);

	fnvlist_remove_nvpair(errors, more_errors);
	fnvlist_add_int32(errors, ZPROP_N_MORE_ERRORS, n);
	ASSERT3U(fnvlist_size(errors), <=, max);
	}

	return (0);
	}

	static int
	put_nvlist(zfs_cmd_t zc, nvlist_t nvl)
	{
	char *packed = NULL;
	int error = 0;
	size_t size;

	size = fnvlist_size(nvl);

	if (size > zc->zc_nvlist_dst_size) {
	error = SET_ERROR(ENOMEM);
	} else {
	packed = fnvlist_pack(nvl, &size);
	if (ddi_copyout(packed, (void *)(uintptr_t)zc->zc_nvlist_dst,
	size, zc->zc_iflags) != 0)
	error = SET_ERROR(EFAULT);
	fnvlist_pack_free(packed, size);
	}

	zc->zc_nvlist_dst_size = size;
	zc->zc_nvlist_dst_filled = B_TRUE;
	return (error);
	}

	int
	getzfsvfs_impl(objset_t os, zfsvfs_t *zfvp)
	{
	int error = 0;
	if (dmu_objset_type(os) != DMU_OST_ZFS) {
	return (SET_ERROR(EINVAL));
	}

	mutex_enter(&os->os_user_ptr_lock);
	*zfvp = dmu_objset_get_user(os);
	/* bump s_active only when non-zero to prevent umount race */
	error = zfs_vfs_ref(zfvp);
	mutex_exit(&os->os_user_ptr_lock);
	return (error);
	}

	int
	getzfsvfs(const char dsname, zfsvfs_t *zfvp)
	{
	objset_t *os;
	int error;

	error = dmu_objset_hold(dsname, FTAG, &os);
	if (error != 0)
	return (error);

	error = getzfsvfs_impl(os, zfvp);
	dmu_objset_rele(os, FTAG);
	return (error);
	}

	/*
	* Find a zfsvfs_t for a mounted filesystem, or create our own, in which
	* case its z_sb will be NULL, and it will be opened as the owner.
	* If 'writer' is set, the z_teardown_lock will be held for RW_WRITER,
	* which prevents all inode ops from running.
	*/
	static int
	zfsvfs_hold(const char name, const void tag, zfsvfs_t **zfvp,
	boolean_t writer)
	{
	int error = 0;

	if (getzfsvfs(name, zfvp) != 0)
	error = zfsvfs_create(name, B_FALSE, zfvp);
	if (error == 0) {
	if (writer)
	ZFS_TEARDOWN_ENTER_WRITE(*zfvp, tag);
	else
	ZFS_TEARDOWN_ENTER_READ(*zfvp, tag);
	if ((*zfvp)->z_unmounted) {
	/*
	* XXX we could probably try again, since the unmounting
	* thread should be just about to disassociate the
	* objset from the zfsvfs.
	*/
	ZFS_TEARDOWN_EXIT(*zfvp, tag);
	return (SET_ERROR(EBUSY));
	}
	}
	return (error);
	}

	static void
	zfsvfs_rele(zfsvfs_t zfsvfs, const void tag)
	{
	ZFS_TEARDOWN_EXIT(zfsvfs, tag);

	if (zfs_vfs_held(zfsvfs)) {
	zfs_vfs_rele(zfsvfs);
	} else {
	dmu_objset_disown(zfsvfs->z_os, B_TRUE, zfsvfs);
	zfsvfs_free(zfsvfs);
	}
	}

	static int
	zfs_ioc_pool_create(zfs_cmd_t *zc)
	{
	int error;
	nvlist_t config, props = NULL;
	nvlist_t *rootprops = NULL;
	nvlist_t *zplprops = NULL;
	dsl_crypto_params_t *dcp = NULL;
	const char *spa_name = zc->zc_name;
	boolean_t unload_wkey = B_TRUE;

	if ((error = get_nvlist(zc->zc_nvlist_conf, zc->zc_nvlist_conf_size,
	zc->zc_iflags, &config)))
	return (error);

	if (zc->zc_nvlist_src_size != 0 && (error =
	get_nvlist(zc->zc_nvlist_src, zc->zc_nvlist_src_size,
	zc->zc_iflags, &props))) {
	nvlist_free(config);
	return (error);
	}

	if (props) {
	nvlist_t *nvl = NULL;
	nvlist_t *hidden_args = NULL;
	uint64_t version = SPA_VERSION;
	const char *tname;

	(void) nvlist_lookup_uint64(props,
	zpool_prop_to_name(ZPOOL_PROP_VERSION), &version);
	if (!SPA_VERSION_IS_SUPPORTED(version)) {
	error = SET_ERROR(EINVAL);
	goto pool_props_bad;
	}
	(void) nvlist_lookup_nvlist(props, ZPOOL_ROOTFS_PROPS, &nvl);
	if (nvl) {
	error = nvlist_dup(nvl, &rootprops, KM_SLEEP);
	if (error != 0)
	goto pool_props_bad;
	(void) nvlist_remove_all(props, ZPOOL_ROOTFS_PROPS);
	}

	(void) nvlist_lookup_nvlist(props, ZPOOL_HIDDEN_ARGS,
	&hidden_args);
	error = dsl_crypto_params_create_nvlist(DCP_CMD_NONE,
	rootprops, hidden_args, &dcp);
	if (error != 0)
	goto pool_props_bad;
	(void) nvlist_remove_all(props, ZPOOL_HIDDEN_ARGS);

	VERIFY(nvlist_alloc(&zplprops, NV_UNIQUE_NAME, KM_SLEEP) == 0);
	error = zfs_fill_zplprops_root(version, rootprops,
	zplprops, NULL);
	if (error != 0)
	goto pool_props_bad;

	if (nvlist_lookup_string(props,
	zpool_prop_to_name(ZPOOL_PROP_TNAME), &tname) == 0)
	spa_name = tname;
	}

	error = spa_create(zc->zc_name, config, props, zplprops, dcp);

	/*
	* Set the remaining root properties
	*/
	if (!error && (error = zfs_set_prop_nvlist(spa_name,
	ZPROP_SRC_LOCAL, rootprops, NULL)) != 0) {
	(void) spa_destroy(spa_name);
	unload_wkey = B_FALSE; /* spa_destroy() unloads wrapping keys */
	}

	pool_props_bad:
	nvlist_free(rootprops);
	nvlist_free(zplprops);
	nvlist_free(config);
	nvlist_free(props);
	dsl_crypto_params_free(dcp, unload_wkey && !!error);

	return (error);
	}

	static int
	zfs_ioc_pool_destroy(zfs_cmd_t *zc)
	{
	int error;
	zfs_log_history(zc);
	error = spa_destroy(zc->zc_name);

	return (error);
	}

	static int
	zfs_ioc_pool_import(zfs_cmd_t *zc)
	{
	nvlist_t config, props = NULL;
	uint64_t guid;
	int error;

	if ((error = get_nvlist(zc->zc_nvlist_conf, zc->zc_nvlist_conf_size,
	zc->zc_iflags, &config)) != 0)
	return (error);

	if (zc->zc_nvlist_src_size != 0 && (error =
	get_nvlist(zc->zc_nvlist_src, zc->zc_nvlist_src_size,
	zc->zc_iflags, &props))) {
	nvlist_free(config);
	return (error);
	}

	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &guid) != 0 \|\|
	guid != zc->zc_guid)
	error = SET_ERROR(EINVAL);
	else
	error = spa_import(zc->zc_name, config, props, zc->zc_cookie);

	if (zc->zc_nvlist_dst != 0) {
	int err;

	if ((err = put_nvlist(zc, config)) != 0)
	error = err;
	}

	nvlist_free(config);
	nvlist_free(props);

	return (error);
	}

	static int
	zfs_ioc_pool_export(zfs_cmd_t *zc)
	{
	int error;
	boolean_t force = (boolean_t)zc->zc_cookie;
	boolean_t hardforce = (boolean_t)zc->zc_guid;

	zfs_log_history(zc);
	error = spa_export(zc->zc_name, NULL, force, hardforce);

	return (error);
	}

	static int
	zfs_ioc_pool_configs(zfs_cmd_t *zc)
	{
	nvlist_t *configs;
	int error;

	error = spa_all_configs(&zc->zc_cookie, &configs);
	if (error)
	return (error);

	error = put_nvlist(zc, configs);

	nvlist_free(configs);

	return (error);
	}

	/*
	* inputs:
	* zc_name name of the pool
	*
	* outputs:
	* zc_cookie real errno
	* zc_nvlist_dst config nvlist
	* zc_nvlist_dst_size size of config nvlist
	*/
	static int
	zfs_ioc_pool_stats(zfs_cmd_t *zc)
	{
	nvlist_t *config;
	int error;
	int ret = 0;

	error = spa_get_stats(zc->zc_name, &config, zc->zc_value,
	sizeof (zc->zc_value));

	if (config != NULL) {
	ret = put_nvlist(zc, config);
	nvlist_free(config);

	/*
	* The config may be present even if 'error' is non-zero.
	* In this case we return success, and preserve the real errno
	* in 'zc_cookie'.
	*/
	zc->zc_cookie = error;
	} else {
	ret = error;
	}

	return (ret);
	}

	/*
	* Try to import the given pool, returning pool stats as appropriate so that
	* user land knows which devices are available and overall pool health.
	*/
	static int
	zfs_ioc_pool_tryimport(zfs_cmd_t *zc)
	{
	nvlist_t tryconfig, config = NULL;
	int error;

	if ((error = get_nvlist(zc->zc_nvlist_conf, zc->zc_nvlist_conf_size,
	zc->zc_iflags, &tryconfig)) != 0)
	return (error);

	config = spa_tryimport(tryconfig);

	nvlist_free(tryconfig);

	if (config == NULL)
	return (SET_ERROR(EINVAL));

	error = put_nvlist(zc, config);
	nvlist_free(config);

	return (error);
	}

	/*
	* inputs:
	* zc_name name of the pool
	* zc_cookie scan func (pool_scan_func_t)
	* zc_flags scrub pause/resume flag (pool_scrub_cmd_t)
	*/
	static int
	zfs_ioc_pool_scan(zfs_cmd_t *zc)
	{
	spa_t *spa;
	int error;

	if (zc->zc_flags >= POOL_SCRUB_FLAGS_END)
	return (SET_ERROR(EINVAL));

	if ((error = spa_open(zc->zc_name, &spa, FTAG)) != 0)
	return (error);

	if (zc->zc_flags == POOL_SCRUB_PAUSE)
	error = spa_scrub_pause_resume(spa, POOL_SCRUB_PAUSE);
	else if (zc->zc_cookie == POOL_SCAN_NONE)
	error = spa_scan_stop(spa);
	else
	error = spa_scan(spa, zc->zc_cookie);

	spa_close(spa, FTAG);

	return (error);
	}

	/*
	* inputs:
	* poolname name of the pool
	* scan_type scan func (pool_scan_func_t)
	* scan_command scrub pause/resume flag (pool_scrub_cmd_t)
	*/
	static const zfs_ioc_key_t zfs_keys_pool_scrub[] = {
	{"scan_type", DATA_TYPE_UINT64, 0},
	{"scan_command", DATA_TYPE_UINT64, 0},
	};

	static int
	zfs_ioc_pool_scrub(const char poolname, nvlist_t innvl, nvlist_t *outnvl)
	{
	spa_t *spa;
	int error;
	uint64_t scan_type, scan_cmd;

	if (nvlist_lookup_uint64(innvl, "scan_type", &scan_type) != 0)
	return (SET_ERROR(EINVAL));
	if (nvlist_lookup_uint64(innvl, "scan_command", &scan_cmd) != 0)
	return (SET_ERROR(EINVAL));

	if (scan_cmd >= POOL_SCRUB_FLAGS_END)
	return (SET_ERROR(EINVAL));

	if ((error = spa_open(poolname, &spa, FTAG)) != 0)
	return (error);

	if (scan_cmd == POOL_SCRUB_PAUSE) {
	error = spa_scrub_pause_resume(spa, POOL_SCRUB_PAUSE);
	} else if (scan_type == POOL_SCAN_NONE) {
	error = spa_scan_stop(spa);
	} else {
	error = spa_scan(spa, scan_type);
	}

	spa_close(spa, FTAG);
	return (error);
	}

	static int
	zfs_ioc_pool_freeze(zfs_cmd_t *zc)
	{
	spa_t *spa;
	int error;

	error = spa_open(zc->zc_name, &spa, FTAG);
	if (error == 0) {
	spa_freeze(spa);
	spa_close(spa, FTAG);
	}
	return (error);
	}

	static int
	zfs_ioc_pool_upgrade(zfs_cmd_t *zc)
	{
	spa_t *spa;
	int error;

	if ((error = spa_open(zc->zc_name, &spa, FTAG)) != 0)
	return (error);

	if (zc->zc_cookie < spa_version(spa) \|\|
	!SPA_VERSION_IS_SUPPORTED(zc->zc_cookie)) {
	spa_close(spa, FTAG);
	return (SET_ERROR(EINVAL));
	}

	spa_upgrade(spa, zc->zc_cookie);
	spa_close(spa, FTAG);

	return (error);
	}

	static int
	zfs_ioc_pool_get_history(zfs_cmd_t *zc)
	{
	spa_t *spa;
	char *hist_buf;
	uint64_t size;
	int error;

	if ((size = zc->zc_history_len) == 0)
	return (SET_ERROR(EINVAL));

	if ((error = spa_open(zc->zc_name, &spa, FTAG)) != 0)
	return (error);

	if (spa_version(spa) < SPA_VERSION_ZPOOL_HISTORY) {
	spa_close(spa, FTAG);
	return (SET_ERROR(ENOTSUP));
	}

	hist_buf = vmem_alloc(size, KM_SLEEP);
	if ((error = spa_history_get(spa, &zc->zc_history_offset,
	&zc->zc_history_len, hist_buf)) == 0) {
	error = ddi_copyout(hist_buf,
	(void *)(uintptr_t)zc->zc_history,
	zc->zc_history_len, zc->zc_iflags);
	}

	spa_close(spa, FTAG);
	vmem_free(hist_buf, size);
	return (error);
	}

	static int
	zfs_ioc_pool_reguid(zfs_cmd_t *zc)
	{
	spa_t *spa;
	int error;

	error = spa_open(zc->zc_name, &spa, FTAG);
	if (error == 0) {
	error = spa_change_guid(spa);
	spa_close(spa, FTAG);
	}
	return (error);
	}

	static int
	zfs_ioc_dsobj_to_dsname(zfs_cmd_t *zc)
	{
	return (dsl_dsobj_to_dsname(zc->zc_name, zc->zc_obj, zc->zc_value));
	}

	/*
	* inputs:
	* zc_name name of filesystem
	* zc_obj object to find
	*
	* outputs:
	* zc_value name of object
	*/
	static int
	zfs_ioc_obj_to_path(zfs_cmd_t *zc)
	{
	objset_t *os;
	int error;

	/* XXX reading from objset not owned */
	if ((error = dmu_objset_hold_flags(zc->zc_name, B_TRUE,
	FTAG, &os)) != 0)
	return (error);
	if (dmu_objset_type(os) != DMU_OST_ZFS) {
	dmu_objset_rele_flags(os, B_TRUE, FTAG);
	return (SET_ERROR(EINVAL));
	}
	error = zfs_obj_to_path(os, zc->zc_obj, zc->zc_value,
	sizeof (zc->zc_value));
	dmu_objset_rele_flags(os, B_TRUE, FTAG);

	return (error);
	}

	/*
	* inputs:
	* zc_name name of filesystem
	* zc_obj object to find
	*
	* outputs:
	* zc_stat stats on object
	* zc_value path to object
	*/
	static int
	zfs_ioc_obj_to_stats(zfs_cmd_t *zc)
	{
	objset_t *os;
	int error;

	/* XXX reading from objset not owned */
	if ((error = dmu_objset_hold_flags(zc->zc_name, B_TRUE,
	FTAG, &os)) != 0)
	return (error);
	if (dmu_objset_type(os) != DMU_OST_ZFS) {
	dmu_objset_rele_flags(os, B_TRUE, FTAG);
	return (SET_ERROR(EINVAL));
	}
	error = zfs_obj_to_stats(os, zc->zc_obj, &zc->zc_stat, zc->zc_value,
	sizeof (zc->zc_value));
	dmu_objset_rele_flags(os, B_TRUE, FTAG);

	return (error);
	}

	static int
	zfs_ioc_vdev_add(zfs_cmd_t *zc)
	{
	spa_t *spa;
	int error;
	nvlist_t *config;

	error = spa_open(zc->zc_name, &spa, FTAG);
	if (error != 0)
	return (error);

	error = get_nvlist(zc->zc_nvlist_conf, zc->zc_nvlist_conf_size,
	zc->zc_iflags, &config);
	if (error == 0) {
	- error = spa_vdev_add(spa, config);
	+ error = spa_vdev_add(spa, config, zc->zc_flags);
	nvlist_free(config);
	}
	spa_close(spa, FTAG);
	return (error);
	}

	/*
	* inputs:
	* zc_name name of the pool
	* zc_guid guid of vdev to remove
	* zc_cookie cancel removal
	*/
	static int
	zfs_ioc_vdev_remove(zfs_cmd_t *zc)
	{
	spa_t *spa;
	int error;

	error = spa_open(zc->zc_name, &spa, FTAG);
	if (error != 0)
	return (error);
	if (zc->zc_cookie != 0) {
	error = spa_vdev_remove_cancel(spa);
	} else {
	error = spa_vdev_remove(spa, zc->zc_guid, B_FALSE);
	}
	spa_close(spa, FTAG);
	return (error);
	}

	static int
	zfs_ioc_vdev_set_state(zfs_cmd_t *zc)
	{
	spa_t *spa;
	int error;
	vdev_state_t newstate = VDEV_STATE_UNKNOWN;

	if ((error = spa_open(zc->zc_name, &spa, FTAG)) != 0)
	return (error);
	switch (zc->zc_cookie) {
	case VDEV_STATE_ONLINE:
	error = vdev_online(spa, zc->zc_guid, zc->zc_obj, &newstate);
	break;

	case VDEV_STATE_OFFLINE:
	error = vdev_offline(spa, zc->zc_guid, zc->zc_obj);
	break;

	case VDEV_STATE_FAULTED:
	if (zc->zc_obj != VDEV_AUX_ERR_EXCEEDED &&
	zc->zc_obj != VDEV_AUX_EXTERNAL &&
	zc->zc_obj != VDEV_AUX_EXTERNAL_PERSIST)
	zc->zc_obj = VDEV_AUX_ERR_EXCEEDED;

	error = vdev_fault(spa, zc->zc_guid, zc->zc_obj);
	break;

	case VDEV_STATE_DEGRADED:
	if (zc->zc_obj != VDEV_AUX_ERR_EXCEEDED &&
	zc->zc_obj != VDEV_AUX_EXTERNAL)
	zc->zc_obj = VDEV_AUX_ERR_EXCEEDED;

	error = vdev_degrade(spa, zc->zc_guid, zc->zc_obj);
	break;

	case VDEV_STATE_REMOVED:
	error = vdev_remove_wanted(spa, zc->zc_guid);
	break;

	default:
	error = SET_ERROR(EINVAL);
	}
	zc->zc_cookie = newstate;
	spa_close(spa, FTAG);
	return (error);
	}

	static int
	zfs_ioc_vdev_attach(zfs_cmd_t *zc)
	{
	spa_t *spa;
	nvlist_t *config;
	int replacing = zc->zc_cookie;
	int rebuild = zc->zc_simple;
	int error;

	if ((error = spa_open(zc->zc_name, &spa, FTAG)) != 0)
	return (error);

	if ((error = get_nvlist(zc->zc_nvlist_conf, zc->zc_nvlist_conf_size,
	zc->zc_iflags, &config)) == 0) {
	error = spa_vdev_attach(spa, zc->zc_guid, config, replacing,
	rebuild);
	nvlist_free(config);
	}

	spa_close(spa, FTAG);
	return (error);
	}

	static int
	zfs_ioc_vdev_detach(zfs_cmd_t *zc)
	{
	spa_t *spa;
	int error;

	if ((error = spa_open(zc->zc_name, &spa, FTAG)) != 0)
	return (error);

	error = spa_vdev_detach(spa, zc->zc_guid, 0, B_FALSE);

	spa_close(spa, FTAG);
	return (error);
	}

	static int
	zfs_ioc_vdev_split(zfs_cmd_t *zc)
	{
	spa_t *spa;
	nvlist_t config, props = NULL;
	int error;
	boolean_t exp = !!(zc->zc_cookie & ZPOOL_EXPORT_AFTER_SPLIT);

	if ((error = spa_open(zc->zc_name, &spa, FTAG)) != 0)
	return (error);

	if ((error = get_nvlist(zc->zc_nvlist_conf, zc->zc_nvlist_conf_size,
	zc->zc_iflags, &config))) {
	spa_close(spa, FTAG);
	return (error);
	}

	if (zc->zc_nvlist_src_size != 0 && (error =
	get_nvlist(zc->zc_nvlist_src, zc->zc_nvlist_src_size,
	zc->zc_iflags, &props))) {
	spa_close(spa, FTAG);
	nvlist_free(config);
	return (error);
	}

	error = spa_vdev_split_mirror(spa, zc->zc_string, config, props, exp);

	spa_close(spa, FTAG);

	nvlist_free(config);
	nvlist_free(props);

	return (error);
	}

	static int
	zfs_ioc_vdev_setpath(zfs_cmd_t *zc)
	{
	spa_t *spa;
	const char *path = zc->zc_value;
	uint64_t guid = zc->zc_guid;
	int error;

	error = spa_open(zc->zc_name, &spa, FTAG);
	if (error != 0)
	return (error);

	error = spa_vdev_setpath(spa, guid, path);
	spa_close(spa, FTAG);
	return (error);
	}

	static int
	zfs_ioc_vdev_setfru(zfs_cmd_t *zc)
	{
	spa_t *spa;
	const char *fru = zc->zc_value;
	uint64_t guid = zc->zc_guid;
	int error;

	error = spa_open(zc->zc_name, &spa, FTAG);
	if (error != 0)
	return (error);

	error = spa_vdev_setfru(spa, guid, fru);
	spa_close(spa, FTAG);
	return (error);
	}

	static int
	zfs_ioc_objset_stats_impl(zfs_cmd_t zc, objset_t os)
	{
	int error = 0;
	nvlist_t *nv;

	dmu_objset_fast_stat(os, &zc->zc_objset_stats);

	if (!zc->zc_simple && zc->zc_nvlist_dst != 0 &&
	(error = dsl_prop_get_all(os, &nv)) == 0) {
	dmu_objset_stats(os, nv);
	/*
	* NB: zvol_get_stats() will read the objset contents,
	* which we aren't supposed to do with a
	* DS_MODE_USER hold, because it could be
	* inconsistent. So this is a bit of a workaround...
	* XXX reading without owning
	*/
	if (!zc->zc_objset_stats.dds_inconsistent &&
	dmu_objset_type(os) == DMU_OST_ZVOL) {
	error = zvol_get_stats(os, nv);
	if (error == EIO) {
	nvlist_free(nv);
	return (error);
	}
	VERIFY0(error);
	}
	if (error == 0)
	error = put_nvlist(zc, nv);
	nvlist_free(nv);
	}

	return (error);
	}

	/*
	* inputs:
	* zc_name name of filesystem
	* zc_nvlist_dst_size size of buffer for property nvlist
	*
	* outputs:
	* zc_objset_stats stats
	* zc_nvlist_dst property nvlist
	* zc_nvlist_dst_size size of property nvlist
	*/
	static int
	zfs_ioc_objset_stats(zfs_cmd_t *zc)
	{
	objset_t *os;
	int error;

	error = dmu_objset_hold(zc->zc_name, FTAG, &os);
	if (error == 0) {
	error = zfs_ioc_objset_stats_impl(zc, os);
	dmu_objset_rele(os, FTAG);
	}

	return (error);
	}

	/*
	* inputs:
	* zc_name name of filesystem
	* zc_nvlist_dst_size size of buffer for property nvlist
	*
	* outputs:
	* zc_nvlist_dst received property nvlist
	* zc_nvlist_dst_size size of received property nvlist
	*
	* Gets received properties (distinct from local properties on or after
	* SPA_VERSION_RECVD_PROPS) for callers who want to differentiate received from
	* local property values.
	*/
	static int
	zfs_ioc_objset_recvd_props(zfs_cmd_t *zc)
	{
	int error = 0;
	nvlist_t *nv;

	/*
	* Without this check, we would return local property values if the
	* caller has not already received properties on or after
	* SPA_VERSION_RECVD_PROPS.
	*/
	if (!dsl_prop_get_hasrecvd(zc->zc_name))
	return (SET_ERROR(ENOTSUP));

	if (zc->zc_nvlist_dst != 0 &&
	(error = dsl_prop_get_received(zc->zc_name, &nv)) == 0) {
	error = put_nvlist(zc, nv);
	nvlist_free(nv);
	}

	return (error);
	}

	static int
	nvl_add_zplprop(objset_t os, nvlist_t props, zfs_prop_t prop)
	{
	uint64_t value;
	int error;

	/*
	* zfs_get_zplprop() will either find a value or give us
	* the default value (if there is one).
	*/
	if ((error = zfs_get_zplprop(os, prop, &value)) != 0)
	return (error);
	VERIFY(nvlist_add_uint64(props, zfs_prop_to_name(prop), value) == 0);
	return (0);
	}

	/*
	* inputs:
	* zc_name name of filesystem
	* zc_nvlist_dst_size size of buffer for zpl property nvlist
	*
	* outputs:
	* zc_nvlist_dst zpl property nvlist
	* zc_nvlist_dst_size size of zpl property nvlist
	*/
	static int
	zfs_ioc_objset_zplprops(zfs_cmd_t *zc)
	{
	objset_t *os;
	int err;

	/* XXX reading without owning */
	if ((err = dmu_objset_hold(zc->zc_name, FTAG, &os)))
	return (err);

	dmu_objset_fast_stat(os, &zc->zc_objset_stats);

	/*
	* NB: nvl_add_zplprop() will read the objset contents,
	* which we aren't supposed to do with a DS_MODE_USER
	* hold, because it could be inconsistent.
	*/
	if (zc->zc_nvlist_dst != 0 &&
	!zc->zc_objset_stats.dds_inconsistent &&
	dmu_objset_type(os) == DMU_OST_ZFS) {
	nvlist_t *nv;

	VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_SLEEP) == 0);
	if ((err = nvl_add_zplprop(os, nv, ZFS_PROP_VERSION)) == 0 &&
	(err = nvl_add_zplprop(os, nv, ZFS_PROP_NORMALIZE)) == 0 &&
	(err = nvl_add_zplprop(os, nv, ZFS_PROP_UTF8ONLY)) == 0 &&
	(err = nvl_add_zplprop(os, nv, ZFS_PROP_CASE)) == 0)
	err = put_nvlist(zc, nv);
	nvlist_free(nv);
	} else {
	err = SET_ERROR(ENOENT);
	}
	dmu_objset_rele(os, FTAG);
	return (err);
	}

	/*
	* inputs:
	* zc_name name of filesystem
	* zc_cookie zap cursor
	* zc_nvlist_dst_size size of buffer for property nvlist
	*
	* outputs:
	* zc_name name of next filesystem
	* zc_cookie zap cursor
	* zc_objset_stats stats
	* zc_nvlist_dst property nvlist
	* zc_nvlist_dst_size size of property nvlist
	*/
	static int
	zfs_ioc_dataset_list_next(zfs_cmd_t *zc)
	{
	objset_t *os;
	int error;
	char *p;
	size_t orig_len = strlen(zc->zc_name);

	top:
	if ((error = dmu_objset_hold(zc->zc_name, FTAG, &os))) {
	if (error == ENOENT)
	error = SET_ERROR(ESRCH);
	return (error);
	}

	p = strrchr(zc->zc_name, '/');
	if (p == NULL \|\| p[1] != '\0')
	(void) strlcat(zc->zc_name, "/", sizeof (zc->zc_name));
	p = zc->zc_name + strlen(zc->zc_name);

	do {
	error = dmu_dir_list_next(os,
	sizeof (zc->zc_name) - (p - zc->zc_name), p,
	NULL, &zc->zc_cookie);
	if (error == ENOENT)
	error = SET_ERROR(ESRCH);
	} while (error == 0 && zfs_dataset_name_hidden(zc->zc_name));
	dmu_objset_rele(os, FTAG);

	/*
	* If it's an internal dataset (ie. with a '$' in its name),
	* don't try to get stats for it, otherwise we'll return ENOENT.
	*/
	if (error == 0 && strchr(zc->zc_name, '$') == NULL) {
	error = zfs_ioc_objset_stats(zc); /* fill in the stats */
	if (error == ENOENT) {
	/* We lost a race with destroy, get the next one. */
	zc->zc_name[orig_len] = '\0';
	goto top;
	}
	}
	return (error);
	}

	/*
	* inputs:
	* zc_name name of filesystem
	* zc_cookie zap cursor
	* zc_nvlist_src iteration range nvlist
	* zc_nvlist_src_size size of iteration range nvlist
	*
	* outputs:
	* zc_name name of next snapshot
	* zc_objset_stats stats
	* zc_nvlist_dst property nvlist
	* zc_nvlist_dst_size size of property nvlist
	*/
	static int
	zfs_ioc_snapshot_list_next(zfs_cmd_t *zc)
	{
	int error;
	objset_t os, ossnap;
	dsl_dataset_t *ds;
	uint64_t min_txg = 0, max_txg = 0;

	if (zc->zc_nvlist_src_size != 0) {
	nvlist_t *props = NULL;
	error = get_nvlist(zc->zc_nvlist_src, zc->zc_nvlist_src_size,
	zc->zc_iflags, &props);
	if (error != 0)
	return (error);
	(void) nvlist_lookup_uint64(props, SNAP_ITER_MIN_TXG,
	&min_txg);
	(void) nvlist_lookup_uint64(props, SNAP_ITER_MAX_TXG,
	&max_txg);
	nvlist_free(props);
	}

	error = dmu_objset_hold(zc->zc_name, FTAG, &os);
	if (error != 0) {
	return (error == ENOENT ? SET_ERROR(ESRCH) : error);
	}

	/*
	* A dataset name of maximum length cannot have any snapshots,
	* so exit immediately.
	*/
	if (strlcat(zc->zc_name, "@", sizeof (zc->zc_name)) >=
	ZFS_MAX_DATASET_NAME_LEN) {
	dmu_objset_rele(os, FTAG);
	return (SET_ERROR(ESRCH));
	}

	while (error == 0) {
	if (issig(JUSTLOOKING) && issig(FORREAL)) {
	error = SET_ERROR(EINTR);
	break;
	}

	error = dmu_snapshot_list_next(os,
	sizeof (zc->zc_name) - strlen(zc->zc_name),
	zc->zc_name + strlen(zc->zc_name), &zc->zc_obj,
	&zc->zc_cookie, NULL);
	if (error == ENOENT) {
	error = SET_ERROR(ESRCH);
	break;
	} else if (error != 0) {
	break;
	}

	error = dsl_dataset_hold_obj(dmu_objset_pool(os), zc->zc_obj,
	FTAG, &ds);
	if (error != 0)
	break;

	if ((min_txg != 0 && dsl_get_creationtxg(ds) < min_txg) \|\|
	(max_txg != 0 && dsl_get_creationtxg(ds) > max_txg)) {
	dsl_dataset_rele(ds, FTAG);
	/* undo snapshot name append */
	*(strchr(zc->zc_name, '@') + 1) = '\0';
	/* skip snapshot */
	continue;
	}

	if (zc->zc_simple) {
	dsl_dataset_fast_stat(ds, &zc->zc_objset_stats);
	dsl_dataset_rele(ds, FTAG);
	break;
	}

	if ((error = dmu_objset_from_ds(ds, &ossnap)) != 0) {
	dsl_dataset_rele(ds, FTAG);
	break;
	}
	if ((error = zfs_ioc_objset_stats_impl(zc, ossnap)) != 0) {
	dsl_dataset_rele(ds, FTAG);
	break;
	}
	dsl_dataset_rele(ds, FTAG);
	break;
	}

	dmu_objset_rele(os, FTAG);
	/* if we failed, undo the @ that we tacked on to zc_name */
	if (error != 0)
	*strchr(zc->zc_name, '@') = '\0';
	return (error);
	}

	static int
	zfs_prop_set_userquota(const char dsname, nvpair_t pair)
	{
	const char *propname = nvpair_name(pair);
	uint64_t *valary;
	unsigned int vallen;
	const char dash, domain;
	zfs_userquota_prop_t type;
	uint64_t rid;
	uint64_t quota;
	zfsvfs_t *zfsvfs;
	int err;

	if (nvpair_type(pair) == DATA_TYPE_NVLIST) {
	nvlist_t *attrs;
	VERIFY(nvpair_value_nvlist(pair, &attrs) == 0);
	if (nvlist_lookup_nvpair(attrs, ZPROP_VALUE,
	&pair) != 0)
	return (SET_ERROR(EINVAL));
	}

	/*
	* A correctly constructed propname is encoded as
	* userquota@<rid>-<domain>.
	*/
	if ((dash = strchr(propname, '-')) == NULL \|\|
	nvpair_value_uint64_array(pair, &valary, &vallen) != 0 \|\|
	vallen != 3)
	return (SET_ERROR(EINVAL));

	domain = dash + 1;
	type = valary[0];
	rid = valary[1];
	quota = valary[2];

	err = zfsvfs_hold(dsname, FTAG, &zfsvfs, B_FALSE);
	if (err == 0) {
	err = zfs_set_userquota(zfsvfs, type, domain, rid, quota);
	zfsvfs_rele(zfsvfs, FTAG);
	}

	return (err);
	}

	/*
	* If the named property is one that has a special function to set its value,
	* return 0 on success and a positive error code on failure; otherwise if it is
	* not one of the special properties handled by this function, return -1.
	*
	* XXX: It would be better for callers of the property interface if we handled
	* these special cases in dsl_prop.c (in the dsl layer).
	*/
	static int
	zfs_prop_set_special(const char *dsname, zprop_source_t source,
	nvpair_t *pair)
	{
	const char *propname = nvpair_name(pair);
	zfs_prop_t prop = zfs_name_to_prop(propname);
	uint64_t intval = 0;
	const char *strval = NULL;
	int err = -1;

	if (prop == ZPROP_USERPROP) {
	if (zfs_prop_userquota(propname))
	return (zfs_prop_set_userquota(dsname, pair));
	return (-1);
	}

	if (nvpair_type(pair) == DATA_TYPE_NVLIST) {
	nvlist_t *attrs;
	VERIFY(nvpair_value_nvlist(pair, &attrs) == 0);
	VERIFY(nvlist_lookup_nvpair(attrs, ZPROP_VALUE,
	&pair) == 0);
	}

	/* all special properties are numeric except for keylocation */
	if (zfs_prop_get_type(prop) == PROP_TYPE_STRING) {
	strval = fnvpair_value_string(pair);
	} else {
	intval = fnvpair_value_uint64(pair);
	}

	switch (prop) {
	case ZFS_PROP_QUOTA:
	err = dsl_dir_set_quota(dsname, source, intval);
	break;
	case ZFS_PROP_REFQUOTA:
	err = dsl_dataset_set_refquota(dsname, source, intval);
	break;
	case ZFS_PROP_FILESYSTEM_LIMIT:
	case ZFS_PROP_SNAPSHOT_LIMIT:
	if (intval == UINT64_MAX) {
	/* clearing the limit, just do it */
	err = 0;
	} else {
	err = dsl_dir_activate_fs_ss_limit(dsname);
	}
	/*
	* Set err to -1 to force the zfs_set_prop_nvlist code down the
	* default path to set the value in the nvlist.
	*/
	if (err == 0)
	err = -1;
	break;
	case ZFS_PROP_KEYLOCATION:
	err = dsl_crypto_can_set_keylocation(dsname, strval);

	/*
	* Set err to -1 to force the zfs_set_prop_nvlist code down the
	* default path to set the value in the nvlist.
	*/
	if (err == 0)
	err = -1;
	break;
	case ZFS_PROP_RESERVATION:
	err = dsl_dir_set_reservation(dsname, source, intval);
	break;
	case ZFS_PROP_REFRESERVATION:
	err = dsl_dataset_set_refreservation(dsname, source, intval);
	break;
	case ZFS_PROP_COMPRESSION:
	err = dsl_dataset_set_compression(dsname, source, intval);
	/*
	* Set err to -1 to force the zfs_set_prop_nvlist code down the
	* default path to set the value in the nvlist.
	*/
	if (err == 0)
	err = -1;
	break;
	case ZFS_PROP_VOLSIZE:
	err = zvol_set_volsize(dsname, intval);
	break;
	case ZFS_PROP_SNAPDEV:
	err = zvol_set_snapdev(dsname, source, intval);
	break;
	case ZFS_PROP_VOLMODE:
	err = zvol_set_volmode(dsname, source, intval);
	break;
	case ZFS_PROP_VERSION:
	{
	zfsvfs_t *zfsvfs;

	if ((err = zfsvfs_hold(dsname, FTAG, &zfsvfs, B_TRUE)) != 0)
	break;

	err = zfs_set_version(zfsvfs, intval);
	zfsvfs_rele(zfsvfs, FTAG);

	if (err == 0 && intval >= ZPL_VERSION_USERSPACE) {
	zfs_cmd_t *zc;

	zc = kmem_zalloc(sizeof (zfs_cmd_t), KM_SLEEP);
	(void) strlcpy(zc->zc_name, dsname,
	sizeof (zc->zc_name));
	(void) zfs_ioc_userspace_upgrade(zc);
	(void) zfs_ioc_id_quota_upgrade(zc);
	kmem_free(zc, sizeof (zfs_cmd_t));
	}
	break;
	}
	default:
	err = -1;
	}

	return (err);
	}

	static boolean_t
	zfs_is_namespace_prop(zfs_prop_t prop)
	{
	switch (prop) {

	case ZFS_PROP_ATIME:
	case ZFS_PROP_RELATIME:
	case ZFS_PROP_DEVICES:
	case ZFS_PROP_EXEC:
	case ZFS_PROP_SETUID:
	case ZFS_PROP_READONLY:
	case ZFS_PROP_XATTR:
	case ZFS_PROP_NBMAND:
	return (B_TRUE);

	default:
	return (B_FALSE);
	}
	}

	/*
	* This function is best effort. If it fails to set any of the given properties,
	* it continues to set as many as it can and returns the last error
	* encountered. If the caller provides a non-NULL errlist, it will be filled in
	* with the list of names of all the properties that failed along with the
	* corresponding error numbers.
	*
	* If every property is set successfully, zero is returned and errlist is not
	* modified.
	*/
	int
	zfs_set_prop_nvlist(const char dsname, zprop_source_t source, nvlist_t nvl,
	nvlist_t *errlist)
	{
	nvpair_t *pair;
	nvpair_t *propval;
	int rv = 0;
	int err;
	uint64_t intval;
	const char *strval;
	boolean_t should_update_mount_cache = B_FALSE;

	nvlist_t *genericnvl = fnvlist_alloc();
	nvlist_t *retrynvl = fnvlist_alloc();
	retry:
	pair = NULL;
	while ((pair = nvlist_next_nvpair(nvl, pair)) != NULL) {
	const char *propname = nvpair_name(pair);
	zfs_prop_t prop = zfs_name_to_prop(propname);
	err = 0;

	/* decode the property value */
	propval = pair;
	if (nvpair_type(pair) == DATA_TYPE_NVLIST) {
	nvlist_t *attrs;
	attrs = fnvpair_value_nvlist(pair);
	if (nvlist_lookup_nvpair(attrs, ZPROP_VALUE,
	&propval) != 0)
	err = SET_ERROR(EINVAL);
	}

	/* Validate value type */
	if (err == 0 && source == ZPROP_SRC_INHERITED) {
	/* inherited properties are expected to be booleans */
	if (nvpair_type(propval) != DATA_TYPE_BOOLEAN)
	err = SET_ERROR(EINVAL);
	} else if (err == 0 && prop == ZPROP_USERPROP) {
	if (zfs_prop_user(propname)) {
	if (nvpair_type(propval) != DATA_TYPE_STRING)
	err = SET_ERROR(EINVAL);
	} else if (zfs_prop_userquota(propname)) {
	if (nvpair_type(propval) !=
	DATA_TYPE_UINT64_ARRAY)
	err = SET_ERROR(EINVAL);
	} else {
	err = SET_ERROR(EINVAL);
	}
	} else if (err == 0) {
	if (nvpair_type(propval) == DATA_TYPE_STRING) {
	if (zfs_prop_get_type(prop) != PROP_TYPE_STRING)
	err = SET_ERROR(EINVAL);
	} else if (nvpair_type(propval) == DATA_TYPE_UINT64) {
	const char *unused;

	intval = fnvpair_value_uint64(propval);

	switch (zfs_prop_get_type(prop)) {
	case PROP_TYPE_NUMBER:
	break;
	case PROP_TYPE_STRING:
	err = SET_ERROR(EINVAL);
	break;
	case PROP_TYPE_INDEX:
	if (zfs_prop_index_to_string(prop,
	intval, &unused) != 0)
	err =
	SET_ERROR(ZFS_ERR_BADPROP);
	break;
	default:
	cmn_err(CE_PANIC,
	"unknown property type");
	}
	} else {
	err = SET_ERROR(EINVAL);
	}
	}

	/* Validate permissions */
	if (err == 0)
	err = zfs_check_settable(dsname, pair, CRED());

	if (err == 0) {
	if (source == ZPROP_SRC_INHERITED)
	err = -1; /* does not need special handling */
	else
	err = zfs_prop_set_special(dsname, source,
	pair);
	if (err == -1) {
	/*
	* For better performance we build up a list of
	* properties to set in a single transaction.
	*/
	err = nvlist_add_nvpair(genericnvl, pair);
	} else if (err != 0 && nvl != retrynvl) {
	/*
	* This may be a spurious error caused by
	* receiving quota and reservation out of order.
	* Try again in a second pass.
	*/
	err = nvlist_add_nvpair(retrynvl, pair);
	}
	}

	if (err != 0) {
	if (errlist != NULL)
	fnvlist_add_int32(errlist, propname, err);
	rv = err;
	}

	if (zfs_is_namespace_prop(prop))
	should_update_mount_cache = B_TRUE;
	}

	if (nvl != retrynvl && !nvlist_empty(retrynvl)) {
	nvl = retrynvl;
	goto retry;
	}

	if (nvlist_empty(genericnvl))
	goto out;

	/*
	* Try to set them all in one batch.
	*/
	err = dsl_props_set(dsname, source, genericnvl);
	if (err == 0)
	goto out;

	/*
	* If batching fails, we still want to set as many properties as we
	* can, so try setting them individually.
	*/
	pair = NULL;
	while ((pair = nvlist_next_nvpair(genericnvl, pair)) != NULL) {
	const char *propname = nvpair_name(pair);

	propval = pair;
	if (nvpair_type(pair) == DATA_TYPE_NVLIST) {
	nvlist_t *attrs;
	attrs = fnvpair_value_nvlist(pair);
	propval = fnvlist_lookup_nvpair(attrs, ZPROP_VALUE);
	}

	if (nvpair_type(propval) == DATA_TYPE_STRING) {
	strval = fnvpair_value_string(propval);
	err = dsl_prop_set_string(dsname, propname,
	source, strval);
	} else if (nvpair_type(propval) == DATA_TYPE_BOOLEAN) {
	err = dsl_prop_inherit(dsname, propname, source);
	} else {
	intval = fnvpair_value_uint64(propval);
	err = dsl_prop_set_int(dsname, propname, source,
	intval);
	}

	if (err != 0) {
	if (errlist != NULL) {
	fnvlist_add_int32(errlist, propname, err);
	}
	rv = err;
	}
	}

	out:
	if (should_update_mount_cache)
	zfs_ioctl_update_mount_cache(dsname);

	nvlist_free(genericnvl);
	nvlist_free(retrynvl);

	return (rv);
	}

	/*
	* Check that all the properties are valid user properties.
	*/
	static int
	zfs_check_userprops(nvlist_t *nvl)
	{
	nvpair_t *pair = NULL;

	while ((pair = nvlist_next_nvpair(nvl, pair)) != NULL) {
	const char *propname = nvpair_name(pair);

	if (!zfs_prop_user(propname) \|\|
	nvpair_type(pair) != DATA_TYPE_STRING)
	return (SET_ERROR(EINVAL));

	if (strlen(propname) >= ZAP_MAXNAMELEN)
	return (SET_ERROR(ENAMETOOLONG));

	if (strlen(fnvpair_value_string(pair)) >= ZAP_MAXVALUELEN)
	return (SET_ERROR(E2BIG));
	}
	return (0);
	}

	static void
	props_skip(nvlist_t props, nvlist_t skipped, nvlist_t **newprops)
	{
	nvpair_t *pair;

	VERIFY(nvlist_alloc(newprops, NV_UNIQUE_NAME, KM_SLEEP) == 0);

	pair = NULL;
	while ((pair = nvlist_next_nvpair(props, pair)) != NULL) {
	if (nvlist_exists(skipped, nvpair_name(pair)))
	continue;

	VERIFY(nvlist_add_nvpair(*newprops, pair) == 0);
	}
	}

	static int
	clear_received_props(const char dsname, nvlist_t props,
	nvlist_t *skipped)
	{
	int err = 0;
	nvlist_t *cleared_props = NULL;
	props_skip(props, skipped, &cleared_props);
	if (!nvlist_empty(cleared_props)) {
	/*
	* Acts on local properties until the dataset has received
	* properties at least once on or after SPA_VERSION_RECVD_PROPS.
	*/
	zprop_source_t flags = (ZPROP_SRC_NONE \|
	(dsl_prop_get_hasrecvd(dsname) ? ZPROP_SRC_RECEIVED : 0));
	err = zfs_set_prop_nvlist(dsname, flags, cleared_props, NULL);
	}
	nvlist_free(cleared_props);
	return (err);
	}

	/*
	* inputs:
	* zc_name name of filesystem
	* zc_value name of property to set
	* zc_nvlist_src{_size} nvlist of properties to apply
	* zc_cookie received properties flag
	*
	* outputs:
	* zc_nvlist_dst{_size} error for each unapplied received property
	*/
	static int
	zfs_ioc_set_prop(zfs_cmd_t *zc)
	{
	nvlist_t *nvl;
	boolean_t received = zc->zc_cookie;
	zprop_source_t source = (received ? ZPROP_SRC_RECEIVED :
	ZPROP_SRC_LOCAL);
	nvlist_t *errors;
	int error;

	if ((error = get_nvlist(zc->zc_nvlist_src, zc->zc_nvlist_src_size,
	zc->zc_iflags, &nvl)) != 0)
	return (error);

	if (received) {
	nvlist_t *origprops;

	if (dsl_prop_get_received(zc->zc_name, &origprops) == 0) {
	(void) clear_received_props(zc->zc_name,
	origprops, nvl);
	nvlist_free(origprops);
	}

	error = dsl_prop_set_hasrecvd(zc->zc_name);
	}

	errors = fnvlist_alloc();
	if (error == 0)
	error = zfs_set_prop_nvlist(zc->zc_name, source, nvl, errors);

	if (zc->zc_nvlist_dst != 0 && errors != NULL) {
	(void) put_nvlist(zc, errors);
	}

	nvlist_free(errors);
	nvlist_free(nvl);
	return (error);
	}

	/*
	* inputs:
	* zc_name name of filesystem
	* zc_value name of property to inherit
	* zc_cookie revert to received value if TRUE
	*
	* outputs: none
	*/
	static int
	zfs_ioc_inherit_prop(zfs_cmd_t *zc)
	{
	const char *propname = zc->zc_value;
	zfs_prop_t prop = zfs_name_to_prop(propname);
	boolean_t received = zc->zc_cookie;
	zprop_source_t source = (received
	? ZPROP_SRC_NONE /* revert to received value, if any */
	: ZPROP_SRC_INHERITED); /* explicitly inherit */
	nvlist_t *dummy;
	nvpair_t *pair;
	zprop_type_t type;
	int err;

	if (!received) {
	/*
	* Only check this in the non-received case. We want to allow
	* 'inherit -S' to revert non-inheritable properties like quota
	* and reservation to the received or default values even though
	* they are not considered inheritable.
	*/
	if (prop != ZPROP_USERPROP && !zfs_prop_inheritable(prop))
	return (SET_ERROR(EINVAL));
	}

	if (prop == ZPROP_USERPROP) {
	if (!zfs_prop_user(propname))
	return (SET_ERROR(EINVAL));

	type = PROP_TYPE_STRING;
	} else if (prop == ZFS_PROP_VOLSIZE \|\| prop == ZFS_PROP_VERSION) {
	return (SET_ERROR(EINVAL));
	} else {
	type = zfs_prop_get_type(prop);
	}

	/*
	* zfs_prop_set_special() expects properties in the form of an
	* nvpair with type info.
	*/
	dummy = fnvlist_alloc();

	switch (type) {
	case PROP_TYPE_STRING:
	VERIFY(0 == nvlist_add_string(dummy, propname, ""));
	break;
	case PROP_TYPE_NUMBER:
	case PROP_TYPE_INDEX:
	VERIFY(0 == nvlist_add_uint64(dummy, propname, 0));
	break;
	default:
	err = SET_ERROR(EINVAL);
	goto errout;
	}

	pair = nvlist_next_nvpair(dummy, NULL);
	if (pair == NULL) {
	err = SET_ERROR(EINVAL);
	} else {
	err = zfs_prop_set_special(zc->zc_name, source, pair);
	if (err == -1) /* property is not "special", needs handling */
	err = dsl_prop_inherit(zc->zc_name, zc->zc_value,
	source);
	}

	errout:
	nvlist_free(dummy);
	return (err);
	}

	static int
	zfs_ioc_pool_set_props(zfs_cmd_t *zc)
	{
	nvlist_t *props;
	spa_t *spa;
	int error;
	nvpair_t *pair;

	if ((error = get_nvlist(zc->zc_nvlist_src, zc->zc_nvlist_src_size,
	zc->zc_iflags, &props)))
	return (error);

	/*
	* If the only property is the configfile, then just do a spa_lookup()
	* to handle the faulted case.
	*/
	pair = nvlist_next_nvpair(props, NULL);
	if (pair != NULL && strcmp(nvpair_name(pair),
	zpool_prop_to_name(ZPOOL_PROP_CACHEFILE)) == 0 &&
	nvlist_next_nvpair(props, pair) == NULL) {
	mutex_enter(&spa_namespace_lock);
	if ((spa = spa_lookup(zc->zc_name)) != NULL) {
	spa_configfile_set(spa, props, B_FALSE);
	spa_write_cachefile(spa, B_FALSE, B_TRUE, B_FALSE);
	}
	mutex_exit(&spa_namespace_lock);
	if (spa != NULL) {
	nvlist_free(props);
	return (0);
	}
	}

	if ((error = spa_open(zc->zc_name, &spa, FTAG)) != 0) {
	nvlist_free(props);
	return (error);
	}

	error = spa_prop_set(spa, props);

	nvlist_free(props);
	spa_close(spa, FTAG);

	return (error);
	}

	static int
	zfs_ioc_pool_get_props(zfs_cmd_t *zc)
	{
	spa_t *spa;
	int error;
	nvlist_t *nvp = NULL;

	if ((error = spa_open(zc->zc_name, &spa, FTAG)) != 0) {
	/*
	* If the pool is faulted, there may be properties we can still
	* get (such as altroot and cachefile), so attempt to get them
	* anyway.
	*/
	mutex_enter(&spa_namespace_lock);
	if ((spa = spa_lookup(zc->zc_name)) != NULL)
	error = spa_prop_get(spa, &nvp);
	mutex_exit(&spa_namespace_lock);
	} else {
	error = spa_prop_get(spa, &nvp);
	spa_close(spa, FTAG);
	}

	if (error == 0 && zc->zc_nvlist_dst != 0)
	error = put_nvlist(zc, nvp);
	else
	error = SET_ERROR(EFAULT);

	nvlist_free(nvp);
	return (error);
	}

	/*
	* innvl: {
	* "vdevprops_set_vdev" -> guid
	* "vdevprops_set_props" -> { prop -> value }
	* }
	*
	* outnvl: propname -> error code (int32)
	*/
	static const zfs_ioc_key_t zfs_keys_vdev_set_props[] = {
	{ZPOOL_VDEV_PROPS_SET_VDEV, DATA_TYPE_UINT64, 0},
	{ZPOOL_VDEV_PROPS_SET_PROPS, DATA_TYPE_NVLIST, 0}
	};

	static int
	zfs_ioc_vdev_set_props(const char poolname, nvlist_t innvl, nvlist_t *outnvl)
	{
	spa_t *spa;
	int error;
	vdev_t *vd;
	uint64_t vdev_guid;

	/* Early validation */
	if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_PROPS_SET_VDEV,
	&vdev_guid) != 0)
	return (SET_ERROR(EINVAL));

	if (outnvl == NULL)
	return (SET_ERROR(EINVAL));

	if ((error = spa_open(poolname, &spa, FTAG)) != 0)
	return (error);

	ASSERT(spa_writeable(spa));

	if ((vd = spa_lookup_by_guid(spa, vdev_guid, B_TRUE)) == NULL) {
	spa_close(spa, FTAG);
	return (SET_ERROR(ENOENT));
	}

	error = vdev_prop_set(vd, innvl, outnvl);

	spa_close(spa, FTAG);

	return (error);
	}

	/*
	* innvl: {
	* "vdevprops_get_vdev" -> guid
	* (optional) "vdevprops_get_props" -> { propname -> propid }
	* }
	*
	* outnvl: propname -> value
	*/
	static const zfs_ioc_key_t zfs_keys_vdev_get_props[] = {
	{ZPOOL_VDEV_PROPS_GET_VDEV, DATA_TYPE_UINT64, 0},
	{ZPOOL_VDEV_PROPS_GET_PROPS, DATA_TYPE_NVLIST, ZK_OPTIONAL}
	};

	static int
	zfs_ioc_vdev_get_props(const char poolname, nvlist_t innvl, nvlist_t *outnvl)
	{
	spa_t *spa;
	int error;
	vdev_t *vd;
	uint64_t vdev_guid;

	/* Early validation */
	if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_PROPS_GET_VDEV,
	&vdev_guid) != 0)
	return (SET_ERROR(EINVAL));

	if (outnvl == NULL)
	return (SET_ERROR(EINVAL));

	if ((error = spa_open(poolname, &spa, FTAG)) != 0)
	return (error);

	if ((vd = spa_lookup_by_guid(spa, vdev_guid, B_TRUE)) == NULL) {
	spa_close(spa, FTAG);
	return (SET_ERROR(ENOENT));
	}

	error = vdev_prop_get(vd, innvl, outnvl);

	spa_close(spa, FTAG);

	return (error);
	}

	/*
	* inputs:
	* zc_name name of filesystem
	* zc_nvlist_src{_size} nvlist of delegated permissions
	* zc_perm_action allow/unallow flag
	*
	* outputs: none
	*/
	static int
	zfs_ioc_set_fsacl(zfs_cmd_t *zc)
	{
	int error;
	nvlist_t *fsaclnv = NULL;

	if ((error = get_nvlist(zc->zc_nvlist_src, zc->zc_nvlist_src_size,
	zc->zc_iflags, &fsaclnv)) != 0)
	return (error);

	/*
	* Verify nvlist is constructed correctly
	*/
	if (zfs_deleg_verify_nvlist(fsaclnv) != 0) {
	nvlist_free(fsaclnv);
	return (SET_ERROR(EINVAL));
	}

	/*
	* If we don't have PRIV_SYS_MOUNT, then validate
	* that user is allowed to hand out each permission in
	* the nvlist(s)
	*/

	error = secpolicy_zfs(CRED());
	if (error != 0) {
	if (zc->zc_perm_action == B_FALSE) {
	error = dsl_deleg_can_allow(zc->zc_name,
	fsaclnv, CRED());
	} else {
	error = dsl_deleg_can_unallow(zc->zc_name,
	fsaclnv, CRED());
	}
	}

	if (error == 0)
	error = dsl_deleg_set(zc->zc_name, fsaclnv, zc->zc_perm_action);

	nvlist_free(fsaclnv);
	return (error);
	}

	/*
	* inputs:
	* zc_name name of filesystem
	*
	* outputs:
	* zc_nvlist_src{_size} nvlist of delegated permissions
	*/
	static int
	zfs_ioc_get_fsacl(zfs_cmd_t *zc)
	{
	nvlist_t *nvp;
	int error;

	if ((error = dsl_deleg_get(zc->zc_name, &nvp)) == 0) {
	error = put_nvlist(zc, nvp);
	nvlist_free(nvp);
	}

	return (error);
	}

	static void
	zfs_create_cb(objset_t os, void arg, cred_t cr, dmu_tx_t tx)
	{
	zfs_creat_t *zct = arg;

	zfs_create_fs(os, cr, zct->zct_zplprops, tx);
	}

	#define ZFS_PROP_UNDEFINED ((uint64_t)-1)

	/*
	* inputs:
	* os parent objset pointer (NULL if root fs)
	* fuids_ok fuids allowed in this version of the spa?
	* sa_ok SAs allowed in this version of the spa?
	* createprops list of properties requested by creator
	*
	* outputs:
	* zplprops values for the zplprops we attach to the master node object
	* is_ci true if requested file system will be purely case-insensitive
	*
	* Determine the settings for utf8only, normalization and
	* casesensitivity. Specific values may have been requested by the
	* creator and/or we can inherit values from the parent dataset. If
	* the file system is of too early a vintage, a creator can not
	* request settings for these properties, even if the requested
	* setting is the default value. We don't actually want to create dsl
	* properties for these, so remove them from the source nvlist after
	* processing.
	*/
	static int
	zfs_fill_zplprops_impl(objset_t *os, uint64_t zplver,
	boolean_t fuids_ok, boolean_t sa_ok, nvlist_t *createprops,
	nvlist_t zplprops, boolean_t is_ci)
	{
	uint64_t sense = ZFS_PROP_UNDEFINED;
	uint64_t norm = ZFS_PROP_UNDEFINED;
	uint64_t u8 = ZFS_PROP_UNDEFINED;
	int error;

	ASSERT(zplprops != NULL);

	/* parent dataset must be a filesystem */
	if (os != NULL && os->os_phys->os_type != DMU_OST_ZFS)
	return (SET_ERROR(ZFS_ERR_WRONG_PARENT));

	/*
	* Pull out creator prop choices, if any.
	*/
	if (createprops) {
	(void) nvlist_lookup_uint64(createprops,
	zfs_prop_to_name(ZFS_PROP_VERSION), &zplver);
	(void) nvlist_lookup_uint64(createprops,
	zfs_prop_to_name(ZFS_PROP_NORMALIZE), &norm);
	(void) nvlist_remove_all(createprops,
	zfs_prop_to_name(ZFS_PROP_NORMALIZE));
	(void) nvlist_lookup_uint64(createprops,
	zfs_prop_to_name(ZFS_PROP_UTF8ONLY), &u8);
	(void) nvlist_remove_all(createprops,
	zfs_prop_to_name(ZFS_PROP_UTF8ONLY));
	(void) nvlist_lookup_uint64(createprops,
	zfs_prop_to_name(ZFS_PROP_CASE), &sense);
	(void) nvlist_remove_all(createprops,
	zfs_prop_to_name(ZFS_PROP_CASE));
	}

	/*
	* If the zpl version requested is whacky or the file system
	* or pool is version is too "young" to support normalization
	* and the creator tried to set a value for one of the props,
	* error out.
	*/
	if ((zplver < ZPL_VERSION_INITIAL \|\| zplver > ZPL_VERSION) \|\|
	(zplver >= ZPL_VERSION_FUID && !fuids_ok) \|\|
	(zplver >= ZPL_VERSION_SA && !sa_ok) \|\|
	(zplver < ZPL_VERSION_NORMALIZATION &&
	(norm != ZFS_PROP_UNDEFINED \|\| u8 != ZFS_PROP_UNDEFINED \|\|
	sense != ZFS_PROP_UNDEFINED)))
	return (SET_ERROR(ENOTSUP));

	/*
	* Put the version in the zplprops
	*/
	VERIFY(nvlist_add_uint64(zplprops,
	zfs_prop_to_name(ZFS_PROP_VERSION), zplver) == 0);

	if (norm == ZFS_PROP_UNDEFINED &&
	(error = zfs_get_zplprop(os, ZFS_PROP_NORMALIZE, &norm)) != 0)
	return (error);
	VERIFY(nvlist_add_uint64(zplprops,
	zfs_prop_to_name(ZFS_PROP_NORMALIZE), norm) == 0);

	/*
	* If we're normalizing, names must always be valid UTF-8 strings.
	*/
	if (norm)
	u8 = 1;
	if (u8 == ZFS_PROP_UNDEFINED &&
	(error = zfs_get_zplprop(os, ZFS_PROP_UTF8ONLY, &u8)) != 0)
	return (error);
	VERIFY(nvlist_add_uint64(zplprops,
	zfs_prop_to_name(ZFS_PROP_UTF8ONLY), u8) == 0);

	if (sense == ZFS_PROP_UNDEFINED &&
	(error = zfs_get_zplprop(os, ZFS_PROP_CASE, &sense)) != 0)
	return (error);
	VERIFY(nvlist_add_uint64(zplprops,
	zfs_prop_to_name(ZFS_PROP_CASE), sense) == 0);

	if (is_ci)
	*is_ci = (sense == ZFS_CASE_INSENSITIVE);

	return (0);
	}

	static int
	zfs_fill_zplprops(const char dataset, nvlist_t createprops,
	nvlist_t zplprops, boolean_t is_ci)
	{
	boolean_t fuids_ok, sa_ok;
	uint64_t zplver = ZPL_VERSION;
	objset_t *os = NULL;
	char parentname[ZFS_MAX_DATASET_NAME_LEN];
	spa_t *spa;
	uint64_t spa_vers;
	int error;

	zfs_get_parent(dataset, parentname, sizeof (parentname));

	if ((error = spa_open(dataset, &spa, FTAG)) != 0)
	return (error);

	spa_vers = spa_version(spa);
	spa_close(spa, FTAG);

	zplver = zfs_zpl_version_map(spa_vers);
	fuids_ok = (zplver >= ZPL_VERSION_FUID);
	sa_ok = (zplver >= ZPL_VERSION_SA);

	/*
	* Open parent object set so we can inherit zplprop values.
	*/
	if ((error = dmu_objset_hold(parentname, FTAG, &os)) != 0)
	return (error);

	error = zfs_fill_zplprops_impl(os, zplver, fuids_ok, sa_ok, createprops,
	zplprops, is_ci);
	dmu_objset_rele(os, FTAG);
	return (error);
	}

	static int
	zfs_fill_zplprops_root(uint64_t spa_vers, nvlist_t *createprops,
	nvlist_t zplprops, boolean_t is_ci)
	{
	boolean_t fuids_ok;
	boolean_t sa_ok;
	uint64_t zplver = ZPL_VERSION;
	int error;

	zplver = zfs_zpl_version_map(spa_vers);
	fuids_ok = (zplver >= ZPL_VERSION_FUID);
	sa_ok = (zplver >= ZPL_VERSION_SA);

	error = zfs_fill_zplprops_impl(NULL, zplver, fuids_ok, sa_ok,
	createprops, zplprops, is_ci);
	return (error);
	}

	/*
	* innvl: {
	* "type" -> dmu_objset_type_t (int32)
	* (optional) "props" -> { prop -> value }
	* (optional) "hidden_args" -> { "wkeydata" -> value }
	* raw uint8_t array of encryption wrapping key data (32 bytes)
	* }
	*
	* outnvl: propname -> error code (int32)
	*/

	static const zfs_ioc_key_t zfs_keys_create[] = {
	{"type", DATA_TYPE_INT32, 0},
	{"props", DATA_TYPE_NVLIST, ZK_OPTIONAL},
	{"hidden_args", DATA_TYPE_NVLIST, ZK_OPTIONAL},
	};

	static int
	zfs_ioc_create(const char fsname, nvlist_t innvl, nvlist_t *outnvl)
	{
	int error = 0;
	zfs_creat_t zct = { 0 };
	nvlist_t *nvprops = NULL;
	nvlist_t *hidden_args = NULL;
	void (cbfunc)(objset_t os, void arg, cred_t cr, dmu_tx_t *tx);
	dmu_objset_type_t type;
	boolean_t is_insensitive = B_FALSE;
	dsl_crypto_params_t *dcp = NULL;

	type = (dmu_objset_type_t)fnvlist_lookup_int32(innvl, "type");
	(void) nvlist_lookup_nvlist(innvl, "props", &nvprops);
	(void) nvlist_lookup_nvlist(innvl, ZPOOL_HIDDEN_ARGS, &hidden_args);

	switch (type) {
	case DMU_OST_ZFS:
	cbfunc = zfs_create_cb;
	break;

	case DMU_OST_ZVOL:
	cbfunc = zvol_create_cb;
	break;

	default:
	cbfunc = NULL;
	break;
	}
	if (strchr(fsname, '@') \|\|
	strchr(fsname, '%'))
	return (SET_ERROR(EINVAL));

	zct.zct_props = nvprops;

	if (cbfunc == NULL)
	return (SET_ERROR(EINVAL));

	if (type == DMU_OST_ZVOL) {
	uint64_t volsize, volblocksize;

	if (nvprops == NULL)
	return (SET_ERROR(EINVAL));
	if (nvlist_lookup_uint64(nvprops,
	zfs_prop_to_name(ZFS_PROP_VOLSIZE), &volsize) != 0)
	return (SET_ERROR(EINVAL));

	if ((error = nvlist_lookup_uint64(nvprops,
	zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE),
	&volblocksize)) != 0 && error != ENOENT)
	return (SET_ERROR(EINVAL));

	if (error != 0)
	volblocksize = zfs_prop_default_numeric(
	ZFS_PROP_VOLBLOCKSIZE);

	if ((error = zvol_check_volblocksize(fsname,
	volblocksize)) != 0 \|\|
	(error = zvol_check_volsize(volsize,
	volblocksize)) != 0)
	return (error);
	} else if (type == DMU_OST_ZFS) {
	int error;

	/*
	* We have to have normalization and
	* case-folding flags correct when we do the
	* file system creation, so go figure them out
	* now.
	*/
	VERIFY(nvlist_alloc(&zct.zct_zplprops,
	NV_UNIQUE_NAME, KM_SLEEP) == 0);
	error = zfs_fill_zplprops(fsname, nvprops,
	zct.zct_zplprops, &is_insensitive);
	if (error != 0) {
	nvlist_free(zct.zct_zplprops);
	return (error);
	}
	}

	error = dsl_crypto_params_create_nvlist(DCP_CMD_NONE, nvprops,
	hidden_args, &dcp);
	if (error != 0) {
	nvlist_free(zct.zct_zplprops);
	return (error);
	}

	error = dmu_objset_create(fsname, type,
	is_insensitive ? DS_FLAG_CI_DATASET : 0, dcp, cbfunc, &zct);

	nvlist_free(zct.zct_zplprops);
	dsl_crypto_params_free(dcp, !!error);

	/*
	* It would be nice to do this atomically.
	*/
	if (error == 0) {
	error = zfs_set_prop_nvlist(fsname, ZPROP_SRC_LOCAL,
	nvprops, outnvl);
	if (error != 0) {
	spa_t *spa;
	int error2;

	/*
	* Volumes will return EBUSY and cannot be destroyed
	* until all asynchronous minor handling (e.g. from
	* setting the volmode property) has completed. Wait for
	* the spa_zvol_taskq to drain then retry.
	*/
	error2 = dsl_destroy_head(fsname);
	while ((error2 == EBUSY) && (type == DMU_OST_ZVOL)) {
	error2 = spa_open(fsname, &spa, FTAG);
	if (error2 == 0) {
	taskq_wait(spa->spa_zvol_taskq);
	spa_close(spa, FTAG);
	}
	error2 = dsl_destroy_head(fsname);
	}
	}
	}
	return (error);
	}

	/*
	* innvl: {
	* "origin" -> name of origin snapshot
	* (optional) "props" -> { prop -> value }
	* (optional) "hidden_args" -> { "wkeydata" -> value }
	* raw uint8_t array of encryption wrapping key data (32 bytes)
	* }
	*
	* outputs:
	* outnvl: propname -> error code (int32)
	*/
	static const zfs_ioc_key_t zfs_keys_clone[] = {
	{"origin", DATA_TYPE_STRING, 0},
	{"props", DATA_TYPE_NVLIST, ZK_OPTIONAL},
	{"hidden_args", DATA_TYPE_NVLIST, ZK_OPTIONAL},
	};

	static int
	zfs_ioc_clone(const char fsname, nvlist_t innvl, nvlist_t *outnvl)
	{
	int error = 0;
	nvlist_t *nvprops = NULL;
	const char *origin_name;

	origin_name = fnvlist_lookup_string(innvl, "origin");
	(void) nvlist_lookup_nvlist(innvl, "props", &nvprops);

	if (strchr(fsname, '@') \|\|
	strchr(fsname, '%'))
	return (SET_ERROR(EINVAL));

	if (dataset_namecheck(origin_name, NULL, NULL) != 0)
	return (SET_ERROR(EINVAL));

	error = dmu_objset_clone(fsname, origin_name);

	/*
	* It would be nice to do this atomically.
	*/
	if (error == 0) {
	error = zfs_set_prop_nvlist(fsname, ZPROP_SRC_LOCAL,
	nvprops, outnvl);
	if (error != 0)
	(void) dsl_destroy_head(fsname);
	}
	return (error);
	}

	static const zfs_ioc_key_t zfs_keys_remap[] = {
	/* no nvl keys */
	};

	static int
	zfs_ioc_remap(const char fsname, nvlist_t innvl, nvlist_t *outnvl)
	{
	/* This IOCTL is no longer supported. */
	(void) fsname, (void) innvl, (void) outnvl;
	return (0);
	}

	/*
	* innvl: {
	* "snaps" -> { snapshot1, snapshot2 }
	* (optional) "props" -> { prop -> value (string) }
	* }
	*
	* outnvl: snapshot -> error code (int32)
	*/
	static const zfs_ioc_key_t zfs_keys_snapshot[] = {
	{"snaps", DATA_TYPE_NVLIST, 0},
	{"props", DATA_TYPE_NVLIST, ZK_OPTIONAL},
	};

	static int
	zfs_ioc_snapshot(const char poolname, nvlist_t innvl, nvlist_t *outnvl)
	{
	nvlist_t *snaps;
	nvlist_t *props = NULL;
	int error, poollen;
	nvpair_t *pair;

	(void) nvlist_lookup_nvlist(innvl, "props", &props);
	if (!nvlist_empty(props) &&
	zfs_earlier_version(poolname, SPA_VERSION_SNAP_PROPS))
	return (SET_ERROR(ENOTSUP));
	if ((error = zfs_check_userprops(props)) != 0)
	return (error);

	snaps = fnvlist_lookup_nvlist(innvl, "snaps");
	poollen = strlen(poolname);
	for (pair = nvlist_next_nvpair(snaps, NULL); pair != NULL;
	pair = nvlist_next_nvpair(snaps, pair)) {
	const char *name = nvpair_name(pair);
	char *cp = strchr(name, '@');

	/*
	* The snap name must contain an @, and the part after it must
	* contain only valid characters.
	*/
	if (cp == NULL \|\|
	zfs_component_namecheck(cp + 1, NULL, NULL) != 0)
	return (SET_ERROR(EINVAL));

	/*
	* The snap must be in the specified pool.
	*/
	if (strncmp(name, poolname, poollen) != 0 \|\|
	(name[poollen] != '/' && name[poollen] != '@'))
	return (SET_ERROR(EXDEV));

	/*
	* Check for permission to set the properties on the fs.
	*/
	if (!nvlist_empty(props)) {
	*cp = '\0';
	error = zfs_secpolicy_write_perms(name,
	ZFS_DELEG_PERM_USERPROP, CRED());
	*cp = '@';
	if (error != 0)
	return (error);
	}

	/* This must be the only snap of this fs. */
	for (nvpair_t *pair2 = nvlist_next_nvpair(snaps, pair);
	pair2 != NULL; pair2 = nvlist_next_nvpair(snaps, pair2)) {
	if (strncmp(name, nvpair_name(pair2), cp - name + 1)
	== 0) {
	return (SET_ERROR(EXDEV));
	}
	}
	}

	error = dsl_dataset_snapshot(snaps, props, outnvl);

	return (error);
	}

	/*
	* innvl: "message" -> string
	*/
	static const zfs_ioc_key_t zfs_keys_log_history[] = {
	{"message", DATA_TYPE_STRING, 0},
	};

	static int
	zfs_ioc_log_history(const char unused, nvlist_t innvl, nvlist_t *outnvl)
	{
	(void) unused, (void) outnvl;
	const char *message;
	char *poolname;
	spa_t *spa;
	int error;

	/*
	* The poolname in the ioctl is not set, we get it from the TSD,
	* which was set at the end of the last successful ioctl that allows
	* logging. The secpolicy func already checked that it is set.
	* Only one log ioctl is allowed after each successful ioctl, so
	* we clear the TSD here.
	*/
	poolname = tsd_get(zfs_allow_log_key);
	if (poolname == NULL)
	return (SET_ERROR(EINVAL));
	(void) tsd_set(zfs_allow_log_key, NULL);
	error = spa_open(poolname, &spa, FTAG);
	kmem_strfree(poolname);
	if (error != 0)
	return (error);

	message = fnvlist_lookup_string(innvl, "message");

	if (spa_version(spa) < SPA_VERSION_ZPOOL_HISTORY) {
	spa_close(spa, FTAG);
	return (SET_ERROR(ENOTSUP));
	}

	error = spa_history_log(spa, message);
	spa_close(spa, FTAG);
	return (error);
	}

	/*
	* This ioctl is used to set the bootenv configuration on the current
	* pool. This configuration is stored in the second padding area of the label,
	* and it is used by the bootloader(s) to store the bootloader and/or system
	* specific data.
	* The data is stored as nvlist data stream, and is protected by
	* an embedded checksum.
	* The version can have two possible values:
	* VB_RAW: nvlist should have key GRUB_ENVMAP, value DATA_TYPE_STRING.
	* VB_NVLIST: nvlist with arbitrary <key, value> pairs.
	*/
	static const zfs_ioc_key_t zfs_keys_set_bootenv[] = {
	{"version", DATA_TYPE_UINT64, 0},
	{"<keys>", DATA_TYPE_ANY, ZK_OPTIONAL \| ZK_WILDCARDLIST},
	};

	static int
	zfs_ioc_set_bootenv(const char name, nvlist_t innvl, nvlist_t *outnvl)
	{
	int error;
	spa_t *spa;

	if ((error = spa_open(name, &spa, FTAG)) != 0)
	return (error);
	spa_vdev_state_enter(spa, SCL_ALL);
	error = vdev_label_write_bootenv(spa->spa_root_vdev, innvl);
	(void) spa_vdev_state_exit(spa, NULL, 0);
	spa_close(spa, FTAG);
	return (error);
	}

	static const zfs_ioc_key_t zfs_keys_get_bootenv[] = {
	/* no nvl keys */
	};

	static int
	zfs_ioc_get_bootenv(const char name, nvlist_t innvl, nvlist_t *outnvl)
	{
	spa_t *spa;
	int error;

	if ((error = spa_open(name, &spa, FTAG)) != 0)
	return (error);
	spa_vdev_state_enter(spa, SCL_ALL);
	error = vdev_label_read_bootenv(spa->spa_root_vdev, outnvl);
	(void) spa_vdev_state_exit(spa, NULL, 0);
	spa_close(spa, FTAG);
	return (error);
	}

	/*
	* The dp_config_rwlock must not be held when calling this, because the
	* unmount may need to write out data.
	*
	* This function is best-effort. Callers must deal gracefully if it
	* remains mounted (or is remounted after this call).
	*
	* Returns 0 if the argument is not a snapshot, or it is not currently a
	* filesystem, or we were able to unmount it. Returns error code otherwise.
	*/
	void
	zfs_unmount_snap(const char *snapname)
	{
	if (strchr(snapname, '@') == NULL)
	return;

	(void) zfsctl_snapshot_unmount(snapname, MNT_FORCE);
	}

	static int
	zfs_unmount_snap_cb(const char snapname, void arg)
	{
	(void) arg;
	zfs_unmount_snap(snapname);
	return (0);
	}

	/*
	* When a clone is destroyed, its origin may also need to be destroyed,
	* in which case it must be unmounted. This routine will do that unmount
	* if necessary.
	*/
	void
	zfs_destroy_unmount_origin(const char *fsname)
	{
	int error;
	objset_t *os;
	dsl_dataset_t *ds;

	error = dmu_objset_hold(fsname, FTAG, &os);
	if (error != 0)
	return;
	ds = dmu_objset_ds(os);
	if (dsl_dir_is_clone(ds->ds_dir) && DS_IS_DEFER_DESTROY(ds->ds_prev)) {
	char originname[ZFS_MAX_DATASET_NAME_LEN];
	dsl_dataset_name(ds->ds_prev, originname);
	dmu_objset_rele(os, FTAG);
	zfs_unmount_snap(originname);
	} else {
	dmu_objset_rele(os, FTAG);
	}
	}

	/*
	* innvl: {
	* "snaps" -> { snapshot1, snapshot2 }
	* (optional boolean) "defer"
	* }
	*
	* outnvl: snapshot -> error code (int32)
	*/
	static const zfs_ioc_key_t zfs_keys_destroy_snaps[] = {
	{"snaps", DATA_TYPE_NVLIST, 0},
	{"defer", DATA_TYPE_BOOLEAN, ZK_OPTIONAL},
	};

	static int
	zfs_ioc_destroy_snaps(const char poolname, nvlist_t innvl, nvlist_t *outnvl)
	{
	int poollen;
	nvlist_t *snaps;
	nvpair_t *pair;
	boolean_t defer;
	spa_t *spa;

	snaps = fnvlist_lookup_nvlist(innvl, "snaps");
	defer = nvlist_exists(innvl, "defer");

	poollen = strlen(poolname);
	for (pair = nvlist_next_nvpair(snaps, NULL); pair != NULL;
	pair = nvlist_next_nvpair(snaps, pair)) {
	const char *name = nvpair_name(pair);

	/*
	* The snap must be in the specified pool to prevent the
	* invalid removal of zvol minors below.
	*/
	if (strncmp(name, poolname, poollen) != 0 \|\|
	(name[poollen] != '/' && name[poollen] != '@'))
	return (SET_ERROR(EXDEV));

	zfs_unmount_snap(nvpair_name(pair));
	if (spa_open(name, &spa, FTAG) == 0) {
	zvol_remove_minors(spa, name, B_TRUE);
	spa_close(spa, FTAG);
	}
	}

	return (dsl_destroy_snapshots_nvl(snaps, defer, outnvl));
	}

	/*
	* Create bookmarks. The bookmark names are of the form <fs>#<bmark>.
	* All bookmarks and snapshots must be in the same pool.
	* dsl_bookmark_create_nvl_validate describes the nvlist schema in more detail.
	*
	* innvl: {
	* new_bookmark1 -> existing_snapshot,
	* new_bookmark2 -> existing_bookmark,
	* }
	*
	* outnvl: bookmark -> error code (int32)
	*
	*/
	static const zfs_ioc_key_t zfs_keys_bookmark[] = {
	{"<bookmark>...", DATA_TYPE_STRING, ZK_WILDCARDLIST},
	};

	static int
	zfs_ioc_bookmark(const char poolname, nvlist_t innvl, nvlist_t *outnvl)
	{
	(void) poolname;
	return (dsl_bookmark_create(innvl, outnvl));
	}

	/*
	* innvl: {
	* property 1, property 2, ...
	* }
	*
	* outnvl: {
	* bookmark name 1 -> { property 1, property 2, ... },
	* bookmark name 2 -> { property 1, property 2, ... }
	* }
	*
	*/
	static const zfs_ioc_key_t zfs_keys_get_bookmarks[] = {
	{"<property>...", DATA_TYPE_BOOLEAN, ZK_WILDCARDLIST \| ZK_OPTIONAL},
	};

	static int
	zfs_ioc_get_bookmarks(const char fsname, nvlist_t innvl, nvlist_t *outnvl)
	{
	return (dsl_get_bookmarks(fsname, innvl, outnvl));
	}

	/*
	* innvl is not used.
	*
	* outnvl: {
	* property 1, property 2, ...
	* }
	*
	*/
	static const zfs_ioc_key_t zfs_keys_get_bookmark_props[] = {
	/* no nvl keys */
	};

	static int
	zfs_ioc_get_bookmark_props(const char bookmark, nvlist_t innvl,
	nvlist_t *outnvl)
	{
	(void) innvl;
	char fsname[ZFS_MAX_DATASET_NAME_LEN];
	char *bmname;

	bmname = strchr(bookmark, '#');
	if (bmname == NULL)
	return (SET_ERROR(EINVAL));
	bmname++;

	(void) strlcpy(fsname, bookmark, sizeof (fsname));
	*(strchr(fsname, '#')) = '\0';

	return (dsl_get_bookmark_props(fsname, bmname, outnvl));
	}

	/*
	* innvl: {
	* bookmark name 1, bookmark name 2
	* }
	*
	* outnvl: bookmark -> error code (int32)
	*
	*/
	static const zfs_ioc_key_t zfs_keys_destroy_bookmarks[] = {
	{"<bookmark>...", DATA_TYPE_BOOLEAN, ZK_WILDCARDLIST},
	};

	static int
	zfs_ioc_destroy_bookmarks(const char poolname, nvlist_t innvl,
	nvlist_t *outnvl)
	{
	int error, poollen;

	poollen = strlen(poolname);
	for (nvpair_t *pair = nvlist_next_nvpair(innvl, NULL);
	pair != NULL; pair = nvlist_next_nvpair(innvl, pair)) {
	const char *name = nvpair_name(pair);
	const char *cp = strchr(name, '#');

	/*
	* The bookmark name must contain an #, and the part after it
	* must contain only valid characters.
	*/
	if (cp == NULL \|\|
	zfs_component_namecheck(cp + 1, NULL, NULL) != 0)
	return (SET_ERROR(EINVAL));

	/*
	* The bookmark must be in the specified pool.
	*/
	if (strncmp(name, poolname, poollen) != 0 \|\|
	(name[poollen] != '/' && name[poollen] != '#'))
	return (SET_ERROR(EXDEV));
	}

	error = dsl_bookmark_destroy(innvl, outnvl);
	return (error);
	}

	static const zfs_ioc_key_t zfs_keys_channel_program[] = {
	{"program", DATA_TYPE_STRING, 0},
	{"arg", DATA_TYPE_ANY, 0},
	{"sync", DATA_TYPE_BOOLEAN_VALUE, ZK_OPTIONAL},
	{"instrlimit", DATA_TYPE_UINT64, ZK_OPTIONAL},
	{"memlimit", DATA_TYPE_UINT64, ZK_OPTIONAL},
	};

	static int
	zfs_ioc_channel_program(const char poolname, nvlist_t innvl,
	nvlist_t *outnvl)
	{
	const char *program;
	uint64_t instrlimit, memlimit;
	boolean_t sync_flag;
	nvpair_t *nvarg = NULL;

	program = fnvlist_lookup_string(innvl, ZCP_ARG_PROGRAM);
	if (0 != nvlist_lookup_boolean_value(innvl, ZCP_ARG_SYNC, &sync_flag)) {
	sync_flag = B_TRUE;
	}
	if (0 != nvlist_lookup_uint64(innvl, ZCP_ARG_INSTRLIMIT, &instrlimit)) {
	instrlimit = ZCP_DEFAULT_INSTRLIMIT;
	}
	if (0 != nvlist_lookup_uint64(innvl, ZCP_ARG_MEMLIMIT, &memlimit)) {
	memlimit = ZCP_DEFAULT_MEMLIMIT;
	}
	nvarg = fnvlist_lookup_nvpair(innvl, ZCP_ARG_ARGLIST);

	if (instrlimit == 0 \|\| instrlimit > zfs_lua_max_instrlimit)
	return (SET_ERROR(EINVAL));
	if (memlimit == 0 \|\| memlimit > zfs_lua_max_memlimit)
	return (SET_ERROR(EINVAL));

	return (zcp_eval(poolname, program, sync_flag, instrlimit, memlimit,
	nvarg, outnvl));
	}

	/*
	* innvl: unused
	* outnvl: empty
	*/
	static const zfs_ioc_key_t zfs_keys_pool_checkpoint[] = {
	/* no nvl keys */
	};

	static int
	zfs_ioc_pool_checkpoint(const char poolname, nvlist_t innvl, nvlist_t *outnvl)
	{
	(void) innvl, (void) outnvl;
	return (spa_checkpoint(poolname));
	}

	/*
	* innvl: unused
	* outnvl: empty
	*/
	static const zfs_ioc_key_t zfs_keys_pool_discard_checkpoint[] = {
	/* no nvl keys */
	};

	static int
	zfs_ioc_pool_discard_checkpoint(const char poolname, nvlist_t innvl,
	nvlist_t *outnvl)
	{
	(void) innvl, (void) outnvl;
	return (spa_checkpoint_discard(poolname));
	}

	/*
	* inputs:
	* zc_name name of dataset to destroy
	* zc_defer_destroy mark for deferred destroy
	*
	* outputs: none
	*/
	static int
	zfs_ioc_destroy(zfs_cmd_t *zc)
	{
	objset_t *os;
	dmu_objset_type_t ost;
	int err;

	err = dmu_objset_hold(zc->zc_name, FTAG, &os);
	if (err != 0)
	return (err);
	ost = dmu_objset_type(os);
	dmu_objset_rele(os, FTAG);

	if (ost == DMU_OST_ZFS)
	zfs_unmount_snap(zc->zc_name);

	if (strchr(zc->zc_name, '@')) {
	err = dsl_destroy_snapshot(zc->zc_name, zc->zc_defer_destroy);
	} else {
	err = dsl_destroy_head(zc->zc_name);
	if (err == EEXIST) {
	/*
	* It is possible that the given DS may have
	* hidden child (%recv) datasets - "leftovers"
	* resulting from the previously interrupted
	* 'zfs receive'.
	*
	* 6 extra bytes for /%recv
	*/
	char namebuf[ZFS_MAX_DATASET_NAME_LEN + 6];

	if (snprintf(namebuf, sizeof (namebuf), "%s/%s",
	zc->zc_name, recv_clone_name) >=
	sizeof (namebuf))
	return (SET_ERROR(EINVAL));

	/*
	* Try to remove the hidden child (%recv) and after
	* that try to remove the target dataset.
	* If the hidden child (%recv) does not exist
	* the original error (EEXIST) will be returned
	*/
	err = dsl_destroy_head(namebuf);
	if (err == 0)
	err = dsl_destroy_head(zc->zc_name);
	else if (err == ENOENT)
	err = SET_ERROR(EEXIST);
	}
	}

	return (err);
	}

	/*
	* innvl: {
	* "initialize_command" -> POOL_INITIALIZE_{CANCEL\|START\|SUSPEND} (uint64)
	* "initialize_vdevs": { -> guids to initialize (nvlist)
	* "vdev_path_1": vdev_guid_1, (uint64),
	* "vdev_path_2": vdev_guid_2, (uint64),
	* ...
	* },
	* }
	*
	* outnvl: {
	* "initialize_vdevs": { -> initialization errors (nvlist)
	* "vdev_path_1": errno, see function body for possible errnos (uint64)
	* "vdev_path_2": errno, ... (uint64)
	* ...
	* }
	* }
	*
	* EINVAL is returned for an unknown commands or if any of the provided vdev
	* guids have be specified with a type other than uint64.
	*/
	static const zfs_ioc_key_t zfs_keys_pool_initialize[] = {
	{ZPOOL_INITIALIZE_COMMAND, DATA_TYPE_UINT64, 0},
	{ZPOOL_INITIALIZE_VDEVS, DATA_TYPE_NVLIST, 0}
	};

	static int
	zfs_ioc_pool_initialize(const char poolname, nvlist_t innvl, nvlist_t *outnvl)
	{
	uint64_t cmd_type;
	if (nvlist_lookup_uint64(innvl, ZPOOL_INITIALIZE_COMMAND,
	&cmd_type) != 0) {
	return (SET_ERROR(EINVAL));
	}

	if (!(cmd_type == POOL_INITIALIZE_CANCEL \|\|
	cmd_type == POOL_INITIALIZE_START \|\|
	cmd_type == POOL_INITIALIZE_SUSPEND \|\|
	cmd_type == POOL_INITIALIZE_UNINIT)) {
	return (SET_ERROR(EINVAL));
	}

	nvlist_t *vdev_guids;
	if (nvlist_lookup_nvlist(innvl, ZPOOL_INITIALIZE_VDEVS,
	&vdev_guids) != 0) {
	return (SET_ERROR(EINVAL));
	}

	for (nvpair_t *pair = nvlist_next_nvpair(vdev_guids, NULL);
	pair != NULL; pair = nvlist_next_nvpair(vdev_guids, pair)) {
	uint64_t vdev_guid;
	if (nvpair_value_uint64(pair, &vdev_guid) != 0) {
	return (SET_ERROR(EINVAL));
	}
	}

	spa_t *spa;
	int error = spa_open(poolname, &spa, FTAG);
	if (error != 0)
	return (error);

	nvlist_t *vdev_errlist = fnvlist_alloc();
	int total_errors = spa_vdev_initialize(spa, vdev_guids, cmd_type,
	vdev_errlist);

	if (fnvlist_size(vdev_errlist) > 0) {
	fnvlist_add_nvlist(outnvl, ZPOOL_INITIALIZE_VDEVS,
	vdev_errlist);
	}
	fnvlist_free(vdev_errlist);

	spa_close(spa, FTAG);
	return (total_errors > 0 ? SET_ERROR(EINVAL) : 0);
	}

	/*
	* innvl: {
	* "trim_command" -> POOL_TRIM_{CANCEL\|START\|SUSPEND} (uint64)
	* "trim_vdevs": { -> guids to TRIM (nvlist)
	* "vdev_path_1": vdev_guid_1, (uint64),
	* "vdev_path_2": vdev_guid_2, (uint64),
	* ...
	* },
	* "trim_rate" -> Target TRIM rate in bytes/sec.
	* "trim_secure" -> Set to request a secure TRIM.
	* }
	*
	* outnvl: {
	* "trim_vdevs": { -> TRIM errors (nvlist)
	* "vdev_path_1": errno, see function body for possible errnos (uint64)
	* "vdev_path_2": errno, ... (uint64)
	* ...
	* }
	* }
	*
	* EINVAL is returned for an unknown commands or if any of the provided vdev
	* guids have be specified with a type other than uint64.
	*/
	static const zfs_ioc_key_t zfs_keys_pool_trim[] = {
	{ZPOOL_TRIM_COMMAND, DATA_TYPE_UINT64, 0},
	{ZPOOL_TRIM_VDEVS, DATA_TYPE_NVLIST, 0},
	{ZPOOL_TRIM_RATE, DATA_TYPE_UINT64, ZK_OPTIONAL},
	{ZPOOL_TRIM_SECURE, DATA_TYPE_BOOLEAN_VALUE, ZK_OPTIONAL},
	};

	static int
	zfs_ioc_pool_trim(const char poolname, nvlist_t innvl, nvlist_t *outnvl)
	{
	uint64_t cmd_type;
	if (nvlist_lookup_uint64(innvl, ZPOOL_TRIM_COMMAND, &cmd_type) != 0)
	return (SET_ERROR(EINVAL));

	if (!(cmd_type == POOL_TRIM_CANCEL \|\|
	cmd_type == POOL_TRIM_START \|\|
	cmd_type == POOL_TRIM_SUSPEND)) {
	return (SET_ERROR(EINVAL));
	}

	nvlist_t *vdev_guids;
	if (nvlist_lookup_nvlist(innvl, ZPOOL_TRIM_VDEVS, &vdev_guids) != 0)
	return (SET_ERROR(EINVAL));

	for (nvpair_t *pair = nvlist_next_nvpair(vdev_guids, NULL);
	pair != NULL; pair = nvlist_next_nvpair(vdev_guids, pair)) {
	uint64_t vdev_guid;
	if (nvpair_value_uint64(pair, &vdev_guid) != 0) {
	return (SET_ERROR(EINVAL));
	}
	}

	/* Optional, defaults to maximum rate when not provided */
	uint64_t rate;
	if (nvlist_lookup_uint64(innvl, ZPOOL_TRIM_RATE, &rate) != 0)
	rate = 0;

	/* Optional, defaults to standard TRIM when not provided */
	boolean_t secure;
	if (nvlist_lookup_boolean_value(innvl, ZPOOL_TRIM_SECURE,
	&secure) != 0) {
	secure = B_FALSE;
	}

	spa_t *spa;
	int error = spa_open(poolname, &spa, FTAG);
	if (error != 0)
	return (error);

	nvlist_t *vdev_errlist = fnvlist_alloc();
	int total_errors = spa_vdev_trim(spa, vdev_guids, cmd_type,
	rate, !!zfs_trim_metaslab_skip, secure, vdev_errlist);

	if (fnvlist_size(vdev_errlist) > 0)
	fnvlist_add_nvlist(outnvl, ZPOOL_TRIM_VDEVS, vdev_errlist);

	fnvlist_free(vdev_errlist);

	spa_close(spa, FTAG);
	return (total_errors > 0 ? SET_ERROR(EINVAL) : 0);
	}

	/*
	* This ioctl waits for activity of a particular type to complete. If there is
	* no activity of that type in progress, it returns immediately, and the
	* returned value "waited" is false. If there is activity in progress, and no
	* tag is passed in, the ioctl blocks until all activity of that type is
	* complete, and then returns with "waited" set to true.
	*
	* If a tag is provided, it identifies a particular instance of an activity to
	* wait for. Currently, this is only valid for use with 'initialize', because
	* that is the only activity for which there can be multiple instances running
	* concurrently. In the case of 'initialize', the tag corresponds to the guid of
	* the vdev on which to wait.
	*
	* If a thread waiting in the ioctl receives a signal, the call will return
	* immediately, and the return value will be EINTR.
	*
	* innvl: {
	* "wait_activity" -> int32_t
	* (optional) "wait_tag" -> uint64_t
	* }
	*
	* outnvl: "waited" -> boolean_t
	*/
	static const zfs_ioc_key_t zfs_keys_pool_wait[] = {
	{ZPOOL_WAIT_ACTIVITY, DATA_TYPE_INT32, 0},
	{ZPOOL_WAIT_TAG, DATA_TYPE_UINT64, ZK_OPTIONAL},
	};

	static int
	zfs_ioc_wait(const char name, nvlist_t innvl, nvlist_t *outnvl)
	{
	int32_t activity;
	uint64_t tag;
	boolean_t waited;
	int error;

	if (nvlist_lookup_int32(innvl, ZPOOL_WAIT_ACTIVITY, &activity) != 0)
	return (EINVAL);

	if (nvlist_lookup_uint64(innvl, ZPOOL_WAIT_TAG, &tag) == 0)
	error = spa_wait_tag(name, activity, tag, &waited);
	else
	error = spa_wait(name, activity, &waited);

	if (error == 0)
	fnvlist_add_boolean_value(outnvl, ZPOOL_WAIT_WAITED, waited);

	return (error);
	}

	/*
	* This ioctl waits for activity of a particular type to complete. If there is
	* no activity of that type in progress, it returns immediately, and the
	* returned value "waited" is false. If there is activity in progress, and no
	* tag is passed in, the ioctl blocks until all activity of that type is
	* complete, and then returns with "waited" set to true.
	*
	* If a thread waiting in the ioctl receives a signal, the call will return
	* immediately, and the return value will be EINTR.
	*
	* innvl: {
	* "wait_activity" -> int32_t
	* }
	*
	* outnvl: "waited" -> boolean_t
	*/
	static const zfs_ioc_key_t zfs_keys_fs_wait[] = {
	{ZFS_WAIT_ACTIVITY, DATA_TYPE_INT32, 0},
	};

	static int
	zfs_ioc_wait_fs(const char name, nvlist_t innvl, nvlist_t *outnvl)
	{
	int32_t activity;
	boolean_t waited = B_FALSE;
	int error;
	dsl_pool_t *dp;
	dsl_dir_t *dd;
	dsl_dataset_t *ds;

	if (nvlist_lookup_int32(innvl, ZFS_WAIT_ACTIVITY, &activity) != 0)
	return (SET_ERROR(EINVAL));

	if (activity >= ZFS_WAIT_NUM_ACTIVITIES \|\| activity < 0)
	return (SET_ERROR(EINVAL));

	if ((error = dsl_pool_hold(name, FTAG, &dp)) != 0)
	return (error);

	if ((error = dsl_dataset_hold(dp, name, FTAG, &ds)) != 0) {
	dsl_pool_rele(dp, FTAG);
	return (error);
	}

	dd = ds->ds_dir;
	mutex_enter(&dd->dd_activity_lock);
	dd->dd_activity_waiters++;

	/*
	* We get a long-hold here so that the dsl_dataset_t and dsl_dir_t
	* aren't evicted while we're waiting. Normally this is prevented by
	* holding the pool, but we can't do that while we're waiting since
	* that would prevent TXGs from syncing out. Some of the functionality
	* of long-holds (e.g. preventing deletion) is unnecessary for this
	* case, since we would cancel the waiters before proceeding with a
	* deletion. An alternative mechanism for keeping the dataset around
	* could be developed but this is simpler.
	*/
	dsl_dataset_long_hold(ds, FTAG);
	dsl_pool_rele(dp, FTAG);

	error = dsl_dir_wait(dd, ds, activity, &waited);

	dsl_dataset_long_rele(ds, FTAG);
	dd->dd_activity_waiters--;
	if (dd->dd_activity_waiters == 0)
	cv_signal(&dd->dd_activity_cv);
	mutex_exit(&dd->dd_activity_lock);

	dsl_dataset_rele(ds, FTAG);

	if (error == 0)
	fnvlist_add_boolean_value(outnvl, ZFS_WAIT_WAITED, waited);

	return (error);
	}

	/*
	* fsname is name of dataset to rollback (to most recent snapshot)
	*
	* innvl may contain name of expected target snapshot
	*
	* outnvl: "target" -> name of most recent snapshot
	* }
	*/
	static const zfs_ioc_key_t zfs_keys_rollback[] = {
	{"target", DATA_TYPE_STRING, ZK_OPTIONAL},
	};

	static int
	zfs_ioc_rollback(const char fsname, nvlist_t innvl, nvlist_t *outnvl)
	{
	zfsvfs_t *zfsvfs;
	zvol_state_handle_t *zv;
	const char *target = NULL;
	int error;

	(void) nvlist_lookup_string(innvl, "target", &target);
	if (target != NULL) {
	const char *cp = strchr(target, '@');

	/*
	* The snap name must contain an @, and the part after it must
	* contain only valid characters.
	*/
	if (cp == NULL \|\|
	zfs_component_namecheck(cp + 1, NULL, NULL) != 0)
	return (SET_ERROR(EINVAL));
	}

	if (getzfsvfs(fsname, &zfsvfs) == 0) {
	dsl_dataset_t *ds;

	ds = dmu_objset_ds(zfsvfs->z_os);
	error = zfs_suspend_fs(zfsvfs);
	if (error == 0) {
	int resume_err;

	error = dsl_dataset_rollback(fsname, target, zfsvfs,
	outnvl);
	resume_err = zfs_resume_fs(zfsvfs, ds);
	error = error ? error : resume_err;
	}
	zfs_vfs_rele(zfsvfs);
	} else if ((zv = zvol_suspend(fsname)) != NULL) {
	error = dsl_dataset_rollback(fsname, target, zvol_tag(zv),
	outnvl);
	zvol_resume(zv);
	} else {
	error = dsl_dataset_rollback(fsname, target, NULL, outnvl);
	}
	return (error);
	}

	static int
	recursive_unmount(const char fsname, void arg)
	{
	const char *snapname = arg;
	char *fullname;

	fullname = kmem_asprintf("%s@%s", fsname, snapname);
	zfs_unmount_snap(fullname);
	kmem_strfree(fullname);

	return (0);
	}

	/*
	*
	* snapname is the snapshot to redact.
	* innvl: {
	* "bookname" -> (string)
	* shortname of the redaction bookmark to generate
	* "snapnv" -> (nvlist, values ignored)
	* snapshots to redact snapname with respect to
	* }
	*
	* outnvl is unused
	*/

	static const zfs_ioc_key_t zfs_keys_redact[] = {
	{"bookname", DATA_TYPE_STRING, 0},
	{"snapnv", DATA_TYPE_NVLIST, 0},
	};

	static int
	zfs_ioc_redact(const char snapname, nvlist_t innvl, nvlist_t *outnvl)
	{
	(void) outnvl;
	nvlist_t *redactnvl = NULL;
	const char *redactbook = NULL;

	if (nvlist_lookup_nvlist(innvl, "snapnv", &redactnvl) != 0)
	return (SET_ERROR(EINVAL));
	if (fnvlist_num_pairs(redactnvl) == 0)
	return (SET_ERROR(ENXIO));
	if (nvlist_lookup_string(innvl, "bookname", &redactbook) != 0)
	return (SET_ERROR(EINVAL));

	return (dmu_redact_snap(snapname, redactnvl, redactbook));
	}

	/*
	* inputs:
	* zc_name old name of dataset
	* zc_value new name of dataset
	* zc_cookie recursive flag (only valid for snapshots)
	*
	* outputs: none
	*/
	static int
	zfs_ioc_rename(zfs_cmd_t *zc)
	{
	objset_t *os;
	dmu_objset_type_t ost;
	boolean_t recursive = zc->zc_cookie & 1;
	boolean_t nounmount = !!(zc->zc_cookie & 2);
	char *at;
	int err;

	/* "zfs rename" from and to ...%recv datasets should both fail */
	zc->zc_name[sizeof (zc->zc_name) - 1] = '\0';
	zc->zc_value[sizeof (zc->zc_value) - 1] = '\0';
	if (dataset_namecheck(zc->zc_name, NULL, NULL) != 0 \|\|
	dataset_namecheck(zc->zc_value, NULL, NULL) != 0 \|\|
	strchr(zc->zc_name, '%') \|\| strchr(zc->zc_value, '%'))
	return (SET_ERROR(EINVAL));

	err = dmu_objset_hold(zc->zc_name, FTAG, &os);
	if (err != 0)
	return (err);
	ost = dmu_objset_type(os);
	dmu_objset_rele(os, FTAG);

	at = strchr(zc->zc_name, '@');
	if (at != NULL) {
	/* snaps must be in same fs */
	int error;

	if (strncmp(zc->zc_name, zc->zc_value, at - zc->zc_name + 1))
	return (SET_ERROR(EXDEV));
	*at = '\0';
	if (ost == DMU_OST_ZFS && !nounmount) {
	error = dmu_objset_find(zc->zc_name,
	recursive_unmount, at + 1,
	recursive ? DS_FIND_CHILDREN : 0);
	if (error != 0) {
	*at = '@';
	return (error);
	}
	}
	error = dsl_dataset_rename_snapshot(zc->zc_name,
	at + 1, strchr(zc->zc_value, '@') + 1, recursive);
	*at = '@';

	return (error);
	} else {
	return (dsl_dir_rename(zc->zc_name, zc->zc_value));
	}
	}

	static int
	zfs_check_settable(const char dsname, nvpair_t pair, cred_t *cr)
	{
	const char *propname = nvpair_name(pair);
	boolean_t issnap = (strchr(dsname, '@') != NULL);
	zfs_prop_t prop = zfs_name_to_prop(propname);
	uint64_t intval, compval;
	int err;

	if (prop == ZPROP_USERPROP) {
	if (zfs_prop_user(propname)) {
	if ((err = zfs_secpolicy_write_perms(dsname,
	ZFS_DELEG_PERM_USERPROP, cr)))
	return (err);
	return (0);
	}

	if (!issnap && zfs_prop_userquota(propname)) {
	const char *perm = NULL;
	const char *uq_prefix =
	zfs_userquota_prop_prefixes[ZFS_PROP_USERQUOTA];
	const char *gq_prefix =
	zfs_userquota_prop_prefixes[ZFS_PROP_GROUPQUOTA];
	const char *uiq_prefix =
	zfs_userquota_prop_prefixes[ZFS_PROP_USEROBJQUOTA];
	const char *giq_prefix =
	zfs_userquota_prop_prefixes[ZFS_PROP_GROUPOBJQUOTA];
	const char *pq_prefix =
	zfs_userquota_prop_prefixes[ZFS_PROP_PROJECTQUOTA];
	const char *piq_prefix = zfs_userquota_prop_prefixes[\
	ZFS_PROP_PROJECTOBJQUOTA];

	if (strncmp(propname, uq_prefix,
	strlen(uq_prefix)) == 0) {
	perm = ZFS_DELEG_PERM_USERQUOTA;
	} else if (strncmp(propname, uiq_prefix,
	strlen(uiq_prefix)) == 0) {
	perm = ZFS_DELEG_PERM_USEROBJQUOTA;
	} else if (strncmp(propname, gq_prefix,
	strlen(gq_prefix)) == 0) {
	perm = ZFS_DELEG_PERM_GROUPQUOTA;
	} else if (strncmp(propname, giq_prefix,
	strlen(giq_prefix)) == 0) {
	perm = ZFS_DELEG_PERM_GROUPOBJQUOTA;
	} else if (strncmp(propname, pq_prefix,
	strlen(pq_prefix)) == 0) {
	perm = ZFS_DELEG_PERM_PROJECTQUOTA;
	} else if (strncmp(propname, piq_prefix,
	strlen(piq_prefix)) == 0) {
	perm = ZFS_DELEG_PERM_PROJECTOBJQUOTA;
	} else {
	/* {USER\|GROUP\|PROJECT}USED are read-only */
	return (SET_ERROR(EINVAL));
	}

	if ((err = zfs_secpolicy_write_perms(dsname, perm, cr)))
	return (err);
	return (0);
	}

	return (SET_ERROR(EINVAL));
	}

	if (issnap)
	return (SET_ERROR(EINVAL));

	if (nvpair_type(pair) == DATA_TYPE_NVLIST) {
	/*
	* dsl_prop_get_all_impl() returns properties in this
	* format.
	*/
	nvlist_t *attrs;
	VERIFY(nvpair_value_nvlist(pair, &attrs) == 0);
	VERIFY(nvlist_lookup_nvpair(attrs, ZPROP_VALUE,
	&pair) == 0);
	}

	/*
	* Check that this value is valid for this pool version
	*/
	switch (prop) {
	case ZFS_PROP_COMPRESSION:
	/*
	* If the user specified gzip compression, make sure
	* the SPA supports it. We ignore any errors here since
	* we'll catch them later.
	*/
	if (nvpair_value_uint64(pair, &intval) == 0) {
	compval = ZIO_COMPRESS_ALGO(intval);
	if (compval >= ZIO_COMPRESS_GZIP_1 &&
	compval <= ZIO_COMPRESS_GZIP_9 &&
	zfs_earlier_version(dsname,
	SPA_VERSION_GZIP_COMPRESSION)) {
	return (SET_ERROR(ENOTSUP));
	}

	if (compval == ZIO_COMPRESS_ZLE &&
	zfs_earlier_version(dsname,
	SPA_VERSION_ZLE_COMPRESSION))
	return (SET_ERROR(ENOTSUP));

	if (compval == ZIO_COMPRESS_LZ4) {
	spa_t *spa;

	if ((err = spa_open(dsname, &spa, FTAG)) != 0)
	return (err);

	if (!spa_feature_is_enabled(spa,
	SPA_FEATURE_LZ4_COMPRESS)) {
	spa_close(spa, FTAG);
	return (SET_ERROR(ENOTSUP));
	}
	spa_close(spa, FTAG);
	}

	if (compval == ZIO_COMPRESS_ZSTD) {
	spa_t *spa;

	if ((err = spa_open(dsname, &spa, FTAG)) != 0)
	return (err);

	if (!spa_feature_is_enabled(spa,
	SPA_FEATURE_ZSTD_COMPRESS)) {
	spa_close(spa, FTAG);
	return (SET_ERROR(ENOTSUP));
	}
	spa_close(spa, FTAG);
	}
	}
	break;

	case ZFS_PROP_COPIES:
	if (zfs_earlier_version(dsname, SPA_VERSION_DITTO_BLOCKS))
	return (SET_ERROR(ENOTSUP));
	break;

	case ZFS_PROP_VOLBLOCKSIZE:
	case ZFS_PROP_RECORDSIZE:
	/* Record sizes above 128k need the feature to be enabled */
	if (nvpair_value_uint64(pair, &intval) == 0 &&
	intval > SPA_OLD_MAXBLOCKSIZE) {
	spa_t *spa;

	/*
	* We don't allow setting the property above 1MB,
	* unless the tunable has been changed.
	*/
	if (intval > zfs_max_recordsize \|\|
	intval > SPA_MAXBLOCKSIZE)
	return (SET_ERROR(ERANGE));

	if ((err = spa_open(dsname, &spa, FTAG)) != 0)
	return (err);

	if (!spa_feature_is_enabled(spa,
	SPA_FEATURE_LARGE_BLOCKS)) {
	spa_close(spa, FTAG);
	return (SET_ERROR(ENOTSUP));
	}
	spa_close(spa, FTAG);
	}
	break;

	case ZFS_PROP_DNODESIZE:
	/* Dnode sizes above 512 need the feature to be enabled */
	if (nvpair_value_uint64(pair, &intval) == 0 &&
	intval != ZFS_DNSIZE_LEGACY) {
	spa_t *spa;

	if ((err = spa_open(dsname, &spa, FTAG)) != 0)
	return (err);

	if (!spa_feature_is_enabled(spa,
	SPA_FEATURE_LARGE_DNODE)) {
	spa_close(spa, FTAG);
	return (SET_ERROR(ENOTSUP));
	}
	spa_close(spa, FTAG);
	}
	break;

	case ZFS_PROP_SPECIAL_SMALL_BLOCKS:
	/*
	* This property could require the allocation classes
	* feature to be active for setting, however we allow
	* it so that tests of settable properties succeed.
	* The CLI will issue a warning in this case.
	*/
	break;

	case ZFS_PROP_SHARESMB:
	if (zpl_earlier_version(dsname, ZPL_VERSION_FUID))
	return (SET_ERROR(ENOTSUP));
	break;

	case ZFS_PROP_ACLINHERIT:
	if (nvpair_type(pair) == DATA_TYPE_UINT64 &&
	nvpair_value_uint64(pair, &intval) == 0) {
	if (intval == ZFS_ACL_PASSTHROUGH_X &&
	zfs_earlier_version(dsname,
	SPA_VERSION_PASSTHROUGH_X))
	return (SET_ERROR(ENOTSUP));
	}
	break;
	case ZFS_PROP_CHECKSUM:
	case ZFS_PROP_DEDUP:
	{
	spa_feature_t feature;
	spa_t *spa;
	int err;

	/* dedup feature version checks */
	if (prop == ZFS_PROP_DEDUP &&
	zfs_earlier_version(dsname, SPA_VERSION_DEDUP))
	return (SET_ERROR(ENOTSUP));

	if (nvpair_type(pair) == DATA_TYPE_UINT64 &&
	nvpair_value_uint64(pair, &intval) == 0) {
	/* check prop value is enabled in features */
	feature = zio_checksum_to_feature(
	intval & ZIO_CHECKSUM_MASK);
	if (feature == SPA_FEATURE_NONE)
	break;

	if ((err = spa_open(dsname, &spa, FTAG)) != 0)
	return (err);

	if (!spa_feature_is_enabled(spa, feature)) {
	spa_close(spa, FTAG);
	return (SET_ERROR(ENOTSUP));
	}
	spa_close(spa, FTAG);
	}
	break;
	}

	default:
	break;
	}

	return (zfs_secpolicy_setprop(dsname, prop, pair, CRED()));
	}

	/*
	* Removes properties from the given props list that fail permission checks
	* needed to clear them and to restore them in case of a receive error. For each
	* property, make sure we have both set and inherit permissions.
	*
	* Returns the first error encountered if any permission checks fail. If the
	* caller provides a non-NULL errlist, it also gives the complete list of names
	* of all the properties that failed a permission check along with the
	* corresponding error numbers. The caller is responsible for freeing the
	* returned errlist.
	*
	* If every property checks out successfully, zero is returned and the list
	* pointed at by errlist is NULL.
	*/
	static int
	zfs_check_clearable(const char dataset, nvlist_t props, nvlist_t **errlist)
	{
	zfs_cmd_t *zc;
	nvpair_t pair, next_pair;
	nvlist_t *errors;
	int err, rv = 0;

	if (props == NULL)
	return (0);

	VERIFY(nvlist_alloc(&errors, NV_UNIQUE_NAME, KM_SLEEP) == 0);

	zc = kmem_alloc(sizeof (zfs_cmd_t), KM_SLEEP);
	(void) strlcpy(zc->zc_name, dataset, sizeof (zc->zc_name));
	pair = nvlist_next_nvpair(props, NULL);
	while (pair != NULL) {
	next_pair = nvlist_next_nvpair(props, pair);

	(void) strlcpy(zc->zc_value, nvpair_name(pair),
	sizeof (zc->zc_value));
	if ((err = zfs_check_settable(dataset, pair, CRED())) != 0 \|\|
	(err = zfs_secpolicy_inherit_prop(zc, NULL, CRED())) != 0) {
	VERIFY(nvlist_remove_nvpair(props, pair) == 0);
	VERIFY(nvlist_add_int32(errors,
	zc->zc_value, err) == 0);
	}
	pair = next_pair;
	}
	kmem_free(zc, sizeof (zfs_cmd_t));

	if ((pair = nvlist_next_nvpair(errors, NULL)) == NULL) {
	nvlist_free(errors);
	errors = NULL;
	} else {
	VERIFY(nvpair_value_int32(pair, &rv) == 0);
	}

	if (errlist == NULL)
	nvlist_free(errors);
	else
	*errlist = errors;

	return (rv);
	}

	static boolean_t
	propval_equals(nvpair_t p1, nvpair_t p2)
	{
	if (nvpair_type(p1) == DATA_TYPE_NVLIST) {
	/* dsl_prop_get_all_impl() format */
	nvlist_t *attrs;
	VERIFY(nvpair_value_nvlist(p1, &attrs) == 0);
	VERIFY(nvlist_lookup_nvpair(attrs, ZPROP_VALUE,
	&p1) == 0);
	}

	if (nvpair_type(p2) == DATA_TYPE_NVLIST) {
	nvlist_t *attrs;
	VERIFY(nvpair_value_nvlist(p2, &attrs) == 0);
	VERIFY(nvlist_lookup_nvpair(attrs, ZPROP_VALUE,
	&p2) == 0);
	}

	if (nvpair_type(p1) != nvpair_type(p2))
	return (B_FALSE);

	if (nvpair_type(p1) == DATA_TYPE_STRING) {
	const char valstr1, valstr2;

	VERIFY(nvpair_value_string(p1, &valstr1) == 0);
	VERIFY(nvpair_value_string(p2, &valstr2) == 0);
	return (strcmp(valstr1, valstr2) == 0);
	} else {
	uint64_t intval1, intval2;

	VERIFY(nvpair_value_uint64(p1, &intval1) == 0);
	VERIFY(nvpair_value_uint64(p2, &intval2) == 0);
	return (intval1 == intval2);
	}
	}

	/*
	* Remove properties from props if they are not going to change (as determined
	* by comparison with origprops). Remove them from origprops as well, since we
	* do not need to clear or restore properties that won't change.
	*/
	static void
	props_reduce(nvlist_t props, nvlist_t origprops)
	{
	nvpair_t pair, next_pair;

	if (origprops == NULL)
	return; /* all props need to be received */

	pair = nvlist_next_nvpair(props, NULL);
	while (pair != NULL) {
	const char *propname = nvpair_name(pair);
	nvpair_t *match;

	next_pair = nvlist_next_nvpair(props, pair);

	if ((nvlist_lookup_nvpair(origprops, propname,
	&match) != 0) \|\| !propval_equals(pair, match))
	goto next; /* need to set received value */

	/* don't clear the existing received value */
	(void) nvlist_remove_nvpair(origprops, match);
	/* don't bother receiving the property */
	(void) nvlist_remove_nvpair(props, pair);
	next:
	pair = next_pair;
	}
	}

	/*
	* Extract properties that cannot be set PRIOR to the receipt of a dataset.
	* For example, refquota cannot be set until after the receipt of a dataset,
	* because in replication streams, an older/earlier snapshot may exceed the
	* refquota. We want to receive the older/earlier snapshot, but setting
	* refquota pre-receipt will set the dsl's ACTUAL quota, which will prevent
	* the older/earlier snapshot from being received (with EDQUOT).
	*
	* The ZFS test "zfs_receive_011_pos" demonstrates such a scenario.
	*
	* libzfs will need to be judicious handling errors encountered by props
	* extracted by this function.
	*/
	static nvlist_t *
	extract_delay_props(nvlist_t *props)
	{
	nvlist_t *delayprops;
	nvpair_t nvp, tmp;
	static const zfs_prop_t delayable[] = {
	ZFS_PROP_REFQUOTA,
	ZFS_PROP_KEYLOCATION,
	/*
	* Setting ZFS_PROP_SHARESMB requires the objset type to be
	* known, which is not possible prior to receipt of raw sends.
	*/
	ZFS_PROP_SHARESMB,
	0
	};
	int i;

	VERIFY(nvlist_alloc(&delayprops, NV_UNIQUE_NAME, KM_SLEEP) == 0);

	for (nvp = nvlist_next_nvpair(props, NULL); nvp != NULL;
	nvp = nvlist_next_nvpair(props, nvp)) {
	/*
	* strcmp() is safe because zfs_prop_to_name() always returns
	* a bounded string.
	*/
	for (i = 0; delayable[i] != 0; i++) {
	if (strcmp(zfs_prop_to_name(delayable[i]),
	nvpair_name(nvp)) == 0) {
	break;
	}
	}
	if (delayable[i] != 0) {
	tmp = nvlist_prev_nvpair(props, nvp);
	VERIFY(nvlist_add_nvpair(delayprops, nvp) == 0);
	VERIFY(nvlist_remove_nvpair(props, nvp) == 0);
	nvp = tmp;
	}
	}

	if (nvlist_empty(delayprops)) {
	nvlist_free(delayprops);
	delayprops = NULL;
	}
	return (delayprops);
	}

	static void
	zfs_allow_log_destroy(void *arg)
	{
	char *poolname = arg;

	if (poolname != NULL)
	kmem_strfree(poolname);
	}

	#ifdef ZFS_DEBUG
	static boolean_t zfs_ioc_recv_inject_err;
	#endif

	/*
	* nvlist 'errors' is always allocated. It will contain descriptions of
	* encountered errors, if any. It's the callers responsibility to free.
	*/
	static int
	zfs_ioc_recv_impl(char tofs, char tosnap, const char *origin,
	nvlist_t recvprops, nvlist_t localprops, nvlist_t *hidden_args,
	boolean_t force, boolean_t heal, boolean_t resumable, int input_fd,
	dmu_replay_record_t begin_record, uint64_t read_bytes,
	uint64_t errflags, nvlist_t *errors)
	{
	dmu_recv_cookie_t drc;
	int error = 0;
	int props_error = 0;
	offset_t off, noff;
	nvlist_t *local_delayprops = NULL;
	nvlist_t *recv_delayprops = NULL;
	nvlist_t *inherited_delayprops = NULL;
	nvlist_t origprops = NULL; / existing properties */
	nvlist_t origrecvd = NULL; / existing received properties */
	boolean_t first_recvd_props = B_FALSE;
	boolean_t tofs_was_redacted;
	zfs_file_t *input_fp;

	*read_bytes = 0;
	*errflags = 0;
	*errors = fnvlist_alloc();
	off = 0;

	if ((input_fp = zfs_file_get(input_fd)) == NULL)
	return (SET_ERROR(EBADF));

	noff = off = zfs_file_off(input_fp);
	error = dmu_recv_begin(tofs, tosnap, begin_record, force, heal,
	resumable, localprops, hidden_args, origin, &drc, input_fp,
	&off);
	if (error != 0)
	goto out;
	tofs_was_redacted = dsl_get_redacted(drc.drc_ds);

	/*
	* Set properties before we receive the stream so that they are applied
	* to the new data. Note that we must call dmu_recv_stream() if
	* dmu_recv_begin() succeeds.
	*/
	if (recvprops != NULL && !drc.drc_newfs) {
	if (spa_version(dsl_dataset_get_spa(drc.drc_ds)) >=
	SPA_VERSION_RECVD_PROPS &&
	!dsl_prop_get_hasrecvd(tofs))
	first_recvd_props = B_TRUE;

	/*
	* If new received properties are supplied, they are to
	* completely replace the existing received properties,
	* so stash away the existing ones.
	*/
	if (dsl_prop_get_received(tofs, &origrecvd) == 0) {
	nvlist_t *errlist = NULL;
	/*
	* Don't bother writing a property if its value won't
	* change (and avoid the unnecessary security checks).
	*
	* The first receive after SPA_VERSION_RECVD_PROPS is a
	* special case where we blow away all local properties
	* regardless.
	*/
	if (!first_recvd_props)
	props_reduce(recvprops, origrecvd);
	if (zfs_check_clearable(tofs, origrecvd, &errlist) != 0)
	(void) nvlist_merge(*errors, errlist, 0);
	nvlist_free(errlist);

	if (clear_received_props(tofs, origrecvd,
	first_recvd_props ? NULL : recvprops) != 0)
	*errflags \|= ZPROP_ERR_NOCLEAR;
	} else {
	*errflags \|= ZPROP_ERR_NOCLEAR;
	}
	}

	/*
	* Stash away existing properties so we can restore them on error unless
	* we're doing the first receive after SPA_VERSION_RECVD_PROPS, in which
	* case "origrecvd" will take care of that.
	*/
	if (localprops != NULL && !drc.drc_newfs && !first_recvd_props) {
	objset_t *os;
	if (dmu_objset_hold(tofs, FTAG, &os) == 0) {
	if (dsl_prop_get_all(os, &origprops) != 0) {
	*errflags \|= ZPROP_ERR_NOCLEAR;
	}
	dmu_objset_rele(os, FTAG);
	} else {
	*errflags \|= ZPROP_ERR_NOCLEAR;
	}
	}

	if (recvprops != NULL) {
	props_error = dsl_prop_set_hasrecvd(tofs);

	if (props_error == 0) {
	recv_delayprops = extract_delay_props(recvprops);
	(void) zfs_set_prop_nvlist(tofs, ZPROP_SRC_RECEIVED,
	recvprops, *errors);
	}
	}

	if (localprops != NULL) {
	nvlist_t *oprops = fnvlist_alloc();
	nvlist_t *xprops = fnvlist_alloc();
	nvpair_t *nvp = NULL;

	while ((nvp = nvlist_next_nvpair(localprops, nvp)) != NULL) {
	if (nvpair_type(nvp) == DATA_TYPE_BOOLEAN) {
	/* -x property */
	const char *name = nvpair_name(nvp);
	zfs_prop_t prop = zfs_name_to_prop(name);
	if (prop != ZPROP_USERPROP) {
	if (!zfs_prop_inheritable(prop))
	continue;
	} else if (!zfs_prop_user(name))
	continue;
	fnvlist_add_boolean(xprops, name);
	} else {
	/* -o property=value */
	fnvlist_add_nvpair(oprops, nvp);
	}
	}

	local_delayprops = extract_delay_props(oprops);
	(void) zfs_set_prop_nvlist(tofs, ZPROP_SRC_LOCAL,
	oprops, *errors);
	inherited_delayprops = extract_delay_props(xprops);
	(void) zfs_set_prop_nvlist(tofs, ZPROP_SRC_INHERITED,
	xprops, *errors);

	nvlist_free(oprops);
	nvlist_free(xprops);
	}

	error = dmu_recv_stream(&drc, &off);

	if (error == 0) {
	zfsvfs_t *zfsvfs = NULL;
	zvol_state_handle_t *zv = NULL;

	if (getzfsvfs(tofs, &zfsvfs) == 0) {
	/* online recv */
	dsl_dataset_t *ds;
	int end_err;
	boolean_t stream_is_redacted = DMU_GET_FEATUREFLAGS(
	begin_record->drr_u.drr_begin.
	drr_versioninfo) & DMU_BACKUP_FEATURE_REDACTED;

	ds = dmu_objset_ds(zfsvfs->z_os);
	error = zfs_suspend_fs(zfsvfs);
	/*
	* If the suspend fails, then the recv_end will
	* likely also fail, and clean up after itself.
	*/
	end_err = dmu_recv_end(&drc, zfsvfs);
	/*
	* If the dataset was not redacted, but we received a
	* redacted stream onto it, we need to unmount the
	* dataset. Otherwise, resume the filesystem.
	*/
	if (error == 0 && !drc.drc_newfs &&
	stream_is_redacted && !tofs_was_redacted) {
	error = zfs_end_fs(zfsvfs, ds);
	} else if (error == 0) {
	error = zfs_resume_fs(zfsvfs, ds);
	}
	error = error ? error : end_err;
	zfs_vfs_rele(zfsvfs);
	} else if ((zv = zvol_suspend(tofs)) != NULL) {
	error = dmu_recv_end(&drc, zvol_tag(zv));
	zvol_resume(zv);
	} else {
	error = dmu_recv_end(&drc, NULL);
	}

	/* Set delayed properties now, after we're done receiving. */
	if (recv_delayprops != NULL && error == 0) {
	(void) zfs_set_prop_nvlist(tofs, ZPROP_SRC_RECEIVED,
	recv_delayprops, *errors);
	}
	if (local_delayprops != NULL && error == 0) {
	(void) zfs_set_prop_nvlist(tofs, ZPROP_SRC_LOCAL,
	local_delayprops, *errors);
	}
	if (inherited_delayprops != NULL && error == 0) {
	(void) zfs_set_prop_nvlist(tofs, ZPROP_SRC_INHERITED,
	inherited_delayprops, *errors);
	}
	}

	/*
	* Merge delayed props back in with initial props, in case
	* we're DEBUG and zfs_ioc_recv_inject_err is set (which means
	* we have to make sure clear_received_props() includes
	* the delayed properties).
	*
	* Since zfs_ioc_recv_inject_err is only in DEBUG kernels,
	* using ASSERT() will be just like a VERIFY.
	*/
	if (recv_delayprops != NULL) {
	ASSERT(nvlist_merge(recvprops, recv_delayprops, 0) == 0);
	nvlist_free(recv_delayprops);
	}
	if (local_delayprops != NULL) {
	ASSERT(nvlist_merge(localprops, local_delayprops, 0) == 0);
	nvlist_free(local_delayprops);
	}
	if (inherited_delayprops != NULL) {
	ASSERT(nvlist_merge(localprops, inherited_delayprops, 0) == 0);
	nvlist_free(inherited_delayprops);
	}
	*read_bytes = off - noff;

	#ifdef ZFS_DEBUG
	if (zfs_ioc_recv_inject_err) {
	zfs_ioc_recv_inject_err = B_FALSE;
	error = 1;
	}
	#endif

	/*
	* On error, restore the original props.
	*/
	if (error != 0 && recvprops != NULL && !drc.drc_newfs) {
	if (clear_received_props(tofs, recvprops, NULL) != 0) {
	/*
	* We failed to clear the received properties.
	* Since we may have left a $recvd value on the
	* system, we can't clear the $hasrecvd flag.
	*/
	*errflags \|= ZPROP_ERR_NORESTORE;
	} else if (first_recvd_props) {
	dsl_prop_unset_hasrecvd(tofs);
	}

	if (origrecvd == NULL && !drc.drc_newfs) {
	/* We failed to stash the original properties. */
	*errflags \|= ZPROP_ERR_NORESTORE;
	}

	/*
	* dsl_props_set() will not convert RECEIVED to LOCAL on or
	* after SPA_VERSION_RECVD_PROPS, so we need to specify LOCAL
	* explicitly if we're restoring local properties cleared in the
	* first new-style receive.
	*/
	if (origrecvd != NULL &&
	zfs_set_prop_nvlist(tofs, (first_recvd_props ?
	ZPROP_SRC_LOCAL : ZPROP_SRC_RECEIVED),
	origrecvd, NULL) != 0) {
	/*
	* We stashed the original properties but failed to
	* restore them.
	*/
	*errflags \|= ZPROP_ERR_NORESTORE;
	}
	}
	if (error != 0 && localprops != NULL && !drc.drc_newfs &&
	!first_recvd_props) {
	nvlist_t *setprops;
	nvlist_t *inheritprops;
	nvpair_t *nvp;

	if (origprops == NULL) {
	/* We failed to stash the original properties. */
	*errflags \|= ZPROP_ERR_NORESTORE;
	goto out;
	}

	/* Restore original props */
	setprops = fnvlist_alloc();
	inheritprops = fnvlist_alloc();
	nvp = NULL;
	while ((nvp = nvlist_next_nvpair(localprops, nvp)) != NULL) {
	const char *name = nvpair_name(nvp);
	const char *source;
	nvlist_t *attrs;

	if (!nvlist_exists(origprops, name)) {
	/*
	* Property was not present or was explicitly
	* inherited before the receive, restore this.
	*/
	fnvlist_add_boolean(inheritprops, name);
	continue;
	}
	attrs = fnvlist_lookup_nvlist(origprops, name);
	source = fnvlist_lookup_string(attrs, ZPROP_SOURCE);

	/* Skip received properties */
	if (strcmp(source, ZPROP_SOURCE_VAL_RECVD) == 0)
	continue;

	if (strcmp(source, tofs) == 0) {
	/* Property was locally set */
	fnvlist_add_nvlist(setprops, name, attrs);
	} else {
	/* Property was implicitly inherited */
	fnvlist_add_boolean(inheritprops, name);
	}
	}

	if (zfs_set_prop_nvlist(tofs, ZPROP_SRC_LOCAL, setprops,
	NULL) != 0)
	*errflags \|= ZPROP_ERR_NORESTORE;
	if (zfs_set_prop_nvlist(tofs, ZPROP_SRC_INHERITED, inheritprops,
	NULL) != 0)
	*errflags \|= ZPROP_ERR_NORESTORE;

	nvlist_free(setprops);
	nvlist_free(inheritprops);
	}
	out:
	zfs_file_put(input_fp);
	nvlist_free(origrecvd);
	nvlist_free(origprops);

	if (error == 0)
	error = props_error;

	return (error);
	}

	/*
	* inputs:
	* zc_name name of containing filesystem (unused)
	* zc_nvlist_src{_size} nvlist of properties to apply
	* zc_nvlist_conf{_size} nvlist of properties to exclude
	* (DATA_TYPE_BOOLEAN) and override (everything else)
	* zc_value name of snapshot to create
	* zc_string name of clone origin (if DRR_FLAG_CLONE)
	* zc_cookie file descriptor to recv from
	* zc_begin_record the BEGIN record of the stream (not byteswapped)
	* zc_guid force flag
	*
	* outputs:
	* zc_cookie number of bytes read
	* zc_obj zprop_errflags_t
	* zc_nvlist_dst{_size} error for each unapplied received property
	*/
	static int
	zfs_ioc_recv(zfs_cmd_t *zc)
	{
	dmu_replay_record_t begin_record;
	nvlist_t *errors = NULL;
	nvlist_t *recvdprops = NULL;
	nvlist_t *localprops = NULL;
	const char *origin = NULL;
	char *tosnap;
	char tofs[ZFS_MAX_DATASET_NAME_LEN];
	int error = 0;

	if (dataset_namecheck(zc->zc_value, NULL, NULL) != 0 \|\|
	strchr(zc->zc_value, '@') == NULL \|\|
	strchr(zc->zc_value, '%'))
	return (SET_ERROR(EINVAL));

	(void) strlcpy(tofs, zc->zc_value, sizeof (tofs));
	tosnap = strchr(tofs, '@');
	*tosnap++ = '\0';

	if (zc->zc_nvlist_src != 0 &&
	(error = get_nvlist(zc->zc_nvlist_src, zc->zc_nvlist_src_size,
	zc->zc_iflags, &recvdprops)) != 0)
	return (error);

	if (zc->zc_nvlist_conf != 0 &&
	(error = get_nvlist(zc->zc_nvlist_conf, zc->zc_nvlist_conf_size,
	zc->zc_iflags, &localprops)) != 0)
	return (error);

	if (zc->zc_string[0])
	origin = zc->zc_string;

	begin_record.drr_type = DRR_BEGIN;
	begin_record.drr_payloadlen = 0;
	begin_record.drr_u.drr_begin = zc->zc_begin_record;

	error = zfs_ioc_recv_impl(tofs, tosnap, origin, recvdprops, localprops,
	NULL, zc->zc_guid, B_FALSE, B_FALSE, zc->zc_cookie, &begin_record,
	&zc->zc_cookie, &zc->zc_obj, &errors);
	nvlist_free(recvdprops);
	nvlist_free(localprops);

	/*
	* Now that all props, initial and delayed, are set, report the prop
	* errors to the caller.
	*/
	if (zc->zc_nvlist_dst_size != 0 && errors != NULL &&
	(nvlist_smush(errors, zc->zc_nvlist_dst_size) != 0 \|\|
	put_nvlist(zc, errors) != 0)) {
	/*
	* Caller made zc->zc_nvlist_dst less than the minimum expected
	* size or supplied an invalid address.
	*/
	error = SET_ERROR(EINVAL);
	}

	nvlist_free(errors);

	return (error);
	}

	/*
	* innvl: {
	* "snapname" -> full name of the snapshot to create
	* (optional) "props" -> received properties to set (nvlist)
	* (optional) "localprops" -> override and exclude properties (nvlist)
	* (optional) "origin" -> name of clone origin (DRR_FLAG_CLONE)
	* "begin_record" -> non-byteswapped dmu_replay_record_t
	* "input_fd" -> file descriptor to read stream from (int32)
	* (optional) "force" -> force flag (value ignored)
	* (optional) "heal" -> use send stream to heal data corruption
	* (optional) "resumable" -> resumable flag (value ignored)
	* (optional) "cleanup_fd" -> unused
	* (optional) "action_handle" -> unused
	* (optional) "hidden_args" -> { "wkeydata" -> value }
	* }
	*
	* outnvl: {
	* "read_bytes" -> number of bytes read
	* "error_flags" -> zprop_errflags_t
	* "errors" -> error for each unapplied received property (nvlist)
	* }
	*/
	static const zfs_ioc_key_t zfs_keys_recv_new[] = {
	{"snapname", DATA_TYPE_STRING, 0},
	{"props", DATA_TYPE_NVLIST, ZK_OPTIONAL},
	{"localprops", DATA_TYPE_NVLIST, ZK_OPTIONAL},
	{"origin", DATA_TYPE_STRING, ZK_OPTIONAL},
	{"begin_record", DATA_TYPE_BYTE_ARRAY, 0},
	{"input_fd", DATA_TYPE_INT32, 0},
	{"force", DATA_TYPE_BOOLEAN, ZK_OPTIONAL},
	{"heal", DATA_TYPE_BOOLEAN, ZK_OPTIONAL},
	{"resumable", DATA_TYPE_BOOLEAN, ZK_OPTIONAL},
	{"cleanup_fd", DATA_TYPE_INT32, ZK_OPTIONAL},
	{"action_handle", DATA_TYPE_UINT64, ZK_OPTIONAL},
	{"hidden_args", DATA_TYPE_NVLIST, ZK_OPTIONAL},
	};

	static int
	zfs_ioc_recv_new(const char fsname, nvlist_t innvl, nvlist_t *outnvl)
	{
	dmu_replay_record_t *begin_record;
	uint_t begin_record_size;
	nvlist_t *errors = NULL;
	nvlist_t *recvprops = NULL;
	nvlist_t *localprops = NULL;
	nvlist_t *hidden_args = NULL;
	const char *snapname;
	const char *origin = NULL;
	char *tosnap;
	char tofs[ZFS_MAX_DATASET_NAME_LEN];
	boolean_t force;
	boolean_t heal;
	boolean_t resumable;
	uint64_t read_bytes = 0;
	uint64_t errflags = 0;
	int input_fd = -1;
	int error;

	snapname = fnvlist_lookup_string(innvl, "snapname");

	if (dataset_namecheck(snapname, NULL, NULL) != 0 \|\|
	strchr(snapname, '@') == NULL \|\|
	strchr(snapname, '%'))
	return (SET_ERROR(EINVAL));

	(void) strlcpy(tofs, snapname, sizeof (tofs));
	tosnap = strchr(tofs, '@');
	*tosnap++ = '\0';

	error = nvlist_lookup_string(innvl, "origin", &origin);
	if (error && error != ENOENT)
	return (error);

	error = nvlist_lookup_byte_array(innvl, "begin_record",
	(uchar_t **)&begin_record, &begin_record_size);
	if (error != 0 \|\| begin_record_size != sizeof (*begin_record))
	return (SET_ERROR(EINVAL));

	input_fd = fnvlist_lookup_int32(innvl, "input_fd");

	force = nvlist_exists(innvl, "force");
	heal = nvlist_exists(innvl, "heal");
	resumable = nvlist_exists(innvl, "resumable");

	/* we still use "props" here for backwards compatibility */
	error = nvlist_lookup_nvlist(innvl, "props", &recvprops);
	if (error && error != ENOENT)
	return (error);

	error = nvlist_lookup_nvlist(innvl, "localprops", &localprops);
	if (error && error != ENOENT)
	return (error);

	error = nvlist_lookup_nvlist(innvl, ZPOOL_HIDDEN_ARGS, &hidden_args);
	if (error && error != ENOENT)
	return (error);

	error = zfs_ioc_recv_impl(tofs, tosnap, origin, recvprops, localprops,
	hidden_args, force, heal, resumable, input_fd, begin_record,
	&read_bytes, &errflags, &errors);

	fnvlist_add_uint64(outnvl, "read_bytes", read_bytes);
	fnvlist_add_uint64(outnvl, "error_flags", errflags);
	fnvlist_add_nvlist(outnvl, "errors", errors);

	nvlist_free(errors);
	nvlist_free(recvprops);
	nvlist_free(localprops);

	return (error);
	}

	typedef struct dump_bytes_io {
	zfs_file_t *dbi_fp;
	caddr_t dbi_buf;
	int dbi_len;
	int dbi_err;
	} dump_bytes_io_t;

	static void
	dump_bytes_cb(void *arg)
	{
	dump_bytes_io_t dbi = (dump_bytes_io_t )arg;
	zfs_file_t *fp;
	caddr_t buf;

	fp = dbi->dbi_fp;
	buf = dbi->dbi_buf;

	dbi->dbi_err = zfs_file_write(fp, buf, dbi->dbi_len, NULL);
	}

	static int
	dump_bytes(objset_t os, void buf, int len, void *arg)
	{
	dump_bytes_io_t dbi;

	dbi.dbi_fp = arg;
	dbi.dbi_buf = buf;
	dbi.dbi_len = len;

	#if defined(HAVE_LARGE_STACKS)
	dump_bytes_cb(&dbi);
	#else
	/*
	* The vn_rdwr() call is performed in a taskq to ensure that there is
	* always enough stack space to write safely to the target filesystem.
	* The ZIO_TYPE_FREE threads are used because there can be a lot of
	* them and they are used in vdev_file.c for a similar purpose.
	*/
	spa_taskq_dispatch_sync(dmu_objset_spa(os), ZIO_TYPE_FREE,
	ZIO_TASKQ_ISSUE, dump_bytes_cb, &dbi, TQ_SLEEP);
	#endif /* HAVE_LARGE_STACKS */

	return (dbi.dbi_err);
	}

	/*
	* inputs:
	* zc_name name of snapshot to send
	* zc_cookie file descriptor to send stream to
	* zc_obj fromorigin flag (mutually exclusive with zc_fromobj)
	* zc_sendobj objsetid of snapshot to send
	* zc_fromobj objsetid of incremental fromsnap (may be zero)
	* zc_guid if set, estimate size of stream only. zc_cookie is ignored.
	* output size in zc_objset_type.
	* zc_flags lzc_send_flags
	*
	* outputs:
	* zc_objset_type estimated size, if zc_guid is set
	*
	* NOTE: This is no longer the preferred interface, any new functionality
	* should be added to zfs_ioc_send_new() instead.
	*/
	static int
	zfs_ioc_send(zfs_cmd_t *zc)
	{
	int error;
	offset_t off;
	boolean_t estimate = (zc->zc_guid != 0);
	boolean_t embedok = (zc->zc_flags & 0x1);
	boolean_t large_block_ok = (zc->zc_flags & 0x2);
	boolean_t compressok = (zc->zc_flags & 0x4);
	boolean_t rawok = (zc->zc_flags & 0x8);
	boolean_t savedok = (zc->zc_flags & 0x10);

	if (zc->zc_obj != 0) {
	dsl_pool_t *dp;
	dsl_dataset_t *tosnap;

	error = dsl_pool_hold(zc->zc_name, FTAG, &dp);
	if (error != 0)
	return (error);

	error = dsl_dataset_hold_obj(dp, zc->zc_sendobj, FTAG, &tosnap);
	if (error != 0) {
	dsl_pool_rele(dp, FTAG);
	return (error);
	}

	if (dsl_dir_is_clone(tosnap->ds_dir))
	zc->zc_fromobj =
	dsl_dir_phys(tosnap->ds_dir)->dd_origin_obj;
	dsl_dataset_rele(tosnap, FTAG);
	dsl_pool_rele(dp, FTAG);
	}

	if (estimate) {
	dsl_pool_t *dp;
	dsl_dataset_t *tosnap;
	dsl_dataset_t *fromsnap = NULL;

	error = dsl_pool_hold(zc->zc_name, FTAG, &dp);
	if (error != 0)
	return (error);

	error = dsl_dataset_hold_obj(dp, zc->zc_sendobj,
	FTAG, &tosnap);
	if (error != 0) {
	dsl_pool_rele(dp, FTAG);
	return (error);
	}

	if (zc->zc_fromobj != 0) {
	error = dsl_dataset_hold_obj(dp, zc->zc_fromobj,
	FTAG, &fromsnap);
	if (error != 0) {
	dsl_dataset_rele(tosnap, FTAG);
	dsl_pool_rele(dp, FTAG);
	return (error);
	}
	}

	error = dmu_send_estimate_fast(tosnap, fromsnap, NULL,
	compressok \|\| rawok, savedok, &zc->zc_objset_type);

	if (fromsnap != NULL)
	dsl_dataset_rele(fromsnap, FTAG);
	dsl_dataset_rele(tosnap, FTAG);
	dsl_pool_rele(dp, FTAG);
	} else {
	zfs_file_t *fp;
	dmu_send_outparams_t out = {0};

	if ((fp = zfs_file_get(zc->zc_cookie)) == NULL)
	return (SET_ERROR(EBADF));

	off = zfs_file_off(fp);
	out.dso_outfunc = dump_bytes;
	out.dso_arg = fp;
	out.dso_dryrun = B_FALSE;
	error = dmu_send_obj(zc->zc_name, zc->zc_sendobj,
	zc->zc_fromobj, embedok, large_block_ok, compressok,
	rawok, savedok, zc->zc_cookie, &off, &out);

	zfs_file_put(fp);
	}
	return (error);
	}

	/*
	* inputs:
	* zc_name name of snapshot on which to report progress
	* zc_cookie file descriptor of send stream
	*
	* outputs:
	* zc_cookie number of bytes written in send stream thus far
	* zc_objset_type logical size of data traversed by send thus far
	*/
	static int
	zfs_ioc_send_progress(zfs_cmd_t *zc)
	{
	dsl_pool_t *dp;
	dsl_dataset_t *ds;
	dmu_sendstatus_t *dsp = NULL;
	int error;

	error = dsl_pool_hold(zc->zc_name, FTAG, &dp);
	if (error != 0)
	return (error);

	error = dsl_dataset_hold(dp, zc->zc_name, FTAG, &ds);
	if (error != 0) {
	dsl_pool_rele(dp, FTAG);
	return (error);
	}

	mutex_enter(&ds->ds_sendstream_lock);

	/*
	* Iterate over all the send streams currently active on this dataset.
	* If there's one which matches the specified file descriptor _and_ the
	* stream was started by the current process, return the progress of
	* that stream.
	*/

	for (dsp = list_head(&ds->ds_sendstreams); dsp != NULL;
	dsp = list_next(&ds->ds_sendstreams, dsp)) {
	if (dsp->dss_outfd == zc->zc_cookie &&
	zfs_proc_is_caller(dsp->dss_proc))
	break;
	}

	if (dsp != NULL) {
	zc->zc_cookie = atomic_cas_64((volatile uint64_t *)dsp->dss_off,
	0, 0);
	/* This is the closest thing we have to atomic_read_64. */
	zc->zc_objset_type = atomic_cas_64(&dsp->dss_blocks, 0, 0);
	} else {
	error = SET_ERROR(ENOENT);
	}

	mutex_exit(&ds->ds_sendstream_lock);
	dsl_dataset_rele(ds, FTAG);
	dsl_pool_rele(dp, FTAG);
	return (error);
	}

	static int
	zfs_ioc_inject_fault(zfs_cmd_t *zc)
	{
	int id, error;

	error = zio_inject_fault(zc->zc_name, (int)zc->zc_guid, &id,
	&zc->zc_inject_record);

	if (error == 0)
	zc->zc_guid = (uint64_t)id;

	return (error);
	}

	static int
	zfs_ioc_clear_fault(zfs_cmd_t *zc)
	{
	return (zio_clear_fault((int)zc->zc_guid));
	}

	static int
	zfs_ioc_inject_list_next(zfs_cmd_t *zc)
	{
	int id = (int)zc->zc_guid;
	int error;

	error = zio_inject_list_next(&id, zc->zc_name, sizeof (zc->zc_name),
	&zc->zc_inject_record);

	zc->zc_guid = id;

	return (error);
	}

	static int
	zfs_ioc_error_log(zfs_cmd_t *zc)
	{
	spa_t *spa;
	int error;

	if ((error = spa_open(zc->zc_name, &spa, FTAG)) != 0)
	return (error);

	error = spa_get_errlog(spa, (void *)(uintptr_t)zc->zc_nvlist_dst,
	&zc->zc_nvlist_dst_size);

	spa_close(spa, FTAG);

	return (error);
	}

	static int
	zfs_ioc_clear(zfs_cmd_t *zc)
	{
	spa_t *spa;
	vdev_t *vd;
	int error;

	/*
	* On zpool clear we also fix up missing slogs
	*/
	mutex_enter(&spa_namespace_lock);
	spa = spa_lookup(zc->zc_name);
	if (spa == NULL) {
	mutex_exit(&spa_namespace_lock);
	return (SET_ERROR(EIO));
	}
	if (spa_get_log_state(spa) == SPA_LOG_MISSING) {
	/* we need to let spa_open/spa_load clear the chains */
	spa_set_log_state(spa, SPA_LOG_CLEAR);
	}
	spa->spa_last_open_failed = 0;
	mutex_exit(&spa_namespace_lock);

	if (zc->zc_cookie & ZPOOL_NO_REWIND) {
	error = spa_open(zc->zc_name, &spa, FTAG);
	} else {
	nvlist_t *policy;
	nvlist_t *config = NULL;

	if (zc->zc_nvlist_src == 0)
	return (SET_ERROR(EINVAL));

	if ((error = get_nvlist(zc->zc_nvlist_src,
	zc->zc_nvlist_src_size, zc->zc_iflags, &policy)) == 0) {
	error = spa_open_rewind(zc->zc_name, &spa, FTAG,
	policy, &config);
	if (config != NULL) {
	int err;

	if ((err = put_nvlist(zc, config)) != 0)
	error = err;
	nvlist_free(config);
	}
	nvlist_free(policy);
	}
	}

	if (error != 0)
	return (error);

	/*
	* If multihost is enabled, resuming I/O is unsafe as another
	- * host may have imported the pool.
	+ * host may have imported the pool. Check for remote activity.
	*/
	- if (spa_multihost(spa) && spa_suspended(spa))
	- return (SET_ERROR(EINVAL));
	+ if (spa_multihost(spa) && spa_suspended(spa) &&
	+ spa_mmp_remote_host_activity(spa)) {
	+ spa_close(spa, FTAG);
	+ return (SET_ERROR(EREMOTEIO));
	+ }

	spa_vdev_state_enter(spa, SCL_NONE);

	if (zc->zc_guid == 0) {
	vd = NULL;
	} else {
	vd = spa_lookup_by_guid(spa, zc->zc_guid, B_TRUE);
	if (vd == NULL) {
	error = SET_ERROR(ENODEV);
	(void) spa_vdev_state_exit(spa, NULL, error);
	spa_close(spa, FTAG);
	return (error);
	}
	}

	vdev_clear(spa, vd);

	(void) spa_vdev_state_exit(spa, spa_suspended(spa) ?
	NULL : spa->spa_root_vdev, 0);

	/*
	* Resume any suspended I/Os.
	*/
	if (zio_resume(spa) != 0)
	error = SET_ERROR(EIO);

	spa_close(spa, FTAG);

	return (error);
	}

	/*
	* Reopen all the vdevs associated with the pool.
	*
	* innvl: {
	* "scrub_restart" -> when true and scrub is running, allow to restart
	* scrub as the side effect of the reopen (boolean).
	* }
	*
	* outnvl is unused
	*/
	static const zfs_ioc_key_t zfs_keys_pool_reopen[] = {
	{"scrub_restart", DATA_TYPE_BOOLEAN_VALUE, ZK_OPTIONAL},
	};

	static int
	zfs_ioc_pool_reopen(const char pool, nvlist_t innvl, nvlist_t *outnvl)
	{
	(void) outnvl;
	spa_t *spa;
	int error;
	boolean_t rc, scrub_restart = B_TRUE;

	if (innvl) {
	error = nvlist_lookup_boolean_value(innvl,
	"scrub_restart", &rc);
	if (error == 0)
	scrub_restart = rc;
	}

	error = spa_open(pool, &spa, FTAG);
	if (error != 0)
	return (error);

	spa_vdev_state_enter(spa, SCL_NONE);

	/*
	* If the scrub_restart flag is B_FALSE and a scrub is already
	* in progress then set spa_scrub_reopen flag to B_TRUE so that
	* we don't restart the scrub as a side effect of the reopen.
	* Otherwise, let vdev_open() decided if a resilver is required.
	*/

	spa->spa_scrub_reopen = (!scrub_restart &&
	dsl_scan_scrubbing(spa->spa_dsl_pool));
	vdev_reopen(spa->spa_root_vdev);
	spa->spa_scrub_reopen = B_FALSE;

	(void) spa_vdev_state_exit(spa, NULL, 0);
	spa_close(spa, FTAG);
	return (0);
	}

	/*
	* inputs:
	* zc_name name of filesystem
	*
	* outputs:
	* zc_string name of conflicting snapshot, if there is one
	*/
	static int
	zfs_ioc_promote(zfs_cmd_t *zc)
	{
	dsl_pool_t *dp;
	dsl_dataset_t ds, ods;
	char origin[ZFS_MAX_DATASET_NAME_LEN];
	char *cp;
	int error;

	zc->zc_name[sizeof (zc->zc_name) - 1] = '\0';
	if (dataset_namecheck(zc->zc_name, NULL, NULL) != 0 \|\|
	strchr(zc->zc_name, '%'))
	return (SET_ERROR(EINVAL));

	error = dsl_pool_hold(zc->zc_name, FTAG, &dp);
	if (error != 0)
	return (error);

	error = dsl_dataset_hold(dp, zc->zc_name, FTAG, &ds);
	if (error != 0) {
	dsl_pool_rele(dp, FTAG);
	return (error);
	}

	if (!dsl_dir_is_clone(ds->ds_dir)) {
	dsl_dataset_rele(ds, FTAG);
	dsl_pool_rele(dp, FTAG);
	return (SET_ERROR(EINVAL));
	}

	error = dsl_dataset_hold_obj(dp,
	dsl_dir_phys(ds->ds_dir)->dd_origin_obj, FTAG, &ods);
	if (error != 0) {
	dsl_dataset_rele(ds, FTAG);
	dsl_pool_rele(dp, FTAG);
	return (error);
	}

	dsl_dataset_name(ods, origin);
	dsl_dataset_rele(ods, FTAG);
	dsl_dataset_rele(ds, FTAG);
	dsl_pool_rele(dp, FTAG);

	/*
	* We don't need to unmount all the origin fs's snapshots, but
	* it's easier.
	*/
	cp = strchr(origin, '@');
	if (cp)
	*cp = '\0';
	(void) dmu_objset_find(origin,
	zfs_unmount_snap_cb, NULL, DS_FIND_SNAPSHOTS);
	return (dsl_dataset_promote(zc->zc_name, zc->zc_string));
	}

	/*
	* Retrieve a single {user\|group\|project}{used\|quota}@... property.
	*
	* inputs:
	* zc_name name of filesystem
	* zc_objset_type zfs_userquota_prop_t
	* zc_value domain name (eg. "S-1-234-567-89")
	* zc_guid RID/UID/GID
	*
	* outputs:
	* zc_cookie property value
	*/
	static int
	zfs_ioc_userspace_one(zfs_cmd_t *zc)
	{
	zfsvfs_t *zfsvfs;
	int error;

	if (zc->zc_objset_type >= ZFS_NUM_USERQUOTA_PROPS)
	return (SET_ERROR(EINVAL));

	error = zfsvfs_hold(zc->zc_name, FTAG, &zfsvfs, B_FALSE);
	if (error != 0)
	return (error);

	error = zfs_userspace_one(zfsvfs,
	zc->zc_objset_type, zc->zc_value, zc->zc_guid, &zc->zc_cookie);
	zfsvfs_rele(zfsvfs, FTAG);

	return (error);
	}

	/*
	* inputs:
	* zc_name name of filesystem
	* zc_cookie zap cursor
	* zc_objset_type zfs_userquota_prop_t
	* zc_nvlist_dst[_size] buffer to fill (not really an nvlist)
	*
	* outputs:
	* zc_nvlist_dst[_size] data buffer (array of zfs_useracct_t)
	* zc_cookie zap cursor
	*/
	static int
	zfs_ioc_userspace_many(zfs_cmd_t *zc)
	{
	zfsvfs_t *zfsvfs;
	int bufsize = zc->zc_nvlist_dst_size;

	if (bufsize <= 0)
	return (SET_ERROR(ENOMEM));

	int error = zfsvfs_hold(zc->zc_name, FTAG, &zfsvfs, B_FALSE);
	if (error != 0)
	return (error);

	void *buf = vmem_alloc(bufsize, KM_SLEEP);

	error = zfs_userspace_many(zfsvfs, zc->zc_objset_type, &zc->zc_cookie,
	buf, &zc->zc_nvlist_dst_size);

	if (error == 0) {
	error = xcopyout(buf,
	(void *)(uintptr_t)zc->zc_nvlist_dst,
	zc->zc_nvlist_dst_size);
	}
	vmem_free(buf, bufsize);
	zfsvfs_rele(zfsvfs, FTAG);

	return (error);
	}

	/*
	* inputs:
	* zc_name name of filesystem
	*
	* outputs:
	* none
	*/
	static int
	zfs_ioc_userspace_upgrade(zfs_cmd_t *zc)
	{
	int error = 0;
	zfsvfs_t *zfsvfs;

	if (getzfsvfs(zc->zc_name, &zfsvfs) == 0) {
	if (!dmu_objset_userused_enabled(zfsvfs->z_os)) {
	/*
	* If userused is not enabled, it may be because the
	* objset needs to be closed & reopened (to grow the
	* objset_phys_t). Suspend/resume the fs will do that.
	*/
	dsl_dataset_t ds, newds;

	ds = dmu_objset_ds(zfsvfs->z_os);
	error = zfs_suspend_fs(zfsvfs);
	if (error == 0) {
	dmu_objset_refresh_ownership(ds, &newds,
	B_TRUE, zfsvfs);
	error = zfs_resume_fs(zfsvfs, newds);
	}
	}
	if (error == 0) {
	mutex_enter(&zfsvfs->z_os->os_upgrade_lock);
	if (zfsvfs->z_os->os_upgrade_id == 0) {
	/* clear potential error code and retry */
	zfsvfs->z_os->os_upgrade_status = 0;
	mutex_exit(&zfsvfs->z_os->os_upgrade_lock);

	dsl_pool_config_enter(
	dmu_objset_pool(zfsvfs->z_os), FTAG);
	dmu_objset_userspace_upgrade(zfsvfs->z_os);
	dsl_pool_config_exit(
	dmu_objset_pool(zfsvfs->z_os), FTAG);
	} else {
	mutex_exit(&zfsvfs->z_os->os_upgrade_lock);
	}

	taskq_wait_id(zfsvfs->z_os->os_spa->spa_upgrade_taskq,
	zfsvfs->z_os->os_upgrade_id);
	error = zfsvfs->z_os->os_upgrade_status;
	}
	zfs_vfs_rele(zfsvfs);
	} else {
	objset_t *os;

	/* XXX kind of reading contents without owning */
	error = dmu_objset_hold_flags(zc->zc_name, B_TRUE, FTAG, &os);
	if (error != 0)
	return (error);

	mutex_enter(&os->os_upgrade_lock);
	if (os->os_upgrade_id == 0) {
	/* clear potential error code and retry */
	os->os_upgrade_status = 0;
	mutex_exit(&os->os_upgrade_lock);

	dmu_objset_userspace_upgrade(os);
	} else {
	mutex_exit(&os->os_upgrade_lock);
	}

	dsl_pool_rele(dmu_objset_pool(os), FTAG);

	taskq_wait_id(os->os_spa->spa_upgrade_taskq, os->os_upgrade_id);
	error = os->os_upgrade_status;

	dsl_dataset_rele_flags(dmu_objset_ds(os), DS_HOLD_FLAG_DECRYPT,
	FTAG);
	}
	return (error);
	}

	/*
	* inputs:
	* zc_name name of filesystem
	*
	* outputs:
	* none
	*/
	static int
	zfs_ioc_id_quota_upgrade(zfs_cmd_t *zc)
	{
	objset_t *os;
	int error;

	error = dmu_objset_hold_flags(zc->zc_name, B_TRUE, FTAG, &os);
	if (error != 0)
	return (error);

	if (dmu_objset_userobjspace_upgradable(os) \|\|
	dmu_objset_projectquota_upgradable(os)) {
	mutex_enter(&os->os_upgrade_lock);
	if (os->os_upgrade_id == 0) {
	/* clear potential error code and retry */
	os->os_upgrade_status = 0;
	mutex_exit(&os->os_upgrade_lock);

	dmu_objset_id_quota_upgrade(os);
	} else {
	mutex_exit(&os->os_upgrade_lock);
	}

	dsl_pool_rele(dmu_objset_pool(os), FTAG);

	taskq_wait_id(os->os_spa->spa_upgrade_taskq, os->os_upgrade_id);
	error = os->os_upgrade_status;
	} else {
	dsl_pool_rele(dmu_objset_pool(os), FTAG);
	}

	dsl_dataset_rele_flags(dmu_objset_ds(os), DS_HOLD_FLAG_DECRYPT, FTAG);

	return (error);
	}

	static int
	zfs_ioc_share(zfs_cmd_t *zc)
	{
	return (SET_ERROR(ENOSYS));
	}

	/*
	* inputs:
	* zc_name name of containing filesystem
	* zc_obj object # beyond which we want next in-use object #
	*
	* outputs:
	* zc_obj next in-use object #
	*/
	static int
	zfs_ioc_next_obj(zfs_cmd_t *zc)
	{
	objset_t *os = NULL;
	int error;

	error = dmu_objset_hold(zc->zc_name, FTAG, &os);
	if (error != 0)
	return (error);

	error = dmu_object_next(os, &zc->zc_obj, B_FALSE, 0);

	dmu_objset_rele(os, FTAG);
	return (error);
	}

	/*
	* inputs:
	* zc_name name of filesystem
	* zc_value prefix name for snapshot
	* zc_cleanup_fd cleanup-on-exit file descriptor for calling process
	*
	* outputs:
	* zc_value short name of new snapshot
	*/
	static int
	zfs_ioc_tmp_snapshot(zfs_cmd_t *zc)
	{
	char *snap_name;
	char *hold_name;
	minor_t minor;

	zfs_file_t *fp = zfs_onexit_fd_hold(zc->zc_cleanup_fd, &minor);
	if (fp == NULL)
	return (SET_ERROR(EBADF));

	snap_name = kmem_asprintf("%s-%016llx", zc->zc_value,
	(u_longlong_t)ddi_get_lbolt64());
	hold_name = kmem_asprintf("%%%s", zc->zc_value);

	int error = dsl_dataset_snapshot_tmp(zc->zc_name, snap_name, minor,
	hold_name);
	if (error == 0)
	(void) strlcpy(zc->zc_value, snap_name,
	sizeof (zc->zc_value));
	kmem_strfree(snap_name);
	kmem_strfree(hold_name);
	zfs_onexit_fd_rele(fp);
	return (error);
	}

	/*
	* inputs:
	* zc_name name of "to" snapshot
	* zc_value name of "from" snapshot
	* zc_cookie file descriptor to write diff data on
	*
	* outputs:
	* dmu_diff_record_t's to the file descriptor
	*/
	static int
	zfs_ioc_diff(zfs_cmd_t *zc)
	{
	zfs_file_t *fp;
	offset_t off;
	int error;

	if ((fp = zfs_file_get(zc->zc_cookie)) == NULL)
	return (SET_ERROR(EBADF));

	off = zfs_file_off(fp);
	error = dmu_diff(zc->zc_name, zc->zc_value, fp, &off);

	zfs_file_put(fp);

	return (error);
	}

	static int
	zfs_ioc_smb_acl(zfs_cmd_t *zc)
	{
	return (SET_ERROR(ENOTSUP));
	}

	/*
	* innvl: {
	* "holds" -> { snapname -> holdname (string), ... }
	* (optional) "cleanup_fd" -> fd (int32)
	* }
	*
	* outnvl: {
	* snapname -> error value (int32)
	* ...
	* }
	*/
	static const zfs_ioc_key_t zfs_keys_hold[] = {
	{"holds", DATA_TYPE_NVLIST, 0},
	{"cleanup_fd", DATA_TYPE_INT32, ZK_OPTIONAL},
	};

	static int
	zfs_ioc_hold(const char pool, nvlist_t args, nvlist_t *errlist)
	{
	(void) pool;
	nvpair_t *pair;
	nvlist_t *holds;
	int cleanup_fd = -1;
	int error;
	minor_t minor = 0;
	zfs_file_t *fp = NULL;

	holds = fnvlist_lookup_nvlist(args, "holds");

	/* make sure the user didn't pass us any invalid (empty) tags */
	for (pair = nvlist_next_nvpair(holds, NULL); pair != NULL;
	pair = nvlist_next_nvpair(holds, pair)) {
	const char *htag;

	error = nvpair_value_string(pair, &htag);
	if (error != 0)
	return (SET_ERROR(error));

	if (strlen(htag) == 0)
	return (SET_ERROR(EINVAL));
	}

	if (nvlist_lookup_int32(args, "cleanup_fd", &cleanup_fd) == 0) {
	fp = zfs_onexit_fd_hold(cleanup_fd, &minor);
	if (fp == NULL)
	return (SET_ERROR(EBADF));
	}

	error = dsl_dataset_user_hold(holds, minor, errlist);
	if (fp != NULL) {
	ASSERT3U(minor, !=, 0);
	zfs_onexit_fd_rele(fp);
	}
	return (SET_ERROR(error));
	}

	/*
	* innvl is not used.
	*
	* outnvl: {
	* holdname -> time added (uint64 seconds since epoch)
	* ...
	* }
	*/
	static const zfs_ioc_key_t zfs_keys_get_holds[] = {
	/* no nvl keys */
	};

	static int
	zfs_ioc_get_holds(const char snapname, nvlist_t args, nvlist_t *outnvl)
	{
	(void) args;
	return (dsl_dataset_get_holds(snapname, outnvl));
	}

	/*
	* innvl: {
	* snapname -> { holdname, ... }
	* ...
	* }
	*
	* outnvl: {
	* snapname -> error value (int32)
	* ...
	* }
	*/
	static const zfs_ioc_key_t zfs_keys_release[] = {
	{"<snapname>...", DATA_TYPE_NVLIST, ZK_WILDCARDLIST},
	};

	static int
	zfs_ioc_release(const char pool, nvlist_t holds, nvlist_t *errlist)
	{
	(void) pool;
	return (dsl_dataset_user_release(holds, errlist));
	}

	/*
	* inputs:
	* zc_guid flags (ZEVENT_NONBLOCK)
	* zc_cleanup_fd zevent file descriptor
	*
	* outputs:
	* zc_nvlist_dst next nvlist event
	* zc_cookie dropped events since last get
	*/
	static int
	zfs_ioc_events_next(zfs_cmd_t *zc)
	{
	zfs_zevent_t *ze;
	nvlist_t *event = NULL;
	minor_t minor;
	uint64_t dropped = 0;
	int error;

	zfs_file_t *fp = zfs_zevent_fd_hold(zc->zc_cleanup_fd, &minor, &ze);
	if (fp == NULL)
	return (SET_ERROR(EBADF));

	do {
	error = zfs_zevent_next(ze, &event,
	&zc->zc_nvlist_dst_size, &dropped);
	if (event != NULL) {
	zc->zc_cookie = dropped;
	error = put_nvlist(zc, event);
	nvlist_free(event);
	}

	if (zc->zc_guid & ZEVENT_NONBLOCK)
	break;

	if ((error == 0) \|\| (error != ENOENT))
	break;

	error = zfs_zevent_wait(ze);
	if (error != 0)
	break;
	} while (1);

	zfs_zevent_fd_rele(fp);

	return (error);
	}

	/*
	* outputs:
	* zc_cookie cleared events count
	*/
	static int
	zfs_ioc_events_clear(zfs_cmd_t *zc)
	{
	uint_t count;

	zfs_zevent_drain_all(&count);
	zc->zc_cookie = count;

	return (0);
	}

	/*
	* inputs:
	* zc_guid eid \| ZEVENT_SEEK_START \| ZEVENT_SEEK_END
	* zc_cleanup zevent file descriptor
	*/
	static int
	zfs_ioc_events_seek(zfs_cmd_t *zc)
	{
	zfs_zevent_t *ze;
	minor_t minor;
	int error;

	zfs_file_t *fp = zfs_zevent_fd_hold(zc->zc_cleanup_fd, &minor, &ze);
	if (fp == NULL)
	return (SET_ERROR(EBADF));

	error = zfs_zevent_seek(ze, zc->zc_guid);
	zfs_zevent_fd_rele(fp);

	return (error);
	}

	/*
	* inputs:
	* zc_name name of later filesystem or snapshot
	* zc_value full name of old snapshot or bookmark
	*
	* outputs:
	* zc_cookie space in bytes
	* zc_objset_type compressed space in bytes
	* zc_perm_action uncompressed space in bytes
	*/
	static int
	zfs_ioc_space_written(zfs_cmd_t *zc)
	{
	int error;
	dsl_pool_t *dp;
	dsl_dataset_t *new;

	error = dsl_pool_hold(zc->zc_name, FTAG, &dp);
	if (error != 0)
	return (error);
	error = dsl_dataset_hold(dp, zc->zc_name, FTAG, &new);
	if (error != 0) {
	dsl_pool_rele(dp, FTAG);
	return (error);
	}
	if (strchr(zc->zc_value, '#') != NULL) {
	zfs_bookmark_phys_t bmp;
	error = dsl_bookmark_lookup(dp, zc->zc_value,
	new, &bmp);
	if (error == 0) {
	error = dsl_dataset_space_written_bookmark(&bmp, new,
	&zc->zc_cookie,
	&zc->zc_objset_type, &zc->zc_perm_action);
	}
	} else {
	dsl_dataset_t *old;
	error = dsl_dataset_hold(dp, zc->zc_value, FTAG, &old);

	if (error == 0) {
	error = dsl_dataset_space_written(old, new,
	&zc->zc_cookie,
	&zc->zc_objset_type, &zc->zc_perm_action);
	dsl_dataset_rele(old, FTAG);
	}
	}
	dsl_dataset_rele(new, FTAG);
	dsl_pool_rele(dp, FTAG);
	return (error);
	}

	/*
	* innvl: {
	* "firstsnap" -> snapshot name
	* }
	*
	* outnvl: {
	* "used" -> space in bytes
	* "compressed" -> compressed space in bytes
	* "uncompressed" -> uncompressed space in bytes
	* }
	*/
	static const zfs_ioc_key_t zfs_keys_space_snaps[] = {
	{"firstsnap", DATA_TYPE_STRING, 0},
	};

	static int
	zfs_ioc_space_snaps(const char lastsnap, nvlist_t innvl, nvlist_t *outnvl)
	{
	int error;
	dsl_pool_t *dp;
	dsl_dataset_t new, old;
	const char *firstsnap;
	uint64_t used, comp, uncomp;

	firstsnap = fnvlist_lookup_string(innvl, "firstsnap");

	error = dsl_pool_hold(lastsnap, FTAG, &dp);
	if (error != 0)
	return (error);

	error = dsl_dataset_hold(dp, lastsnap, FTAG, &new);
	if (error == 0 && !new->ds_is_snapshot) {
	dsl_dataset_rele(new, FTAG);
	error = SET_ERROR(EINVAL);
	}
	if (error != 0) {
	dsl_pool_rele(dp, FTAG);
	return (error);
	}
	error = dsl_dataset_hold(dp, firstsnap, FTAG, &old);
	if (error == 0 && !old->ds_is_snapshot) {
	dsl_dataset_rele(old, FTAG);
	error = SET_ERROR(EINVAL);
	}
	if (error != 0) {
	dsl_dataset_rele(new, FTAG);
	dsl_pool_rele(dp, FTAG);
	return (error);
	}

	error = dsl_dataset_space_wouldfree(old, new, &used, &comp, &uncomp);
	dsl_dataset_rele(old, FTAG);
	dsl_dataset_rele(new, FTAG);
	dsl_pool_rele(dp, FTAG);
	fnvlist_add_uint64(outnvl, "used", used);
	fnvlist_add_uint64(outnvl, "compressed", comp);
	fnvlist_add_uint64(outnvl, "uncompressed", uncomp);
	return (error);
	}

	/*
	* innvl: {
	* "fd" -> file descriptor to write stream to (int32)
	* (optional) "fromsnap" -> full snap name to send an incremental from
	* (optional) "largeblockok" -> (value ignored)
	* indicates that blocks > 128KB are permitted
	* (optional) "embedok" -> (value ignored)
	* presence indicates DRR_WRITE_EMBEDDED records are permitted
	* (optional) "compressok" -> (value ignored)
	* presence indicates compressed DRR_WRITE records are permitted
	* (optional) "rawok" -> (value ignored)
	* presence indicates raw encrypted records should be used.
	* (optional) "savedok" -> (value ignored)
	* presence indicates we should send a partially received snapshot
	* (optional) "resume_object" and "resume_offset" -> (uint64)
	* if present, resume send stream from specified object and offset.
	* (optional) "redactbook" -> (string)
	* if present, use this bookmark's redaction list to generate a redacted
	* send stream
	* }
	*
	* outnvl is unused
	*/
	static const zfs_ioc_key_t zfs_keys_send_new[] = {
	{"fd", DATA_TYPE_INT32, 0},
	{"fromsnap", DATA_TYPE_STRING, ZK_OPTIONAL},
	{"largeblockok", DATA_TYPE_BOOLEAN, ZK_OPTIONAL},
	{"embedok", DATA_TYPE_BOOLEAN, ZK_OPTIONAL},
	{"compressok", DATA_TYPE_BOOLEAN, ZK_OPTIONAL},
	{"rawok", DATA_TYPE_BOOLEAN, ZK_OPTIONAL},
	{"savedok", DATA_TYPE_BOOLEAN, ZK_OPTIONAL},
	{"resume_object", DATA_TYPE_UINT64, ZK_OPTIONAL},
	{"resume_offset", DATA_TYPE_UINT64, ZK_OPTIONAL},
	{"redactbook", DATA_TYPE_STRING, ZK_OPTIONAL},
	};

	static int
	zfs_ioc_send_new(const char snapname, nvlist_t innvl, nvlist_t *outnvl)
	{
	(void) outnvl;
	int error;
	offset_t off;
	const char *fromname = NULL;
	int fd;
	zfs_file_t *fp;
	boolean_t largeblockok;
	boolean_t embedok;
	boolean_t compressok;
	boolean_t rawok;
	boolean_t savedok;
	uint64_t resumeobj = 0;
	uint64_t resumeoff = 0;
	const char *redactbook = NULL;

	fd = fnvlist_lookup_int32(innvl, "fd");

	(void) nvlist_lookup_string(innvl, "fromsnap", &fromname);

	largeblockok = nvlist_exists(innvl, "largeblockok");
	embedok = nvlist_exists(innvl, "embedok");
	compressok = nvlist_exists(innvl, "compressok");
	rawok = nvlist_exists(innvl, "rawok");
	savedok = nvlist_exists(innvl, "savedok");

	(void) nvlist_lookup_uint64(innvl, "resume_object", &resumeobj);
	(void) nvlist_lookup_uint64(innvl, "resume_offset", &resumeoff);

	(void) nvlist_lookup_string(innvl, "redactbook", &redactbook);

	if ((fp = zfs_file_get(fd)) == NULL)
	return (SET_ERROR(EBADF));

	off = zfs_file_off(fp);

	dmu_send_outparams_t out = {0};
	out.dso_outfunc = dump_bytes;
	out.dso_arg = fp;
	out.dso_dryrun = B_FALSE;
	error = dmu_send(snapname, fromname, embedok, largeblockok,
	compressok, rawok, savedok, resumeobj, resumeoff,
	redactbook, fd, &off, &out);

	zfs_file_put(fp);
	return (error);
	}

	static int
	send_space_sum(objset_t os, void buf, int len, void *arg)
	{
	(void) os, (void) buf;
	uint64_t *size = arg;

	*size += len;
	return (0);
	}

	/*
	* Determine approximately how large a zfs send stream will be -- the number
	* of bytes that will be written to the fd supplied to zfs_ioc_send_new().
	*
	* innvl: {
	* (optional) "from" -> full snap or bookmark name to send an incremental
	* from
	* (optional) "largeblockok" -> (value ignored)
	* indicates that blocks > 128KB are permitted
	* (optional) "embedok" -> (value ignored)
	* presence indicates DRR_WRITE_EMBEDDED records are permitted
	* (optional) "compressok" -> (value ignored)
	* presence indicates compressed DRR_WRITE records are permitted
	* (optional) "rawok" -> (value ignored)
	* presence indicates raw encrypted records should be used.
	* (optional) "resume_object" and "resume_offset" -> (uint64)
	* if present, resume send stream from specified object and offset.
	* (optional) "fd" -> file descriptor to use as a cookie for progress
	* tracking (int32)
	* }
	*
	* outnvl: {
	* "space" -> bytes of space (uint64)
	* }
	*/
	static const zfs_ioc_key_t zfs_keys_send_space[] = {
	{"from", DATA_TYPE_STRING, ZK_OPTIONAL},
	{"fromsnap", DATA_TYPE_STRING, ZK_OPTIONAL},
	{"largeblockok", DATA_TYPE_BOOLEAN, ZK_OPTIONAL},
	{"embedok", DATA_TYPE_BOOLEAN, ZK_OPTIONAL},
	{"compressok", DATA_TYPE_BOOLEAN, ZK_OPTIONAL},
	{"rawok", DATA_TYPE_BOOLEAN, ZK_OPTIONAL},
	{"fd", DATA_TYPE_INT32, ZK_OPTIONAL},
	{"redactbook", DATA_TYPE_STRING, ZK_OPTIONAL},
	{"resume_object", DATA_TYPE_UINT64, ZK_OPTIONAL},
	{"resume_offset", DATA_TYPE_UINT64, ZK_OPTIONAL},
	{"bytes", DATA_TYPE_UINT64, ZK_OPTIONAL},
	};

	static int
	zfs_ioc_send_space(const char snapname, nvlist_t innvl, nvlist_t *outnvl)
	{
	dsl_pool_t *dp;
	dsl_dataset_t *tosnap;
	dsl_dataset_t *fromsnap = NULL;
	int error;
	const char *fromname = NULL;
	const char *redactlist_book = NULL;
	boolean_t largeblockok;
	boolean_t embedok;
	boolean_t compressok;
	boolean_t rawok;
	boolean_t savedok;
	uint64_t space = 0;
	boolean_t full_estimate = B_FALSE;
	uint64_t resumeobj = 0;
	uint64_t resumeoff = 0;
	uint64_t resume_bytes = 0;
	int32_t fd = -1;
	zfs_bookmark_phys_t zbm = {0};

	error = dsl_pool_hold(snapname, FTAG, &dp);
	if (error != 0)
	return (error);

	error = dsl_dataset_hold(dp, snapname, FTAG, &tosnap);
	if (error != 0) {
	dsl_pool_rele(dp, FTAG);
	return (error);
	}
	(void) nvlist_lookup_int32(innvl, "fd", &fd);

	largeblockok = nvlist_exists(innvl, "largeblockok");
	embedok = nvlist_exists(innvl, "embedok");
	compressok = nvlist_exists(innvl, "compressok");
	rawok = nvlist_exists(innvl, "rawok");
	savedok = nvlist_exists(innvl, "savedok");
	boolean_t from = (nvlist_lookup_string(innvl, "from", &fromname) == 0);
	boolean_t altbook = (nvlist_lookup_string(innvl, "redactbook",
	&redactlist_book) == 0);

	(void) nvlist_lookup_uint64(innvl, "resume_object", &resumeobj);
	(void) nvlist_lookup_uint64(innvl, "resume_offset", &resumeoff);
	(void) nvlist_lookup_uint64(innvl, "bytes", &resume_bytes);

	if (altbook) {
	full_estimate = B_TRUE;
	} else if (from) {
	if (strchr(fromname, '#')) {
	error = dsl_bookmark_lookup(dp, fromname, tosnap, &zbm);

	/*
	* dsl_bookmark_lookup() will fail with EXDEV if
	* the from-bookmark and tosnap are at the same txg.
	* However, it's valid to do a send (and therefore,
	* a send estimate) from and to the same time point,
	* if the bookmark is redacted (the incremental send
	* can change what's redacted on the target). In
	* this case, dsl_bookmark_lookup() fills in zbm
	* but returns EXDEV. Ignore this error.
	*/
	if (error == EXDEV && zbm.zbm_redaction_obj != 0 &&
	zbm.zbm_guid ==
	dsl_dataset_phys(tosnap)->ds_guid)
	error = 0;

	if (error != 0) {
	dsl_dataset_rele(tosnap, FTAG);
	dsl_pool_rele(dp, FTAG);
	return (error);
	}
	if (zbm.zbm_redaction_obj != 0 \|\| !(zbm.zbm_flags &
	ZBM_FLAG_HAS_FBN)) {
	full_estimate = B_TRUE;
	}
	} else if (strchr(fromname, '@')) {
	error = dsl_dataset_hold(dp, fromname, FTAG, &fromsnap);
	if (error != 0) {
	dsl_dataset_rele(tosnap, FTAG);
	dsl_pool_rele(dp, FTAG);
	return (error);
	}

	if (!dsl_dataset_is_before(tosnap, fromsnap, 0)) {
	full_estimate = B_TRUE;
	dsl_dataset_rele(fromsnap, FTAG);
	}
	} else {
	/*
	* from is not properly formatted as a snapshot or
	* bookmark
	*/
	dsl_dataset_rele(tosnap, FTAG);
	dsl_pool_rele(dp, FTAG);
	return (SET_ERROR(EINVAL));
	}
	}

	if (full_estimate) {
	dmu_send_outparams_t out = {0};
	offset_t off = 0;
	out.dso_outfunc = send_space_sum;
	out.dso_arg = &space;
	out.dso_dryrun = B_TRUE;
	/*
	* We have to release these holds so dmu_send can take them. It
	* will do all the error checking we need.
	*/
	dsl_dataset_rele(tosnap, FTAG);
	dsl_pool_rele(dp, FTAG);
	error = dmu_send(snapname, fromname, embedok, largeblockok,
	compressok, rawok, savedok, resumeobj, resumeoff,
	redactlist_book, fd, &off, &out);
	} else {
	error = dmu_send_estimate_fast(tosnap, fromsnap,
	(from && strchr(fromname, '#') != NULL ? &zbm : NULL),
	compressok \|\| rawok, savedok, &space);
	space -= resume_bytes;
	if (fromsnap != NULL)
	dsl_dataset_rele(fromsnap, FTAG);
	dsl_dataset_rele(tosnap, FTAG);
	dsl_pool_rele(dp, FTAG);
	}

	fnvlist_add_uint64(outnvl, "space", space);

	return (error);
	}

	/*
	* Sync the currently open TXG to disk for the specified pool.
	* This is somewhat similar to 'zfs_sync()'.
	* For cases that do not result in error this ioctl will wait for
	* the currently open TXG to commit before returning back to the caller.
	*
	* innvl: {
	* "force" -> when true, force uberblock update even if there is no dirty data.
	* In addition this will cause the vdev configuration to be written
	* out including updating the zpool cache file. (boolean_t)
	* }
	*
	* onvl is unused
	*/
	static const zfs_ioc_key_t zfs_keys_pool_sync[] = {
	{"force", DATA_TYPE_BOOLEAN_VALUE, 0},
	};

	static int
	zfs_ioc_pool_sync(const char pool, nvlist_t innvl, nvlist_t *onvl)
	{
	(void) onvl;
	int err;
	boolean_t rc, force = B_FALSE;
	spa_t *spa;

	if ((err = spa_open(pool, &spa, FTAG)) != 0)
	return (err);

	if (innvl) {
	err = nvlist_lookup_boolean_value(innvl, "force", &rc);
	if (err == 0)
	force = rc;
	}

	if (force) {
	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_WRITER);
	vdev_config_dirty(spa->spa_root_vdev);
	spa_config_exit(spa, SCL_CONFIG, FTAG);
	}
	txg_wait_synced(spa_get_dsl(spa), 0);

	spa_close(spa, FTAG);

	return (0);
	}

	/*
	* Load a user's wrapping key into the kernel.
	* innvl: {
	* "hidden_args" -> { "wkeydata" -> value }
	* raw uint8_t array of encryption wrapping key data (32 bytes)
	* (optional) "noop" -> (value ignored)
	* presence indicated key should only be verified, not loaded
	* }
	*/
	static const zfs_ioc_key_t zfs_keys_load_key[] = {
	{"hidden_args", DATA_TYPE_NVLIST, 0},
	{"noop", DATA_TYPE_BOOLEAN, ZK_OPTIONAL},
	};

	static int
	zfs_ioc_load_key(const char dsname, nvlist_t innvl, nvlist_t *outnvl)
	{
	(void) outnvl;
	int ret;
	dsl_crypto_params_t *dcp = NULL;
	nvlist_t *hidden_args;
	boolean_t noop = nvlist_exists(innvl, "noop");

	if (strchr(dsname, '@') != NULL \|\| strchr(dsname, '%') != NULL) {
	ret = SET_ERROR(EINVAL);
	goto error;
	}

	hidden_args = fnvlist_lookup_nvlist(innvl, ZPOOL_HIDDEN_ARGS);

	ret = dsl_crypto_params_create_nvlist(DCP_CMD_NONE, NULL,
	hidden_args, &dcp);
	if (ret != 0)
	goto error;

	ret = spa_keystore_load_wkey(dsname, dcp, noop);
	if (ret != 0)
	goto error;

	dsl_crypto_params_free(dcp, noop);

	return (0);

	error:
	dsl_crypto_params_free(dcp, B_TRUE);
	return (ret);
	}

	/*
	* Unload a user's wrapping key from the kernel.
	* Both innvl and outnvl are unused.
	*/
	static const zfs_ioc_key_t zfs_keys_unload_key[] = {
	/* no nvl keys */
	};

	static int
	zfs_ioc_unload_key(const char dsname, nvlist_t innvl, nvlist_t *outnvl)
	{
	(void) innvl, (void) outnvl;
	int ret = 0;

	if (strchr(dsname, '@') != NULL \|\| strchr(dsname, '%') != NULL) {
	ret = (SET_ERROR(EINVAL));
	goto out;
	}

	ret = spa_keystore_unload_wkey(dsname);
	if (ret != 0)
	goto out;

	out:
	return (ret);
	}

	/*
	* Changes a user's wrapping key used to decrypt a dataset. The keyformat,
	* keylocation, pbkdf2salt, and pbkdf2iters properties can also be specified
	* here to change how the key is derived in userspace.
	*
	* innvl: {
	* "hidden_args" (optional) -> { "wkeydata" -> value }
	* raw uint8_t array of new encryption wrapping key data (32 bytes)
	* "props" (optional) -> { prop -> value }
	* }
	*
	* outnvl is unused
	*/
	static const zfs_ioc_key_t zfs_keys_change_key[] = {
	{"crypt_cmd", DATA_TYPE_UINT64, ZK_OPTIONAL},
	{"hidden_args", DATA_TYPE_NVLIST, ZK_OPTIONAL},
	{"props", DATA_TYPE_NVLIST, ZK_OPTIONAL},
	};

	static int
	zfs_ioc_change_key(const char dsname, nvlist_t innvl, nvlist_t *outnvl)
	{
	(void) outnvl;
	int ret;
	uint64_t cmd = DCP_CMD_NONE;
	dsl_crypto_params_t *dcp = NULL;
	nvlist_t args = NULL, hidden_args = NULL;

	if (strchr(dsname, '@') != NULL \|\| strchr(dsname, '%') != NULL) {
	ret = (SET_ERROR(EINVAL));
	goto error;
	}

	(void) nvlist_lookup_uint64(innvl, "crypt_cmd", &cmd);
	(void) nvlist_lookup_nvlist(innvl, "props", &args);
	(void) nvlist_lookup_nvlist(innvl, ZPOOL_HIDDEN_ARGS, &hidden_args);

	ret = dsl_crypto_params_create_nvlist(cmd, args, hidden_args, &dcp);
	if (ret != 0)
	goto error;

	ret = spa_keystore_change_key(dsname, dcp);
	if (ret != 0)
	goto error;

	dsl_crypto_params_free(dcp, B_FALSE);

	return (0);

	error:
	dsl_crypto_params_free(dcp, B_TRUE);
	return (ret);
	}

	static zfs_ioc_vec_t zfs_ioc_vec[ZFS_IOC_LAST - ZFS_IOC_FIRST];

	static void
	zfs_ioctl_register_legacy(zfs_ioc_t ioc, zfs_ioc_legacy_func_t *func,
	zfs_secpolicy_func_t *secpolicy, zfs_ioc_namecheck_t namecheck,
	boolean_t log_history, zfs_ioc_poolcheck_t pool_check)
	{
	zfs_ioc_vec_t *vec = &zfs_ioc_vec[ioc - ZFS_IOC_FIRST];

	ASSERT3U(ioc, >=, ZFS_IOC_FIRST);
	ASSERT3U(ioc, <, ZFS_IOC_LAST);
	ASSERT3P(vec->zvec_legacy_func, ==, NULL);
	ASSERT3P(vec->zvec_func, ==, NULL);

	vec->zvec_legacy_func = func;
	vec->zvec_secpolicy = secpolicy;
	vec->zvec_namecheck = namecheck;
	vec->zvec_allow_log = log_history;
	vec->zvec_pool_check = pool_check;
	}

	/*
	* See the block comment at the beginning of this file for details on
	* each argument to this function.
	*/
	void
	zfs_ioctl_register(const char name, zfs_ioc_t ioc, zfs_ioc_func_t func,
	zfs_secpolicy_func_t *secpolicy, zfs_ioc_namecheck_t namecheck,
	zfs_ioc_poolcheck_t pool_check, boolean_t smush_outnvlist,
	boolean_t allow_log, const zfs_ioc_key_t *nvl_keys, size_t num_keys)
	{
	zfs_ioc_vec_t *vec = &zfs_ioc_vec[ioc - ZFS_IOC_FIRST];

	ASSERT3U(ioc, >=, ZFS_IOC_FIRST);
	ASSERT3U(ioc, <, ZFS_IOC_LAST);
	ASSERT3P(vec->zvec_legacy_func, ==, NULL);
	ASSERT3P(vec->zvec_func, ==, NULL);

	/* if we are logging, the name must be valid */
	ASSERT(!allow_log \|\| namecheck != NO_NAME);

	vec->zvec_name = name;
	vec->zvec_func = func;
	vec->zvec_secpolicy = secpolicy;
	vec->zvec_namecheck = namecheck;
	vec->zvec_pool_check = pool_check;
	vec->zvec_smush_outnvlist = smush_outnvlist;
	vec->zvec_allow_log = allow_log;
	vec->zvec_nvl_keys = nvl_keys;
	vec->zvec_nvl_key_count = num_keys;
	}

	static void
	zfs_ioctl_register_pool(zfs_ioc_t ioc, zfs_ioc_legacy_func_t *func,
	zfs_secpolicy_func_t *secpolicy, boolean_t log_history,
	zfs_ioc_poolcheck_t pool_check)
	{
	zfs_ioctl_register_legacy(ioc, func, secpolicy,
	POOL_NAME, log_history, pool_check);
	}

	void
	zfs_ioctl_register_dataset_nolog(zfs_ioc_t ioc, zfs_ioc_legacy_func_t *func,
	zfs_secpolicy_func_t *secpolicy, zfs_ioc_poolcheck_t pool_check)
	{
	zfs_ioctl_register_legacy(ioc, func, secpolicy,
	DATASET_NAME, B_FALSE, pool_check);
	}

	static void
	zfs_ioctl_register_pool_modify(zfs_ioc_t ioc, zfs_ioc_legacy_func_t *func)
	{
	zfs_ioctl_register_legacy(ioc, func, zfs_secpolicy_config,
	POOL_NAME, B_TRUE, POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY);
	}

	static void
	zfs_ioctl_register_pool_meta(zfs_ioc_t ioc, zfs_ioc_legacy_func_t *func,
	zfs_secpolicy_func_t *secpolicy)
	{
	zfs_ioctl_register_legacy(ioc, func, secpolicy,
	NO_NAME, B_FALSE, POOL_CHECK_NONE);
	}

	static void
	zfs_ioctl_register_dataset_read_secpolicy(zfs_ioc_t ioc,
	zfs_ioc_legacy_func_t func, zfs_secpolicy_func_t secpolicy)
	{
	zfs_ioctl_register_legacy(ioc, func, secpolicy,
	DATASET_NAME, B_FALSE, POOL_CHECK_SUSPENDED);
	}

	static void
	zfs_ioctl_register_dataset_read(zfs_ioc_t ioc, zfs_ioc_legacy_func_t *func)
	{
	zfs_ioctl_register_dataset_read_secpolicy(ioc, func,
	zfs_secpolicy_read);
	}

	static void
	zfs_ioctl_register_dataset_modify(zfs_ioc_t ioc, zfs_ioc_legacy_func_t *func,
	zfs_secpolicy_func_t *secpolicy)
	{
	zfs_ioctl_register_legacy(ioc, func, secpolicy,
	DATASET_NAME, B_TRUE, POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY);
	}

	static void
	zfs_ioctl_init(void)
	{
	zfs_ioctl_register("snapshot", ZFS_IOC_SNAPSHOT,
	zfs_ioc_snapshot, zfs_secpolicy_snapshot, POOL_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_TRUE, B_TRUE,
	zfs_keys_snapshot, ARRAY_SIZE(zfs_keys_snapshot));

	zfs_ioctl_register("log_history", ZFS_IOC_LOG_HISTORY,
	zfs_ioc_log_history, zfs_secpolicy_log_history, NO_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_FALSE, B_FALSE,
	zfs_keys_log_history, ARRAY_SIZE(zfs_keys_log_history));

	zfs_ioctl_register("space_snaps", ZFS_IOC_SPACE_SNAPS,
	zfs_ioc_space_snaps, zfs_secpolicy_read, DATASET_NAME,
	POOL_CHECK_SUSPENDED, B_FALSE, B_FALSE,
	zfs_keys_space_snaps, ARRAY_SIZE(zfs_keys_space_snaps));

	zfs_ioctl_register("send", ZFS_IOC_SEND_NEW,
	zfs_ioc_send_new, zfs_secpolicy_send_new, DATASET_NAME,
	POOL_CHECK_SUSPENDED, B_FALSE, B_FALSE,
	zfs_keys_send_new, ARRAY_SIZE(zfs_keys_send_new));

	zfs_ioctl_register("send_space", ZFS_IOC_SEND_SPACE,
	zfs_ioc_send_space, zfs_secpolicy_read, DATASET_NAME,
	POOL_CHECK_SUSPENDED, B_FALSE, B_FALSE,
	zfs_keys_send_space, ARRAY_SIZE(zfs_keys_send_space));

	zfs_ioctl_register("create", ZFS_IOC_CREATE,
	zfs_ioc_create, zfs_secpolicy_create_clone, DATASET_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_TRUE, B_TRUE,
	zfs_keys_create, ARRAY_SIZE(zfs_keys_create));

	zfs_ioctl_register("clone", ZFS_IOC_CLONE,
	zfs_ioc_clone, zfs_secpolicy_create_clone, DATASET_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_TRUE, B_TRUE,
	zfs_keys_clone, ARRAY_SIZE(zfs_keys_clone));

	zfs_ioctl_register("remap", ZFS_IOC_REMAP,
	zfs_ioc_remap, zfs_secpolicy_none, DATASET_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_FALSE, B_TRUE,
	zfs_keys_remap, ARRAY_SIZE(zfs_keys_remap));

	zfs_ioctl_register("destroy_snaps", ZFS_IOC_DESTROY_SNAPS,
	zfs_ioc_destroy_snaps, zfs_secpolicy_destroy_snaps, POOL_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_TRUE, B_TRUE,
	zfs_keys_destroy_snaps, ARRAY_SIZE(zfs_keys_destroy_snaps));

	zfs_ioctl_register("hold", ZFS_IOC_HOLD,
	zfs_ioc_hold, zfs_secpolicy_hold, POOL_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_TRUE, B_TRUE,
	zfs_keys_hold, ARRAY_SIZE(zfs_keys_hold));
	zfs_ioctl_register("release", ZFS_IOC_RELEASE,
	zfs_ioc_release, zfs_secpolicy_release, POOL_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_TRUE, B_TRUE,
	zfs_keys_release, ARRAY_SIZE(zfs_keys_release));

	zfs_ioctl_register("get_holds", ZFS_IOC_GET_HOLDS,
	zfs_ioc_get_holds, zfs_secpolicy_read, DATASET_NAME,
	POOL_CHECK_SUSPENDED, B_FALSE, B_FALSE,
	zfs_keys_get_holds, ARRAY_SIZE(zfs_keys_get_holds));

	zfs_ioctl_register("rollback", ZFS_IOC_ROLLBACK,
	zfs_ioc_rollback, zfs_secpolicy_rollback, DATASET_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_FALSE, B_TRUE,
	zfs_keys_rollback, ARRAY_SIZE(zfs_keys_rollback));

	zfs_ioctl_register("bookmark", ZFS_IOC_BOOKMARK,
	zfs_ioc_bookmark, zfs_secpolicy_bookmark, POOL_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_TRUE, B_TRUE,
	zfs_keys_bookmark, ARRAY_SIZE(zfs_keys_bookmark));

	zfs_ioctl_register("get_bookmarks", ZFS_IOC_GET_BOOKMARKS,
	zfs_ioc_get_bookmarks, zfs_secpolicy_read, DATASET_NAME,
	POOL_CHECK_SUSPENDED, B_FALSE, B_FALSE,
	zfs_keys_get_bookmarks, ARRAY_SIZE(zfs_keys_get_bookmarks));

	zfs_ioctl_register("get_bookmark_props", ZFS_IOC_GET_BOOKMARK_PROPS,
	zfs_ioc_get_bookmark_props, zfs_secpolicy_read, ENTITY_NAME,
	POOL_CHECK_SUSPENDED, B_FALSE, B_FALSE, zfs_keys_get_bookmark_props,
	ARRAY_SIZE(zfs_keys_get_bookmark_props));

	zfs_ioctl_register("destroy_bookmarks", ZFS_IOC_DESTROY_BOOKMARKS,
	zfs_ioc_destroy_bookmarks, zfs_secpolicy_destroy_bookmarks,
	POOL_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_TRUE, B_TRUE,
	zfs_keys_destroy_bookmarks,
	ARRAY_SIZE(zfs_keys_destroy_bookmarks));

	zfs_ioctl_register("receive", ZFS_IOC_RECV_NEW,
	zfs_ioc_recv_new, zfs_secpolicy_recv, DATASET_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_TRUE, B_TRUE,
	zfs_keys_recv_new, ARRAY_SIZE(zfs_keys_recv_new));
	zfs_ioctl_register("load-key", ZFS_IOC_LOAD_KEY,
	zfs_ioc_load_key, zfs_secpolicy_load_key,
	DATASET_NAME, POOL_CHECK_SUSPENDED, B_TRUE, B_TRUE,
	zfs_keys_load_key, ARRAY_SIZE(zfs_keys_load_key));
	zfs_ioctl_register("unload-key", ZFS_IOC_UNLOAD_KEY,
	zfs_ioc_unload_key, zfs_secpolicy_load_key,
	DATASET_NAME, POOL_CHECK_SUSPENDED, B_TRUE, B_TRUE,
	zfs_keys_unload_key, ARRAY_SIZE(zfs_keys_unload_key));
	zfs_ioctl_register("change-key", ZFS_IOC_CHANGE_KEY,
	zfs_ioc_change_key, zfs_secpolicy_change_key,
	DATASET_NAME, POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY,
	B_TRUE, B_TRUE, zfs_keys_change_key,
	ARRAY_SIZE(zfs_keys_change_key));

	zfs_ioctl_register("sync", ZFS_IOC_POOL_SYNC,
	zfs_ioc_pool_sync, zfs_secpolicy_none, POOL_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_FALSE, B_FALSE,
	zfs_keys_pool_sync, ARRAY_SIZE(zfs_keys_pool_sync));
	zfs_ioctl_register("reopen", ZFS_IOC_POOL_REOPEN, zfs_ioc_pool_reopen,
	zfs_secpolicy_config, POOL_NAME, POOL_CHECK_SUSPENDED, B_TRUE,
	B_TRUE, zfs_keys_pool_reopen, ARRAY_SIZE(zfs_keys_pool_reopen));

	zfs_ioctl_register("channel_program", ZFS_IOC_CHANNEL_PROGRAM,
	zfs_ioc_channel_program, zfs_secpolicy_config,
	POOL_NAME, POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_TRUE,
	B_TRUE, zfs_keys_channel_program,
	ARRAY_SIZE(zfs_keys_channel_program));

	zfs_ioctl_register("redact", ZFS_IOC_REDACT,
	zfs_ioc_redact, zfs_secpolicy_config, DATASET_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_TRUE, B_TRUE,
	zfs_keys_redact, ARRAY_SIZE(zfs_keys_redact));

	zfs_ioctl_register("zpool_checkpoint", ZFS_IOC_POOL_CHECKPOINT,
	zfs_ioc_pool_checkpoint, zfs_secpolicy_config, POOL_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_TRUE, B_TRUE,
	zfs_keys_pool_checkpoint, ARRAY_SIZE(zfs_keys_pool_checkpoint));

	zfs_ioctl_register("zpool_discard_checkpoint",
	ZFS_IOC_POOL_DISCARD_CHECKPOINT, zfs_ioc_pool_discard_checkpoint,
	zfs_secpolicy_config, POOL_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_TRUE, B_TRUE,
	zfs_keys_pool_discard_checkpoint,
	ARRAY_SIZE(zfs_keys_pool_discard_checkpoint));

	zfs_ioctl_register("initialize", ZFS_IOC_POOL_INITIALIZE,
	zfs_ioc_pool_initialize, zfs_secpolicy_config, POOL_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_TRUE, B_TRUE,
	zfs_keys_pool_initialize, ARRAY_SIZE(zfs_keys_pool_initialize));

	zfs_ioctl_register("trim", ZFS_IOC_POOL_TRIM,
	zfs_ioc_pool_trim, zfs_secpolicy_config, POOL_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_TRUE, B_TRUE,
	zfs_keys_pool_trim, ARRAY_SIZE(zfs_keys_pool_trim));

	zfs_ioctl_register("wait", ZFS_IOC_WAIT,
	zfs_ioc_wait, zfs_secpolicy_none, POOL_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_FALSE, B_FALSE,
	zfs_keys_pool_wait, ARRAY_SIZE(zfs_keys_pool_wait));

	zfs_ioctl_register("wait_fs", ZFS_IOC_WAIT_FS,
	zfs_ioc_wait_fs, zfs_secpolicy_none, DATASET_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_FALSE, B_FALSE,
	zfs_keys_fs_wait, ARRAY_SIZE(zfs_keys_fs_wait));

	zfs_ioctl_register("set_bootenv", ZFS_IOC_SET_BOOTENV,
	zfs_ioc_set_bootenv, zfs_secpolicy_config, POOL_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_FALSE, B_TRUE,
	zfs_keys_set_bootenv, ARRAY_SIZE(zfs_keys_set_bootenv));

	zfs_ioctl_register("get_bootenv", ZFS_IOC_GET_BOOTENV,
	zfs_ioc_get_bootenv, zfs_secpolicy_none, POOL_NAME,
	POOL_CHECK_SUSPENDED, B_FALSE, B_TRUE,
	zfs_keys_get_bootenv, ARRAY_SIZE(zfs_keys_get_bootenv));

	zfs_ioctl_register("zpool_vdev_get_props", ZFS_IOC_VDEV_GET_PROPS,
	zfs_ioc_vdev_get_props, zfs_secpolicy_read, POOL_NAME,
	POOL_CHECK_NONE, B_FALSE, B_FALSE, zfs_keys_vdev_get_props,
	ARRAY_SIZE(zfs_keys_vdev_get_props));

	zfs_ioctl_register("zpool_vdev_set_props", ZFS_IOC_VDEV_SET_PROPS,
	zfs_ioc_vdev_set_props, zfs_secpolicy_config, POOL_NAME,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY, B_FALSE, B_FALSE,
	zfs_keys_vdev_set_props, ARRAY_SIZE(zfs_keys_vdev_set_props));

	zfs_ioctl_register("scrub", ZFS_IOC_POOL_SCRUB,
	zfs_ioc_pool_scrub, zfs_secpolicy_config, POOL_NAME,
	POOL_CHECK_NONE, B_TRUE, B_TRUE,
	zfs_keys_pool_scrub, ARRAY_SIZE(zfs_keys_pool_scrub));

	/* IOCTLS that use the legacy function signature */

	zfs_ioctl_register_legacy(ZFS_IOC_POOL_FREEZE, zfs_ioc_pool_freeze,
	zfs_secpolicy_config, NO_NAME, B_FALSE, POOL_CHECK_READONLY);

	zfs_ioctl_register_pool(ZFS_IOC_POOL_CREATE, zfs_ioc_pool_create,
	zfs_secpolicy_config, B_TRUE, POOL_CHECK_NONE);
	zfs_ioctl_register_pool_modify(ZFS_IOC_POOL_SCAN,
	zfs_ioc_pool_scan);
	zfs_ioctl_register_pool_modify(ZFS_IOC_POOL_UPGRADE,
	zfs_ioc_pool_upgrade);
	zfs_ioctl_register_pool_modify(ZFS_IOC_VDEV_ADD,
	zfs_ioc_vdev_add);
	zfs_ioctl_register_pool_modify(ZFS_IOC_VDEV_REMOVE,
	zfs_ioc_vdev_remove);
	zfs_ioctl_register_pool_modify(ZFS_IOC_VDEV_SET_STATE,
	zfs_ioc_vdev_set_state);
	zfs_ioctl_register_pool_modify(ZFS_IOC_VDEV_ATTACH,
	zfs_ioc_vdev_attach);
	zfs_ioctl_register_pool_modify(ZFS_IOC_VDEV_DETACH,
	zfs_ioc_vdev_detach);
	zfs_ioctl_register_pool_modify(ZFS_IOC_VDEV_SETPATH,
	zfs_ioc_vdev_setpath);
	zfs_ioctl_register_pool_modify(ZFS_IOC_VDEV_SETFRU,
	zfs_ioc_vdev_setfru);
	zfs_ioctl_register_pool_modify(ZFS_IOC_POOL_SET_PROPS,
	zfs_ioc_pool_set_props);
	zfs_ioctl_register_pool_modify(ZFS_IOC_VDEV_SPLIT,
	zfs_ioc_vdev_split);
	zfs_ioctl_register_pool_modify(ZFS_IOC_POOL_REGUID,
	zfs_ioc_pool_reguid);

	zfs_ioctl_register_pool_meta(ZFS_IOC_POOL_CONFIGS,
	zfs_ioc_pool_configs, zfs_secpolicy_none);
	zfs_ioctl_register_pool_meta(ZFS_IOC_POOL_TRYIMPORT,
	zfs_ioc_pool_tryimport, zfs_secpolicy_config);
	zfs_ioctl_register_pool_meta(ZFS_IOC_INJECT_FAULT,
	zfs_ioc_inject_fault, zfs_secpolicy_inject);
	zfs_ioctl_register_pool_meta(ZFS_IOC_CLEAR_FAULT,
	zfs_ioc_clear_fault, zfs_secpolicy_inject);
	zfs_ioctl_register_pool_meta(ZFS_IOC_INJECT_LIST_NEXT,
	zfs_ioc_inject_list_next, zfs_secpolicy_inject);

	/*
	* pool destroy, and export don't log the history as part of
	* zfsdev_ioctl, but rather zfs_ioc_pool_export
	* does the logging of those commands.
	*/
	zfs_ioctl_register_pool(ZFS_IOC_POOL_DESTROY, zfs_ioc_pool_destroy,
	zfs_secpolicy_config, B_FALSE, POOL_CHECK_SUSPENDED);
	zfs_ioctl_register_pool(ZFS_IOC_POOL_EXPORT, zfs_ioc_pool_export,
	zfs_secpolicy_config, B_FALSE, POOL_CHECK_SUSPENDED);

	zfs_ioctl_register_pool(ZFS_IOC_POOL_STATS, zfs_ioc_pool_stats,
	zfs_secpolicy_read, B_FALSE, POOL_CHECK_NONE);
	zfs_ioctl_register_pool(ZFS_IOC_POOL_GET_PROPS, zfs_ioc_pool_get_props,
	zfs_secpolicy_read, B_FALSE, POOL_CHECK_NONE);

	zfs_ioctl_register_pool(ZFS_IOC_ERROR_LOG, zfs_ioc_error_log,
	zfs_secpolicy_inject, B_FALSE, POOL_CHECK_SUSPENDED);
	zfs_ioctl_register_pool(ZFS_IOC_DSOBJ_TO_DSNAME,
	zfs_ioc_dsobj_to_dsname,
	zfs_secpolicy_diff, B_FALSE, POOL_CHECK_SUSPENDED);
	zfs_ioctl_register_pool(ZFS_IOC_POOL_GET_HISTORY,
	zfs_ioc_pool_get_history,
	zfs_secpolicy_config, B_FALSE, POOL_CHECK_SUSPENDED);

	zfs_ioctl_register_pool(ZFS_IOC_POOL_IMPORT, zfs_ioc_pool_import,
	zfs_secpolicy_config, B_TRUE, POOL_CHECK_NONE);

	zfs_ioctl_register_pool(ZFS_IOC_CLEAR, zfs_ioc_clear,
	zfs_secpolicy_config, B_TRUE, POOL_CHECK_READONLY);

	zfs_ioctl_register_dataset_read(ZFS_IOC_SPACE_WRITTEN,
	zfs_ioc_space_written);
	zfs_ioctl_register_dataset_read(ZFS_IOC_OBJSET_RECVD_PROPS,
	zfs_ioc_objset_recvd_props);
	zfs_ioctl_register_dataset_read(ZFS_IOC_NEXT_OBJ,
	zfs_ioc_next_obj);
	zfs_ioctl_register_dataset_read(ZFS_IOC_GET_FSACL,
	zfs_ioc_get_fsacl);
	zfs_ioctl_register_dataset_read(ZFS_IOC_OBJSET_STATS,
	zfs_ioc_objset_stats);
	zfs_ioctl_register_dataset_read(ZFS_IOC_OBJSET_ZPLPROPS,
	zfs_ioc_objset_zplprops);
	zfs_ioctl_register_dataset_read(ZFS_IOC_DATASET_LIST_NEXT,
	zfs_ioc_dataset_list_next);
	zfs_ioctl_register_dataset_read(ZFS_IOC_SNAPSHOT_LIST_NEXT,
	zfs_ioc_snapshot_list_next);
	zfs_ioctl_register_dataset_read(ZFS_IOC_SEND_PROGRESS,
	zfs_ioc_send_progress);

	zfs_ioctl_register_dataset_read_secpolicy(ZFS_IOC_DIFF,
	zfs_ioc_diff, zfs_secpolicy_diff);
	zfs_ioctl_register_dataset_read_secpolicy(ZFS_IOC_OBJ_TO_STATS,
	zfs_ioc_obj_to_stats, zfs_secpolicy_diff);
	zfs_ioctl_register_dataset_read_secpolicy(ZFS_IOC_OBJ_TO_PATH,
	zfs_ioc_obj_to_path, zfs_secpolicy_diff);
	zfs_ioctl_register_dataset_read_secpolicy(ZFS_IOC_USERSPACE_ONE,
	zfs_ioc_userspace_one, zfs_secpolicy_userspace_one);
	zfs_ioctl_register_dataset_read_secpolicy(ZFS_IOC_USERSPACE_MANY,
	zfs_ioc_userspace_many, zfs_secpolicy_userspace_many);
	zfs_ioctl_register_dataset_read_secpolicy(ZFS_IOC_SEND,
	zfs_ioc_send, zfs_secpolicy_send);

	zfs_ioctl_register_dataset_modify(ZFS_IOC_SET_PROP, zfs_ioc_set_prop,
	zfs_secpolicy_none);
	zfs_ioctl_register_dataset_modify(ZFS_IOC_DESTROY, zfs_ioc_destroy,
	zfs_secpolicy_destroy);
	zfs_ioctl_register_dataset_modify(ZFS_IOC_RENAME, zfs_ioc_rename,
	zfs_secpolicy_rename);
	zfs_ioctl_register_dataset_modify(ZFS_IOC_RECV, zfs_ioc_recv,
	zfs_secpolicy_recv);
	zfs_ioctl_register_dataset_modify(ZFS_IOC_PROMOTE, zfs_ioc_promote,
	zfs_secpolicy_promote);
	zfs_ioctl_register_dataset_modify(ZFS_IOC_INHERIT_PROP,
	zfs_ioc_inherit_prop, zfs_secpolicy_inherit_prop);
	zfs_ioctl_register_dataset_modify(ZFS_IOC_SET_FSACL, zfs_ioc_set_fsacl,
	zfs_secpolicy_set_fsacl);

	zfs_ioctl_register_dataset_nolog(ZFS_IOC_SHARE, zfs_ioc_share,
	zfs_secpolicy_share, POOL_CHECK_NONE);
	zfs_ioctl_register_dataset_nolog(ZFS_IOC_SMB_ACL, zfs_ioc_smb_acl,
	zfs_secpolicy_smb_acl, POOL_CHECK_NONE);
	zfs_ioctl_register_dataset_nolog(ZFS_IOC_USERSPACE_UPGRADE,
	zfs_ioc_userspace_upgrade, zfs_secpolicy_userspace_upgrade,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY);
	zfs_ioctl_register_dataset_nolog(ZFS_IOC_TMP_SNAPSHOT,
	zfs_ioc_tmp_snapshot, zfs_secpolicy_tmp_snapshot,
	POOL_CHECK_SUSPENDED \| POOL_CHECK_READONLY);

	zfs_ioctl_register_legacy(ZFS_IOC_EVENTS_NEXT, zfs_ioc_events_next,
	zfs_secpolicy_config, NO_NAME, B_FALSE, POOL_CHECK_NONE);
	zfs_ioctl_register_legacy(ZFS_IOC_EVENTS_CLEAR, zfs_ioc_events_clear,
	zfs_secpolicy_config, NO_NAME, B_FALSE, POOL_CHECK_NONE);
	zfs_ioctl_register_legacy(ZFS_IOC_EVENTS_SEEK, zfs_ioc_events_seek,
	zfs_secpolicy_config, NO_NAME, B_FALSE, POOL_CHECK_NONE);

	zfs_ioctl_init_os();
	}

	/*
	* Verify that for non-legacy ioctls the input nvlist
	* pairs match against the expected input.
	*
	* Possible errors are:
	* ZFS_ERR_IOC_ARG_UNAVAIL An unrecognized nvpair was encountered
	* ZFS_ERR_IOC_ARG_REQUIRED A required nvpair is missing
	* ZFS_ERR_IOC_ARG_BADTYPE Invalid type for nvpair
	*/
	static int
	zfs_check_input_nvpairs(nvlist_t innvl, const zfs_ioc_vec_t vec)
	{
	const zfs_ioc_key_t *nvl_keys = vec->zvec_nvl_keys;
	boolean_t required_keys_found = B_FALSE;

	/*
	* examine each input pair
	*/
	for (nvpair_t *pair = nvlist_next_nvpair(innvl, NULL);
	pair != NULL; pair = nvlist_next_nvpair(innvl, pair)) {
	const char *name = nvpair_name(pair);
	data_type_t type = nvpair_type(pair);
	boolean_t identified = B_FALSE;

	/*
	* check pair against the documented names and type
	*/
	for (int k = 0; k < vec->zvec_nvl_key_count; k++) {
	/* if not a wild card name, check for an exact match */
	if ((nvl_keys[k].zkey_flags & ZK_WILDCARDLIST) == 0 &&
	strcmp(nvl_keys[k].zkey_name, name) != 0)
	continue;

	identified = B_TRUE;

	if (nvl_keys[k].zkey_type != DATA_TYPE_ANY &&
	nvl_keys[k].zkey_type != type) {
	return (SET_ERROR(ZFS_ERR_IOC_ARG_BADTYPE));
	}

	if (nvl_keys[k].zkey_flags & ZK_OPTIONAL)
	continue;

	required_keys_found = B_TRUE;
	break;
	}

	/* allow an 'optional' key, everything else is invalid */
	if (!identified &&
	(strcmp(name, "optional") != 0 \|\|
	type != DATA_TYPE_NVLIST)) {
	return (SET_ERROR(ZFS_ERR_IOC_ARG_UNAVAIL));
	}
	}

	/* verify that all required keys were found */
	for (int k = 0; k < vec->zvec_nvl_key_count; k++) {
	if (nvl_keys[k].zkey_flags & ZK_OPTIONAL)
	continue;

	if (nvl_keys[k].zkey_flags & ZK_WILDCARDLIST) {
	/* at least one non-optional key is expected here */
	if (!required_keys_found)
	return (SET_ERROR(ZFS_ERR_IOC_ARG_REQUIRED));
	continue;
	}

	if (!nvlist_exists(innvl, nvl_keys[k].zkey_name))
	return (SET_ERROR(ZFS_ERR_IOC_ARG_REQUIRED));
	}

	return (0);
	}

	static int
	pool_status_check(const char *name, zfs_ioc_namecheck_t type,
	zfs_ioc_poolcheck_t check)
	{
	spa_t *spa;
	int error;

	ASSERT(type == POOL_NAME \|\| type == DATASET_NAME \|\|
	type == ENTITY_NAME);

	if (check & POOL_CHECK_NONE)
	return (0);

	error = spa_open(name, &spa, FTAG);
	if (error == 0) {
	if ((check & POOL_CHECK_SUSPENDED) && spa_suspended(spa))
	error = SET_ERROR(EAGAIN);
	else if ((check & POOL_CHECK_READONLY) && !spa_writeable(spa))
	error = SET_ERROR(EROFS);
	spa_close(spa, FTAG);
	}
	return (error);
	}

	int
	zfsdev_getminor(zfs_file_t fp, minor_t minorp)
	{
	zfsdev_state_t zs, fpd;

	ASSERT(!MUTEX_HELD(&zfsdev_state_lock));

	fpd = zfs_file_private(fp);
	if (fpd == NULL)
	return (SET_ERROR(EBADF));

	mutex_enter(&zfsdev_state_lock);

	for (zs = &zfsdev_state_listhead; zs != NULL; zs = zs->zs_next) {

	if (zs->zs_minor == -1)
	continue;

	if (fpd == zs) {
	*minorp = fpd->zs_minor;
	mutex_exit(&zfsdev_state_lock);
	return (0);
	}
	}

	mutex_exit(&zfsdev_state_lock);

	return (SET_ERROR(EBADF));
	}

	void *
	zfsdev_get_state(minor_t minor, enum zfsdev_state_type which)
	{
	zfsdev_state_t *zs;

	for (zs = &zfsdev_state_listhead; zs != NULL; zs = zs->zs_next) {
	if (zs->zs_minor == minor) {
	membar_consumer();
	switch (which) {
	case ZST_ONEXIT:
	return (zs->zs_onexit);
	case ZST_ZEVENT:
	return (zs->zs_zevent);
	case ZST_ALL:
	return (zs);
	}
	}
	}

	return (NULL);
	}

	/*
	* Find a free minor number. The zfsdev_state_list is expected to
	* be short since it is only a list of currently open file handles.
	*/
	static minor_t
	zfsdev_minor_alloc(void)
	{
	static minor_t last_minor = 0;
	minor_t m;

	ASSERT(MUTEX_HELD(&zfsdev_state_lock));

	for (m = last_minor + 1; m != last_minor; m++) {
	if (m > ZFSDEV_MAX_MINOR)
	m = 1;
	if (zfsdev_get_state(m, ZST_ALL) == NULL) {
	last_minor = m;
	return (m);
	}
	}

	return (0);
	}

	int
	zfsdev_state_init(void *priv)
	{
	zfsdev_state_t zs, zsprev = NULL;
	minor_t minor;
	boolean_t newzs = B_FALSE;

	ASSERT(MUTEX_HELD(&zfsdev_state_lock));

	minor = zfsdev_minor_alloc();
	if (minor == 0)
	return (SET_ERROR(ENXIO));

	for (zs = &zfsdev_state_listhead; zs != NULL; zs = zs->zs_next) {
	if (zs->zs_minor == -1)
	break;
	zsprev = zs;
	}

	if (!zs) {
	zs = kmem_zalloc(sizeof (zfsdev_state_t), KM_SLEEP);
	newzs = B_TRUE;
	}

	zfsdev_private_set_state(priv, zs);

	zfs_onexit_init((zfs_onexit_t **)&zs->zs_onexit);
	zfs_zevent_init((zfs_zevent_t **)&zs->zs_zevent);

	/*
	* In order to provide for lock-free concurrent read access
	* to the minor list in zfsdev_get_state(), new entries
	* must be completely written before linking them into the
	* list whereas existing entries are already linked; the last
	* operation must be updating zs_minor (from -1 to the new
	* value).
	*/
	if (newzs) {
	zs->zs_minor = minor;
	membar_producer();
	zsprev->zs_next = zs;
	} else {
	membar_producer();
	zs->zs_minor = minor;
	}

	return (0);
	}

	void
	zfsdev_state_destroy(void *priv)
	{
	zfsdev_state_t *zs = zfsdev_private_get_state(priv);

	ASSERT(zs != NULL);
	ASSERT3S(zs->zs_minor, >, 0);

	/*
	* The last reference to this zfsdev file descriptor is being dropped.
	* We don't have to worry about lookup grabbing this state object, and
	* zfsdev_state_init() will not try to reuse this object until it is
	* invalidated by setting zs_minor to -1. Invalidation must be done
	* last, with a memory barrier to ensure ordering. This lets us avoid
	* taking the global zfsdev state lock around destruction.
	*/
	zfs_onexit_destroy(zs->zs_onexit);
	zfs_zevent_destroy(zs->zs_zevent);
	zs->zs_onexit = NULL;
	zs->zs_zevent = NULL;
	membar_producer();
	zs->zs_minor = -1;
	}

	long
	zfsdev_ioctl_common(uint_t vecnum, zfs_cmd_t *zc, int flag)
	{
	int error, cmd;
	const zfs_ioc_vec_t *vec;
	char *saved_poolname = NULL;
	uint64_t max_nvlist_src_size;
	size_t saved_poolname_len = 0;
	nvlist_t *innvl = NULL;
	fstrans_cookie_t cookie;
	hrtime_t start_time = gethrtime();

	cmd = vecnum;
	error = 0;
	if (vecnum >= sizeof (zfs_ioc_vec) / sizeof (zfs_ioc_vec[0]))
	return (SET_ERROR(ZFS_ERR_IOC_CMD_UNAVAIL));

	vec = &zfs_ioc_vec[vecnum];

	/*
	* The registered ioctl list may be sparse, verify that either
	* a normal or legacy handler are registered.
	*/
	if (vec->zvec_func == NULL && vec->zvec_legacy_func == NULL)
	return (SET_ERROR(ZFS_ERR_IOC_CMD_UNAVAIL));

	zc->zc_iflags = flag & FKIOCTL;
	max_nvlist_src_size = zfs_max_nvlist_src_size_os();
	if (zc->zc_nvlist_src_size > max_nvlist_src_size) {
	/*
	* Make sure the user doesn't pass in an insane value for
	* zc_nvlist_src_size. We have to check, since we will end
	* up allocating that much memory inside of get_nvlist(). This
	* prevents a nefarious user from allocating tons of kernel
	* memory.
	*
	* Also, we return EINVAL instead of ENOMEM here. The reason
	* being that returning ENOMEM from an ioctl() has a special
	* connotation; that the user's size value is too small and
	* needs to be expanded to hold the nvlist. See
	* zcmd_expand_dst_nvlist() for details.
	*/
	error = SET_ERROR(EINVAL); /* User's size too big */

	} else if (zc->zc_nvlist_src_size != 0) {
	error = get_nvlist(zc->zc_nvlist_src, zc->zc_nvlist_src_size,
	zc->zc_iflags, &innvl);
	if (error != 0)
	goto out;
	}

	/*
	* Ensure that all pool/dataset names are valid before we pass down to
	* the lower layers.
	*/
	zc->zc_name[sizeof (zc->zc_name) - 1] = '\0';
	switch (vec->zvec_namecheck) {
	case POOL_NAME:
	if (pool_namecheck(zc->zc_name, NULL, NULL) != 0)
	error = SET_ERROR(EINVAL);
	else
	error = pool_status_check(zc->zc_name,
	vec->zvec_namecheck, vec->zvec_pool_check);
	break;

	case DATASET_NAME:
	if (dataset_namecheck(zc->zc_name, NULL, NULL) != 0)
	error = SET_ERROR(EINVAL);
	else
	error = pool_status_check(zc->zc_name,
	vec->zvec_namecheck, vec->zvec_pool_check);
	break;

	case ENTITY_NAME:
	if (entity_namecheck(zc->zc_name, NULL, NULL) != 0) {
	error = SET_ERROR(EINVAL);
	} else {
	error = pool_status_check(zc->zc_name,
	vec->zvec_namecheck, vec->zvec_pool_check);
	}
	break;

	case NO_NAME:
	break;
	}
	/*
	* Ensure that all input pairs are valid before we pass them down
	* to the lower layers.
	*
	* The vectored functions can use fnvlist_lookup_{type} for any
	* required pairs since zfs_check_input_nvpairs() confirmed that
	* they exist and are of the correct type.
	*/
	if (error == 0 && vec->zvec_func != NULL) {
	error = zfs_check_input_nvpairs(innvl, vec);
	if (error != 0)
	goto out;
	}

	if (error == 0) {
	cookie = spl_fstrans_mark();
	error = vec->zvec_secpolicy(zc, innvl, CRED());
	spl_fstrans_unmark(cookie);
	}

	if (error != 0)
	goto out;

	/* legacy ioctls can modify zc_name */
	/*
	* Can't use kmem_strdup() as we might truncate the string and
	* kmem_strfree() would then free with incorrect size.
	*/
	saved_poolname_len = strlen(zc->zc_name) + 1;
	saved_poolname = kmem_alloc(saved_poolname_len, KM_SLEEP);

	strlcpy(saved_poolname, zc->zc_name, saved_poolname_len);
	saved_poolname[strcspn(saved_poolname, "/@#")] = '\0';

	if (vec->zvec_func != NULL) {
	nvlist_t *outnvl;
	int puterror = 0;
	spa_t *spa;
	nvlist_t *lognv = NULL;

	ASSERT(vec->zvec_legacy_func == NULL);

	/*
	* Add the innvl to the lognv before calling the func,
	* in case the func changes the innvl.
	*/
	if (vec->zvec_allow_log) {
	lognv = fnvlist_alloc();
	fnvlist_add_string(lognv, ZPOOL_HIST_IOCTL,
	vec->zvec_name);
	if (!nvlist_empty(innvl)) {
	fnvlist_add_nvlist(lognv, ZPOOL_HIST_INPUT_NVL,
	innvl);
	}
	}

	outnvl = fnvlist_alloc();
	cookie = spl_fstrans_mark();
	error = vec->zvec_func(zc->zc_name, innvl, outnvl);
	spl_fstrans_unmark(cookie);

	/*
	* Some commands can partially execute, modify state, and still
	* return an error. In these cases, attempt to record what
	* was modified.
	*/
	if ((error == 0 \|\|
	(cmd == ZFS_IOC_CHANNEL_PROGRAM && error != EINVAL)) &&
	vec->zvec_allow_log &&
	spa_open(zc->zc_name, &spa, FTAG) == 0) {
	if (!nvlist_empty(outnvl)) {
	size_t out_size = fnvlist_size(outnvl);
	if (out_size > zfs_history_output_max) {
	fnvlist_add_int64(lognv,
	ZPOOL_HIST_OUTPUT_SIZE, out_size);
	} else {
	fnvlist_add_nvlist(lognv,
	ZPOOL_HIST_OUTPUT_NVL, outnvl);
	}
	}
	if (error != 0) {
	fnvlist_add_int64(lognv, ZPOOL_HIST_ERRNO,
	error);
	}
	fnvlist_add_int64(lognv, ZPOOL_HIST_ELAPSED_NS,
	gethrtime() - start_time);
	(void) spa_history_log_nvl(spa, lognv);
	spa_close(spa, FTAG);
	}
	fnvlist_free(lognv);

	if (!nvlist_empty(outnvl) \|\| zc->zc_nvlist_dst_size != 0) {
	int smusherror = 0;
	if (vec->zvec_smush_outnvlist) {
	smusherror = nvlist_smush(outnvl,
	zc->zc_nvlist_dst_size);
	}
	if (smusherror == 0)
	puterror = put_nvlist(zc, outnvl);
	}

	if (puterror != 0)
	error = puterror;

	nvlist_free(outnvl);
	} else {
	cookie = spl_fstrans_mark();
	error = vec->zvec_legacy_func(zc);
	spl_fstrans_unmark(cookie);
	}

	out:
	nvlist_free(innvl);
	if (error == 0 && vec->zvec_allow_log) {
	char *s = tsd_get(zfs_allow_log_key);
	if (s != NULL)
	kmem_strfree(s);
	(void) tsd_set(zfs_allow_log_key, kmem_strdup(saved_poolname));
	}
	if (saved_poolname != NULL)
	kmem_free(saved_poolname, saved_poolname_len);

	return (error);
	}

	int
	zfs_kmod_init(void)
	{
	int error;

	if ((error = zvol_init()) != 0)
	return (error);

	spa_init(SPA_MODE_READ \| SPA_MODE_WRITE);
	zfs_init();

	zfs_ioctl_init();

	mutex_init(&zfsdev_state_lock, NULL, MUTEX_DEFAULT, NULL);
	zfsdev_state_listhead.zs_minor = -1;

	if ((error = zfsdev_attach()) != 0)
	goto out;

	tsd_create(&zfs_fsyncer_key, NULL);
	tsd_create(&rrw_tsd_key, rrw_tsd_destroy);
	tsd_create(&zfs_allow_log_key, zfs_allow_log_destroy);

	return (0);
	out:
	zfs_fini();
	spa_fini();
	zvol_fini();

	return (error);
	}

	void
	zfs_kmod_fini(void)
	{
	zfsdev_state_t zs, zsnext = NULL;

	zfsdev_detach();

	mutex_destroy(&zfsdev_state_lock);

	for (zs = &zfsdev_state_listhead; zs != NULL; zs = zsnext) {
	zsnext = zs->zs_next;
	if (zs->zs_onexit)
	zfs_onexit_destroy(zs->zs_onexit);
	if (zs->zs_zevent)
	zfs_zevent_destroy(zs->zs_zevent);
	if (zs != &zfsdev_state_listhead)
	kmem_free(zs, sizeof (zfsdev_state_t));
	}

	zfs_ereport_taskq_fini(); /* run before zfs_fini() on Linux */
	zfs_fini();
	spa_fini();
	zvol_fini();

	tsd_destroy(&zfs_fsyncer_key);
	tsd_destroy(&rrw_tsd_key);
	tsd_destroy(&zfs_allow_log_key);
	}

	ZFS_MODULE_PARAM(zfs, zfs_, max_nvlist_src_size, U64, ZMOD_RW,
	"Maximum size in bytes allowed for src nvlist passed with ZFS ioctls");

	ZFS_MODULE_PARAM(zfs, zfs_, history_output_max, U64, ZMOD_RW,
	"Maximum size in bytes of ZFS ioctl output that will be logged");
	diff --git a/sys/contrib/openzfs/module/zfs/zfs_vnops.c b/sys/contrib/openzfs/module/zfs/zfs_vnops.c
	index 2b37834d5c56..7020f88ecf93 100644
	--- a/sys/contrib/openzfs/module/zfs/zfs_vnops.c
	+++ b/sys/contrib/openzfs/module/zfs/zfs_vnops.c
	@@ -1,1558 +1,1562 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/

	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2012, 2018 by Delphix. All rights reserved.
	* Copyright (c) 2015 by Chunwei Chen. All rights reserved.
	* Copyright 2017 Nexenta Systems, Inc.
	* Copyright (c) 2021, 2022 by Pawel Jakub Dawidek
	*/

	/* Portions Copyright 2007 Jeremy Teo */
	/* Portions Copyright 2010 Robert Milkowski */

	#include <sys/types.h>
	#include <sys/param.h>
	#include <sys/time.h>
	#include <sys/sysmacros.h>
	#include <sys/vfs.h>
	#include <sys/uio_impl.h>
	#include <sys/file.h>
	#include <sys/stat.h>
	#include <sys/kmem.h>
	#include <sys/cmn_err.h>
	#include <sys/errno.h>
	#include <sys/zfs_dir.h>
	#include <sys/zfs_acl.h>
	#include <sys/zfs_ioctl.h>
	#include <sys/fs/zfs.h>
	#include <sys/dmu.h>
	#include <sys/dmu_objset.h>
	#include <sys/dsl_crypt.h>
	#include <sys/spa.h>
	#include <sys/txg.h>
	#include <sys/dbuf.h>
	#include <sys/policy.h>
	#include <sys/zfeature.h>
	#include <sys/zfs_vnops.h>
	#include <sys/zfs_quota.h>
	#include <sys/zfs_vfsops.h>
	#include <sys/zfs_znode.h>

	/*
	* Enable the experimental block cloning feature. If this setting is 0, then
	* even if feature@block_cloning is enabled, attempts to clone blocks will act
	* as though the feature is disabled.
	*/
	int zfs_bclone_enabled = 0;

	/*
	* When set zfs_clone_range() waits for dirty data to be written to disk.
	* This allows the clone operation to reliably succeed when a file is modified
	* and then immediately cloned. For small files this may be slower than making
	* a copy of the file and is therefore not the default. However, in certain
	* scenarios this behavior may be desirable so a tunable is provided.
	*/
	static int zfs_bclone_wait_dirty = 0;

	/*
	* Maximum bytes to read per chunk in zfs_read().
	*/
	static uint64_t zfs_vnops_read_chunk_size = 1024 * 1024;

	static ulong_t zfs_fsync_sync_cnt = 4;

	int
	zfs_fsync(znode_t zp, int syncflag, cred_t cr)
	{
	int error = 0;
	zfsvfs_t *zfsvfs = ZTOZSB(zp);

	(void) tsd_set(zfs_fsyncer_key, (void *)(uintptr_t)zfs_fsync_sync_cnt);

	if (zfsvfs->z_os->os_sync != ZFS_SYNC_DISABLED) {
	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	goto out;
	atomic_inc_32(&zp->z_sync_writes_cnt);
	zil_commit(zfsvfs->z_log, zp->z_id);
	atomic_dec_32(&zp->z_sync_writes_cnt);
	zfs_exit(zfsvfs, FTAG);
	}
	out:
	tsd_set(zfs_fsyncer_key, NULL);

	return (error);
	}


	#if defined(SEEK_HOLE) && defined(SEEK_DATA)
	/*
	* Lseek support for finding holes (cmd == SEEK_HOLE) and
	* data (cmd == SEEK_DATA). "off" is an in/out parameter.
	*/
	static int
	zfs_holey_common(znode_t zp, ulong_t cmd, loff_t off)
	{
	zfs_locked_range_t *lr;
	uint64_t noff = (uint64_t)off; / new offset */
	uint64_t file_sz;
	int error;
	boolean_t hole;

	file_sz = zp->z_size;
	if (noff >= file_sz) {
	return (SET_ERROR(ENXIO));
	}

	if (cmd == F_SEEK_HOLE)
	hole = B_TRUE;
	else
	hole = B_FALSE;

	/* Flush any mmap()'d data to disk */
	if (zn_has_cached_data(zp, 0, file_sz - 1))
	- zn_flush_cached_data(zp, B_FALSE);
	+ zn_flush_cached_data(zp, B_TRUE);

	lr = zfs_rangelock_enter(&zp->z_rangelock, 0, UINT64_MAX, RL_READER);
	error = dmu_offset_next(ZTOZSB(zp)->z_os, zp->z_id, hole, &noff);
	zfs_rangelock_exit(lr);

	if (error == ESRCH)
	return (SET_ERROR(ENXIO));

	/* File was dirty, so fall back to using generic logic */
	if (error == EBUSY) {
	if (hole)
	*off = file_sz;

	return (0);
	}

	/*
	* We could find a hole that begins after the logical end-of-file,
	* because dmu_offset_next() only works on whole blocks. If the
	* EOF falls mid-block, then indicate that the "virtual hole"
	* at the end of the file begins at the logical EOF, rather than
	* at the end of the last block.
	*/
	if (noff > file_sz) {
	ASSERT(hole);
	noff = file_sz;
	}

	if (noff < *off)
	return (error);
	*off = noff;
	return (error);
	}

	int
	zfs_holey(znode_t zp, ulong_t cmd, loff_t off)
	{
	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	int error;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	error = zfs_holey_common(zp, cmd, off);

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}
	#endif /* SEEK_HOLE && SEEK_DATA */

	int
	zfs_access(znode_t zp, int mode, int flag, cred_t cr)
	{
	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	int error;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	if (flag & V_ACE_MASK)
	#if defined(__linux__)
	error = zfs_zaccess(zp, mode, flag, B_FALSE, cr,
	zfs_init_idmap);
	#else
	error = zfs_zaccess(zp, mode, flag, B_FALSE, cr,
	NULL);
	#endif
	else
	#if defined(__linux__)
	error = zfs_zaccess_rwx(zp, mode, flag, cr, zfs_init_idmap);
	#else
	error = zfs_zaccess_rwx(zp, mode, flag, cr, NULL);
	#endif

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	/*
	* Read bytes from specified file into supplied buffer.
	*
	* IN: zp - inode of file to be read from.
	* uio - structure supplying read location, range info,
	* and return buffer.
	* ioflag - O_SYNC flags; used to provide FRSYNC semantics.
	* O_DIRECT flag; used to bypass page cache.
	* cr - credentials of caller.
	*
	* OUT: uio - updated offset and range, buffer filled.
	*
	* RETURN: 0 on success, error code on failure.
	*
	* Side Effects:
	* inode - atime updated if byte count > 0
	*/
	int
	zfs_read(struct znode zp, zfs_uio_t uio, int ioflag, cred_t *cr)
	{
	(void) cr;
	int error = 0;
	boolean_t frsync = B_FALSE;

	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	if (zp->z_pflags & ZFS_AV_QUARANTINED) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EACCES));
	}

	/* We don't copy out anything useful for directories. */
	if (Z_ISDIR(ZTOTYPE(zp))) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EISDIR));
	}

	/*
	* Validate file offset
	*/
	if (zfs_uio_offset(uio) < (offset_t)0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	/*
	* Fasttrack empty reads
	*/
	if (zfs_uio_resid(uio) == 0) {
	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	#ifdef FRSYNC
	/*
	* If we're in FRSYNC mode, sync out this znode before reading it.
	* Only do this for non-snapshots.
	*
	* Some platforms do not support FRSYNC and instead map it
	* to O_SYNC, which results in unnecessary calls to zil_commit. We
	* only honor FRSYNC requests on platforms which support it.
	*/
	frsync = !!(ioflag & FRSYNC);
	#endif
	if (zfsvfs->z_log &&
	(frsync \|\| zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS))
	zil_commit(zfsvfs->z_log, zp->z_id);

	/*
	* Lock the range against changes.
	*/
	zfs_locked_range_t *lr = zfs_rangelock_enter(&zp->z_rangelock,
	zfs_uio_offset(uio), zfs_uio_resid(uio), RL_READER);

	/*
	* If we are reading past end-of-file we can skip
	* to the end; but we might still need to set atime.
	*/
	if (zfs_uio_offset(uio) >= zp->z_size) {
	error = 0;
	goto out;
	}

	ASSERT(zfs_uio_offset(uio) < zp->z_size);
	#if defined(__linux__)
	ssize_t start_offset = zfs_uio_offset(uio);
	#endif
	ssize_t n = MIN(zfs_uio_resid(uio), zp->z_size - zfs_uio_offset(uio));
	ssize_t start_resid = n;

	while (n > 0) {
	ssize_t nbytes = MIN(n, zfs_vnops_read_chunk_size -
	P2PHASE(zfs_uio_offset(uio), zfs_vnops_read_chunk_size));
	#ifdef UIO_NOCOPY
	if (zfs_uio_segflg(uio) == UIO_NOCOPY)
	error = mappedread_sf(zp, nbytes, uio);
	else
	#endif
	if (zn_has_cached_data(zp, zfs_uio_offset(uio),
	zfs_uio_offset(uio) + nbytes - 1) && !(ioflag & O_DIRECT)) {
	error = mappedread(zp, nbytes, uio);
	} else {
	error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl),
	uio, nbytes);
	}

	if (error) {
	/* convert checksum errors into IO errors */
	if (error == ECKSUM)
	error = SET_ERROR(EIO);

	#if defined(__linux__)
	/*
	* if we actually read some bytes, bubbling EFAULT
	* up to become EAGAIN isn't what we want here...
	*
	* ...on Linux, at least. On FBSD, doing this breaks.
	*/
	if (error == EFAULT &&
	(zfs_uio_offset(uio) - start_offset) != 0)
	error = 0;
	#endif
	break;
	}

	n -= nbytes;
	}

	int64_t nread = start_resid - n;
	dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, nread);
	task_io_account_read(nread);
	out:
	zfs_rangelock_exit(lr);

	ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	static void
	zfs_clear_setid_bits_if_necessary(zfsvfs_t zfsvfs, znode_t zp, cred_t *cr,
	uint64_t clear_setid_bits_txgp, dmu_tx_t tx)
	{
	zilog_t *zilog = zfsvfs->z_log;
	const uint64_t uid = KUID_TO_SUID(ZTOUID(zp));

	ASSERT(clear_setid_bits_txgp != NULL);
	ASSERT(tx != NULL);

	/*
	* Clear Set-UID/Set-GID bits on successful write if not
	* privileged and at least one of the execute bits is set.
	*
	* It would be nice to do this after all writes have
	* been done, but that would still expose the ISUID/ISGID
	* to another app after the partial write is committed.
	*
	* Note: we don't call zfs_fuid_map_id() here because
	* user 0 is not an ephemeral uid.
	*/
	mutex_enter(&zp->z_acl_lock);
	if ((zp->z_mode & (S_IXUSR \| (S_IXUSR >> 3) \| (S_IXUSR >> 6))) != 0 &&
	(zp->z_mode & (S_ISUID \| S_ISGID)) != 0 &&
	secpolicy_vnode_setid_retain(zp, cr,
	((zp->z_mode & S_ISUID) != 0 && uid == 0)) != 0) {
	uint64_t newmode;

	zp->z_mode &= ~(S_ISUID \| S_ISGID);
	newmode = zp->z_mode;
	(void) sa_update(zp->z_sa_hdl, SA_ZPL_MODE(zfsvfs),
	(void *)&newmode, sizeof (uint64_t), tx);

	mutex_exit(&zp->z_acl_lock);

	/*
	* Make sure SUID/SGID bits will be removed when we replay the
	* log. If the setid bits are keep coming back, don't log more
	* than one TX_SETATTR per transaction group.
	*/
	if (*clear_setid_bits_txgp != dmu_tx_get_txg(tx)) {
	vattr_t va = {0};

	va.va_mask = ATTR_MODE;
	va.va_nodeid = zp->z_id;
	va.va_mode = newmode;
	zfs_log_setattr(zilog, tx, TX_SETATTR, zp, &va,
	ATTR_MODE, NULL);
	*clear_setid_bits_txgp = dmu_tx_get_txg(tx);
	}
	} else {
	mutex_exit(&zp->z_acl_lock);
	}
	}

	/*
	* Write the bytes to a file.
	*
	* IN: zp - znode of file to be written to.
	* uio - structure supplying write location, range info,
	* and data buffer.
	* ioflag - O_APPEND flag set if in append mode.
	* O_DIRECT flag; used to bypass page cache.
	* cr - credentials of caller.
	*
	* OUT: uio - updated offset and range.
	*
	* RETURN: 0 if success
	* error code if failure
	*
	* Timestamps:
	* ip - ctime\|mtime updated if byte count > 0
	*/
	int
	zfs_write(znode_t zp, zfs_uio_t uio, int ioflag, cred_t *cr)
	{
	int error = 0, error1;
	ssize_t start_resid = zfs_uio_resid(uio);
	uint64_t clear_setid_bits_txg = 0;

	/*
	* Fasttrack empty write
	*/
	ssize_t n = start_resid;
	if (n == 0)
	return (0);

	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	sa_bulk_attr_t bulk[4];
	int count = 0;
	uint64_t mtime[2], ctime[2];
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
	&zp->z_size, 8);
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
	&zp->z_pflags, 8);

	/*
	* Callers might not be able to detect properly that we are read-only,
	* so check it explicitly here.
	*/
	if (zfs_is_readonly(zfsvfs)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EROFS));
	}

	/*
	* If immutable or not appending then return EPERM.
	* Intentionally allow ZFS_READONLY through here.
	* See zfs_zaccess_common()
	*/
	if ((zp->z_pflags & ZFS_IMMUTABLE) \|\|
	((zp->z_pflags & ZFS_APPENDONLY) && !(ioflag & O_APPEND) &&
	(zfs_uio_offset(uio) < zp->z_size))) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EPERM));
	}

	/*
	* Validate file offset
	*/
	offset_t woff = ioflag & O_APPEND ? zp->z_size : zfs_uio_offset(uio);
	if (woff < 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	/*
	* Pre-fault the pages to ensure slow (eg NFS) pages
	* don't hold up txg.
	*/
	ssize_t pfbytes = MIN(n, DMU_MAX_ACCESS >> 1);
	if (zfs_uio_prefaultpages(pfbytes, uio)) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EFAULT));
	}

	/*
	* If in append mode, set the io offset pointer to eof.
	*/
	zfs_locked_range_t *lr;
	if (ioflag & O_APPEND) {
	/*
	* Obtain an appending range lock to guarantee file append
	* semantics. We reset the write offset once we have the lock.
	*/
	lr = zfs_rangelock_enter(&zp->z_rangelock, 0, n, RL_APPEND);
	woff = lr->lr_offset;
	if (lr->lr_length == UINT64_MAX) {
	/*
	* We overlocked the file because this write will cause
	* the file block size to increase.
	* Note that zp_size cannot change with this lock held.
	*/
	woff = zp->z_size;
	}
	zfs_uio_setoffset(uio, woff);
	} else {
	/*
	* Note that if the file block size will change as a result of
	* this write, then this range lock will lock the entire file
	* so that we can re-write the block safely.
	*/
	lr = zfs_rangelock_enter(&zp->z_rangelock, woff, n, RL_WRITER);
	}

	if (zn_rlimit_fsize_uio(zp, uio)) {
	zfs_rangelock_exit(lr);
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EFBIG));
	}

	const rlim64_t limit = MAXOFFSET_T;

	if (woff >= limit) {
	zfs_rangelock_exit(lr);
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EFBIG));
	}

	if (n > limit - woff)
	n = limit - woff;

	uint64_t end_size = MAX(zp->z_size, woff + n);
	zilog_t *zilog = zfsvfs->z_log;

	const uint64_t uid = KUID_TO_SUID(ZTOUID(zp));
	const uint64_t gid = KGID_TO_SGID(ZTOGID(zp));
	const uint64_t projid = zp->z_projid;

	/*
	* Write the file in reasonable size chunks. Each chunk is written
	* in a separate transaction; this keeps the intent log records small
	* and allows us to do more fine-grained space accounting.
	*/
	while (n > 0) {
	woff = zfs_uio_offset(uio);

	if (zfs_id_overblockquota(zfsvfs, DMU_USERUSED_OBJECT, uid) \|\|
	zfs_id_overblockquota(zfsvfs, DMU_GROUPUSED_OBJECT, gid) \|\|
	(projid != ZFS_DEFAULT_PROJID &&
	zfs_id_overblockquota(zfsvfs, DMU_PROJECTUSED_OBJECT,
	projid))) {
	error = SET_ERROR(EDQUOT);
	break;
	}

	uint64_t blksz;
	if (lr->lr_length == UINT64_MAX && zp->z_size <= zp->z_blksz) {
	if (zp->z_blksz > zfsvfs->z_max_blksz &&
	!ISP2(zp->z_blksz)) {
	/*
	* File's blocksize is already larger than the
	* "recordsize" property. Only let it grow to
	* the next power of 2.
	*/
	blksz = 1 << highbit64(zp->z_blksz);
	} else {
	blksz = zfsvfs->z_max_blksz;
	}
	blksz = MIN(blksz, P2ROUNDUP(end_size,
	SPA_MINBLOCKSIZE));
	blksz = MAX(blksz, zp->z_blksz);
	} else {
	blksz = zp->z_blksz;
	}

	arc_buf_t *abuf = NULL;
	ssize_t nbytes = n;
	if (n >= blksz && woff >= zp->z_size &&
	P2PHASE(woff, blksz) == 0 &&
	(blksz >= SPA_OLD_MAXBLOCKSIZE \|\| n < 4 * blksz)) {
	/*
	* This write covers a full block. "Borrow" a buffer
	* from the dmu so that we can fill it before we enter
	* a transaction. This avoids the possibility of
	* holding up the transaction if the data copy hangs
	* up on a pagefault (e.g., from an NFS server mapping).
	*/
	abuf = dmu_request_arcbuf(sa_get_db(zp->z_sa_hdl),
	blksz);
	ASSERT(abuf != NULL);
	ASSERT(arc_buf_size(abuf) == blksz);
	if ((error = zfs_uiocopy(abuf->b_data, blksz,
	UIO_WRITE, uio, &nbytes))) {
	dmu_return_arcbuf(abuf);
	break;
	}
	ASSERT3S(nbytes, ==, blksz);
	} else {
	nbytes = MIN(n, (DMU_MAX_ACCESS >> 1) -
	P2PHASE(woff, blksz));
	if (pfbytes < nbytes) {
	if (zfs_uio_prefaultpages(nbytes, uio)) {
	error = SET_ERROR(EFAULT);
	break;
	}
	pfbytes = nbytes;
	}
	}

	/*
	* Start a transaction.
	*/
	dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os);
	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
	dmu_buf_impl_t db = (dmu_buf_impl_t )sa_get_db(zp->z_sa_hdl);
	DB_DNODE_ENTER(db);
	dmu_tx_hold_write_by_dnode(tx, DB_DNODE(db), woff, nbytes);
	DB_DNODE_EXIT(db);
	zfs_sa_upgrade_txholds(tx, zp);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error) {
	dmu_tx_abort(tx);
	if (abuf != NULL)
	dmu_return_arcbuf(abuf);
	break;
	}

	/*
	* NB: We must call zfs_clear_setid_bits_if_necessary before
	* committing the transaction!
	*/

	/*
	* If rangelock_enter() over-locked we grow the blocksize
	* and then reduce the lock range. This will only happen
	* on the first iteration since rangelock_reduce() will
	* shrink down lr_length to the appropriate size.
	*/
	if (lr->lr_length == UINT64_MAX) {
	zfs_grow_blocksize(zp, blksz, tx);
	zfs_rangelock_reduce(lr, woff, n);
	}

	ssize_t tx_bytes;
	if (abuf == NULL) {
	tx_bytes = zfs_uio_resid(uio);
	zfs_uio_fault_disable(uio, B_TRUE);
	error = dmu_write_uio_dbuf(sa_get_db(zp->z_sa_hdl),
	uio, nbytes, tx);
	zfs_uio_fault_disable(uio, B_FALSE);
	#ifdef __linux__
	if (error == EFAULT) {
	zfs_clear_setid_bits_if_necessary(zfsvfs, zp,
	cr, &clear_setid_bits_txg, tx);
	dmu_tx_commit(tx);
	/*
	* Account for partial writes before
	* continuing the loop.
	* Update needs to occur before the next
	* zfs_uio_prefaultpages, or prefaultpages may
	* error, and we may break the loop early.
	*/
	n -= tx_bytes - zfs_uio_resid(uio);
	pfbytes -= tx_bytes - zfs_uio_resid(uio);
	continue;
	}
	#endif
	/*
	* On FreeBSD, EFAULT should be propagated back to the
	* VFS, which will handle faulting and will retry.
	*/
	if (error != 0 && error != EFAULT) {
	zfs_clear_setid_bits_if_necessary(zfsvfs, zp,
	cr, &clear_setid_bits_txg, tx);
	dmu_tx_commit(tx);
	break;
	}
	tx_bytes -= zfs_uio_resid(uio);
	} else {
	/*
	* Thus, we're writing a full block at a block-aligned
	* offset and extending the file past EOF.
	*
	* dmu_assign_arcbuf_by_dbuf() will directly assign the
	* arc buffer to a dbuf.
	*/
	error = dmu_assign_arcbuf_by_dbuf(
	sa_get_db(zp->z_sa_hdl), woff, abuf, tx);
	if (error != 0) {
	/*
	* XXX This might not be necessary if
	* dmu_assign_arcbuf_by_dbuf is guaranteed
	* to be atomic.
	*/
	zfs_clear_setid_bits_if_necessary(zfsvfs, zp,
	cr, &clear_setid_bits_txg, tx);
	dmu_return_arcbuf(abuf);
	dmu_tx_commit(tx);
	break;
	}
	ASSERT3S(nbytes, <=, zfs_uio_resid(uio));
	zfs_uioskip(uio, nbytes);
	tx_bytes = nbytes;
	}
	if (tx_bytes &&
	zn_has_cached_data(zp, woff, woff + tx_bytes - 1) &&
	!(ioflag & O_DIRECT)) {
	update_pages(zp, woff, tx_bytes, zfsvfs->z_os);
	}

	/*
	* If we made no progress, we're done. If we made even
	* partial progress, update the znode and ZIL accordingly.
	*/
	if (tx_bytes == 0) {
	(void) sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(zfsvfs),
	(void *)&zp->z_size, sizeof (uint64_t), tx);
	dmu_tx_commit(tx);
	ASSERT(error != 0);
	break;
	}

	zfs_clear_setid_bits_if_necessary(zfsvfs, zp, cr,
	&clear_setid_bits_txg, tx);

	zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);

	/*
	* Update the file size (zp_size) if it has changed;
	* account for possible concurrent updates.
	*/
	while ((end_size = zp->z_size) < zfs_uio_offset(uio)) {
	(void) atomic_cas_64(&zp->z_size, end_size,
	zfs_uio_offset(uio));
	ASSERT(error == 0 \|\| error == EFAULT);
	}
	/*
	* If we are replaying and eof is non zero then force
	* the file size to the specified eof. Note, there's no
	* concurrency during replay.
	*/
	if (zfsvfs->z_replay && zfsvfs->z_replay_eof != 0)
	zp->z_size = zfsvfs->z_replay_eof;

	error1 = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
	if (error1 != 0)
	/* Avoid clobbering EFAULT. */
	error = error1;

	/*
	* NB: During replay, the TX_SETATTR record logged by
	* zfs_clear_setid_bits_if_necessary must precede any of
	* the TX_WRITE records logged here.
	*/
	zfs_log_write(zilog, tx, TX_WRITE, zp, woff, tx_bytes, ioflag,
	NULL, NULL);

	dmu_tx_commit(tx);

	if (error != 0)
	break;
	ASSERT3S(tx_bytes, ==, nbytes);
	n -= nbytes;
	pfbytes -= nbytes;
	}

	zfs_znode_update_vfs(zp);
	zfs_rangelock_exit(lr);

	/*
	* If we're in replay mode, or we made no progress, or the
	* uio data is inaccessible return an error. Otherwise, it's
	* at least a partial write, so it's successful.
	*/
	if (zfsvfs->z_replay \|\| zfs_uio_resid(uio) == start_resid \|\|
	error == EFAULT) {
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	if (ioflag & (O_SYNC \| O_DSYNC) \|\|
	zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, zp->z_id);

	const int64_t nwritten = start_resid - zfs_uio_resid(uio);
	dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, nwritten);
	task_io_account_write(nwritten);

	zfs_exit(zfsvfs, FTAG);
	return (0);
	}

	int
	zfs_getsecattr(znode_t zp, vsecattr_t vsecp, int flag, cred_t *cr)
	{
	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	int error;
	boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);
	error = zfs_getacl(zp, vsecp, skipaclchk, cr);
	zfs_exit(zfsvfs, FTAG);

	return (error);
	}

	int
	zfs_setsecattr(znode_t zp, vsecattr_t vsecp, int flag, cred_t *cr)
	{
	zfsvfs_t *zfsvfs = ZTOZSB(zp);
	int error;
	boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
	zilog_t *zilog;

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);
	zilog = zfsvfs->z_log;
	error = zfs_setacl(zp, vsecp, skipaclchk, cr);

	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
	zil_commit(zilog, 0);

	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	#ifdef ZFS_DEBUG
	static int zil_fault_io = 0;
	#endif

	static void zfs_get_done(zgd_t *zgd, int error);

	/*
	* Get data to generate a TX_WRITE intent log record.
	*/
	int
	zfs_get_data(void arg, uint64_t gen, lr_write_t lr, char *buf,
	struct lwb lwb, zio_t zio)
	{
	zfsvfs_t *zfsvfs = arg;
	objset_t *os = zfsvfs->z_os;
	znode_t *zp;
	uint64_t object = lr->lr_foid;
	uint64_t offset = lr->lr_offset;
	uint64_t size = lr->lr_length;
	dmu_buf_t *db;
	zgd_t *zgd;
	int error = 0;
	uint64_t zp_gen;

	ASSERT3P(lwb, !=, NULL);
	ASSERT3U(size, !=, 0);

	/*
	* Nothing to do if the file has been removed
	*/
	if (zfs_zget(zfsvfs, object, &zp) != 0)
	return (SET_ERROR(ENOENT));
	if (zp->z_unlinked) {
	/*
	* Release the vnode asynchronously as we currently have the
	* txg stopped from syncing.
	*/
	zfs_zrele_async(zp);
	return (SET_ERROR(ENOENT));
	}
	/* check if generation number matches */
	if (sa_lookup(zp->z_sa_hdl, SA_ZPL_GEN(zfsvfs), &zp_gen,
	sizeof (zp_gen)) != 0) {
	zfs_zrele_async(zp);
	return (SET_ERROR(EIO));
	}
	if (zp_gen != gen) {
	zfs_zrele_async(zp);
	return (SET_ERROR(ENOENT));
	}

	zgd = kmem_zalloc(sizeof (zgd_t), KM_SLEEP);
	zgd->zgd_lwb = lwb;
	zgd->zgd_private = zp;

	/*
	* Write records come in two flavors: immediate and indirect.
	* For small writes it's cheaper to store the data with the
	* log record (immediate); for large writes it's cheaper to
	* sync the data and get a pointer to it (indirect) so that
	* we don't have to write the data twice.
	*/
	if (buf != NULL) { /* immediate write */
	zgd->zgd_lr = zfs_rangelock_enter(&zp->z_rangelock,
	offset, size, RL_READER);
	/* test for truncation needs to be done while range locked */
	if (offset >= zp->z_size) {
	error = SET_ERROR(ENOENT);
	} else {
	error = dmu_read(os, object, offset, size, buf,
	DMU_READ_NO_PREFETCH);
	}
	ASSERT(error == 0 \|\| error == ENOENT);
	} else { /* indirect write */
	ASSERT3P(zio, !=, NULL);
	/*
	* Have to lock the whole block to ensure when it's
	* written out and its checksum is being calculated
	* that no one can change the data. We need to re-check
	* blocksize after we get the lock in case it's changed!
	*/
	for (;;) {
	uint64_t blkoff;
	size = zp->z_blksz;
	blkoff = ISP2(size) ? P2PHASE(offset, size) : offset;
	offset -= blkoff;
	zgd->zgd_lr = zfs_rangelock_enter(&zp->z_rangelock,
	offset, size, RL_READER);
	if (zp->z_blksz == size)
	break;
	offset += blkoff;
	zfs_rangelock_exit(zgd->zgd_lr);
	}
	/* test for truncation needs to be done while range locked */
	if (lr->lr_offset >= zp->z_size)
	error = SET_ERROR(ENOENT);
	#ifdef ZFS_DEBUG
	if (zil_fault_io) {
	error = SET_ERROR(EIO);
	zil_fault_io = 0;
	}
	#endif
	if (error == 0)
	error = dmu_buf_hold_noread(os, object, offset, zgd,
	&db);

	if (error == 0) {
	blkptr_t *bp = &lr->lr_blkptr;

	zgd->zgd_db = db;
	zgd->zgd_bp = bp;

	ASSERT(db->db_offset == offset);
	ASSERT(db->db_size == size);

	error = dmu_sync(zio, lr->lr_common.lrc_txg,
	zfs_get_done, zgd);
	ASSERT(error \|\| lr->lr_length <= size);

	/*
	* On success, we need to wait for the write I/O
	* initiated by dmu_sync() to complete before we can
	* release this dbuf. We will finish everything up
	* in the zfs_get_done() callback.
	*/
	if (error == 0)
	return (0);

	if (error == EALREADY) {
	lr->lr_common.lrc_txtype = TX_WRITE2;
	/*
	* TX_WRITE2 relies on the data previously
	* written by the TX_WRITE that caused
	* EALREADY. We zero out the BP because
	* it is the old, currently-on-disk BP.
	*/
	zgd->zgd_bp = NULL;
	BP_ZERO(bp);
	error = 0;
	}
	}
	}

	zfs_get_done(zgd, error);

	return (error);
	}


	static void
	zfs_get_done(zgd_t *zgd, int error)
	{
	(void) error;
	znode_t *zp = zgd->zgd_private;

	if (zgd->zgd_db)
	dmu_buf_rele(zgd->zgd_db, zgd);

	zfs_rangelock_exit(zgd->zgd_lr);

	/*
	* Release the vnode asynchronously as we currently have the
	* txg stopped from syncing.
	*/
	zfs_zrele_async(zp);

	kmem_free(zgd, sizeof (zgd_t));
	}

	static int
	zfs_enter_two(zfsvfs_t zfsvfs1, zfsvfs_t zfsvfs2, const char *tag)
	{
	int error;

	/* Swap. Not sure if the order of zfs_enter()s is important. */
	if (zfsvfs1 > zfsvfs2) {
	zfsvfs_t *tmpzfsvfs;

	tmpzfsvfs = zfsvfs2;
	zfsvfs2 = zfsvfs1;
	zfsvfs1 = tmpzfsvfs;
	}

	error = zfs_enter(zfsvfs1, tag);
	if (error != 0)
	return (error);
	if (zfsvfs1 != zfsvfs2) {
	error = zfs_enter(zfsvfs2, tag);
	if (error != 0) {
	zfs_exit(zfsvfs1, tag);
	return (error);
	}
	}

	return (0);
	}

	static void
	zfs_exit_two(zfsvfs_t zfsvfs1, zfsvfs_t zfsvfs2, const char *tag)
	{

	zfs_exit(zfsvfs1, tag);
	if (zfsvfs1 != zfsvfs2)
	zfs_exit(zfsvfs2, tag);
	}

	/*
	* We split each clone request in chunks that can fit into a single ZIL
	* log entry. Each ZIL log entry can fit 130816 bytes for a block cloning
	* operation (see zil_max_log_data() and zfs_log_clone_range()). This gives
	* us room for storing 1022 block pointers.
	*
	* On success, the function return the number of bytes copied in *lenp.
	* Note, it doesn't return how much bytes are left to be copied.
	* On errors which are caused by any file system limitations or
	* brt limitations `EINVAL` is returned. In the most cases a user
	* requested bad parameters, it could be possible to clone the file but
	* some parameters don't match the requirements.
	*/
	int
	zfs_clone_range(znode_t inzp, uint64_t inoffp, znode_t *outzp,
	uint64_t outoffp, uint64_t lenp, cred_t *cr)
	{
	zfsvfs_t inzfsvfs, outzfsvfs;
	objset_t inos, outos;
	zfs_locked_range_t inlr, outlr;
	dmu_buf_impl_t *db;
	dmu_tx_t *tx;
	zilog_t *zilog;
	uint64_t inoff, outoff, len, done;
	uint64_t outsize, size;
	int error;
	int count = 0;
	sa_bulk_attr_t bulk[3];
	uint64_t mtime[2], ctime[2];
	uint64_t uid, gid, projid;
	blkptr_t *bps;
	size_t maxblocks, nbps;
	uint_t inblksz;
	uint64_t clear_setid_bits_txg = 0;
	uint64_t last_synced_txg = 0;

	inoff = *inoffp;
	outoff = *outoffp;
	len = *lenp;
	done = 0;

	inzfsvfs = ZTOZSB(inzp);
	outzfsvfs = ZTOZSB(outzp);

	/*
	* We need to call zfs_enter() potentially on two different datasets,
	* so we need a dedicated function for that.
	*/
	error = zfs_enter_two(inzfsvfs, outzfsvfs, FTAG);
	if (error != 0)
	return (error);

	inos = inzfsvfs->z_os;
	outos = outzfsvfs->z_os;

	/*
	* Both source and destination have to belong to the same storage pool.
	*/
	if (dmu_objset_spa(inos) != dmu_objset_spa(outos)) {
	zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
	return (SET_ERROR(EXDEV));
	}

	/*
	* outos and inos belongs to the same storage pool.
	* see a few lines above, only one check.
	*/
	if (!spa_feature_is_enabled(dmu_objset_spa(outos),
	SPA_FEATURE_BLOCK_CLONING)) {
	zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
	return (SET_ERROR(EOPNOTSUPP));
	}

	ASSERT(!outzfsvfs->z_replay);

	/*
	* Block cloning from an unencrypted dataset into an encrypted
	* dataset and vice versa is not supported.
	*/
	if (inos->os_encrypted != outos->os_encrypted) {
	zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
	return (SET_ERROR(EXDEV));
	}

	/*
	* Cloning across encrypted datasets is possible only if they
	* share the same master key.
	*/
	if (inos != outos && inos->os_encrypted &&
	!dmu_objset_crypto_key_equal(inos, outos)) {
	zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
	return (SET_ERROR(EXDEV));
	}

	error = zfs_verify_zp(inzp);
	if (error == 0)
	error = zfs_verify_zp(outzp);
	if (error != 0) {
	zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
	return (error);
	}

	/*
	* We don't copy source file's flags that's why we don't allow to clone
	* files that are in quarantine.
	*/
	if (inzp->z_pflags & ZFS_AV_QUARANTINED) {
	zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
	return (SET_ERROR(EACCES));
	}

	if (inoff >= inzp->z_size) {
	*lenp = 0;
	zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
	return (0);
	}
	if (len > inzp->z_size - inoff) {
	len = inzp->z_size - inoff;
	}
	if (len == 0) {
	*lenp = 0;
	zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
	return (0);
	}

	/*
	* Callers might not be able to detect properly that we are read-only,
	* so check it explicitly here.
	*/
	if (zfs_is_readonly(outzfsvfs)) {
	zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
	return (SET_ERROR(EROFS));
	}

	/*
	* If immutable or not appending then return EPERM.
	* Intentionally allow ZFS_READONLY through here.
	* See zfs_zaccess_common()
	*/
	if ((outzp->z_pflags & ZFS_IMMUTABLE) != 0) {
	zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
	return (SET_ERROR(EPERM));
	}

	/*
	* No overlapping if we are cloning within the same file.
	*/
	if (inzp == outzp) {
	if (inoff < outoff + len && outoff < inoff + len) {
	zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}
	}

	+ /* Flush any mmap()'d data to disk */
	+ if (zn_has_cached_data(inzp, inoff, inoff + len - 1))
	+ zn_flush_cached_data(inzp, B_TRUE);
	+
	/*
	* Maintain predictable lock order.
	*/
	if (inzp < outzp \|\| (inzp == outzp && inoff < outoff)) {
	inlr = zfs_rangelock_enter(&inzp->z_rangelock, inoff, len,
	RL_READER);
	outlr = zfs_rangelock_enter(&outzp->z_rangelock, outoff, len,
	RL_WRITER);
	} else {
	outlr = zfs_rangelock_enter(&outzp->z_rangelock, outoff, len,
	RL_WRITER);
	inlr = zfs_rangelock_enter(&inzp->z_rangelock, inoff, len,
	RL_READER);
	}

	inblksz = inzp->z_blksz;

	/*
	* We cannot clone into a file with different block size if we can't
	* grow it (block size is already bigger, has more than one block, or
	* not locked for growth). There are other possible reasons for the
	* grow to fail, but we cover what we can before opening transaction
	* and the rest detect after we try to do it.
	*/
	if (inblksz < outzp->z_blksz) {
	error = SET_ERROR(EINVAL);
	goto unlock;
	}
	if (inblksz != outzp->z_blksz && (outzp->z_size > outzp->z_blksz \|\|
	outlr->lr_length != UINT64_MAX)) {
	error = SET_ERROR(EINVAL);
	goto unlock;
	}

	/*
	* Block size must be power-of-2 if destination offset != 0.
	* There can be no multiple blocks of non-power-of-2 size.
	*/
	if (outoff != 0 && !ISP2(inblksz)) {
	error = SET_ERROR(EINVAL);
	goto unlock;
	}

	/*
	* Offsets and len must be at block boundries.
	*/
	if ((inoff % inblksz) != 0 \|\| (outoff % inblksz) != 0) {
	error = SET_ERROR(EINVAL);
	goto unlock;
	}
	/*
	* Length must be multipe of blksz, except for the end of the file.
	*/
	if ((len % inblksz) != 0 &&
	(len < inzp->z_size - inoff \|\| len < outzp->z_size - outoff)) {
	error = SET_ERROR(EINVAL);
	goto unlock;
	}

	/*
	* If we are copying only one block and it is smaller than recordsize
	* property, do not allow destination to grow beyond one block if it
	* is not there yet. Otherwise the destination will get stuck with
	* that block size forever, that can be as small as 512 bytes, no
	* matter how big the destination grow later.
	*/
	if (len <= inblksz && inblksz < outzfsvfs->z_max_blksz &&
	outzp->z_size <= inblksz && outoff + len > inblksz) {
	error = SET_ERROR(EINVAL);
	goto unlock;
	}

	error = zn_rlimit_fsize(outoff + len);
	if (error != 0) {
	goto unlock;
	}

	if (inoff >= MAXOFFSET_T \|\| outoff >= MAXOFFSET_T) {
	error = SET_ERROR(EFBIG);
	goto unlock;
	}

	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(outzfsvfs), NULL,
	&mtime, 16);
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(outzfsvfs), NULL,
	&ctime, 16);
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(outzfsvfs), NULL,
	&outzp->z_size, 8);

	zilog = outzfsvfs->z_log;
	maxblocks = zil_max_log_data(zilog, sizeof (lr_clone_range_t)) /
	sizeof (bps[0]);

	uid = KUID_TO_SUID(ZTOUID(outzp));
	gid = KGID_TO_SGID(ZTOGID(outzp));
	projid = outzp->z_projid;

	bps = vmem_alloc(sizeof (bps[0]) * maxblocks, KM_SLEEP);

	/*
	* Clone the file in reasonable size chunks. Each chunk is cloned
	* in a separate transaction; this keeps the intent log records small
	* and allows us to do more fine-grained space accounting.
	*/
	while (len > 0) {
	size = MIN(inblksz * maxblocks, len);

	if (zfs_id_overblockquota(outzfsvfs, DMU_USERUSED_OBJECT,
	uid) \|\|
	zfs_id_overblockquota(outzfsvfs, DMU_GROUPUSED_OBJECT,
	gid) \|\|
	(projid != ZFS_DEFAULT_PROJID &&
	zfs_id_overblockquota(outzfsvfs, DMU_PROJECTUSED_OBJECT,
	projid))) {
	error = SET_ERROR(EDQUOT);
	break;
	}

	nbps = maxblocks;
	last_synced_txg = spa_last_synced_txg(dmu_objset_spa(inos));
	error = dmu_read_l0_bps(inos, inzp->z_id, inoff, size, bps,
	&nbps);
	if (error != 0) {
	/*
	* If we are trying to clone a block that was created
	* in the current transaction group, the error will be
	* EAGAIN here. Based on zfs_bclone_wait_dirty either
	* return a shortened range to the caller so it can
	* fallback, or wait for the next TXG and check again.
	*/
	if (error == EAGAIN && zfs_bclone_wait_dirty) {
	txg_wait_synced(dmu_objset_pool(inos),
	last_synced_txg + 1);
	continue;
	}

	break;
	}

	/*
	* Start a transaction.
	*/
	tx = dmu_tx_create(outos);
	dmu_tx_hold_sa(tx, outzp->z_sa_hdl, B_FALSE);
	db = (dmu_buf_impl_t *)sa_get_db(outzp->z_sa_hdl);
	DB_DNODE_ENTER(db);
	dmu_tx_hold_clone_by_dnode(tx, DB_DNODE(db), outoff, size);
	DB_DNODE_EXIT(db);
	zfs_sa_upgrade_txholds(tx, outzp);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error != 0) {
	dmu_tx_abort(tx);
	break;
	}

	/*
	* Copy source znode's block size. This is done only if the
	* whole znode is locked (see zfs_rangelock_cb()) and only
	* on the first iteration since zfs_rangelock_reduce() will
	* shrink down lr_length to the appropriate size.
	*/
	if (outlr->lr_length == UINT64_MAX) {
	zfs_grow_blocksize(outzp, inblksz, tx);

	/*
	* Block growth may fail for many reasons we can not
	* predict here. If it happen the cloning is doomed.
	*/
	if (inblksz != outzp->z_blksz) {
	error = SET_ERROR(EINVAL);
	dmu_tx_abort(tx);
	break;
	}

	/*
	* Round range lock up to the block boundary, so we
	* prevent appends until we are done.
	*/
	zfs_rangelock_reduce(outlr, outoff,
	((len - 1) / inblksz + 1) * inblksz);
	}

	error = dmu_brt_clone(outos, outzp->z_id, outoff, size, tx,
	bps, nbps);
	if (error != 0) {
	dmu_tx_commit(tx);
	break;
	}

	if (zn_has_cached_data(outzp, outoff, outoff + size - 1)) {
	update_pages(outzp, outoff, size, outos);
	}

	zfs_clear_setid_bits_if_necessary(outzfsvfs, outzp, cr,
	&clear_setid_bits_txg, tx);

	zfs_tstamp_update_setup(outzp, CONTENT_MODIFIED, mtime, ctime);

	/*
	* Update the file size (zp_size) if it has changed;
	* account for possible concurrent updates.
	*/
	while ((outsize = outzp->z_size) < outoff + size) {
	(void) atomic_cas_64(&outzp->z_size, outsize,
	outoff + size);
	}

	error = sa_bulk_update(outzp->z_sa_hdl, bulk, count, tx);

	zfs_log_clone_range(zilog, tx, TX_CLONE_RANGE, outzp, outoff,
	size, inblksz, bps, nbps);

	dmu_tx_commit(tx);

	if (error != 0)
	break;

	inoff += size;
	outoff += size;
	len -= size;
	done += size;
	}

	vmem_free(bps, sizeof (bps[0]) * maxblocks);
	zfs_znode_update_vfs(outzp);

	unlock:
	zfs_rangelock_exit(outlr);
	zfs_rangelock_exit(inlr);

	if (done > 0) {
	/*
	* If we have made at least partial progress, reset the error.
	*/
	error = 0;

	ZFS_ACCESSTIME_STAMP(inzfsvfs, inzp);

	if (outos->os_sync == ZFS_SYNC_ALWAYS) {
	zil_commit(zilog, outzp->z_id);
	}

	*inoffp += done;
	*outoffp += done;
	*lenp = done;
	} else {
	/*
	* If we made no progress, there must be a good reason.
	* EOF is handled explicitly above, before the loop.
	*/
	ASSERT3S(error, !=, 0);
	}

	zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);

	return (error);
	}

	/*
	* Usual pattern would be to call zfs_clone_range() from zfs_replay_clone(),
	* but we cannot do that, because when replaying we don't have source znode
	* available. This is why we need a dedicated replay function.
	*/
	int
	zfs_clone_range_replay(znode_t *zp, uint64_t off, uint64_t len, uint64_t blksz,
	const blkptr_t *bps, size_t nbps)
	{
	zfsvfs_t *zfsvfs;
	dmu_buf_impl_t *db;
	dmu_tx_t *tx;
	int error;
	int count = 0;
	sa_bulk_attr_t bulk[3];
	uint64_t mtime[2], ctime[2];

	ASSERT3U(off, <, MAXOFFSET_T);
	ASSERT3U(len, >, 0);
	ASSERT3U(nbps, >, 0);

	zfsvfs = ZTOZSB(zp);

	ASSERT(spa_feature_is_enabled(dmu_objset_spa(zfsvfs->z_os),
	SPA_FEATURE_BLOCK_CLONING));

	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
	return (error);

	ASSERT(zfsvfs->z_replay);
	ASSERT(!zfs_is_readonly(zfsvfs));

	if ((off % blksz) != 0) {
	zfs_exit(zfsvfs, FTAG);
	return (SET_ERROR(EINVAL));
	}

	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
	&zp->z_size, 8);

	/*
	* Start a transaction.
	*/
	tx = dmu_tx_create(zfsvfs->z_os);

	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
	db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl);
	DB_DNODE_ENTER(db);
	dmu_tx_hold_clone_by_dnode(tx, DB_DNODE(db), off, len);
	DB_DNODE_EXIT(db);
	zfs_sa_upgrade_txholds(tx, zp);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error != 0) {
	dmu_tx_abort(tx);
	zfs_exit(zfsvfs, FTAG);
	return (error);
	}

	if (zp->z_blksz < blksz)
	zfs_grow_blocksize(zp, blksz, tx);

	dmu_brt_clone(zfsvfs->z_os, zp->z_id, off, len, tx, bps, nbps);

	zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);

	if (zp->z_size < off + len)
	zp->z_size = off + len;

	error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);

	/*
	* zil_replaying() not only check if we are replaying ZIL, but also
	* updates the ZIL header to record replay progress.
	*/
	VERIFY(zil_replaying(zfsvfs->z_log, tx));

	dmu_tx_commit(tx);

	zfs_znode_update_vfs(zp);

	zfs_exit(zfsvfs, FTAG);

	return (error);
	}

	EXPORT_SYMBOL(zfs_access);
	EXPORT_SYMBOL(zfs_fsync);
	EXPORT_SYMBOL(zfs_holey);
	EXPORT_SYMBOL(zfs_read);
	EXPORT_SYMBOL(zfs_write);
	EXPORT_SYMBOL(zfs_getsecattr);
	EXPORT_SYMBOL(zfs_setsecattr);
	EXPORT_SYMBOL(zfs_clone_range);
	EXPORT_SYMBOL(zfs_clone_range_replay);

	ZFS_MODULE_PARAM(zfs_vnops, zfs_vnops_, read_chunk_size, U64, ZMOD_RW,
	"Bytes to read per chunk");

	ZFS_MODULE_PARAM(zfs, zfs_, bclone_enabled, INT, ZMOD_RW,
	"Enable block cloning");

	ZFS_MODULE_PARAM(zfs, zfs_, bclone_wait_dirty, INT, ZMOD_RW,
	"Wait for dirty blocks when cloning");
	diff --git a/sys/contrib/openzfs/module/zfs/zil.c b/sys/contrib/openzfs/module/zfs/zil.c
	index 5642f082bdb8..9b5d866a8c22 100644
	--- a/sys/contrib/openzfs/module/zfs/zil.c
	+++ b/sys/contrib/openzfs/module/zfs/zil.c
	@@ -1,4269 +1,4383 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2011, 2018 by Delphix. All rights reserved.
	* Copyright (c) 2014 Integros [integros.com]
	* Copyright (c) 2018 Datto Inc.
	*/

	/* Portions Copyright 2010 Robert Milkowski */

	#include <sys/zfs_context.h>
	#include <sys/spa.h>
	#include <sys/spa_impl.h>
	#include <sys/dmu.h>
	#include <sys/zap.h>
	#include <sys/arc.h>
	#include <sys/stat.h>
	#include <sys/zil.h>
	#include <sys/zil_impl.h>
	#include <sys/dsl_dataset.h>
	#include <sys/vdev_impl.h>
	#include <sys/dmu_tx.h>
	#include <sys/dsl_pool.h>
	#include <sys/metaslab.h>
	#include <sys/trace_zfs.h>
	#include <sys/abd.h>
	#include <sys/brt.h>
	#include <sys/wmsum.h>

	/*
	* The ZFS Intent Log (ZIL) saves "transaction records" (itxs) of system
	* calls that change the file system. Each itx has enough information to
	* be able to replay them after a system crash, power loss, or
	* equivalent failure mode. These are stored in memory until either:
	*
	* 1. they are committed to the pool by the DMU transaction group
	* (txg), at which point they can be discarded; or
	* 2. they are committed to the on-disk ZIL for the dataset being
	* modified (e.g. due to an fsync, O_DSYNC, or other synchronous
	* requirement).
	*
	* In the event of a crash or power loss, the itxs contained by each
	* dataset's on-disk ZIL will be replayed when that dataset is first
	* instantiated (e.g. if the dataset is a normal filesystem, when it is
	* first mounted).
	*
	* As hinted at above, there is one ZIL per dataset (both the in-memory
	* representation, and the on-disk representation). The on-disk format
	* consists of 3 parts:
	*
	* - a single, per-dataset, ZIL header; which points to a chain of
	* - zero or more ZIL blocks; each of which contains
	* - zero or more ZIL records
	*
	* A ZIL record holds the information necessary to replay a single
	* system call transaction. A ZIL block can hold many ZIL records, and
	* the blocks are chained together, similarly to a singly linked list.
	*
	* Each ZIL block contains a block pointer (blkptr_t) to the next ZIL
	* block in the chain, and the ZIL header points to the first block in
	* the chain.
	*
	* Note, there is not a fixed place in the pool to hold these ZIL
	* blocks; they are dynamically allocated and freed as needed from the
	* blocks available on the pool, though they can be preferentially
	* allocated from a dedicated "log" vdev.
	*/

	/*
	* This controls the amount of time that a ZIL block (lwb) will remain
	* "open" when it isn't "full", and it has a thread waiting for it to be
	* committed to stable storage. Please refer to the zil_commit_waiter()
	* function (and the comments within it) for more details.
	*/
	-static uint_t zfs_commit_timeout_pct = 5;
	-
	-/*
	- * Minimal time we care to delay commit waiting for more ZIL records.
	- * At least FreeBSD kernel can't sleep for less than 2us at its best.
	- * So requests to sleep for less then 5us is a waste of CPU time with
	- * a risk of significant log latency increase due to oversleep.
	- */
	-static uint64_t zil_min_commit_timeout = 5000;
	+static uint_t zfs_commit_timeout_pct = 10;

	/*
	* See zil.h for more information about these fields.
	*/
	static zil_kstat_values_t zil_stats = {
	{ "zil_commit_count", KSTAT_DATA_UINT64 },
	{ "zil_commit_writer_count", KSTAT_DATA_UINT64 },
	{ "zil_itx_count", KSTAT_DATA_UINT64 },
	{ "zil_itx_indirect_count", KSTAT_DATA_UINT64 },
	{ "zil_itx_indirect_bytes", KSTAT_DATA_UINT64 },
	{ "zil_itx_copied_count", KSTAT_DATA_UINT64 },
	{ "zil_itx_copied_bytes", KSTAT_DATA_UINT64 },
	{ "zil_itx_needcopy_count", KSTAT_DATA_UINT64 },
	{ "zil_itx_needcopy_bytes", KSTAT_DATA_UINT64 },
	{ "zil_itx_metaslab_normal_count", KSTAT_DATA_UINT64 },
	{ "zil_itx_metaslab_normal_bytes", KSTAT_DATA_UINT64 },
	{ "zil_itx_metaslab_normal_write", KSTAT_DATA_UINT64 },
	{ "zil_itx_metaslab_normal_alloc", KSTAT_DATA_UINT64 },
	{ "zil_itx_metaslab_slog_count", KSTAT_DATA_UINT64 },
	{ "zil_itx_metaslab_slog_bytes", KSTAT_DATA_UINT64 },
	{ "zil_itx_metaslab_slog_write", KSTAT_DATA_UINT64 },
	{ "zil_itx_metaslab_slog_alloc", KSTAT_DATA_UINT64 },
	};

	static zil_sums_t zil_sums_global;
	static kstat_t *zil_kstats_global;

	/*
	* Disable intent logging replay. This global ZIL switch affects all pools.
	*/
	int zil_replay_disable = 0;

	/*
	* Disable the DKIOCFLUSHWRITECACHE commands that are normally sent to
	* the disk(s) by the ZIL after an LWB write has completed. Setting this
	* will cause ZIL corruption on power loss if a volatile out-of-order
	* write cache is enabled.
	*/
	static int zil_nocacheflush = 0;

	/*
	* Limit SLOG write size per commit executed with synchronous priority.
	* Any writes above that will be executed with lower (asynchronous) priority
	* to limit potential SLOG device abuse by single active ZIL writer.
	*/
	static uint64_t zil_slog_bulk = 64 * 1024 * 1024;

	static kmem_cache_t *zil_lwb_cache;
	static kmem_cache_t *zil_zcw_cache;

	static void zil_lwb_commit(zilog_t zilog, lwb_t lwb, itx_t *itx);
	static itx_t zil_itx_clone(itx_t oitx);
	+static uint64_t zil_max_waste_space(zilog_t *zilog);

	static int
	zil_bp_compare(const void x1, const void x2)
	{
	const dva_t dva1 = &((zil_bp_node_t )x1)->zn_dva;
	const dva_t dva2 = &((zil_bp_node_t )x2)->zn_dva;

	int cmp = TREE_CMP(DVA_GET_VDEV(dva1), DVA_GET_VDEV(dva2));
	if (likely(cmp))
	return (cmp);

	return (TREE_CMP(DVA_GET_OFFSET(dva1), DVA_GET_OFFSET(dva2)));
	}

	static void
	zil_bp_tree_init(zilog_t *zilog)
	{
	avl_create(&zilog->zl_bp_tree, zil_bp_compare,
	sizeof (zil_bp_node_t), offsetof(zil_bp_node_t, zn_node));
	}

	static void
	zil_bp_tree_fini(zilog_t *zilog)
	{
	avl_tree_t *t = &zilog->zl_bp_tree;
	zil_bp_node_t *zn;
	void *cookie = NULL;

	while ((zn = avl_destroy_nodes(t, &cookie)) != NULL)
	kmem_free(zn, sizeof (zil_bp_node_t));

	avl_destroy(t);
	}

	int
	zil_bp_tree_add(zilog_t zilog, const blkptr_t bp)
	{
	avl_tree_t *t = &zilog->zl_bp_tree;
	const dva_t *dva;
	zil_bp_node_t *zn;
	avl_index_t where;

	if (BP_IS_EMBEDDED(bp))
	return (0);

	dva = BP_IDENTITY(bp);

	if (avl_find(t, dva, &where) != NULL)
	return (SET_ERROR(EEXIST));

	zn = kmem_alloc(sizeof (zil_bp_node_t), KM_SLEEP);
	zn->zn_dva = *dva;
	avl_insert(t, zn, where);

	return (0);
	}

	static zil_header_t *
	zil_header_in_syncing_context(zilog_t *zilog)
	{
	return ((zil_header_t *)zilog->zl_header);
	}

	static void
	zil_init_log_chain(zilog_t zilog, blkptr_t bp)
	{
	zio_cksum_t *zc = &bp->blk_cksum;

	(void) random_get_pseudo_bytes((void *)&zc->zc_word[ZIL_ZC_GUID_0],
	sizeof (zc->zc_word[ZIL_ZC_GUID_0]));
	(void) random_get_pseudo_bytes((void *)&zc->zc_word[ZIL_ZC_GUID_1],
	sizeof (zc->zc_word[ZIL_ZC_GUID_1]));
	zc->zc_word[ZIL_ZC_OBJSET] = dmu_objset_id(zilog->zl_os);
	zc->zc_word[ZIL_ZC_SEQ] = 1ULL;
	}

	static int
	zil_kstats_global_update(kstat_t *ksp, int rw)
	{
	zil_kstat_values_t *zs = ksp->ks_data;
	ASSERT3P(&zil_stats, ==, zs);

	if (rw == KSTAT_WRITE) {
	return (SET_ERROR(EACCES));
	}

	zil_kstat_values_update(zs, &zil_sums_global);

	return (0);
	}

	/*
	* Read a log block and make sure it's valid.
	*/
	static int
	zil_read_log_block(zilog_t zilog, boolean_t decrypt, const blkptr_t bp,
	blkptr_t nbp, char begin, char end, arc_buf_t *abuf)
	{
	zio_flag_t zio_flags = ZIO_FLAG_CANFAIL;
	arc_flags_t aflags = ARC_FLAG_WAIT;
	zbookmark_phys_t zb;
	int error;

	if (zilog->zl_header->zh_claim_txg == 0)
	zio_flags \|= ZIO_FLAG_SPECULATIVE \| ZIO_FLAG_SCRUB;

	if (!(zilog->zl_header->zh_flags & ZIL_CLAIM_LR_SEQ_VALID))
	zio_flags \|= ZIO_FLAG_SPECULATIVE;

	if (!decrypt)
	zio_flags \|= ZIO_FLAG_RAW;

	SET_BOOKMARK(&zb, bp->blk_cksum.zc_word[ZIL_ZC_OBJSET],
	ZB_ZIL_OBJECT, ZB_ZIL_LEVEL, bp->blk_cksum.zc_word[ZIL_ZC_SEQ]);

	error = arc_read(NULL, zilog->zl_spa, bp, arc_getbuf_func,
	abuf, ZIO_PRIORITY_SYNC_READ, zio_flags, &aflags, &zb);

	if (error == 0) {
	zio_cksum_t cksum = bp->blk_cksum;

	/*
	* Validate the checksummed log block.
	*
	* Sequence numbers should be... sequential. The checksum
	* verifier for the next block should be bp's checksum plus 1.
	*
	* Also check the log chain linkage and size used.
	*/
	cksum.zc_word[ZIL_ZC_SEQ]++;

	uint64_t size = BP_GET_LSIZE(bp);
	if (BP_GET_CHECKSUM(bp) == ZIO_CHECKSUM_ZILOG2) {
	zil_chain_t zilc = (abuf)->b_data;
	char lr = (char )(zilc + 1);

	if (memcmp(&cksum, &zilc->zc_next_blk.blk_cksum,
	sizeof (cksum)) \|\|
	zilc->zc_nused < sizeof (*zilc) \|\|
	zilc->zc_nused > size) {
	error = SET_ERROR(ECKSUM);
	} else {
	*begin = lr;
	end = lr + zilc->zc_nused - sizeof (zilc);
	*nbp = zilc->zc_next_blk;
	}
	} else {
	char lr = (abuf)->b_data;
	zil_chain_t zilc = (zil_chain_t )(lr + size) - 1;

	if (memcmp(&cksum, &zilc->zc_next_blk.blk_cksum,
	sizeof (cksum)) \|\|
	(zilc->zc_nused > (size - sizeof (*zilc)))) {
	error = SET_ERROR(ECKSUM);
	} else {
	*begin = lr;
	*end = lr + zilc->zc_nused;
	*nbp = zilc->zc_next_blk;
	}
	}
	}

	return (error);
	}

	/*
	* Read a TX_WRITE log data block.
	*/
	static int
	zil_read_log_data(zilog_t zilog, const lr_write_t lr, void *wbuf)
	{
	zio_flag_t zio_flags = ZIO_FLAG_CANFAIL;
	const blkptr_t *bp = &lr->lr_blkptr;
	arc_flags_t aflags = ARC_FLAG_WAIT;
	arc_buf_t *abuf = NULL;
	zbookmark_phys_t zb;
	int error;

	if (BP_IS_HOLE(bp)) {
	if (wbuf != NULL)
	memset(wbuf, 0, MAX(BP_GET_LSIZE(bp), lr->lr_length));
	return (0);
	}

	if (zilog->zl_header->zh_claim_txg == 0)
	zio_flags \|= ZIO_FLAG_SPECULATIVE \| ZIO_FLAG_SCRUB;

	/*
	* If we are not using the resulting data, we are just checking that
	* it hasn't been corrupted so we don't need to waste CPU time
	* decompressing and decrypting it.
	*/
	if (wbuf == NULL)
	zio_flags \|= ZIO_FLAG_RAW;

	ASSERT3U(BP_GET_LSIZE(bp), !=, 0);
	SET_BOOKMARK(&zb, dmu_objset_id(zilog->zl_os), lr->lr_foid,
	ZB_ZIL_LEVEL, lr->lr_offset / BP_GET_LSIZE(bp));

	error = arc_read(NULL, zilog->zl_spa, bp, arc_getbuf_func, &abuf,
	ZIO_PRIORITY_SYNC_READ, zio_flags, &aflags, &zb);

	if (error == 0) {
	if (wbuf != NULL)
	memcpy(wbuf, abuf->b_data, arc_buf_size(abuf));
	arc_buf_destroy(abuf, &abuf);
	}

	return (error);
	}

	void
	zil_sums_init(zil_sums_t *zs)
	{
	wmsum_init(&zs->zil_commit_count, 0);
	wmsum_init(&zs->zil_commit_writer_count, 0);
	wmsum_init(&zs->zil_itx_count, 0);
	wmsum_init(&zs->zil_itx_indirect_count, 0);
	wmsum_init(&zs->zil_itx_indirect_bytes, 0);
	wmsum_init(&zs->zil_itx_copied_count, 0);
	wmsum_init(&zs->zil_itx_copied_bytes, 0);
	wmsum_init(&zs->zil_itx_needcopy_count, 0);
	wmsum_init(&zs->zil_itx_needcopy_bytes, 0);
	wmsum_init(&zs->zil_itx_metaslab_normal_count, 0);
	wmsum_init(&zs->zil_itx_metaslab_normal_bytes, 0);
	wmsum_init(&zs->zil_itx_metaslab_normal_write, 0);
	wmsum_init(&zs->zil_itx_metaslab_normal_alloc, 0);
	wmsum_init(&zs->zil_itx_metaslab_slog_count, 0);
	wmsum_init(&zs->zil_itx_metaslab_slog_bytes, 0);
	wmsum_init(&zs->zil_itx_metaslab_slog_write, 0);
	wmsum_init(&zs->zil_itx_metaslab_slog_alloc, 0);
	}

	void
	zil_sums_fini(zil_sums_t *zs)
	{
	wmsum_fini(&zs->zil_commit_count);
	wmsum_fini(&zs->zil_commit_writer_count);
	wmsum_fini(&zs->zil_itx_count);
	wmsum_fini(&zs->zil_itx_indirect_count);
	wmsum_fini(&zs->zil_itx_indirect_bytes);
	wmsum_fini(&zs->zil_itx_copied_count);
	wmsum_fini(&zs->zil_itx_copied_bytes);
	wmsum_fini(&zs->zil_itx_needcopy_count);
	wmsum_fini(&zs->zil_itx_needcopy_bytes);
	wmsum_fini(&zs->zil_itx_metaslab_normal_count);
	wmsum_fini(&zs->zil_itx_metaslab_normal_bytes);
	wmsum_fini(&zs->zil_itx_metaslab_normal_write);
	wmsum_fini(&zs->zil_itx_metaslab_normal_alloc);
	wmsum_fini(&zs->zil_itx_metaslab_slog_count);
	wmsum_fini(&zs->zil_itx_metaslab_slog_bytes);
	wmsum_fini(&zs->zil_itx_metaslab_slog_write);
	wmsum_fini(&zs->zil_itx_metaslab_slog_alloc);
	}

	void
	zil_kstat_values_update(zil_kstat_values_t zs, zil_sums_t zil_sums)
	{
	zs->zil_commit_count.value.ui64 =
	wmsum_value(&zil_sums->zil_commit_count);
	zs->zil_commit_writer_count.value.ui64 =
	wmsum_value(&zil_sums->zil_commit_writer_count);
	zs->zil_itx_count.value.ui64 =
	wmsum_value(&zil_sums->zil_itx_count);
	zs->zil_itx_indirect_count.value.ui64 =
	wmsum_value(&zil_sums->zil_itx_indirect_count);
	zs->zil_itx_indirect_bytes.value.ui64 =
	wmsum_value(&zil_sums->zil_itx_indirect_bytes);
	zs->zil_itx_copied_count.value.ui64 =
	wmsum_value(&zil_sums->zil_itx_copied_count);
	zs->zil_itx_copied_bytes.value.ui64 =
	wmsum_value(&zil_sums->zil_itx_copied_bytes);
	zs->zil_itx_needcopy_count.value.ui64 =
	wmsum_value(&zil_sums->zil_itx_needcopy_count);
	zs->zil_itx_needcopy_bytes.value.ui64 =
	wmsum_value(&zil_sums->zil_itx_needcopy_bytes);
	zs->zil_itx_metaslab_normal_count.value.ui64 =
	wmsum_value(&zil_sums->zil_itx_metaslab_normal_count);
	zs->zil_itx_metaslab_normal_bytes.value.ui64 =
	wmsum_value(&zil_sums->zil_itx_metaslab_normal_bytes);
	zs->zil_itx_metaslab_normal_write.value.ui64 =
	wmsum_value(&zil_sums->zil_itx_metaslab_normal_write);
	zs->zil_itx_metaslab_normal_alloc.value.ui64 =
	wmsum_value(&zil_sums->zil_itx_metaslab_normal_alloc);
	zs->zil_itx_metaslab_slog_count.value.ui64 =
	wmsum_value(&zil_sums->zil_itx_metaslab_slog_count);
	zs->zil_itx_metaslab_slog_bytes.value.ui64 =
	wmsum_value(&zil_sums->zil_itx_metaslab_slog_bytes);
	zs->zil_itx_metaslab_slog_write.value.ui64 =
	wmsum_value(&zil_sums->zil_itx_metaslab_slog_write);
	zs->zil_itx_metaslab_slog_alloc.value.ui64 =
	wmsum_value(&zil_sums->zil_itx_metaslab_slog_alloc);
	}

	/*
	* Parse the intent log, and call parse_func for each valid record within.
	*/
	int
	zil_parse(zilog_t zilog, zil_parse_blk_func_t parse_blk_func,
	zil_parse_lr_func_t parse_lr_func, void arg, uint64_t txg,
	boolean_t decrypt)
	{
	const zil_header_t *zh = zilog->zl_header;
	boolean_t claimed = !!zh->zh_claim_txg;
	uint64_t claim_blk_seq = claimed ? zh->zh_claim_blk_seq : UINT64_MAX;
	uint64_t claim_lr_seq = claimed ? zh->zh_claim_lr_seq : UINT64_MAX;
	uint64_t max_blk_seq = 0;
	uint64_t max_lr_seq = 0;
	uint64_t blk_count = 0;
	uint64_t lr_count = 0;
	blkptr_t blk, next_blk = {{{{0}}}};
	int error = 0;

	/*
	* Old logs didn't record the maximum zh_claim_lr_seq.
	*/
	if (!(zh->zh_flags & ZIL_CLAIM_LR_SEQ_VALID))
	claim_lr_seq = UINT64_MAX;

	/*
	* Starting at the block pointed to by zh_log we read the log chain.
	* For each block in the chain we strongly check that block to
	* ensure its validity. We stop when an invalid block is found.
	* For each block pointer in the chain we call parse_blk_func().
	* For each record in each valid block we call parse_lr_func().
	* If the log has been claimed, stop if we encounter a sequence
	* number greater than the highest claimed sequence number.
	*/
	zil_bp_tree_init(zilog);

	for (blk = zh->zh_log; !BP_IS_HOLE(&blk); blk = next_blk) {
	uint64_t blk_seq = blk.blk_cksum.zc_word[ZIL_ZC_SEQ];
	int reclen;
	char lrp, end;
	arc_buf_t *abuf = NULL;

	if (blk_seq > claim_blk_seq)
	break;

	error = parse_blk_func(zilog, &blk, arg, txg);
	if (error != 0)
	break;
	ASSERT3U(max_blk_seq, <, blk_seq);
	max_blk_seq = blk_seq;
	blk_count++;

	if (max_lr_seq == claim_lr_seq && max_blk_seq == claim_blk_seq)
	break;

	error = zil_read_log_block(zilog, decrypt, &blk, &next_blk,
	&lrp, &end, &abuf);
	if (error != 0) {
	if (abuf)
	arc_buf_destroy(abuf, &abuf);
	if (claimed) {
	char name[ZFS_MAX_DATASET_NAME_LEN];

	dmu_objset_name(zilog->zl_os, name);

	cmn_err(CE_WARN, "ZFS read log block error %d, "
	"dataset %s, seq 0x%llx\n", error, name,
	(u_longlong_t)blk_seq);
	}
	break;
	}

	for (; lrp < end; lrp += reclen) {
	lr_t lr = (lr_t )lrp;
	reclen = lr->lrc_reclen;
	ASSERT3U(reclen, >=, sizeof (lr_t));
	ASSERT3U(reclen, <=, end - lrp);
	if (lr->lrc_seq > claim_lr_seq) {
	arc_buf_destroy(abuf, &abuf);
	goto done;
	}

	error = parse_lr_func(zilog, lr, arg, txg);
	if (error != 0) {
	arc_buf_destroy(abuf, &abuf);
	goto done;
	}
	ASSERT3U(max_lr_seq, <, lr->lrc_seq);
	max_lr_seq = lr->lrc_seq;
	lr_count++;
	}
	arc_buf_destroy(abuf, &abuf);
	}
	done:
	zilog->zl_parse_error = error;
	zilog->zl_parse_blk_seq = max_blk_seq;
	zilog->zl_parse_lr_seq = max_lr_seq;
	zilog->zl_parse_blk_count = blk_count;
	zilog->zl_parse_lr_count = lr_count;

	zil_bp_tree_fini(zilog);

	return (error);
	}

	static int
	zil_clear_log_block(zilog_t zilog, const blkptr_t bp, void *tx,
	uint64_t first_txg)
	{
	(void) tx;
	ASSERT(!BP_IS_HOLE(bp));

	/*
	* As we call this function from the context of a rewind to a
	* checkpoint, each ZIL block whose txg is later than the txg
	* that we rewind to is invalid. Thus, we return -1 so
	* zil_parse() doesn't attempt to read it.
	*/
	if (bp->blk_birth >= first_txg)
	return (-1);

	if (zil_bp_tree_add(zilog, bp) != 0)
	return (0);

	zio_free(zilog->zl_spa, first_txg, bp);
	return (0);
	}

	static int
	zil_noop_log_record(zilog_t zilog, const lr_t lrc, void *tx,
	uint64_t first_txg)
	{
	(void) zilog, (void) lrc, (void) tx, (void) first_txg;
	return (0);
	}

	static int
	zil_claim_log_block(zilog_t zilog, const blkptr_t bp, void *tx,
	uint64_t first_txg)
	{
	/*
	* Claim log block if not already committed and not already claimed.
	* If tx == NULL, just verify that the block is claimable.
	*/
	if (BP_IS_HOLE(bp) \|\| bp->blk_birth < first_txg \|\|
	zil_bp_tree_add(zilog, bp) != 0)
	return (0);

	return (zio_wait(zio_claim(NULL, zilog->zl_spa,
	tx == NULL ? 0 : first_txg, bp, spa_claim_notify, NULL,
	ZIO_FLAG_CANFAIL \| ZIO_FLAG_SPECULATIVE \| ZIO_FLAG_SCRUB)));
	}

	static int
	zil_claim_write(zilog_t zilog, const lr_t lrc, void *tx, uint64_t first_txg)
	{
	lr_write_t lr = (lr_write_t )lrc;
	int error;

	ASSERT3U(lrc->lrc_reclen, >=, sizeof (*lr));

	/*
	* If the block is not readable, don't claim it. This can happen
	* in normal operation when a log block is written to disk before
	* some of the dmu_sync() blocks it points to. In this case, the
	* transaction cannot have been committed to anyone (we would have
	* waited for all writes to be stable first), so it is semantically
	* correct to declare this the end of the log.
	*/
	if (lr->lr_blkptr.blk_birth >= first_txg) {
	error = zil_read_log_data(zilog, lr, NULL);
	if (error != 0)
	return (error);
	}

	return (zil_claim_log_block(zilog, &lr->lr_blkptr, tx, first_txg));
	}

	static int
	zil_claim_clone_range(zilog_t zilog, const lr_t lrc, void *tx,
	uint64_t first_txg)
	{
	const lr_clone_range_t lr = (const lr_clone_range_t )lrc;
	const blkptr_t *bp;
	spa_t *spa = zilog->zl_spa;
	uint_t ii;

	ASSERT3U(lrc->lrc_reclen, >=, sizeof (*lr));
	ASSERT3U(lrc->lrc_reclen, >=, offsetof(lr_clone_range_t,
	lr_bps[lr->lr_nbps]));

	if (tx == NULL) {
	return (0);
	}

	/*
	* XXX: Do we need to byteswap lr?
	*/

	for (ii = 0; ii < lr->lr_nbps; ii++) {
	bp = &lr->lr_bps[ii];

	/*
	* When data is embedded into the BP there is no need to create
	* BRT entry as there is no data block. Just copy the BP as it
	* contains the data.
	*/
	if (BP_IS_HOLE(bp) \|\| BP_IS_EMBEDDED(bp))
	continue;

	/*
	* We can not handle block pointers from the future, since they
	* are not yet allocated. It should not normally happen, but
	* just in case lets be safe and just stop here now instead of
	* corrupting the pool.
	*/
	if (BP_PHYSICAL_BIRTH(bp) >= first_txg)
	return (SET_ERROR(ENOENT));

	/*
	* Assert the block is really allocated before we reference it.
	*/
	metaslab_check_free(spa, bp);
	}

	for (ii = 0; ii < lr->lr_nbps; ii++) {
	bp = &lr->lr_bps[ii];
	if (!BP_IS_HOLE(bp) && !BP_IS_EMBEDDED(bp))
	brt_pending_add(spa, bp, tx);
	}

	return (0);
	}

	static int
	zil_claim_log_record(zilog_t zilog, const lr_t lrc, void *tx,
	uint64_t first_txg)
	{

	switch (lrc->lrc_txtype) {
	case TX_WRITE:
	return (zil_claim_write(zilog, lrc, tx, first_txg));
	case TX_CLONE_RANGE:
	return (zil_claim_clone_range(zilog, lrc, tx, first_txg));
	default:
	return (0);
	}
	}

	static int
	zil_free_log_block(zilog_t zilog, const blkptr_t bp, void *tx,
	uint64_t claim_txg)
	{
	(void) claim_txg;

	zio_free(zilog->zl_spa, dmu_tx_get_txg(tx), bp);

	return (0);
	}

	static int
	zil_free_write(zilog_t zilog, const lr_t lrc, void *tx, uint64_t claim_txg)
	{
	lr_write_t lr = (lr_write_t )lrc;
	blkptr_t *bp = &lr->lr_blkptr;

	ASSERT3U(lrc->lrc_reclen, >=, sizeof (*lr));

	/*
	* If we previously claimed it, we need to free it.
	*/
	if (bp->blk_birth >= claim_txg && zil_bp_tree_add(zilog, bp) == 0 &&
	!BP_IS_HOLE(bp)) {
	zio_free(zilog->zl_spa, dmu_tx_get_txg(tx), bp);
	}

	return (0);
	}

	static int
	zil_free_clone_range(zilog_t zilog, const lr_t lrc, void *tx)
	{
	const lr_clone_range_t lr = (const lr_clone_range_t )lrc;
	const blkptr_t *bp;
	spa_t *spa;
	uint_t ii;

	ASSERT3U(lrc->lrc_reclen, >=, sizeof (*lr));
	ASSERT3U(lrc->lrc_reclen, >=, offsetof(lr_clone_range_t,
	lr_bps[lr->lr_nbps]));

	if (tx == NULL) {
	return (0);
	}

	spa = zilog->zl_spa;

	for (ii = 0; ii < lr->lr_nbps; ii++) {
	bp = &lr->lr_bps[ii];

	if (!BP_IS_HOLE(bp)) {
	zio_free(spa, dmu_tx_get_txg(tx), bp);
	}
	}

	return (0);
	}

	static int
	zil_free_log_record(zilog_t zilog, const lr_t lrc, void *tx,
	uint64_t claim_txg)
	{

	if (claim_txg == 0) {
	return (0);
	}

	switch (lrc->lrc_txtype) {
	case TX_WRITE:
	return (zil_free_write(zilog, lrc, tx, claim_txg));
	case TX_CLONE_RANGE:
	return (zil_free_clone_range(zilog, lrc, tx));
	default:
	return (0);
	}
	}

	static int
	zil_lwb_vdev_compare(const void x1, const void x2)
	{
	const uint64_t v1 = ((zil_vdev_node_t *)x1)->zv_vdev;
	const uint64_t v2 = ((zil_vdev_node_t *)x2)->zv_vdev;

	return (TREE_CMP(v1, v2));
	}

	/*
	* Allocate a new lwb. We may already have a block pointer for it, in which
	* case we get size and version from there. Or we may not yet, in which case
	* we choose them here and later make the block allocation match.
	*/
	static lwb_t *
	zil_alloc_lwb(zilog_t zilog, int sz, blkptr_t bp, boolean_t slog,
	uint64_t txg, lwb_state_t state)
	{
	lwb_t *lwb;

	lwb = kmem_cache_alloc(zil_lwb_cache, KM_SLEEP);
	lwb->lwb_zilog = zilog;
	if (bp) {
	lwb->lwb_blk = *bp;
	lwb->lwb_slim = (BP_GET_CHECKSUM(bp) == ZIO_CHECKSUM_ZILOG2);
	sz = BP_GET_LSIZE(bp);
	} else {
	BP_ZERO(&lwb->lwb_blk);
	lwb->lwb_slim = (spa_version(zilog->zl_spa) >=
	SPA_VERSION_SLIM_ZIL);
	}
	lwb->lwb_slog = slog;
	lwb->lwb_error = 0;
	if (lwb->lwb_slim) {
	lwb->lwb_nmax = sz;
	lwb->lwb_nused = lwb->lwb_nfilled = sizeof (zil_chain_t);
	} else {
	lwb->lwb_nmax = sz - sizeof (zil_chain_t);
	lwb->lwb_nused = lwb->lwb_nfilled = 0;
	}
	lwb->lwb_sz = sz;
	lwb->lwb_state = state;
	lwb->lwb_buf = zio_buf_alloc(sz);
	lwb->lwb_child_zio = NULL;
	lwb->lwb_write_zio = NULL;
	lwb->lwb_root_zio = NULL;
	lwb->lwb_issued_timestamp = 0;
	lwb->lwb_issued_txg = 0;
	lwb->lwb_alloc_txg = txg;
	lwb->lwb_max_txg = 0;

	mutex_enter(&zilog->zl_lock);
	list_insert_tail(&zilog->zl_lwb_list, lwb);
	if (state != LWB_STATE_NEW)
	zilog->zl_last_lwb_opened = lwb;
	mutex_exit(&zilog->zl_lock);

	return (lwb);
	}

	static void
	zil_free_lwb(zilog_t zilog, lwb_t lwb)
	{
	ASSERT(MUTEX_HELD(&zilog->zl_lock));
	ASSERT(lwb->lwb_state == LWB_STATE_NEW \|\|
	lwb->lwb_state == LWB_STATE_FLUSH_DONE);
	ASSERT3P(lwb->lwb_child_zio, ==, NULL);
	ASSERT3P(lwb->lwb_write_zio, ==, NULL);
	ASSERT3P(lwb->lwb_root_zio, ==, NULL);
	ASSERT3U(lwb->lwb_alloc_txg, <=, spa_syncing_txg(zilog->zl_spa));
	ASSERT3U(lwb->lwb_max_txg, <=, spa_syncing_txg(zilog->zl_spa));
	VERIFY(list_is_empty(&lwb->lwb_itxs));
	VERIFY(list_is_empty(&lwb->lwb_waiters));
	ASSERT(avl_is_empty(&lwb->lwb_vdev_tree));
	ASSERT(!MUTEX_HELD(&lwb->lwb_vdev_lock));

	/*
	* Clear the zilog's field to indicate this lwb is no longer
	* valid, and prevent use-after-free errors.
	*/
	if (zilog->zl_last_lwb_opened == lwb)
	zilog->zl_last_lwb_opened = NULL;

	kmem_cache_free(zil_lwb_cache, lwb);
	}

	/*
	* Called when we create in-memory log transactions so that we know
	* to cleanup the itxs at the end of spa_sync().
	*/
	static void
	zilog_dirty(zilog_t *zilog, uint64_t txg)
	{
	dsl_pool_t *dp = zilog->zl_dmu_pool;
	dsl_dataset_t *ds = dmu_objset_ds(zilog->zl_os);

	ASSERT(spa_writeable(zilog->zl_spa));

	if (ds->ds_is_snapshot)
	panic("dirtying snapshot!");

	if (txg_list_add(&dp->dp_dirty_zilogs, zilog, txg)) {
	/* up the hold count until we can be written out */
	dmu_buf_add_ref(ds->ds_dbuf, zilog);

	zilog->zl_dirty_max_txg = MAX(txg, zilog->zl_dirty_max_txg);
	}
	}

	/*
	* Determine if the zil is dirty in the specified txg. Callers wanting to
	* ensure that the dirty state does not change must hold the itxg_lock for
	* the specified txg. Holding the lock will ensure that the zil cannot be
	* dirtied (zil_itx_assign) or cleaned (zil_clean) while we check its current
	* state.
	*/
	static boolean_t __maybe_unused
	zilog_is_dirty_in_txg(zilog_t *zilog, uint64_t txg)
	{
	dsl_pool_t *dp = zilog->zl_dmu_pool;

	if (txg_list_member(&dp->dp_dirty_zilogs, zilog, txg & TXG_MASK))
	return (B_TRUE);
	return (B_FALSE);
	}

	/*
	* Determine if the zil is dirty. The zil is considered dirty if it has
	* any pending itx records that have not been cleaned by zil_clean().
	*/
	static boolean_t
	zilog_is_dirty(zilog_t *zilog)
	{
	dsl_pool_t *dp = zilog->zl_dmu_pool;

	for (int t = 0; t < TXG_SIZE; t++) {
	if (txg_list_member(&dp->dp_dirty_zilogs, zilog, t))
	return (B_TRUE);
	}
	return (B_FALSE);
	}

	/*
	* Its called in zil_commit context (zil_process_commit_list()/zil_create()).
	* It activates SPA_FEATURE_ZILSAXATTR feature, if its enabled.
	* Check dsl_dataset_feature_is_active to avoid txg_wait_synced() on every
	* zil_commit.
	*/
	static void
	zil_commit_activate_saxattr_feature(zilog_t *zilog)
	{
	dsl_dataset_t *ds = dmu_objset_ds(zilog->zl_os);
	uint64_t txg = 0;
	dmu_tx_t *tx = NULL;

	if (spa_feature_is_enabled(zilog->zl_spa, SPA_FEATURE_ZILSAXATTR) &&
	dmu_objset_type(zilog->zl_os) != DMU_OST_ZVOL &&
	!dsl_dataset_feature_is_active(ds, SPA_FEATURE_ZILSAXATTR)) {
	tx = dmu_tx_create(zilog->zl_os);
	VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
	dsl_dataset_dirty(ds, tx);
	txg = dmu_tx_get_txg(tx);

	mutex_enter(&ds->ds_lock);
	ds->ds_feature_activation[SPA_FEATURE_ZILSAXATTR] =
	(void *)B_TRUE;
	mutex_exit(&ds->ds_lock);
	dmu_tx_commit(tx);
	txg_wait_synced(zilog->zl_dmu_pool, txg);
	}
	}

	/*
	* Create an on-disk intent log.
	*/
	static lwb_t *
	zil_create(zilog_t *zilog)
	{
	const zil_header_t *zh = zilog->zl_header;
	lwb_t *lwb = NULL;
	uint64_t txg = 0;
	dmu_tx_t *tx = NULL;
	blkptr_t blk;
	int error = 0;
	boolean_t slog = FALSE;
	dsl_dataset_t *ds = dmu_objset_ds(zilog->zl_os);


	/*
	* Wait for any previous destroy to complete.
	*/
	txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg);

	ASSERT(zh->zh_claim_txg == 0);
	ASSERT(zh->zh_replay_seq == 0);

	blk = zh->zh_log;

	/*
	* Allocate an initial log block if:
	* - there isn't one already
	* - the existing block is the wrong endianness
	*/
	if (BP_IS_HOLE(&blk) \|\| BP_SHOULD_BYTESWAP(&blk)) {
	tx = dmu_tx_create(zilog->zl_os);
	VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
	dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx);
	txg = dmu_tx_get_txg(tx);

	if (!BP_IS_HOLE(&blk)) {
	zio_free(zilog->zl_spa, txg, &blk);
	BP_ZERO(&blk);
	}

	error = zio_alloc_zil(zilog->zl_spa, zilog->zl_os, txg, &blk,
	ZIL_MIN_BLKSZ, &slog);
	if (error == 0)
	zil_init_log_chain(zilog, &blk);
	}

	/*
	* Allocate a log write block (lwb) for the first log block.
	*/
	if (error == 0)
	lwb = zil_alloc_lwb(zilog, 0, &blk, slog, txg, LWB_STATE_NEW);

	/*
	* If we just allocated the first log block, commit our transaction
	* and wait for zil_sync() to stuff the block pointer into zh_log.
	* (zh is part of the MOS, so we cannot modify it in open context.)
	*/
	if (tx != NULL) {
	/*
	* If "zilsaxattr" feature is enabled on zpool, then activate
	* it now when we're creating the ZIL chain. We can't wait with
	* this until we write the first xattr log record because we
	* need to wait for the feature activation to sync out.
	*/
	if (spa_feature_is_enabled(zilog->zl_spa,
	SPA_FEATURE_ZILSAXATTR) && dmu_objset_type(zilog->zl_os) !=
	DMU_OST_ZVOL) {
	mutex_enter(&ds->ds_lock);
	ds->ds_feature_activation[SPA_FEATURE_ZILSAXATTR] =
	(void *)B_TRUE;
	mutex_exit(&ds->ds_lock);
	}

	dmu_tx_commit(tx);
	txg_wait_synced(zilog->zl_dmu_pool, txg);
	} else {
	/*
	* This branch covers the case where we enable the feature on a
	* zpool that has existing ZIL headers.
	*/
	zil_commit_activate_saxattr_feature(zilog);
	}
	IMPLY(spa_feature_is_enabled(zilog->zl_spa, SPA_FEATURE_ZILSAXATTR) &&
	dmu_objset_type(zilog->zl_os) != DMU_OST_ZVOL,
	dsl_dataset_feature_is_active(ds, SPA_FEATURE_ZILSAXATTR));

	ASSERT(error != 0 \|\| memcmp(&blk, &zh->zh_log, sizeof (blk)) == 0);
	IMPLY(error == 0, lwb != NULL);

	return (lwb);
	}

	/*
	* In one tx, free all log blocks and clear the log header. If keep_first
	* is set, then we're replaying a log with no content. We want to keep the
	* first block, however, so that the first synchronous transaction doesn't
	* require a txg_wait_synced() in zil_create(). We don't need to
	* txg_wait_synced() here either when keep_first is set, because both
	* zil_create() and zil_destroy() will wait for any in-progress destroys
	* to complete.
	* Return B_TRUE if there were any entries to replay.
	*/
	boolean_t
	zil_destroy(zilog_t *zilog, boolean_t keep_first)
	{
	const zil_header_t *zh = zilog->zl_header;
	lwb_t *lwb;
	dmu_tx_t *tx;
	uint64_t txg;

	/*
	* Wait for any previous destroy to complete.
	*/
	txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg);

	zilog->zl_old_header = zh; / debugging aid */

	if (BP_IS_HOLE(&zh->zh_log))
	return (B_FALSE);

	tx = dmu_tx_create(zilog->zl_os);
	VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
	dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx);
	txg = dmu_tx_get_txg(tx);

	mutex_enter(&zilog->zl_lock);

	ASSERT3U(zilog->zl_destroy_txg, <, txg);
	zilog->zl_destroy_txg = txg;
	zilog->zl_keep_first = keep_first;

	if (!list_is_empty(&zilog->zl_lwb_list)) {
	ASSERT(zh->zh_claim_txg == 0);
	VERIFY(!keep_first);
	while ((lwb = list_remove_head(&zilog->zl_lwb_list)) != NULL) {
	if (lwb->lwb_buf != NULL)
	zio_buf_free(lwb->lwb_buf, lwb->lwb_sz);
	if (!BP_IS_HOLE(&lwb->lwb_blk))
	zio_free(zilog->zl_spa, txg, &lwb->lwb_blk);
	zil_free_lwb(zilog, lwb);
	}
	} else if (!keep_first) {
	zil_destroy_sync(zilog, tx);
	}
	mutex_exit(&zilog->zl_lock);

	dmu_tx_commit(tx);

	return (B_TRUE);
	}

	void
	zil_destroy_sync(zilog_t zilog, dmu_tx_t tx)
	{
	ASSERT(list_is_empty(&zilog->zl_lwb_list));
	(void) zil_parse(zilog, zil_free_log_block,
	zil_free_log_record, tx, zilog->zl_header->zh_claim_txg, B_FALSE);
	}

	int
	zil_claim(dsl_pool_t dp, dsl_dataset_t ds, void *txarg)
	{
	dmu_tx_t *tx = txarg;
	zilog_t *zilog;
	uint64_t first_txg;
	zil_header_t *zh;
	objset_t *os;
	int error;

	error = dmu_objset_own_obj(dp, ds->ds_object,
	DMU_OST_ANY, B_FALSE, B_FALSE, FTAG, &os);
	if (error != 0) {
	/*
	* EBUSY indicates that the objset is inconsistent, in which
	* case it can not have a ZIL.
	*/
	if (error != EBUSY) {
	cmn_err(CE_WARN, "can't open objset for %llu, error %u",
	(unsigned long long)ds->ds_object, error);
	}

	return (0);
	}

	zilog = dmu_objset_zil(os);
	zh = zil_header_in_syncing_context(zilog);
	ASSERT3U(tx->tx_txg, ==, spa_first_txg(zilog->zl_spa));
	first_txg = spa_min_claim_txg(zilog->zl_spa);

	/*
	* If the spa_log_state is not set to be cleared, check whether
	* the current uberblock is a checkpoint one and if the current
	* header has been claimed before moving on.
	*
	* If the current uberblock is a checkpointed uberblock then
	* one of the following scenarios took place:
	*
	* 1] We are currently rewinding to the checkpoint of the pool.
	* 2] We crashed in the middle of a checkpoint rewind but we
	* did manage to write the checkpointed uberblock to the
	* vdev labels, so when we tried to import the pool again
	* the checkpointed uberblock was selected from the import
	* procedure.
	*
	* In both cases we want to zero out all the ZIL blocks, except
	* the ones that have been claimed at the time of the checkpoint
	* (their zh_claim_txg != 0). The reason is that these blocks
	* may be corrupted since we may have reused their locations on
	* disk after we took the checkpoint.
	*
	* We could try to set spa_log_state to SPA_LOG_CLEAR earlier
	* when we first figure out whether the current uberblock is
	* checkpointed or not. Unfortunately, that would discard all
	* the logs, including the ones that are claimed, and we would
	* leak space.
	*/
	if (spa_get_log_state(zilog->zl_spa) == SPA_LOG_CLEAR \|\|
	(zilog->zl_spa->spa_uberblock.ub_checkpoint_txg != 0 &&
	zh->zh_claim_txg == 0)) {
	if (!BP_IS_HOLE(&zh->zh_log)) {
	(void) zil_parse(zilog, zil_clear_log_block,
	zil_noop_log_record, tx, first_txg, B_FALSE);
	}
	BP_ZERO(&zh->zh_log);
	if (os->os_encrypted)
	os->os_next_write_raw[tx->tx_txg & TXG_MASK] = B_TRUE;
	dsl_dataset_dirty(dmu_objset_ds(os), tx);
	dmu_objset_disown(os, B_FALSE, FTAG);
	return (0);
	}

	/*
	* If we are not rewinding and opening the pool normally, then
	* the min_claim_txg should be equal to the first txg of the pool.
	*/
	ASSERT3U(first_txg, ==, spa_first_txg(zilog->zl_spa));

	/*
	* Claim all log blocks if we haven't already done so, and remember
	* the highest claimed sequence number. This ensures that if we can
	* read only part of the log now (e.g. due to a missing device),
	* but we can read the entire log later, we will not try to replay
	* or destroy beyond the last block we successfully claimed.
	*/
	ASSERT3U(zh->zh_claim_txg, <=, first_txg);
	if (zh->zh_claim_txg == 0 && !BP_IS_HOLE(&zh->zh_log)) {
	(void) zil_parse(zilog, zil_claim_log_block,
	zil_claim_log_record, tx, first_txg, B_FALSE);
	zh->zh_claim_txg = first_txg;
	zh->zh_claim_blk_seq = zilog->zl_parse_blk_seq;
	zh->zh_claim_lr_seq = zilog->zl_parse_lr_seq;
	if (zilog->zl_parse_lr_count \|\| zilog->zl_parse_blk_count > 1)
	zh->zh_flags \|= ZIL_REPLAY_NEEDED;
	zh->zh_flags \|= ZIL_CLAIM_LR_SEQ_VALID;
	if (os->os_encrypted)
	os->os_next_write_raw[tx->tx_txg & TXG_MASK] = B_TRUE;
	dsl_dataset_dirty(dmu_objset_ds(os), tx);
	}

	ASSERT3U(first_txg, ==, (spa_last_synced_txg(zilog->zl_spa) + 1));
	dmu_objset_disown(os, B_FALSE, FTAG);
	return (0);
	}

	/*
	* Check the log by walking the log chain.
	* Checksum errors are ok as they indicate the end of the chain.
	* Any other error (no device or read failure) returns an error.
	*/
	int
	zil_check_log_chain(dsl_pool_t dp, dsl_dataset_t ds, void *tx)
	{
	(void) dp;
	zilog_t *zilog;
	objset_t *os;
	blkptr_t *bp;
	int error;

	ASSERT(tx == NULL);

	error = dmu_objset_from_ds(ds, &os);
	if (error != 0) {
	cmn_err(CE_WARN, "can't open objset %llu, error %d",
	(unsigned long long)ds->ds_object, error);
	return (0);
	}

	zilog = dmu_objset_zil(os);
	bp = (blkptr_t *)&zilog->zl_header->zh_log;

	if (!BP_IS_HOLE(bp)) {
	vdev_t *vd;
	boolean_t valid = B_TRUE;

	/*
	* Check the first block and determine if it's on a log device
	* which may have been removed or faulted prior to loading this
	* pool. If so, there's no point in checking the rest of the
	* log as its content should have already been synced to the
	* pool.
	*/
	spa_config_enter(os->os_spa, SCL_STATE, FTAG, RW_READER);
	vd = vdev_lookup_top(os->os_spa, DVA_GET_VDEV(&bp->blk_dva[0]));
	if (vd->vdev_islog && vdev_is_dead(vd))
	valid = vdev_log_state_valid(vd);
	spa_config_exit(os->os_spa, SCL_STATE, FTAG);

	if (!valid)
	return (0);

	/*
	* Check whether the current uberblock is checkpointed (e.g.
	* we are rewinding) and whether the current header has been
	* claimed or not. If it hasn't then skip verifying it. We
	* do this because its ZIL blocks may be part of the pool's
	* state before the rewind, which is no longer valid.
	*/
	zil_header_t *zh = zil_header_in_syncing_context(zilog);
	if (zilog->zl_spa->spa_uberblock.ub_checkpoint_txg != 0 &&
	zh->zh_claim_txg == 0)
	return (0);
	}

	/*
	* Because tx == NULL, zil_claim_log_block() will not actually claim
	* any blocks, but just determine whether it is possible to do so.
	* In addition to checking the log chain, zil_claim_log_block()
	* will invoke zio_claim() with a done func of spa_claim_notify(),
	* which will update spa_max_claim_txg. See spa_load() for details.
	*/
	error = zil_parse(zilog, zil_claim_log_block, zil_claim_log_record, tx,
	zilog->zl_header->zh_claim_txg ? -1ULL :
	spa_min_claim_txg(os->os_spa), B_FALSE);

	return ((error == ECKSUM \|\| error == ENOENT) ? 0 : error);
	}

	/*
	* When an itx is "skipped", this function is used to properly mark the
	* waiter as "done, and signal any thread(s) waiting on it. An itx can
	* be skipped (and not committed to an lwb) for a variety of reasons,
	* one of them being that the itx was committed via spa_sync(), prior to
	* it being committed to an lwb; this can happen if a thread calling
	* zil_commit() is racing with spa_sync().
	*/
	static void
	zil_commit_waiter_skip(zil_commit_waiter_t *zcw)
	{
	mutex_enter(&zcw->zcw_lock);
	ASSERT3B(zcw->zcw_done, ==, B_FALSE);
	zcw->zcw_done = B_TRUE;
	cv_broadcast(&zcw->zcw_cv);
	mutex_exit(&zcw->zcw_lock);
	}

	/*
	* This function is used when the given waiter is to be linked into an
	* lwb's "lwb_waiter" list; i.e. when the itx is committed to the lwb.
	* At this point, the waiter will no longer be referenced by the itx,
	* and instead, will be referenced by the lwb.
	*/
	static void
	zil_commit_waiter_link_lwb(zil_commit_waiter_t zcw, lwb_t lwb)
	{
	/*
	* The lwb_waiters field of the lwb is protected by the zilog's
	* zl_issuer_lock while the lwb is open and zl_lock otherwise.
	* zl_issuer_lock also protects leaving the open state.
	* zcw_lwb setting is protected by zl_issuer_lock and state !=
	* flush_done, which transition is protected by zl_lock.
	*/
	ASSERT(MUTEX_HELD(&lwb->lwb_zilog->zl_issuer_lock));
	IMPLY(lwb->lwb_state != LWB_STATE_OPENED,
	MUTEX_HELD(&lwb->lwb_zilog->zl_lock));
	ASSERT3S(lwb->lwb_state, !=, LWB_STATE_NEW);
	ASSERT3S(lwb->lwb_state, !=, LWB_STATE_FLUSH_DONE);

	ASSERT(!list_link_active(&zcw->zcw_node));
	list_insert_tail(&lwb->lwb_waiters, zcw);
	ASSERT3P(zcw->zcw_lwb, ==, NULL);
	zcw->zcw_lwb = lwb;
	}

	/*
	* This function is used when zio_alloc_zil() fails to allocate a ZIL
	* block, and the given waiter must be linked to the "nolwb waiters"
	* list inside of zil_process_commit_list().
	*/
	static void
	zil_commit_waiter_link_nolwb(zil_commit_waiter_t zcw, list_t nolwb)
	{
	ASSERT(!list_link_active(&zcw->zcw_node));
	list_insert_tail(nolwb, zcw);
	ASSERT3P(zcw->zcw_lwb, ==, NULL);
	}

	void
	zil_lwb_add_block(lwb_t lwb, const blkptr_t bp)
	{
	avl_tree_t *t = &lwb->lwb_vdev_tree;
	avl_index_t where;
	zil_vdev_node_t *zv, zvsearch;
	int ndvas = BP_GET_NDVAS(bp);
	int i;

	ASSERT3S(lwb->lwb_state, !=, LWB_STATE_WRITE_DONE);
	ASSERT3S(lwb->lwb_state, !=, LWB_STATE_FLUSH_DONE);

	if (zil_nocacheflush)
	return;

	mutex_enter(&lwb->lwb_vdev_lock);
	for (i = 0; i < ndvas; i++) {
	zvsearch.zv_vdev = DVA_GET_VDEV(&bp->blk_dva[i]);
	if (avl_find(t, &zvsearch, &where) == NULL) {
	zv = kmem_alloc(sizeof (*zv), KM_SLEEP);
	zv->zv_vdev = zvsearch.zv_vdev;
	avl_insert(t, zv, where);
	}
	}
	mutex_exit(&lwb->lwb_vdev_lock);
	}

	static void
	zil_lwb_flush_defer(lwb_t lwb, lwb_t nlwb)
	{
	avl_tree_t *src = &lwb->lwb_vdev_tree;
	avl_tree_t *dst = &nlwb->lwb_vdev_tree;
	void *cookie = NULL;
	zil_vdev_node_t *zv;

	ASSERT3S(lwb->lwb_state, ==, LWB_STATE_WRITE_DONE);
	ASSERT3S(nlwb->lwb_state, !=, LWB_STATE_WRITE_DONE);
	ASSERT3S(nlwb->lwb_state, !=, LWB_STATE_FLUSH_DONE);

	/*
	* While 'lwb' is at a point in its lifetime where lwb_vdev_tree does
	* not need the protection of lwb_vdev_lock (it will only be modified
	* while holding zilog->zl_lock) as its writes and those of its
	* children have all completed. The younger 'nlwb' may be waiting on
	* future writes to additional vdevs.
	*/
	mutex_enter(&nlwb->lwb_vdev_lock);
	/*
	* Tear down the 'lwb' vdev tree, ensuring that entries which do not
	* exist in 'nlwb' are moved to it, freeing any would-be duplicates.
	*/
	while ((zv = avl_destroy_nodes(src, &cookie)) != NULL) {
	avl_index_t where;

	if (avl_find(dst, zv, &where) == NULL) {
	avl_insert(dst, zv, where);
	} else {
	kmem_free(zv, sizeof (*zv));
	}
	}
	mutex_exit(&nlwb->lwb_vdev_lock);
	}

	void
	zil_lwb_add_txg(lwb_t *lwb, uint64_t txg)
	{
	lwb->lwb_max_txg = MAX(lwb->lwb_max_txg, txg);
	}

	/*
	* This function is a called after all vdevs associated with a given lwb
	* write have completed their DKIOCFLUSHWRITECACHE command; or as soon
	* as the lwb write completes, if "zil_nocacheflush" is set. Further,
	* all "previous" lwb's will have completed before this function is
	* called; i.e. this function is called for all previous lwbs before
	* it's called for "this" lwb (enforced via zio the dependencies
	* configured in zil_lwb_set_zio_dependency()).
	*
	* The intention is for this function to be called as soon as the
	* contents of an lwb are considered "stable" on disk, and will survive
	* any sudden loss of power. At this point, any threads waiting for the
	* lwb to reach this state are signalled, and the "waiter" structures
	* are marked "done".
	*/
	static void
	zil_lwb_flush_vdevs_done(zio_t *zio)
	{
	lwb_t *lwb = zio->io_private;
	zilog_t *zilog = lwb->lwb_zilog;
	zil_commit_waiter_t *zcw;
	itx_t *itx;

	spa_config_exit(zilog->zl_spa, SCL_STATE, lwb);

	hrtime_t t = gethrtime() - lwb->lwb_issued_timestamp;

	mutex_enter(&zilog->zl_lock);

	zilog->zl_last_lwb_latency = (zilog->zl_last_lwb_latency * 7 + t) / 8;

	lwb->lwb_root_zio = NULL;

	ASSERT3S(lwb->lwb_state, ==, LWB_STATE_WRITE_DONE);
	lwb->lwb_state = LWB_STATE_FLUSH_DONE;

	if (zilog->zl_last_lwb_opened == lwb) {
	/*
	* Remember the highest committed log sequence number
	* for ztest. We only update this value when all the log
	* writes succeeded, because ztest wants to ASSERT that
	* it got the whole log chain.
	*/
	zilog->zl_commit_lr_seq = zilog->zl_lr_seq;
	}

	while ((itx = list_remove_head(&lwb->lwb_itxs)) != NULL)
	zil_itx_destroy(itx);

	while ((zcw = list_remove_head(&lwb->lwb_waiters)) != NULL) {
	mutex_enter(&zcw->zcw_lock);

	ASSERT3P(zcw->zcw_lwb, ==, lwb);
	zcw->zcw_lwb = NULL;
	/*
	* We expect any ZIO errors from child ZIOs to have been
	* propagated "up" to this specific LWB's root ZIO, in
	* order for this error handling to work correctly. This
	* includes ZIO errors from either this LWB's write or
	* flush, as well as any errors from other dependent LWBs
	* (e.g. a root LWB ZIO that might be a child of this LWB).
	*
	* With that said, it's important to note that LWB flush
	* errors are not propagated up to the LWB root ZIO.
	* This is incorrect behavior, and results in VDEV flush
	* errors not being handled correctly here. See the
	* comment above the call to "zio_flush" for details.
	*/

	zcw->zcw_zio_error = zio->io_error;

	ASSERT3B(zcw->zcw_done, ==, B_FALSE);
	zcw->zcw_done = B_TRUE;
	cv_broadcast(&zcw->zcw_cv);

	mutex_exit(&zcw->zcw_lock);
	}

	uint64_t txg = lwb->lwb_issued_txg;

	/* Once we drop the lock, lwb may be freed by zil_sync(). */
	mutex_exit(&zilog->zl_lock);

	mutex_enter(&zilog->zl_lwb_io_lock);
	ASSERT3U(zilog->zl_lwb_inflight[txg & TXG_MASK], >, 0);
	zilog->zl_lwb_inflight[txg & TXG_MASK]--;
	if (zilog->zl_lwb_inflight[txg & TXG_MASK] == 0)
	cv_broadcast(&zilog->zl_lwb_io_cv);
	mutex_exit(&zilog->zl_lwb_io_lock);
	}

	/*
	* Wait for the completion of all issued write/flush of that txg provided.
	* It guarantees zil_lwb_flush_vdevs_done() is called and returned.
	*/
	static void
	zil_lwb_flush_wait_all(zilog_t *zilog, uint64_t txg)
	{
	ASSERT3U(txg, ==, spa_syncing_txg(zilog->zl_spa));

	mutex_enter(&zilog->zl_lwb_io_lock);
	while (zilog->zl_lwb_inflight[txg & TXG_MASK] > 0)
	cv_wait(&zilog->zl_lwb_io_cv, &zilog->zl_lwb_io_lock);
	mutex_exit(&zilog->zl_lwb_io_lock);

	#ifdef ZFS_DEBUG
	mutex_enter(&zilog->zl_lock);
	mutex_enter(&zilog->zl_lwb_io_lock);
	lwb_t *lwb = list_head(&zilog->zl_lwb_list);
	while (lwb != NULL) {
	if (lwb->lwb_issued_txg <= txg) {
	ASSERT(lwb->lwb_state != LWB_STATE_ISSUED);
	ASSERT(lwb->lwb_state != LWB_STATE_WRITE_DONE);
	IMPLY(lwb->lwb_issued_txg > 0,
	lwb->lwb_state == LWB_STATE_FLUSH_DONE);
	}
	IMPLY(lwb->lwb_state == LWB_STATE_WRITE_DONE \|\|
	lwb->lwb_state == LWB_STATE_FLUSH_DONE,
	lwb->lwb_buf == NULL);
	lwb = list_next(&zilog->zl_lwb_list, lwb);
	}
	mutex_exit(&zilog->zl_lwb_io_lock);
	mutex_exit(&zilog->zl_lock);
	#endif
	}

	/*
	* This is called when an lwb's write zio completes. The callback's
	* purpose is to issue the DKIOCFLUSHWRITECACHE commands for the vdevs
	* in the lwb's lwb_vdev_tree. The tree will contain the vdevs involved
	* in writing out this specific lwb's data, and in the case that cache
	* flushes have been deferred, vdevs involved in writing the data for
	* previous lwbs. The writes corresponding to all the vdevs in the
	* lwb_vdev_tree will have completed by the time this is called, due to
	* the zio dependencies configured in zil_lwb_set_zio_dependency(),
	* which takes deferred flushes into account. The lwb will be "done"
	* once zil_lwb_flush_vdevs_done() is called, which occurs in the zio
	* completion callback for the lwb's root zio.
	*/
	static void
	zil_lwb_write_done(zio_t *zio)
	{
	lwb_t *lwb = zio->io_private;
	spa_t *spa = zio->io_spa;
	zilog_t *zilog = lwb->lwb_zilog;
	avl_tree_t *t = &lwb->lwb_vdev_tree;
	void *cookie = NULL;
	zil_vdev_node_t *zv;
	lwb_t *nlwb;

	ASSERT3S(spa_config_held(spa, SCL_STATE, RW_READER), !=, 0);

	abd_free(zio->io_abd);
	zio_buf_free(lwb->lwb_buf, lwb->lwb_sz);
	lwb->lwb_buf = NULL;

	mutex_enter(&zilog->zl_lock);
	ASSERT3S(lwb->lwb_state, ==, LWB_STATE_ISSUED);
	lwb->lwb_state = LWB_STATE_WRITE_DONE;
	lwb->lwb_child_zio = NULL;
	lwb->lwb_write_zio = NULL;

	/*
	* If nlwb is not yet issued, zil_lwb_set_zio_dependency() is not
	* called for it yet, and when it will be, it won't be able to make
	* its write ZIO a parent this ZIO. In such case we can not defer
	* our flushes or below may be a race between the done callbacks.
	*/
	nlwb = list_next(&zilog->zl_lwb_list, lwb);
	if (nlwb && nlwb->lwb_state != LWB_STATE_ISSUED)
	nlwb = NULL;
	mutex_exit(&zilog->zl_lock);

	if (avl_numnodes(t) == 0)
	return;

	/*
	* If there was an IO error, we're not going to call zio_flush()
	* on these vdevs, so we simply empty the tree and free the
	* nodes. We avoid calling zio_flush() since there isn't any
	* good reason for doing so, after the lwb block failed to be
	* written out.
	*
	* Additionally, we don't perform any further error handling at
	* this point (e.g. setting "zcw_zio_error" appropriately), as
	* we expect that to occur in "zil_lwb_flush_vdevs_done" (thus,
	* we expect any error seen here, to have been propagated to
	* that function).
	*/
	if (zio->io_error != 0) {
	while ((zv = avl_destroy_nodes(t, &cookie)) != NULL)
	kmem_free(zv, sizeof (*zv));
	return;
	}

	/*
	* If this lwb does not have any threads waiting for it to
	* complete, we want to defer issuing the DKIOCFLUSHWRITECACHE
	* command to the vdevs written to by "this" lwb, and instead
	* rely on the "next" lwb to handle the DKIOCFLUSHWRITECACHE
	* command for those vdevs. Thus, we merge the vdev tree of
	* "this" lwb with the vdev tree of the "next" lwb in the list,
	* and assume the "next" lwb will handle flushing the vdevs (or
	* deferring the flush(s) again).
	*
	* This is a useful performance optimization, especially for
	* workloads with lots of async write activity and few sync
	* write and/or fsync activity, as it has the potential to
	* coalesce multiple flush commands to a vdev into one.
	*/
	if (list_is_empty(&lwb->lwb_waiters) && nlwb != NULL) {
	zil_lwb_flush_defer(lwb, nlwb);
	ASSERT(avl_is_empty(&lwb->lwb_vdev_tree));
	return;
	}

	while ((zv = avl_destroy_nodes(t, &cookie)) != NULL) {
	vdev_t *vd = vdev_lookup_top(spa, zv->zv_vdev);
	- if (vd != NULL && !vd->vdev_nowritecache) {
	+ if (vd != NULL) {
	/*
	* The "ZIO_FLAG_DONT_PROPAGATE" is currently
	* always used within "zio_flush". This means,
	* any errors when flushing the vdev(s), will
	* (unfortunately) not be handled correctly,
	* since these "zio_flush" errors will not be
	* propagated up to "zil_lwb_flush_vdevs_done".
	*/
	zio_flush(lwb->lwb_root_zio, vd);
	}
	kmem_free(zv, sizeof (*zv));
	}
	}

	/*
	* Build the zio dependency chain, which is used to preserve the ordering of
	* lwb completions that is required by the semantics of the ZIL. Each new lwb
	* zio becomes a parent of the previous lwb zio, such that the new lwb's zio
	* cannot complete until the previous lwb's zio completes.
	*
	* This is required by the semantics of zil_commit(): the commit waiters
	* attached to the lwbs will be woken in the lwb zio's completion callback,
	* so this zio dependency graph ensures the waiters are woken in the correct
	* order (the same order the lwbs were created).
	*/
	static void
	zil_lwb_set_zio_dependency(zilog_t zilog, lwb_t lwb)
	{
	ASSERT(MUTEX_HELD(&zilog->zl_lock));

	lwb_t *prev_lwb = list_prev(&zilog->zl_lwb_list, lwb);
	if (prev_lwb == NULL \|\|
	prev_lwb->lwb_state == LWB_STATE_FLUSH_DONE)
	return;

	/*
	* If the previous lwb's write hasn't already completed, we also want
	* to order the completion of the lwb write zios (above, we only order
	* the completion of the lwb root zios). This is required because of
	* how we can defer the DKIOCFLUSHWRITECACHE commands for each lwb.
	*
	* When the DKIOCFLUSHWRITECACHE commands are deferred, the previous
	* lwb will rely on this lwb to flush the vdevs written to by that
	* previous lwb. Thus, we need to ensure this lwb doesn't issue the
	* flush until after the previous lwb's write completes. We ensure
	* this ordering by setting the zio parent/child relationship here.
	*
	* Without this relationship on the lwb's write zio, it's possible
	* for this lwb's write to complete prior to the previous lwb's write
	* completing; and thus, the vdevs for the previous lwb would be
	* flushed prior to that lwb's data being written to those vdevs (the
	* vdevs are flushed in the lwb write zio's completion handler,
	* zil_lwb_write_done()).
	*/
	if (prev_lwb->lwb_state == LWB_STATE_ISSUED) {
	ASSERT3P(prev_lwb->lwb_write_zio, !=, NULL);
	zio_add_child(lwb->lwb_write_zio, prev_lwb->lwb_write_zio);
	} else {
	ASSERT3S(prev_lwb->lwb_state, ==, LWB_STATE_WRITE_DONE);
	}

	ASSERT3P(prev_lwb->lwb_root_zio, !=, NULL);
	zio_add_child(lwb->lwb_root_zio, prev_lwb->lwb_root_zio);
	}


	/*
	* This function's purpose is to "open" an lwb such that it is ready to
	* accept new itxs being committed to it. This function is idempotent; if
	* the passed in lwb has already been opened, it is essentially a no-op.
	*/
	static void
	zil_lwb_write_open(zilog_t zilog, lwb_t lwb)
	{
	ASSERT(MUTEX_HELD(&zilog->zl_issuer_lock));

	if (lwb->lwb_state != LWB_STATE_NEW) {
	ASSERT3S(lwb->lwb_state, ==, LWB_STATE_OPENED);
	return;
	}

	mutex_enter(&zilog->zl_lock);
	lwb->lwb_state = LWB_STATE_OPENED;
	zilog->zl_last_lwb_opened = lwb;
	mutex_exit(&zilog->zl_lock);
	}

	-/*
	- * Define a limited set of intent log block sizes.
	- *
	- * These must be a multiple of 4KB. Note only the amount used (again
	- * aligned to 4KB) actually gets written. However, we can't always just
	- * allocate SPA_OLD_MAXBLOCKSIZE as the slog space could be exhausted.
	- */
	-static const struct {
	- uint64_t limit;
	- uint64_t blksz;
	-} zil_block_buckets[] = {
	- { 4096, 4096 }, /* non TX_WRITE */
	- { 8192 + 4096, 8192 + 4096 }, /* database */
	- { 32768 + 4096, 32768 + 4096 }, /* NFS writes */
	- { 65536 + 4096, 65536 + 4096 }, /* 64KB writes */
	- { UINT64_MAX, SPA_OLD_MAXBLOCKSIZE}, /* > 128KB writes */
	-};
	-
	/*
	* Maximum block size used by the ZIL. This is picked up when the ZIL is
	* initialized. Otherwise this should not be used directly; see
	* zl_max_block_size instead.
	*/
	static uint_t zil_maxblocksize = SPA_OLD_MAXBLOCKSIZE;

	+/*
	+ * Plan splitting of the provided burst size between several blocks.
	+ */
	+static uint_t
	+zil_lwb_plan(zilog_t zilog, uint64_t size, uint_t minsize)
	+{
	+ uint_t md = zilog->zl_max_block_size - sizeof (zil_chain_t);
	+
	+ if (size <= md) {
	+ /*
	+ * Small bursts are written as-is in one block.
	+ */
	+ *minsize = size;
	+ return (size);
	+ } else if (size > 8 * md) {
	+ /*
	+ * Big bursts use maximum blocks. The first block size
	+ * is hard to predict, but it does not really matter.
	+ */
	+ *minsize = 0;
	+ return (md);
	+ }
	+
	+ /*
	+ * Medium bursts try to divide evenly to better utilize several SLOG
	+ * VDEVs. The first block size we predict assuming the worst case of
	+ * maxing out others. Fall back to using maximum blocks if due to
	+ * large records or wasted space we can not predict anything better.
	+ */
	+ uint_t s = size;
	+ uint_t n = DIV_ROUND_UP(s, md - sizeof (lr_write_t));
	+ uint_t chunk = DIV_ROUND_UP(s, n);
	+ uint_t waste = zil_max_waste_space(zilog);
	+ waste = MAX(waste, zilog->zl_cur_max);
	+ if (chunk <= md - waste) {
	+ minsize = MAX(s - (md - waste) (n - 1), waste);
	+ return (chunk);
	+ } else {
	+ *minsize = 0;
	+ return (md);
	+ }
	+}
	+
	+/*
	+ * Try to predict next block size based on previous history. Make prediction
	+ * sufficient for 7 of 8 previous bursts. Don't try to save if the saving is
	+ * less then 50%, extra writes may cost more, but we don't want single spike
	+ * to badly affect our predictions.
	+ */
	+static uint_t
	+zil_lwb_predict(zilog_t *zilog)
	+{
	+ uint_t m, o;
	+
	+ /* If we are in the middle of a burst, take it into account also. */
	+ if (zilog->zl_cur_size > 0) {
	+ o = zil_lwb_plan(zilog, zilog->zl_cur_size, &m);
	+ } else {
	+ o = UINT_MAX;
	+ m = 0;
	+ }
	+
	+ /* Find minimum optimal size. We don't need to go below that. */
	+ for (int i = 0; i < ZIL_BURSTS; i++)
	+ o = MIN(o, zilog->zl_prev_opt[i]);
	+
	+ /* Find two biggest minimal first block sizes above the optimal. */
	+ uint_t m1 = MAX(m, o), m2 = o;
	+ for (int i = 0; i < ZIL_BURSTS; i++) {
	+ m = zilog->zl_prev_min[i];
	+ if (m >= m1) {
	+ m2 = m1;
	+ m1 = m;
	+ } else if (m > m2) {
	+ m2 = m;
	+ }
	+ }
	+
	+ /*
	+ * If second minimum size gives 50% saving -- use it. It may cost us
	+ * one additional write later, but the space saving is just too big.
	+ */
	+ return ((m1 < m2 * 2) ? m1 : m2);
	+}
	+
	/*
	* Close the log block for being issued and allocate the next one.
	* Has to be called under zl_issuer_lock to chain more lwbs.
	*/
	static lwb_t *
	zil_lwb_write_close(zilog_t zilog, lwb_t lwb, lwb_state_t state)
	{
	- int i;
	+ uint64_t blksz, plan, plan2;

	ASSERT(MUTEX_HELD(&zilog->zl_issuer_lock));
	ASSERT3S(lwb->lwb_state, ==, LWB_STATE_OPENED);
	lwb->lwb_state = LWB_STATE_CLOSED;

	/*
	* If there was an allocation failure then returned NULL will trigger
	* zil_commit_writer_stall() at the caller. This is inherently racy,
	* since allocation may not have happened yet.
	*/
	if (lwb->lwb_error != 0)
	return (NULL);

	/*
	- * Log blocks are pre-allocated. Here we select the size of the next
	- * block, based on size used in the last block.
	- * - first find the smallest bucket that will fit the block from a
	- * limited set of block sizes. This is because it's faster to write
	- * blocks allocated from the same metaslab as they are adjacent or
	- * close.
	- * - next find the maximum from the new suggested size and an array of
	- * previous sizes. This lessens a picket fence effect of wrongly
	- * guessing the size if we have a stream of say 2k, 64k, 2k, 64k
	- * requests.
	- *
	- * Note we only write what is used, but we can't just allocate
	- * the maximum block size because we can exhaust the available
	- * pool log space.
	+ * Log blocks are pre-allocated. Here we select the size of the next
	+ * block, based on what's left of this burst and the previous history.
	+ * While we try to only write used part of the block, we can't just
	+ * always allocate the maximum block size because we can exhaust all
	+ * available pool log space, so we try to be reasonable.
	*/
	- uint64_t zil_blksz = zilog->zl_cur_used + sizeof (zil_chain_t);
	- for (i = 0; zil_blksz > zil_block_buckets[i].limit; i++)
	- continue;
	- zil_blksz = MIN(zil_block_buckets[i].blksz, zilog->zl_max_block_size);
	- zilog->zl_prev_blks[zilog->zl_prev_rotor] = zil_blksz;
	- for (i = 0; i < ZIL_PREV_BLKS; i++)
	- zil_blksz = MAX(zil_blksz, zilog->zl_prev_blks[i]);
	- DTRACE_PROBE3(zil__block__size, zilog_t *, zilog,
	- uint64_t, zil_blksz,
	- uint64_t, zilog->zl_prev_blks[zilog->zl_prev_rotor]);
	- zilog->zl_prev_rotor = (zilog->zl_prev_rotor + 1) & (ZIL_PREV_BLKS - 1);
	-
	- return (zil_alloc_lwb(zilog, zil_blksz, NULL, 0, 0, state));
	+ if (zilog->zl_cur_left > 0) {
	+ /*
	+ * We are in the middle of a burst and know how much is left.
	+ * But if workload is multi-threaded there may be more soon.
	+ * Try to predict what can it be and plan for the worst case.
	+ */
	+ uint_t m;
	+ plan = zil_lwb_plan(zilog, zilog->zl_cur_left, &m);
	+ if (zilog->zl_parallel) {
	+ plan2 = zil_lwb_plan(zilog, zilog->zl_cur_left +
	+ zil_lwb_predict(zilog), &m);
	+ if (plan < plan2)
	+ plan = plan2;
	+ }
	+ } else {
	+ /*
	+ * The previous burst is done and we can only predict what
	+ * will come next.
	+ */
	+ plan = zil_lwb_predict(zilog);
	+ }
	+ blksz = plan + sizeof (zil_chain_t);
	+ blksz = P2ROUNDUP_TYPED(blksz, ZIL_MIN_BLKSZ, uint64_t);
	+ blksz = MIN(blksz, zilog->zl_max_block_size);
	+ DTRACE_PROBE3(zil__block__size, zilog_t *, zilog, uint64_t, blksz,
	+ uint64_t, plan);
	+
	+ return (zil_alloc_lwb(zilog, blksz, NULL, 0, 0, state));
	}

	/*
	* Finalize previously closed block and issue the write zio.
	*/
	static void
	zil_lwb_write_issue(zilog_t zilog, lwb_t lwb)
	{
	spa_t *spa = zilog->zl_spa;
	zil_chain_t *zilc;
	boolean_t slog;
	zbookmark_phys_t zb;
	zio_priority_t prio;
	int error;

	ASSERT3S(lwb->lwb_state, ==, LWB_STATE_CLOSED);

	/* Actually fill the lwb with the data. */
	for (itx_t *itx = list_head(&lwb->lwb_itxs); itx;
	itx = list_next(&lwb->lwb_itxs, itx))
	zil_lwb_commit(zilog, lwb, itx);
	lwb->lwb_nused = lwb->lwb_nfilled;
	ASSERT3U(lwb->lwb_nused, <=, lwb->lwb_nmax);

	lwb->lwb_root_zio = zio_root(spa, zil_lwb_flush_vdevs_done, lwb,
	ZIO_FLAG_CANFAIL);

	/*
	* The lwb is now ready to be issued, but it can be only if it already
	* got its block pointer allocated or the allocation has failed.
	* Otherwise leave it as-is, relying on some other thread to issue it
	* after allocating its block pointer via calling zil_lwb_write_issue()
	* for the previous lwb(s) in the chain.
	*/
	mutex_enter(&zilog->zl_lock);
	lwb->lwb_state = LWB_STATE_READY;
	if (BP_IS_HOLE(&lwb->lwb_blk) && lwb->lwb_error == 0) {
	mutex_exit(&zilog->zl_lock);
	return;
	}
	mutex_exit(&zilog->zl_lock);

	next_lwb:
	if (lwb->lwb_slim)
	zilc = (zil_chain_t *)lwb->lwb_buf;
	else
	zilc = (zil_chain_t *)(lwb->lwb_buf + lwb->lwb_nmax);
	int wsz = lwb->lwb_sz;
	if (lwb->lwb_error == 0) {
	abd_t *lwb_abd = abd_get_from_buf(lwb->lwb_buf, lwb->lwb_sz);
	- if (!lwb->lwb_slog \|\| zilog->zl_cur_used <= zil_slog_bulk)
	+ if (!lwb->lwb_slog \|\| zilog->zl_cur_size <= zil_slog_bulk)
	prio = ZIO_PRIORITY_SYNC_WRITE;
	else
	prio = ZIO_PRIORITY_ASYNC_WRITE;
	SET_BOOKMARK(&zb, lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_OBJSET],
	ZB_ZIL_OBJECT, ZB_ZIL_LEVEL,
	lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_SEQ]);
	lwb->lwb_write_zio = zio_rewrite(lwb->lwb_root_zio, spa, 0,
	&lwb->lwb_blk, lwb_abd, lwb->lwb_sz, zil_lwb_write_done,
	lwb, prio, ZIO_FLAG_CANFAIL, &zb);
	zil_lwb_add_block(lwb, &lwb->lwb_blk);

	if (lwb->lwb_slim) {
	/* For Slim ZIL only write what is used. */
	wsz = P2ROUNDUP_TYPED(lwb->lwb_nused, ZIL_MIN_BLKSZ,
	int);
	ASSERT3S(wsz, <=, lwb->lwb_sz);
	zio_shrink(lwb->lwb_write_zio, wsz);
	wsz = lwb->lwb_write_zio->io_size;
	}
	memset(lwb->lwb_buf + lwb->lwb_nused, 0, wsz - lwb->lwb_nused);
	zilc->zc_pad = 0;
	zilc->zc_nused = lwb->lwb_nused;
	zilc->zc_eck.zec_cksum = lwb->lwb_blk.blk_cksum;
	} else {
	/*
	* We can't write the lwb if there was an allocation failure,
	* so create a null zio instead just to maintain dependencies.
	*/
	lwb->lwb_write_zio = zio_null(lwb->lwb_root_zio, spa, NULL,
	zil_lwb_write_done, lwb, ZIO_FLAG_CANFAIL);
	lwb->lwb_write_zio->io_error = lwb->lwb_error;
	}
	if (lwb->lwb_child_zio)
	zio_add_child(lwb->lwb_write_zio, lwb->lwb_child_zio);

	/*
	* Open transaction to allocate the next block pointer.
	*/
	dmu_tx_t *tx = dmu_tx_create(zilog->zl_os);
	VERIFY0(dmu_tx_assign(tx, TXG_WAIT \| TXG_NOTHROTTLE));
	dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx);
	uint64_t txg = dmu_tx_get_txg(tx);

	/*
	* Allocate next the block pointer unless we are already in error.
	*/
	lwb_t *nlwb = list_next(&zilog->zl_lwb_list, lwb);
	blkptr_t *bp = &zilc->zc_next_blk;
	BP_ZERO(bp);
	error = lwb->lwb_error;
	if (error == 0) {
	error = zio_alloc_zil(spa, zilog->zl_os, txg, bp, nlwb->lwb_sz,
	&slog);
	}
	if (error == 0) {
	ASSERT3U(bp->blk_birth, ==, txg);
	BP_SET_CHECKSUM(bp, nlwb->lwb_slim ? ZIO_CHECKSUM_ZILOG2 :
	ZIO_CHECKSUM_ZILOG);
	bp->blk_cksum = lwb->lwb_blk.blk_cksum;
	bp->blk_cksum.zc_word[ZIL_ZC_SEQ]++;
	}

	/*
	* Reduce TXG open time by incrementing inflight counter and committing
	* the transaciton. zil_sync() will wait for it to return to zero.
	*/
	mutex_enter(&zilog->zl_lwb_io_lock);
	lwb->lwb_issued_txg = txg;
	zilog->zl_lwb_inflight[txg & TXG_MASK]++;
	zilog->zl_lwb_max_issued_txg = MAX(txg, zilog->zl_lwb_max_issued_txg);
	mutex_exit(&zilog->zl_lwb_io_lock);
	dmu_tx_commit(tx);

	spa_config_enter(spa, SCL_STATE, lwb, RW_READER);

	/*
	* We've completed all potentially blocking operations. Update the
	* nlwb and allow it proceed without possible lock order reversals.
	*/
	mutex_enter(&zilog->zl_lock);
	zil_lwb_set_zio_dependency(zilog, lwb);
	lwb->lwb_state = LWB_STATE_ISSUED;

	if (nlwb) {
	nlwb->lwb_blk = *bp;
	nlwb->lwb_error = error;
	nlwb->lwb_slog = slog;
	nlwb->lwb_alloc_txg = txg;
	if (nlwb->lwb_state != LWB_STATE_READY)
	nlwb = NULL;
	}
	mutex_exit(&zilog->zl_lock);

	if (lwb->lwb_slog) {
	ZIL_STAT_BUMP(zilog, zil_itx_metaslab_slog_count);
	ZIL_STAT_INCR(zilog, zil_itx_metaslab_slog_bytes,
	lwb->lwb_nused);
	ZIL_STAT_INCR(zilog, zil_itx_metaslab_slog_write,
	wsz);
	ZIL_STAT_INCR(zilog, zil_itx_metaslab_slog_alloc,
	BP_GET_LSIZE(&lwb->lwb_blk));
	} else {
	ZIL_STAT_BUMP(zilog, zil_itx_metaslab_normal_count);
	ZIL_STAT_INCR(zilog, zil_itx_metaslab_normal_bytes,
	lwb->lwb_nused);
	ZIL_STAT_INCR(zilog, zil_itx_metaslab_normal_write,
	wsz);
	ZIL_STAT_INCR(zilog, zil_itx_metaslab_normal_alloc,
	BP_GET_LSIZE(&lwb->lwb_blk));
	}
	lwb->lwb_issued_timestamp = gethrtime();
	if (lwb->lwb_child_zio)
	zio_nowait(lwb->lwb_child_zio);
	zio_nowait(lwb->lwb_write_zio);
	zio_nowait(lwb->lwb_root_zio);

	/*
	* If nlwb was ready when we gave it the block pointer,
	* it is on us to issue it and possibly following ones.
	*/
	lwb = nlwb;
	if (lwb)
	goto next_lwb;
	}

	/*
	* Maximum amount of data that can be put into single log block.
	*/
	uint64_t
	zil_max_log_data(zilog_t *zilog, size_t hdrsize)
	{
	return (zilog->zl_max_block_size - sizeof (zil_chain_t) - hdrsize);
	}

	/*
	* Maximum amount of log space we agree to waste to reduce number of
	* WR_NEED_COPY chunks to reduce zl_get_data() overhead (~6%).
	*/
	static inline uint64_t
	zil_max_waste_space(zilog_t *zilog)
	{
	return (zil_max_log_data(zilog, sizeof (lr_write_t)) / 16);
	}

	/*
	* Maximum amount of write data for WR_COPIED. For correctness, consumers
	* must fall back to WR_NEED_COPY if we can't fit the entire record into one
	* maximum sized log block, because each WR_COPIED record must fit in a
	* single log block. Below that it is a tradeoff of additional memory copy
	* and possibly worse log space efficiency vs additional range lock/unlock.
	*/
	static uint_t zil_maxcopied = 7680;

	uint64_t
	zil_max_copied_data(zilog_t *zilog)
	{
	uint64_t max_data = zil_max_log_data(zilog, sizeof (lr_write_t));
	return (MIN(max_data, zil_maxcopied));
	}

	+static uint64_t
	+zil_itx_record_size(itx_t *itx)
	+{
	+ lr_t *lr = &itx->itx_lr;
	+
	+ if (lr->lrc_txtype == TX_COMMIT)
	+ return (0);
	+ ASSERT3U(lr->lrc_reclen, >=, sizeof (lr_t));
	+ return (lr->lrc_reclen);
	+}
	+
	+static uint64_t
	+zil_itx_data_size(itx_t *itx)
	+{
	+ lr_t *lr = &itx->itx_lr;
	+ lr_write_t lrw = (lr_write_t )lr;
	+
	+ if (lr->lrc_txtype == TX_WRITE && itx->itx_wr_state == WR_NEED_COPY) {
	+ ASSERT3U(lr->lrc_reclen, ==, sizeof (lr_write_t));
	+ return (P2ROUNDUP_TYPED(lrw->lr_length, sizeof (uint64_t),
	+ uint64_t));
	+ }
	+ return (0);
	+}
	+
	+static uint64_t
	+zil_itx_full_size(itx_t *itx)
	+{
	+ lr_t *lr = &itx->itx_lr;
	+
	+ if (lr->lrc_txtype == TX_COMMIT)
	+ return (0);
	+ ASSERT3U(lr->lrc_reclen, >=, sizeof (lr_t));
	+ return (lr->lrc_reclen + zil_itx_data_size(itx));
	+}
	+
	/*
	* Estimate space needed in the lwb for the itx. Allocate more lwbs or
	* split the itx as needed, but don't touch the actual transaction data.
	* Has to be called under zl_issuer_lock to call zil_lwb_write_close()
	* to chain more lwbs.
	*/
	static lwb_t *
	zil_lwb_assign(zilog_t zilog, lwb_t lwb, itx_t itx, list_t ilwbs)
	{
	itx_t *citx;
	lr_t lr, clr;
	lr_write_t *lrw;
	uint64_t dlen, dnow, lwb_sp, reclen, max_log_data;

	ASSERT(MUTEX_HELD(&zilog->zl_issuer_lock));
	ASSERT3P(lwb, !=, NULL);
	ASSERT3P(lwb->lwb_buf, !=, NULL);

	zil_lwb_write_open(zilog, lwb);

	lr = &itx->itx_lr;
	lrw = (lr_write_t *)lr;

	/*
	* A commit itx doesn't represent any on-disk state; instead
	* it's simply used as a place holder on the commit list, and
	* provides a mechanism for attaching a "commit waiter" onto the
	* correct lwb (such that the waiter can be signalled upon
	* completion of that lwb). Thus, we don't process this itx's
	* log record if it's a commit itx (these itx's don't have log
	* records), and instead link the itx's waiter onto the lwb's
	* list of waiters.
	*
	* For more details, see the comment above zil_commit().
	*/
	if (lr->lrc_txtype == TX_COMMIT) {
	zil_commit_waiter_link_lwb(itx->itx_private, lwb);
	list_insert_tail(&lwb->lwb_itxs, itx);
	return (lwb);
	}

	reclen = lr->lrc_reclen;
	- if (lr->lrc_txtype == TX_WRITE && itx->itx_wr_state == WR_NEED_COPY) {
	- ASSERT3U(reclen, ==, sizeof (lr_write_t));
	- dlen = P2ROUNDUP_TYPED(
	- lrw->lr_length, sizeof (uint64_t), uint64_t);
	- } else {
	- ASSERT3U(reclen, >=, sizeof (lr_t));
	- dlen = 0;
	- }
	+ ASSERT3U(reclen, >=, sizeof (lr_t));
	ASSERT3U(reclen, <=, zil_max_log_data(zilog, 0));
	- zilog->zl_cur_used += (reclen + dlen);
	+ dlen = zil_itx_data_size(itx);

	cont:
	/*
	* If this record won't fit in the current log block, start a new one.
	* For WR_NEED_COPY optimize layout for minimal number of chunks.
	*/
	lwb_sp = lwb->lwb_nmax - lwb->lwb_nused;
	max_log_data = zil_max_log_data(zilog, sizeof (lr_write_t));
	if (reclen > lwb_sp \|\| (reclen + dlen > lwb_sp &&
	lwb_sp < zil_max_waste_space(zilog) &&
	(dlen % max_log_data == 0 \|\|
	lwb_sp < reclen + dlen % max_log_data))) {
	list_insert_tail(ilwbs, lwb);
	lwb = zil_lwb_write_close(zilog, lwb, LWB_STATE_OPENED);
	if (lwb == NULL)
	return (NULL);
	lwb_sp = lwb->lwb_nmax - lwb->lwb_nused;
	}

	/*
	* There must be enough space in the log block to hold reclen.
	* For WR_COPIED, we need to fit the whole record in one block,
	* and reclen is the write record header size + the data size.
	* For WR_NEED_COPY, we can create multiple records, splitting
	* the data into multiple blocks, so we only need to fit one
	* word of data per block; in this case reclen is just the header
	* size (no data).
	*/
	ASSERT3U(reclen + MIN(dlen, sizeof (uint64_t)), <=, lwb_sp);

	dnow = MIN(dlen, lwb_sp - reclen);
	if (dlen > dnow) {
	ASSERT3U(lr->lrc_txtype, ==, TX_WRITE);
	ASSERT3U(itx->itx_wr_state, ==, WR_NEED_COPY);
	citx = zil_itx_clone(itx);
	clr = &citx->itx_lr;
	lr_write_t clrw = (lr_write_t )clr;
	clrw->lr_length = dnow;
	lrw->lr_offset += dnow;
	lrw->lr_length -= dnow;
	+ zilog->zl_cur_left -= dnow;
	} else {
	citx = itx;
	clr = lr;
	}

	/*
	* We're actually making an entry, so update lrc_seq to be the
	* log record sequence number. Note that this is generally not
	* equal to the itx sequence number because not all transactions
	* are synchronous, and sometimes spa_sync() gets there first.
	*/
	clr->lrc_seq = ++zilog->zl_lr_seq;

	lwb->lwb_nused += reclen + dnow;
	ASSERT3U(lwb->lwb_nused, <=, lwb->lwb_nmax);
	ASSERT0(P2PHASE(lwb->lwb_nused, sizeof (uint64_t)));

	zil_lwb_add_txg(lwb, lr->lrc_txg);
	list_insert_tail(&lwb->lwb_itxs, citx);

	dlen -= dnow;
	- if (dlen > 0) {
	- zilog->zl_cur_used += reclen;
	+ if (dlen > 0)
	goto cont;
	- }

	if (lr->lrc_txtype == TX_WRITE &&
	lr->lrc_txg > spa_freeze_txg(zilog->zl_spa))
	txg_wait_synced(zilog->zl_dmu_pool, lr->lrc_txg);

	return (lwb);
	}

	/*
	* Fill the actual transaction data into the lwb, following zil_lwb_assign().
	* Does not require locking.
	*/
	static void
	zil_lwb_commit(zilog_t zilog, lwb_t lwb, itx_t *itx)
	{
	lr_t lr, lrb;
	lr_write_t lrw, lrwb;
	char *lr_buf;
	uint64_t dlen, reclen;

	lr = &itx->itx_lr;
	lrw = (lr_write_t *)lr;

	if (lr->lrc_txtype == TX_COMMIT)
	return;

	- if (lr->lrc_txtype == TX_WRITE && itx->itx_wr_state == WR_NEED_COPY) {
	- dlen = P2ROUNDUP_TYPED(
	- lrw->lr_length, sizeof (uint64_t), uint64_t);
	- } else {
	- dlen = 0;
	- }
	reclen = lr->lrc_reclen;
	+ dlen = zil_itx_data_size(itx);
	ASSERT3U(reclen + dlen, <=, lwb->lwb_nused - lwb->lwb_nfilled);

	lr_buf = lwb->lwb_buf + lwb->lwb_nfilled;
	memcpy(lr_buf, lr, reclen);
	lrb = (lr_t )lr_buf; / Like lr, but inside lwb. */
	lrwb = (lr_write_t )lrb; / Like lrw, but inside lwb. */

	ZIL_STAT_BUMP(zilog, zil_itx_count);

	/*
	* If it's a write, fetch the data or get its blkptr as appropriate.
	*/
	if (lr->lrc_txtype == TX_WRITE) {
	if (itx->itx_wr_state == WR_COPIED) {
	ZIL_STAT_BUMP(zilog, zil_itx_copied_count);
	ZIL_STAT_INCR(zilog, zil_itx_copied_bytes,
	lrw->lr_length);
	} else {
	char *dbuf;
	int error;

	if (itx->itx_wr_state == WR_NEED_COPY) {
	dbuf = lr_buf + reclen;
	lrb->lrc_reclen += dlen;
	ZIL_STAT_BUMP(zilog, zil_itx_needcopy_count);
	ZIL_STAT_INCR(zilog, zil_itx_needcopy_bytes,
	dlen);
	} else {
	ASSERT3S(itx->itx_wr_state, ==, WR_INDIRECT);
	dbuf = NULL;
	ZIL_STAT_BUMP(zilog, zil_itx_indirect_count);
	ZIL_STAT_INCR(zilog, zil_itx_indirect_bytes,
	lrw->lr_length);
	if (lwb->lwb_child_zio == NULL) {
	lwb->lwb_child_zio = zio_root(
	zilog->zl_spa, NULL, NULL,
	ZIO_FLAG_CANFAIL);
	}
	}

	/*
	* The "lwb_child_zio" we pass in will become a child of
	* "lwb_write_zio", when one is created, so one will be
	* a parent of any zio's created by the "zl_get_data".
	* This way "lwb_write_zio" will first wait for children
	* block pointers before own writing, and then for their
	* writing completion before the vdev cache flushing.
	*/
	error = zilog->zl_get_data(itx->itx_private,
	itx->itx_gen, lrwb, dbuf, lwb,
	lwb->lwb_child_zio);
	if (dbuf != NULL && error == 0) {
	/* Zero any padding bytes in the last block. */
	memset((char *)dbuf + lrwb->lr_length, 0,
	dlen - lrwb->lr_length);
	}

	/*
	* Typically, the only return values we should see from
	* ->zl_get_data() are 0, EIO, ENOENT, EEXIST or
	* EALREADY. However, it is also possible to see other
	* error values such as ENOSPC or EINVAL from
	* dmu_read() -> dnode_hold() -> dnode_hold_impl() or
	* ENXIO as well as a multitude of others from the
	* block layer through dmu_buf_hold() -> dbuf_read()
	* -> zio_wait(), as well as through dmu_read() ->
	* dnode_hold() -> dnode_hold_impl() -> dbuf_read() ->
	* zio_wait(). When these errors happen, we can assume
	* that neither an immediate write nor an indirect
	* write occurred, so we need to fall back to
	* txg_wait_synced(). This is unusual, so we print to
	* dmesg whenever one of these errors occurs.
	*/
	switch (error) {
	case 0:
	break;
	default:
	cmn_err(CE_WARN, "zil_lwb_commit() received "
	"unexpected error %d from ->zl_get_data()"
	". Falling back to txg_wait_synced().",
	error);
	zfs_fallthrough;
	case EIO:
	txg_wait_synced(zilog->zl_dmu_pool,
	lr->lrc_txg);
	zfs_fallthrough;
	case ENOENT:
	zfs_fallthrough;
	case EEXIST:
	zfs_fallthrough;
	case EALREADY:
	return;
	}
	}
	}

	lwb->lwb_nfilled += reclen + dlen;
	ASSERT3S(lwb->lwb_nfilled, <=, lwb->lwb_nused);
	ASSERT0(P2PHASE(lwb->lwb_nfilled, sizeof (uint64_t)));
	}

	itx_t *
	zil_itx_create(uint64_t txtype, size_t olrsize)
	{
	size_t itxsize, lrsize;
	itx_t *itx;

	ASSERT3U(olrsize, >=, sizeof (lr_t));
	lrsize = P2ROUNDUP_TYPED(olrsize, sizeof (uint64_t), size_t);
	ASSERT3U(lrsize, >=, olrsize);
	itxsize = offsetof(itx_t, itx_lr) + lrsize;

	itx = zio_data_buf_alloc(itxsize);
	itx->itx_lr.lrc_txtype = txtype;
	itx->itx_lr.lrc_reclen = lrsize;
	itx->itx_lr.lrc_seq = 0; /* defensive */
	memset((char *)&itx->itx_lr + olrsize, 0, lrsize - olrsize);
	itx->itx_sync = B_TRUE; /* default is synchronous */
	itx->itx_callback = NULL;
	itx->itx_callback_data = NULL;
	itx->itx_size = itxsize;

	return (itx);
	}

	static itx_t *
	zil_itx_clone(itx_t *oitx)
	{
	ASSERT3U(oitx->itx_size, >=, sizeof (itx_t));
	ASSERT3U(oitx->itx_size, ==,
	offsetof(itx_t, itx_lr) + oitx->itx_lr.lrc_reclen);

	itx_t *itx = zio_data_buf_alloc(oitx->itx_size);
	memcpy(itx, oitx, oitx->itx_size);
	itx->itx_callback = NULL;
	itx->itx_callback_data = NULL;
	return (itx);
	}

	void
	zil_itx_destroy(itx_t *itx)
	{
	ASSERT3U(itx->itx_size, >=, sizeof (itx_t));
	ASSERT3U(itx->itx_lr.lrc_reclen, ==,
	itx->itx_size - offsetof(itx_t, itx_lr));
	IMPLY(itx->itx_lr.lrc_txtype == TX_COMMIT, itx->itx_callback == NULL);
	IMPLY(itx->itx_callback != NULL, itx->itx_lr.lrc_txtype != TX_COMMIT);

	if (itx->itx_callback != NULL)
	itx->itx_callback(itx->itx_callback_data);

	zio_data_buf_free(itx, itx->itx_size);
	}

	/*
	* Free up the sync and async itxs. The itxs_t has already been detached
	* so no locks are needed.
	*/
	static void
	zil_itxg_clean(void *arg)
	{
	itx_t *itx;
	list_t *list;
	avl_tree_t *t;
	void *cookie;
	itxs_t *itxs = arg;
	itx_async_node_t *ian;

	list = &itxs->i_sync_list;
	while ((itx = list_remove_head(list)) != NULL) {
	/*
	* In the general case, commit itxs will not be found
	* here, as they'll be committed to an lwb via
	* zil_lwb_assign(), and free'd in that function. Having
	* said that, it is still possible for commit itxs to be
	* found here, due to the following race:
	*
	* - a thread calls zil_commit() which assigns the
	* commit itx to a per-txg i_sync_list
	* - zil_itxg_clean() is called (e.g. via spa_sync())
	* while the waiter is still on the i_sync_list
	*
	* There's nothing to prevent syncing the txg while the
	* waiter is on the i_sync_list. This normally doesn't
	* happen because spa_sync() is slower than zil_commit(),
	* but if zil_commit() calls txg_wait_synced() (e.g.
	* because zil_create() or zil_commit_writer_stall() is
	* called) we will hit this case.
	*/
	if (itx->itx_lr.lrc_txtype == TX_COMMIT)
	zil_commit_waiter_skip(itx->itx_private);

	zil_itx_destroy(itx);
	}

	cookie = NULL;
	t = &itxs->i_async_tree;
	while ((ian = avl_destroy_nodes(t, &cookie)) != NULL) {
	list = &ian->ia_list;
	while ((itx = list_remove_head(list)) != NULL) {
	/* commit itxs should never be on the async lists. */
	ASSERT3U(itx->itx_lr.lrc_txtype, !=, TX_COMMIT);
	zil_itx_destroy(itx);
	}
	list_destroy(list);
	kmem_free(ian, sizeof (itx_async_node_t));
	}
	avl_destroy(t);

	kmem_free(itxs, sizeof (itxs_t));
	}

	static int
	zil_aitx_compare(const void x1, const void x2)
	{
	const uint64_t o1 = ((itx_async_node_t *)x1)->ia_foid;
	const uint64_t o2 = ((itx_async_node_t *)x2)->ia_foid;

	return (TREE_CMP(o1, o2));
	}

	/*
	* Remove all async itx with the given oid.
	*/
	void
	zil_remove_async(zilog_t *zilog, uint64_t oid)
	{
	uint64_t otxg, txg;
	itx_async_node_t *ian, ian_search;
	avl_tree_t *t;
	avl_index_t where;
	list_t clean_list;
	itx_t *itx;

	ASSERT(oid != 0);
	list_create(&clean_list, sizeof (itx_t), offsetof(itx_t, itx_node));

	if (spa_freeze_txg(zilog->zl_spa) != UINT64_MAX) /* ziltest support */
	otxg = ZILTEST_TXG;
	else
	otxg = spa_last_synced_txg(zilog->zl_spa) + 1;

	for (txg = otxg; txg < (otxg + TXG_CONCURRENT_STATES); txg++) {
	itxg_t *itxg = &zilog->zl_itxg[txg & TXG_MASK];

	mutex_enter(&itxg->itxg_lock);
	if (itxg->itxg_txg != txg) {
	mutex_exit(&itxg->itxg_lock);
	continue;
	}

	/*
	* Locate the object node and append its list.
	*/
	t = &itxg->itxg_itxs->i_async_tree;
	ian_search.ia_foid = oid;
	ian = avl_find(t, &ian_search, &where);
	if (ian != NULL)
	list_move_tail(&clean_list, &ian->ia_list);
	mutex_exit(&itxg->itxg_lock);
	}
	while ((itx = list_remove_head(&clean_list)) != NULL) {
	/* commit itxs should never be on the async lists. */
	ASSERT3U(itx->itx_lr.lrc_txtype, !=, TX_COMMIT);
	zil_itx_destroy(itx);
	}
	list_destroy(&clean_list);
	}

	void
	zil_itx_assign(zilog_t zilog, itx_t itx, dmu_tx_t *tx)
	{
	uint64_t txg;
	itxg_t *itxg;
	itxs_t itxs, clean = NULL;

	/*
	* Ensure the data of a renamed file is committed before the rename.
	*/
	if ((itx->itx_lr.lrc_txtype & ~TX_CI) == TX_RENAME)
	zil_async_to_sync(zilog, itx->itx_oid);

	if (spa_freeze_txg(zilog->zl_spa) != UINT64_MAX)
	txg = ZILTEST_TXG;
	else
	txg = dmu_tx_get_txg(tx);

	itxg = &zilog->zl_itxg[txg & TXG_MASK];
	mutex_enter(&itxg->itxg_lock);
	itxs = itxg->itxg_itxs;
	if (itxg->itxg_txg != txg) {
	if (itxs != NULL) {
	/*
	* The zil_clean callback hasn't got around to cleaning
	* this itxg. Save the itxs for release below.
	* This should be rare.
	*/
	zfs_dbgmsg("zil_itx_assign: missed itx cleanup for "
	"txg %llu", (u_longlong_t)itxg->itxg_txg);
	clean = itxg->itxg_itxs;
	}
	itxg->itxg_txg = txg;
	itxs = itxg->itxg_itxs = kmem_zalloc(sizeof (itxs_t),
	KM_SLEEP);

	list_create(&itxs->i_sync_list, sizeof (itx_t),
	offsetof(itx_t, itx_node));
	avl_create(&itxs->i_async_tree, zil_aitx_compare,
	sizeof (itx_async_node_t),
	offsetof(itx_async_node_t, ia_node));
	}
	if (itx->itx_sync) {
	list_insert_tail(&itxs->i_sync_list, itx);
	} else {
	avl_tree_t *t = &itxs->i_async_tree;
	uint64_t foid =
	LR_FOID_GET_OBJ(((lr_ooo_t *)&itx->itx_lr)->lr_foid);
	itx_async_node_t *ian;
	avl_index_t where;

	ian = avl_find(t, &foid, &where);
	if (ian == NULL) {
	ian = kmem_alloc(sizeof (itx_async_node_t),
	KM_SLEEP);
	list_create(&ian->ia_list, sizeof (itx_t),
	offsetof(itx_t, itx_node));
	ian->ia_foid = foid;
	avl_insert(t, ian, where);
	}
	list_insert_tail(&ian->ia_list, itx);
	}

	itx->itx_lr.lrc_txg = dmu_tx_get_txg(tx);

	/*
	* We don't want to dirty the ZIL using ZILTEST_TXG, because
	* zil_clean() will never be called using ZILTEST_TXG. Thus, we
	* need to be careful to always dirty the ZIL using the "real"
	* TXG (not itxg_txg) even when the SPA is frozen.
	*/
	zilog_dirty(zilog, dmu_tx_get_txg(tx));
	mutex_exit(&itxg->itxg_lock);

	/* Release the old itxs now we've dropped the lock */
	if (clean != NULL)
	zil_itxg_clean(clean);
	}

	/*
	* If there are any in-memory intent log transactions which have now been
	* synced then start up a taskq to free them. We should only do this after we
	* have written out the uberblocks (i.e. txg has been committed) so that
	* don't inadvertently clean out in-memory log records that would be required
	* by zil_commit().
	*/
	void
	zil_clean(zilog_t *zilog, uint64_t synced_txg)
	{
	itxg_t *itxg = &zilog->zl_itxg[synced_txg & TXG_MASK];
	itxs_t *clean_me;

	ASSERT3U(synced_txg, <, ZILTEST_TXG);

	mutex_enter(&itxg->itxg_lock);
	if (itxg->itxg_itxs == NULL \|\| itxg->itxg_txg == ZILTEST_TXG) {
	mutex_exit(&itxg->itxg_lock);
	return;
	}
	ASSERT3U(itxg->itxg_txg, <=, synced_txg);
	ASSERT3U(itxg->itxg_txg, !=, 0);
	clean_me = itxg->itxg_itxs;
	itxg->itxg_itxs = NULL;
	itxg->itxg_txg = 0;
	mutex_exit(&itxg->itxg_lock);
	/*
	* Preferably start a task queue to free up the old itxs but
	* if taskq_dispatch can't allocate resources to do that then
	* free it in-line. This should be rare. Note, using TQ_SLEEP
	* created a bad performance problem.
	*/
	ASSERT3P(zilog->zl_dmu_pool, !=, NULL);
	ASSERT3P(zilog->zl_dmu_pool->dp_zil_clean_taskq, !=, NULL);
	taskqid_t id = taskq_dispatch(zilog->zl_dmu_pool->dp_zil_clean_taskq,
	zil_itxg_clean, clean_me, TQ_NOSLEEP);
	if (id == TASKQID_INVALID)
	zil_itxg_clean(clean_me);
	}

	/*
	* This function will traverse the queue of itxs that need to be
	* committed, and move them onto the ZIL's zl_itx_commit_list.
	*/
	static uint64_t
	zil_get_commit_list(zilog_t *zilog)
	{
	uint64_t otxg, txg, wtxg = 0;
	list_t *commit_list = &zilog->zl_itx_commit_list;

	ASSERT(MUTEX_HELD(&zilog->zl_issuer_lock));

	if (spa_freeze_txg(zilog->zl_spa) != UINT64_MAX) /* ziltest support */
	otxg = ZILTEST_TXG;
	else
	otxg = spa_last_synced_txg(zilog->zl_spa) + 1;

	/*
	* This is inherently racy, since there is nothing to prevent
	* the last synced txg from changing. That's okay since we'll
	* only commit things in the future.
	*/
	for (txg = otxg; txg < (otxg + TXG_CONCURRENT_STATES); txg++) {
	itxg_t *itxg = &zilog->zl_itxg[txg & TXG_MASK];

	mutex_enter(&itxg->itxg_lock);
	if (itxg->itxg_txg != txg) {
	mutex_exit(&itxg->itxg_lock);
	continue;
	}

	/*
	* If we're adding itx records to the zl_itx_commit_list,
	* then the zil better be dirty in this "txg". We can assert
	* that here since we're holding the itxg_lock which will
	* prevent spa_sync from cleaning it. Once we add the itxs
	* to the zl_itx_commit_list we must commit it to disk even
	* if it's unnecessary (i.e. the txg was synced).
	*/
	ASSERT(zilog_is_dirty_in_txg(zilog, txg) \|\|
	spa_freeze_txg(zilog->zl_spa) != UINT64_MAX);
	list_t *sync_list = &itxg->itxg_itxs->i_sync_list;
	+ itx_t *itx = NULL;
	if (unlikely(zilog->zl_suspend > 0)) {
	/*
	* ZIL was just suspended, but we lost the race.
	* Allow all earlier itxs to be committed, but ask
	* caller to do txg_wait_synced(txg) for any new.
	*/
	if (!list_is_empty(sync_list))
	wtxg = MAX(wtxg, txg);
	} else {
	+ itx = list_head(sync_list);
	list_move_tail(commit_list, sync_list);
	}

	mutex_exit(&itxg->itxg_lock);
	+
	+ while (itx != NULL) {
	+ uint64_t s = zil_itx_full_size(itx);
	+ zilog->zl_cur_size += s;
	+ zilog->zl_cur_left += s;
	+ s = zil_itx_record_size(itx);
	+ zilog->zl_cur_max = MAX(zilog->zl_cur_max, s);
	+ itx = list_next(commit_list, itx);
	+ }
	}
	return (wtxg);
	}

	/*
	* Move the async itxs for a specified object to commit into sync lists.
	*/
	void
	zil_async_to_sync(zilog_t *zilog, uint64_t foid)
	{
	uint64_t otxg, txg;
	itx_async_node_t *ian, ian_search;
	avl_tree_t *t;
	avl_index_t where;

	if (spa_freeze_txg(zilog->zl_spa) != UINT64_MAX) /* ziltest support */
	otxg = ZILTEST_TXG;
	else
	otxg = spa_last_synced_txg(zilog->zl_spa) + 1;

	/*
	* This is inherently racy, since there is nothing to prevent
	* the last synced txg from changing.
	*/
	for (txg = otxg; txg < (otxg + TXG_CONCURRENT_STATES); txg++) {
	itxg_t *itxg = &zilog->zl_itxg[txg & TXG_MASK];

	mutex_enter(&itxg->itxg_lock);
	if (itxg->itxg_txg != txg) {
	mutex_exit(&itxg->itxg_lock);
	continue;
	}

	/*
	* If a foid is specified then find that node and append its
	* list. Otherwise walk the tree appending all the lists
	* to the sync list. We add to the end rather than the
	* beginning to ensure the create has happened.
	*/
	t = &itxg->itxg_itxs->i_async_tree;
	if (foid != 0) {
	ian_search.ia_foid = foid;
	ian = avl_find(t, &ian_search, &where);
	if (ian != NULL) {
	list_move_tail(&itxg->itxg_itxs->i_sync_list,
	&ian->ia_list);
	}
	} else {
	void *cookie = NULL;

	while ((ian = avl_destroy_nodes(t, &cookie)) != NULL) {
	list_move_tail(&itxg->itxg_itxs->i_sync_list,
	&ian->ia_list);
	list_destroy(&ian->ia_list);
	kmem_free(ian, sizeof (itx_async_node_t));
	}
	}
	mutex_exit(&itxg->itxg_lock);
	}
	}

	/*
	* This function will prune commit itxs that are at the head of the
	* commit list (it won't prune past the first non-commit itx), and
	* either: a) attach them to the last lwb that's still pending
	* completion, or b) skip them altogether.
	*
	* This is used as a performance optimization to prevent commit itxs
	* from generating new lwbs when it's unnecessary to do so.
	*/
	static void
	zil_prune_commit_list(zilog_t *zilog)
	{
	itx_t *itx;

	ASSERT(MUTEX_HELD(&zilog->zl_issuer_lock));

	while ((itx = list_head(&zilog->zl_itx_commit_list)) != NULL) {
	lr_t *lrc = &itx->itx_lr;
	if (lrc->lrc_txtype != TX_COMMIT)
	break;

	mutex_enter(&zilog->zl_lock);

	lwb_t *last_lwb = zilog->zl_last_lwb_opened;
	if (last_lwb == NULL \|\|
	last_lwb->lwb_state == LWB_STATE_FLUSH_DONE) {
	/*
	* All of the itxs this waiter was waiting on
	* must have already completed (or there were
	* never any itx's for it to wait on), so it's
	* safe to skip this waiter and mark it done.
	*/
	zil_commit_waiter_skip(itx->itx_private);
	} else {
	zil_commit_waiter_link_lwb(itx->itx_private, last_lwb);
	}

	mutex_exit(&zilog->zl_lock);

	list_remove(&zilog->zl_itx_commit_list, itx);
	zil_itx_destroy(itx);
	}

	IMPLY(itx != NULL, itx->itx_lr.lrc_txtype != TX_COMMIT);
	}

	static void
	zil_commit_writer_stall(zilog_t *zilog)
	{
	/*
	* When zio_alloc_zil() fails to allocate the next lwb block on
	* disk, we must call txg_wait_synced() to ensure all of the
	* lwbs in the zilog's zl_lwb_list are synced and then freed (in
	* zil_sync()), such that any subsequent ZIL writer (i.e. a call
	* to zil_process_commit_list()) will have to call zil_create(),
	* and start a new ZIL chain.
	*
	* Since zil_alloc_zil() failed, the lwb that was previously
	* issued does not have a pointer to the "next" lwb on disk.
	* Thus, if another ZIL writer thread was to allocate the "next"
	* on-disk lwb, that block could be leaked in the event of a
	* crash (because the previous lwb on-disk would not point to
	* it).
	*
	* We must hold the zilog's zl_issuer_lock while we do this, to
	* ensure no new threads enter zil_process_commit_list() until
	* all lwb's in the zl_lwb_list have been synced and freed
	* (which is achieved via the txg_wait_synced() call).
	*/
	ASSERT(MUTEX_HELD(&zilog->zl_issuer_lock));
	txg_wait_synced(zilog->zl_dmu_pool, 0);
	ASSERT(list_is_empty(&zilog->zl_lwb_list));
	}

	+static void
	+zil_burst_done(zilog_t *zilog)
	+{
	+ if (!list_is_empty(&zilog->zl_itx_commit_list) \|\|
	+ zilog->zl_cur_size == 0)
	+ return;
	+
	+ if (zilog->zl_parallel)
	+ zilog->zl_parallel--;
	+
	+ uint_t r = (zilog->zl_prev_rotor + 1) & (ZIL_BURSTS - 1);
	+ zilog->zl_prev_rotor = r;
	+ zilog->zl_prev_opt[r] = zil_lwb_plan(zilog, zilog->zl_cur_size,
	+ &zilog->zl_prev_min[r]);
	+
	+ zilog->zl_cur_size = 0;
	+ zilog->zl_cur_max = 0;
	+ zilog->zl_cur_left = 0;
	+}
	+
	/*
	* This function will traverse the commit list, creating new lwbs as
	* needed, and committing the itxs from the commit list to these newly
	* created lwbs. Additionally, as a new lwb is created, the previous
	* lwb will be issued to the zio layer to be written to disk.
	*/
	static void
	zil_process_commit_list(zilog_t zilog, zil_commit_waiter_t zcw, list_t *ilwbs)
	{
	spa_t *spa = zilog->zl_spa;
	list_t nolwb_itxs;
	list_t nolwb_waiters;
	lwb_t lwb, plwb;
	itx_t *itx;
	- boolean_t first = B_TRUE;

	ASSERT(MUTEX_HELD(&zilog->zl_issuer_lock));

	/*
	* Return if there's nothing to commit before we dirty the fs by
	* calling zil_create().
	*/
	if (list_is_empty(&zilog->zl_itx_commit_list))
	return;

	list_create(&nolwb_itxs, sizeof (itx_t), offsetof(itx_t, itx_node));
	list_create(&nolwb_waiters, sizeof (zil_commit_waiter_t),
	offsetof(zil_commit_waiter_t, zcw_node));

	lwb = list_tail(&zilog->zl_lwb_list);
	if (lwb == NULL) {
	lwb = zil_create(zilog);
	} else {
	/*
	* Activate SPA_FEATURE_ZILSAXATTR for the cases where ZIL will
	* have already been created (zl_lwb_list not empty).
	*/
	zil_commit_activate_saxattr_feature(zilog);
	ASSERT(lwb->lwb_state == LWB_STATE_NEW \|\|
	lwb->lwb_state == LWB_STATE_OPENED);
	- first = (lwb->lwb_state == LWB_STATE_NEW) &&
	- ((plwb = list_prev(&zilog->zl_lwb_list, lwb)) == NULL \|\|
	- plwb->lwb_state == LWB_STATE_FLUSH_DONE);
	+
	+ /*
	+ * If the lwb is still opened, it means the workload is really
	+ * multi-threaded and we won the chance of write aggregation.
	+ * If it is not opened yet, but previous lwb is still not
	+ * flushed, it still means the workload is multi-threaded, but
	+ * there was too much time between the commits to aggregate, so
	+ * we try aggregation next times, but without too much hopes.
	+ */
	+ if (lwb->lwb_state == LWB_STATE_OPENED) {
	+ zilog->zl_parallel = ZIL_BURSTS;
	+ } else if ((plwb = list_prev(&zilog->zl_lwb_list, lwb))
	+ != NULL && plwb->lwb_state != LWB_STATE_FLUSH_DONE) {
	+ zilog->zl_parallel = MAX(zilog->zl_parallel,
	+ ZIL_BURSTS / 2);
	+ }
	}

	while ((itx = list_remove_head(&zilog->zl_itx_commit_list)) != NULL) {
	lr_t *lrc = &itx->itx_lr;
	uint64_t txg = lrc->lrc_txg;

	ASSERT3U(txg, !=, 0);

	if (lrc->lrc_txtype == TX_COMMIT) {
	DTRACE_PROBE2(zil__process__commit__itx,
	zilog_t , zilog, itx_t , itx);
	} else {
	DTRACE_PROBE2(zil__process__normal__itx,
	zilog_t , zilog, itx_t , itx);
	}

	boolean_t synced = txg <= spa_last_synced_txg(spa);
	boolean_t frozen = txg > spa_freeze_txg(spa);

	/*
	* If the txg of this itx has already been synced out, then
	* we don't need to commit this itx to an lwb. This is
	* because the data of this itx will have already been
	* written to the main pool. This is inherently racy, and
	* it's still ok to commit an itx whose txg has already
	* been synced; this will result in a write that's
	* unnecessary, but will do no harm.
	*
	* With that said, we always want to commit TX_COMMIT itxs
	* to an lwb, regardless of whether or not that itx's txg
	* has been synced out. We do this to ensure any OPENED lwb
	* will always have at least one zil_commit_waiter_t linked
	* to the lwb.
	*
	* As a counter-example, if we skipped TX_COMMIT itx's
	* whose txg had already been synced, the following
	* situation could occur if we happened to be racing with
	* spa_sync:
	*
	* 1. We commit a non-TX_COMMIT itx to an lwb, where the
	* itx's txg is 10 and the last synced txg is 9.
	* 2. spa_sync finishes syncing out txg 10.
	* 3. We move to the next itx in the list, it's a TX_COMMIT
	* whose txg is 10, so we skip it rather than committing
	* it to the lwb used in (1).
	*
	* If the itx that is skipped in (3) is the last TX_COMMIT
	* itx in the commit list, than it's possible for the lwb
	* used in (1) to remain in the OPENED state indefinitely.
	*
	* To prevent the above scenario from occurring, ensuring
	* that once an lwb is OPENED it will transition to ISSUED
	* and eventually DONE, we always commit TX_COMMIT itx's to
	* an lwb here, even if that itx's txg has already been
	* synced.
	*
	* Finally, if the pool is frozen, we _always_ commit the
	* itx. The point of freezing the pool is to prevent data
	* from being written to the main pool via spa_sync, and
	* instead rely solely on the ZIL to persistently store the
	* data; i.e. when the pool is frozen, the last synced txg
	* value can't be trusted.
	*/
	if (frozen \|\| !synced \|\| lrc->lrc_txtype == TX_COMMIT) {
	if (lwb != NULL) {
	lwb = zil_lwb_assign(zilog, lwb, itx, ilwbs);
	if (lwb == NULL) {
	list_insert_tail(&nolwb_itxs, itx);
	} else if ((zcw->zcw_lwb != NULL &&
	zcw->zcw_lwb != lwb) \|\| zcw->zcw_done) {
	/*
	* Our lwb is done, leave the rest of
	* itx list to somebody else who care.
	*/
	- first = B_FALSE;
	+ zilog->zl_parallel = ZIL_BURSTS;
	+ zilog->zl_cur_left -=
	+ zil_itx_full_size(itx);
	break;
	}
	} else {
	if (lrc->lrc_txtype == TX_COMMIT) {
	zil_commit_waiter_link_nolwb(
	itx->itx_private, &nolwb_waiters);
	}
	list_insert_tail(&nolwb_itxs, itx);
	}
	+ zilog->zl_cur_left -= zil_itx_full_size(itx);
	} else {
	ASSERT3S(lrc->lrc_txtype, !=, TX_COMMIT);
	+ zilog->zl_cur_left -= zil_itx_full_size(itx);
	zil_itx_destroy(itx);
	}
	}

	if (lwb == NULL) {
	/*
	* This indicates zio_alloc_zil() failed to allocate the
	* "next" lwb on-disk. When this happens, we must stall
	* the ZIL write pipeline; see the comment within
	* zil_commit_writer_stall() for more details.
	*/
	while ((lwb = list_remove_head(ilwbs)) != NULL)
	zil_lwb_write_issue(zilog, lwb);
	zil_commit_writer_stall(zilog);

	/*
	* Additionally, we have to signal and mark the "nolwb"
	* waiters as "done" here, since without an lwb, we
	* can't do this via zil_lwb_flush_vdevs_done() like
	* normal.
	*/
	zil_commit_waiter_t *zcw;
	while ((zcw = list_remove_head(&nolwb_waiters)) != NULL)
	zil_commit_waiter_skip(zcw);

	/*
	* And finally, we have to destroy the itx's that
	* couldn't be committed to an lwb; this will also call
	* the itx's callback if one exists for the itx.
	*/
	while ((itx = list_remove_head(&nolwb_itxs)) != NULL)
	zil_itx_destroy(itx);
	} else {
	ASSERT(list_is_empty(&nolwb_waiters));
	ASSERT3P(lwb, !=, NULL);
	ASSERT(lwb->lwb_state == LWB_STATE_NEW \|\|
	lwb->lwb_state == LWB_STATE_OPENED);

	/*
	* At this point, the ZIL block pointed at by the "lwb"
	* variable is in "new" or "opened" state.
	*
	* If it's "new", then no itxs have been committed to it, so
	* there's no point in issuing its zio (i.e. it's "empty").
	*
	* If it's "opened", then it contains one or more itxs that
	* eventually need to be committed to stable storage. In
	* this case we intentionally do not issue the lwb's zio
	* to disk yet, and instead rely on one of the following
	* two mechanisms for issuing the zio:
	*
	* 1. Ideally, there will be more ZIL activity occurring on
	* the system, such that this function will be immediately
	* called again by different thread and this lwb will be
	* closed by zil_lwb_assign(). This way, the lwb will be
	* "full" when it is issued to disk, and we'll make use of
	* the lwb's size the best we can.
	*
	* 2. If there isn't sufficient ZIL activity occurring on
	* the system, zil_commit_waiter() will close it and issue
	* the zio. If this occurs, the lwb is not guaranteed
	* to be "full" by the time its zio is issued, and means
	* the size of the lwb was "too large" given the amount
	* of ZIL activity occurring on the system at that time.
	*
	* We do this for a couple of reasons:
	*
	* 1. To try and reduce the number of IOPs needed to
	* write the same number of itxs. If an lwb has space
	* available in its buffer for more itxs, and more itxs
	* will be committed relatively soon (relative to the
	* latency of performing a write), then it's beneficial
	* to wait for these "next" itxs. This way, more itxs
	* can be committed to stable storage with fewer writes.
	*
	* 2. To try and use the largest lwb block size that the
	* incoming rate of itxs can support. Again, this is to
	* try and pack as many itxs into as few lwbs as
	* possible, without significantly impacting the latency
	* of each individual itx.
	- *
	- * If we had no already running or open LWBs, it can be
	- * the workload is single-threaded. And if the ZIL write
	- * latency is very small or if the LWB is almost full, it
	- * may be cheaper to bypass the delay.
	*/
	- if (lwb->lwb_state == LWB_STATE_OPENED && first) {
	- hrtime_t sleep = zilog->zl_last_lwb_latency *
	- zfs_commit_timeout_pct / 100;
	- if (sleep < zil_min_commit_timeout \|\|
	- lwb->lwb_nmax - lwb->lwb_nused <
	- lwb->lwb_nmax / 8) {
	- list_insert_tail(ilwbs, lwb);
	- lwb = zil_lwb_write_close(zilog, lwb,
	- LWB_STATE_NEW);
	- zilog->zl_cur_used = 0;
	- if (lwb == NULL) {
	- while ((lwb = list_remove_head(ilwbs))
	- != NULL)
	- zil_lwb_write_issue(zilog, lwb);
	- zil_commit_writer_stall(zilog);
	- }
	+ if (lwb->lwb_state == LWB_STATE_OPENED && !zilog->zl_parallel) {
	+ zil_burst_done(zilog);
	+ list_insert_tail(ilwbs, lwb);
	+ lwb = zil_lwb_write_close(zilog, lwb, LWB_STATE_NEW);
	+ if (lwb == NULL) {
	+ while ((lwb = list_remove_head(ilwbs)) != NULL)
	+ zil_lwb_write_issue(zilog, lwb);
	+ zil_commit_writer_stall(zilog);
	}
	}
	}
	}

	/*
	* This function is responsible for ensuring the passed in commit waiter
	* (and associated commit itx) is committed to an lwb. If the waiter is
	* not already committed to an lwb, all itxs in the zilog's queue of
	* itxs will be processed. The assumption is the passed in waiter's
	* commit itx will found in the queue just like the other non-commit
	* itxs, such that when the entire queue is processed, the waiter will
	* have been committed to an lwb.
	*
	* The lwb associated with the passed in waiter is not guaranteed to
	* have been issued by the time this function completes. If the lwb is
	* not issued, we rely on future calls to zil_commit_writer() to issue
	* the lwb, or the timeout mechanism found in zil_commit_waiter().
	*/
	static uint64_t
	zil_commit_writer(zilog_t zilog, zil_commit_waiter_t zcw)
	{
	list_t ilwbs;
	lwb_t *lwb;
	uint64_t wtxg = 0;

	ASSERT(!MUTEX_HELD(&zilog->zl_lock));
	ASSERT(spa_writeable(zilog->zl_spa));

	list_create(&ilwbs, sizeof (lwb_t), offsetof(lwb_t, lwb_issue_node));
	mutex_enter(&zilog->zl_issuer_lock);

	if (zcw->zcw_lwb != NULL \|\| zcw->zcw_done) {
	/*
	* It's possible that, while we were waiting to acquire
	* the "zl_issuer_lock", another thread committed this
	* waiter to an lwb. If that occurs, we bail out early,
	* without processing any of the zilog's queue of itxs.
	*
	* On certain workloads and system configurations, the
	* "zl_issuer_lock" can become highly contended. In an
	* attempt to reduce this contention, we immediately drop
	* the lock if the waiter has already been processed.
	*
	* We've measured this optimization to reduce CPU spent
	* contending on this lock by up to 5%, using a system
	* with 32 CPUs, low latency storage (~50 usec writes),
	* and 1024 threads performing sync writes.
	*/
	goto out;
	}

	ZIL_STAT_BUMP(zilog, zil_commit_writer_count);

	wtxg = zil_get_commit_list(zilog);
	zil_prune_commit_list(zilog);
	zil_process_commit_list(zilog, zcw, &ilwbs);

	out:
	mutex_exit(&zilog->zl_issuer_lock);
	while ((lwb = list_remove_head(&ilwbs)) != NULL)
	zil_lwb_write_issue(zilog, lwb);
	list_destroy(&ilwbs);
	return (wtxg);
	}

	static void
	zil_commit_waiter_timeout(zilog_t zilog, zil_commit_waiter_t zcw)
	{
	ASSERT(!MUTEX_HELD(&zilog->zl_issuer_lock));
	ASSERT(MUTEX_HELD(&zcw->zcw_lock));
	ASSERT3B(zcw->zcw_done, ==, B_FALSE);

	lwb_t *lwb = zcw->zcw_lwb;
	ASSERT3P(lwb, !=, NULL);
	ASSERT3S(lwb->lwb_state, !=, LWB_STATE_NEW);

	/*
	* If the lwb has already been issued by another thread, we can
	* immediately return since there's no work to be done (the
	* point of this function is to issue the lwb). Additionally, we
	* do this prior to acquiring the zl_issuer_lock, to avoid
	* acquiring it when it's not necessary to do so.
	*/
	if (lwb->lwb_state != LWB_STATE_OPENED)
	return;

	/*
	* In order to call zil_lwb_write_close() we must hold the
	* zilog's "zl_issuer_lock". We can't simply acquire that lock,
	* since we're already holding the commit waiter's "zcw_lock",
	* and those two locks are acquired in the opposite order
	* elsewhere.
	*/
	mutex_exit(&zcw->zcw_lock);
	mutex_enter(&zilog->zl_issuer_lock);
	mutex_enter(&zcw->zcw_lock);

	/*
	* Since we just dropped and re-acquired the commit waiter's
	* lock, we have to re-check to see if the waiter was marked
	* "done" during that process. If the waiter was marked "done",
	* the "lwb" pointer is no longer valid (it can be free'd after
	* the waiter is marked "done"), so without this check we could
	* wind up with a use-after-free error below.
	*/
	if (zcw->zcw_done) {
	mutex_exit(&zilog->zl_issuer_lock);
	return;
	}

	ASSERT3P(lwb, ==, zcw->zcw_lwb);

	/*
	* We've already checked this above, but since we hadn't acquired
	* the zilog's zl_issuer_lock, we have to perform this check a
	* second time while holding the lock.
	*
	* We don't need to hold the zl_lock since the lwb cannot transition
	* from OPENED to CLOSED while we hold the zl_issuer_lock. The lwb
	* _can_ transition from CLOSED to DONE, but it's OK to race with
	* that transition since we treat the lwb the same, whether it's in
	* the CLOSED, ISSUED or DONE states.
	*
	* The important thing, is we treat the lwb differently depending on
	* if it's OPENED or CLOSED, and block any other threads that might
	* attempt to close/issue this lwb. For that reason we hold the
	* zl_issuer_lock when checking the lwb_state; we must not call
	* zil_lwb_write_close() if the lwb had already been closed/issued.
	*
	* See the comment above the lwb_state_t structure definition for
	* more details on the lwb states, and locking requirements.
	*/
	if (lwb->lwb_state != LWB_STATE_OPENED) {
	mutex_exit(&zilog->zl_issuer_lock);
	return;
	}

	/*
	* We do not need zcw_lock once we hold zl_issuer_lock and know lwb
	* is still open. But we have to drop it to avoid a deadlock in case
	* callback of zio issued by zil_lwb_write_issue() try to get it,
	* while zil_lwb_write_issue() is blocked on attempt to issue next
	* lwb it found in LWB_STATE_READY state.
	*/
	mutex_exit(&zcw->zcw_lock);

	/*
	* As described in the comments above zil_commit_waiter() and
	* zil_process_commit_list(), we need to issue this lwb's zio
	* since we've reached the commit waiter's timeout and it still
	* hasn't been issued.
	*/
	+ zil_burst_done(zilog);
	lwb_t *nlwb = zil_lwb_write_close(zilog, lwb, LWB_STATE_NEW);

	ASSERT3S(lwb->lwb_state, ==, LWB_STATE_CLOSED);

	- /*
	- * Since the lwb's zio hadn't been issued by the time this thread
	- * reached its timeout, we reset the zilog's "zl_cur_used" field
	- * to influence the zil block size selection algorithm.
	- *
	- * By having to issue the lwb's zio here, it means the size of the
	- * lwb was too large, given the incoming throughput of itxs. By
	- * setting "zl_cur_used" to zero, we communicate this fact to the
	- * block size selection algorithm, so it can take this information
	- * into account, and potentially select a smaller size for the
	- * next lwb block that is allocated.
	- */
	- zilog->zl_cur_used = 0;
	-
	if (nlwb == NULL) {
	/*
	* When zil_lwb_write_close() returns NULL, this
	* indicates zio_alloc_zil() failed to allocate the
	* "next" lwb on-disk. When this occurs, the ZIL write
	* pipeline must be stalled; see the comment within the
	* zil_commit_writer_stall() function for more details.
	*/
	zil_lwb_write_issue(zilog, lwb);
	zil_commit_writer_stall(zilog);
	mutex_exit(&zilog->zl_issuer_lock);
	} else {
	mutex_exit(&zilog->zl_issuer_lock);
	zil_lwb_write_issue(zilog, lwb);
	}
	mutex_enter(&zcw->zcw_lock);
	}

	/*
	* This function is responsible for performing the following two tasks:
	*
	* 1. its primary responsibility is to block until the given "commit
	* waiter" is considered "done".
	*
	* 2. its secondary responsibility is to issue the zio for the lwb that
	* the given "commit waiter" is waiting on, if this function has
	* waited "long enough" and the lwb is still in the "open" state.
	*
	* Given a sufficient amount of itxs being generated and written using
	* the ZIL, the lwb's zio will be issued via the zil_lwb_assign()
	* function. If this does not occur, this secondary responsibility will
	* ensure the lwb is issued even if there is not other synchronous
	* activity on the system.
	*
	* For more details, see zil_process_commit_list(); more specifically,
	* the comment at the bottom of that function.
	*/
	static void
	zil_commit_waiter(zilog_t zilog, zil_commit_waiter_t zcw)
	{
	ASSERT(!MUTEX_HELD(&zilog->zl_lock));
	ASSERT(!MUTEX_HELD(&zilog->zl_issuer_lock));
	ASSERT(spa_writeable(zilog->zl_spa));

	mutex_enter(&zcw->zcw_lock);

	/*
	* The timeout is scaled based on the lwb latency to avoid
	* significantly impacting the latency of each individual itx.
	* For more details, see the comment at the bottom of the
	* zil_process_commit_list() function.
	*/
	int pct = MAX(zfs_commit_timeout_pct, 1);
	hrtime_t sleep = (zilog->zl_last_lwb_latency * pct) / 100;
	hrtime_t wakeup = gethrtime() + sleep;
	boolean_t timedout = B_FALSE;

	while (!zcw->zcw_done) {
	ASSERT(MUTEX_HELD(&zcw->zcw_lock));

	lwb_t *lwb = zcw->zcw_lwb;

	/*
	* Usually, the waiter will have a non-NULL lwb field here,
	* but it's possible for it to be NULL as a result of
	* zil_commit() racing with spa_sync().
	*
	* When zil_clean() is called, it's possible for the itxg
	* list (which may be cleaned via a taskq) to contain
	* commit itxs. When this occurs, the commit waiters linked
	* off of these commit itxs will not be committed to an
	* lwb. Additionally, these commit waiters will not be
	* marked done until zil_commit_waiter_skip() is called via
	* zil_itxg_clean().
	*
	* Thus, it's possible for this commit waiter (i.e. the
	* "zcw" variable) to be found in this "in between" state;
	* where it's "zcw_lwb" field is NULL, and it hasn't yet
	* been skipped, so it's "zcw_done" field is still B_FALSE.
	*/
	IMPLY(lwb != NULL, lwb->lwb_state != LWB_STATE_NEW);

	if (lwb != NULL && lwb->lwb_state == LWB_STATE_OPENED) {
	ASSERT3B(timedout, ==, B_FALSE);

	/*
	* If the lwb hasn't been issued yet, then we
	* need to wait with a timeout, in case this
	* function needs to issue the lwb after the
	* timeout is reached; responsibility (2) from
	* the comment above this function.
	*/
	int rc = cv_timedwait_hires(&zcw->zcw_cv,
	&zcw->zcw_lock, wakeup, USEC2NSEC(1),
	CALLOUT_FLAG_ABSOLUTE);

	if (rc != -1 \|\| zcw->zcw_done)
	continue;

	timedout = B_TRUE;
	zil_commit_waiter_timeout(zilog, zcw);

	if (!zcw->zcw_done) {
	/*
	* If the commit waiter has already been
	* marked "done", it's possible for the
	* waiter's lwb structure to have already
	* been freed. Thus, we can only reliably
	* make these assertions if the waiter
	* isn't done.
	*/
	ASSERT3P(lwb, ==, zcw->zcw_lwb);
	ASSERT3S(lwb->lwb_state, !=, LWB_STATE_OPENED);
	}
	} else {
	/*
	* If the lwb isn't open, then it must have already
	* been issued. In that case, there's no need to
	* use a timeout when waiting for the lwb to
	* complete.
	*
	* Additionally, if the lwb is NULL, the waiter
	* will soon be signaled and marked done via
	* zil_clean() and zil_itxg_clean(), so no timeout
	* is required.
	*/

	IMPLY(lwb != NULL,
	lwb->lwb_state == LWB_STATE_CLOSED \|\|
	lwb->lwb_state == LWB_STATE_READY \|\|
	lwb->lwb_state == LWB_STATE_ISSUED \|\|
	lwb->lwb_state == LWB_STATE_WRITE_DONE \|\|
	lwb->lwb_state == LWB_STATE_FLUSH_DONE);
	cv_wait(&zcw->zcw_cv, &zcw->zcw_lock);
	}
	}

	mutex_exit(&zcw->zcw_lock);
	}

	static zil_commit_waiter_t *
	zil_alloc_commit_waiter(void)
	{
	zil_commit_waiter_t *zcw = kmem_cache_alloc(zil_zcw_cache, KM_SLEEP);

	cv_init(&zcw->zcw_cv, NULL, CV_DEFAULT, NULL);
	mutex_init(&zcw->zcw_lock, NULL, MUTEX_DEFAULT, NULL);
	list_link_init(&zcw->zcw_node);
	zcw->zcw_lwb = NULL;
	zcw->zcw_done = B_FALSE;
	zcw->zcw_zio_error = 0;

	return (zcw);
	}

	static void
	zil_free_commit_waiter(zil_commit_waiter_t *zcw)
	{
	ASSERT(!list_link_active(&zcw->zcw_node));
	ASSERT3P(zcw->zcw_lwb, ==, NULL);
	ASSERT3B(zcw->zcw_done, ==, B_TRUE);
	mutex_destroy(&zcw->zcw_lock);
	cv_destroy(&zcw->zcw_cv);
	kmem_cache_free(zil_zcw_cache, zcw);
	}

	/*
	* This function is used to create a TX_COMMIT itx and assign it. This
	* way, it will be linked into the ZIL's list of synchronous itxs, and
	* then later committed to an lwb (or skipped) when
	* zil_process_commit_list() is called.
	*/
	static void
	zil_commit_itx_assign(zilog_t zilog, zil_commit_waiter_t zcw)
	{
	dmu_tx_t *tx = dmu_tx_create(zilog->zl_os);

	/*
	* Since we are not going to create any new dirty data, and we
	* can even help with clearing the existing dirty data, we
	* should not be subject to the dirty data based delays. We
	* use TXG_NOTHROTTLE to bypass the delay mechanism.
	*/
	VERIFY0(dmu_tx_assign(tx, TXG_WAIT \| TXG_NOTHROTTLE));

	itx_t *itx = zil_itx_create(TX_COMMIT, sizeof (lr_t));
	itx->itx_sync = B_TRUE;
	itx->itx_private = zcw;

	zil_itx_assign(zilog, itx, tx);

	dmu_tx_commit(tx);
	}

	/*
	* Commit ZFS Intent Log transactions (itxs) to stable storage.
	*
	* When writing ZIL transactions to the on-disk representation of the
	* ZIL, the itxs are committed to a Log Write Block (lwb). Multiple
	* itxs can be committed to a single lwb. Once a lwb is written and
	* committed to stable storage (i.e. the lwb is written, and vdevs have
	* been flushed), each itx that was committed to that lwb is also
	* considered to be committed to stable storage.
	*
	* When an itx is committed to an lwb, the log record (lr_t) contained
	* by the itx is copied into the lwb's zio buffer, and once this buffer
	* is written to disk, it becomes an on-disk ZIL block.
	*
	* As itxs are generated, they're inserted into the ZIL's queue of
	* uncommitted itxs. The semantics of zil_commit() are such that it will
	* block until all itxs that were in the queue when it was called, are
	* committed to stable storage.
	*
	* If "foid" is zero, this means all "synchronous" and "asynchronous"
	* itxs, for all objects in the dataset, will be committed to stable
	* storage prior to zil_commit() returning. If "foid" is non-zero, all
	* "synchronous" itxs for all objects, but only "asynchronous" itxs
	* that correspond to the foid passed in, will be committed to stable
	* storage prior to zil_commit() returning.
	*
	* Generally speaking, when zil_commit() is called, the consumer doesn't
	* actually care about _all_ of the uncommitted itxs. Instead, they're
	* simply trying to waiting for a specific itx to be committed to disk,
	* but the interface(s) for interacting with the ZIL don't allow such
	* fine-grained communication. A better interface would allow a consumer
	* to create and assign an itx, and then pass a reference to this itx to
	* zil_commit(); such that zil_commit() would return as soon as that
	* specific itx was committed to disk (instead of waiting for _all_
	* itxs to be committed).
	*
	* When a thread calls zil_commit() a special "commit itx" will be
	* generated, along with a corresponding "waiter" for this commit itx.
	* zil_commit() will wait on this waiter's CV, such that when the waiter
	* is marked done, and signaled, zil_commit() will return.
	*
	* This commit itx is inserted into the queue of uncommitted itxs. This
	* provides an easy mechanism for determining which itxs were in the
	* queue prior to zil_commit() having been called, and which itxs were
	* added after zil_commit() was called.
	*
	* The commit itx is special; it doesn't have any on-disk representation.
	* When a commit itx is "committed" to an lwb, the waiter associated
	* with it is linked onto the lwb's list of waiters. Then, when that lwb
	* completes, each waiter on the lwb's list is marked done and signaled
	* -- allowing the thread waiting on the waiter to return from zil_commit().
	*
	* It's important to point out a few critical factors that allow us
	* to make use of the commit itxs, commit waiters, per-lwb lists of
	* commit waiters, and zio completion callbacks like we're doing:
	*
	* 1. The list of waiters for each lwb is traversed, and each commit
	* waiter is marked "done" and signaled, in the zio completion
	* callback of the lwb's zio[*].
	*
	* * Actually, the waiters are signaled in the zio completion
	* callback of the root zio for the DKIOCFLUSHWRITECACHE commands
	* that are sent to the vdevs upon completion of the lwb zio.
	*
	* 2. When the itxs are inserted into the ZIL's queue of uncommitted
	* itxs, the order in which they are inserted is preserved[*]; as
	* itxs are added to the queue, they are added to the tail of
	* in-memory linked lists.
	*
	* When committing the itxs to lwbs (to be written to disk), they
	* are committed in the same order in which the itxs were added to
	* the uncommitted queue's linked list(s); i.e. the linked list of
	* itxs to commit is traversed from head to tail, and each itx is
	* committed to an lwb in that order.
	*
	* * To clarify:
	*
	* - the order of "sync" itxs is preserved w.r.t. other
	* "sync" itxs, regardless of the corresponding objects.
	* - the order of "async" itxs is preserved w.r.t. other
	* "async" itxs corresponding to the same object.
	* - the order of "async" itxs is not preserved w.r.t. other
	* "async" itxs corresponding to different objects.
	* - the order of "sync" itxs w.r.t. "async" itxs (or vice
	* versa) is not preserved, even for itxs that correspond
	* to the same object.
	*
	* For more details, see: zil_itx_assign(), zil_async_to_sync(),
	* zil_get_commit_list(), and zil_process_commit_list().
	*
	* 3. The lwbs represent a linked list of blocks on disk. Thus, any
	* lwb cannot be considered committed to stable storage, until its
	* "previous" lwb is also committed to stable storage. This fact,
	* coupled with the fact described above, means that itxs are
	* committed in (roughly) the order in which they were generated.
	* This is essential because itxs are dependent on prior itxs.
	* Thus, we must not deem an itx as being committed to stable
	* storage, until all prior itxs have also been committed to
	* stable storage.
	*
	* To enforce this ordering of lwb zio's, while still leveraging as
	* much of the underlying storage performance as possible, we rely
	* on two fundamental concepts:
	*
	* 1. The creation and issuance of lwb zio's is protected by
	* the zilog's "zl_issuer_lock", which ensures only a single
	* thread is creating and/or issuing lwb's at a time
	* 2. The "previous" lwb is a child of the "current" lwb
	* (leveraging the zio parent-child dependency graph)
	*
	* By relying on this parent-child zio relationship, we can have
	* many lwb zio's concurrently issued to the underlying storage,
	* but the order in which they complete will be the same order in
	* which they were created.
	*/
	void
	zil_commit(zilog_t *zilog, uint64_t foid)
	{
	/*
	* We should never attempt to call zil_commit on a snapshot for
	* a couple of reasons:
	*
	* 1. A snapshot may never be modified, thus it cannot have any
	* in-flight itxs that would have modified the dataset.
	*
	* 2. By design, when zil_commit() is called, a commit itx will
	* be assigned to this zilog; as a result, the zilog will be
	* dirtied. We must not dirty the zilog of a snapshot; there's
	* checks in the code that enforce this invariant, and will
	* cause a panic if it's not upheld.
	*/
	ASSERT3B(dmu_objset_is_snapshot(zilog->zl_os), ==, B_FALSE);

	if (zilog->zl_sync == ZFS_SYNC_DISABLED)
	return;

	if (!spa_writeable(zilog->zl_spa)) {
	/*
	* If the SPA is not writable, there should never be any
	* pending itxs waiting to be committed to disk. If that
	* weren't true, we'd skip writing those itxs out, and
	* would break the semantics of zil_commit(); thus, we're
	* verifying that truth before we return to the caller.
	*/
	ASSERT(list_is_empty(&zilog->zl_lwb_list));
	ASSERT3P(zilog->zl_last_lwb_opened, ==, NULL);
	for (int i = 0; i < TXG_SIZE; i++)
	ASSERT3P(zilog->zl_itxg[i].itxg_itxs, ==, NULL);
	return;
	}

	/*
	* If the ZIL is suspended, we don't want to dirty it by calling
	* zil_commit_itx_assign() below, nor can we write out
	* lwbs like would be done in zil_commit_write(). Thus, we
	* simply rely on txg_wait_synced() to maintain the necessary
	* semantics, and avoid calling those functions altogether.
	*/
	if (zilog->zl_suspend > 0) {
	txg_wait_synced(zilog->zl_dmu_pool, 0);
	return;
	}

	zil_commit_impl(zilog, foid);
	}

	void
	zil_commit_impl(zilog_t *zilog, uint64_t foid)
	{
	ZIL_STAT_BUMP(zilog, zil_commit_count);

	/*
	* Move the "async" itxs for the specified foid to the "sync"
	* queues, such that they will be later committed (or skipped)
	* to an lwb when zil_process_commit_list() is called.
	*
	* Since these "async" itxs must be committed prior to this
	* call to zil_commit returning, we must perform this operation
	* before we call zil_commit_itx_assign().
	*/
	zil_async_to_sync(zilog, foid);

	/*
	* We allocate a new "waiter" structure which will initially be
	* linked to the commit itx using the itx's "itx_private" field.
	* Since the commit itx doesn't represent any on-disk state,
	* when it's committed to an lwb, rather than copying the its
	* lr_t into the lwb's buffer, the commit itx's "waiter" will be
	* added to the lwb's list of waiters. Then, when the lwb is
	* committed to stable storage, each waiter in the lwb's list of
	* waiters will be marked "done", and signalled.
	*
	* We must create the waiter and assign the commit itx prior to
	* calling zil_commit_writer(), or else our specific commit itx
	* is not guaranteed to be committed to an lwb prior to calling
	* zil_commit_waiter().
	*/
	zil_commit_waiter_t *zcw = zil_alloc_commit_waiter();
	zil_commit_itx_assign(zilog, zcw);

	uint64_t wtxg = zil_commit_writer(zilog, zcw);
	zil_commit_waiter(zilog, zcw);

	if (zcw->zcw_zio_error != 0) {
	/*
	* If there was an error writing out the ZIL blocks that
	* this thread is waiting on, then we fallback to
	* relying on spa_sync() to write out the data this
	* thread is waiting on. Obviously this has performance
	* implications, but the expectation is for this to be
	* an exceptional case, and shouldn't occur often.
	*/
	DTRACE_PROBE2(zil__commit__io__error,
	zilog_t , zilog, zil_commit_waiter_t , zcw);
	txg_wait_synced(zilog->zl_dmu_pool, 0);
	} else if (wtxg != 0) {
	txg_wait_synced(zilog->zl_dmu_pool, wtxg);
	}

	zil_free_commit_waiter(zcw);
	}

	/*
	* Called in syncing context to free committed log blocks and update log header.
	*/
	void
	zil_sync(zilog_t zilog, dmu_tx_t tx)
	{
	zil_header_t *zh = zil_header_in_syncing_context(zilog);
	uint64_t txg = dmu_tx_get_txg(tx);
	spa_t *spa = zilog->zl_spa;
	uint64_t *replayed_seq = &zilog->zl_replayed_seq[txg & TXG_MASK];
	lwb_t *lwb;

	/*
	* We don't zero out zl_destroy_txg, so make sure we don't try
	* to destroy it twice.
	*/
	if (spa_sync_pass(spa) != 1)
	return;

	zil_lwb_flush_wait_all(zilog, txg);

	mutex_enter(&zilog->zl_lock);

	ASSERT(zilog->zl_stop_sync == 0);

	if (*replayed_seq != 0) {
	ASSERT(zh->zh_replay_seq < *replayed_seq);
	zh->zh_replay_seq = *replayed_seq;
	*replayed_seq = 0;
	}

	if (zilog->zl_destroy_txg == txg) {
	blkptr_t blk = zh->zh_log;
	dsl_dataset_t *ds = dmu_objset_ds(zilog->zl_os);

	ASSERT(list_is_empty(&zilog->zl_lwb_list));

	memset(zh, 0, sizeof (zil_header_t));
	memset(zilog->zl_replayed_seq, 0,
	sizeof (zilog->zl_replayed_seq));

	if (zilog->zl_keep_first) {
	/*
	* If this block was part of log chain that couldn't
	* be claimed because a device was missing during
	* zil_claim(), but that device later returns,
	* then this block could erroneously appear valid.
	* To guard against this, assign a new GUID to the new
	* log chain so it doesn't matter what blk points to.
	*/
	zil_init_log_chain(zilog, &blk);
	zh->zh_log = blk;
	} else {
	/*
	* A destroyed ZIL chain can't contain any TX_SETSAXATTR
	* records. So, deactivate the feature for this dataset.
	* We activate it again when we start a new ZIL chain.
	*/
	if (dsl_dataset_feature_is_active(ds,
	SPA_FEATURE_ZILSAXATTR))
	dsl_dataset_deactivate_feature(ds,
	SPA_FEATURE_ZILSAXATTR, tx);
	}
	}

	while ((lwb = list_head(&zilog->zl_lwb_list)) != NULL) {
	zh->zh_log = lwb->lwb_blk;
	if (lwb->lwb_state != LWB_STATE_FLUSH_DONE \|\|
	lwb->lwb_alloc_txg > txg \|\| lwb->lwb_max_txg > txg)
	break;
	list_remove(&zilog->zl_lwb_list, lwb);
	if (!BP_IS_HOLE(&lwb->lwb_blk))
	zio_free(spa, txg, &lwb->lwb_blk);
	zil_free_lwb(zilog, lwb);

	/*
	* If we don't have anything left in the lwb list then
	* we've had an allocation failure and we need to zero
	* out the zil_header blkptr so that we don't end
	* up freeing the same block twice.
	*/
	if (list_is_empty(&zilog->zl_lwb_list))
	BP_ZERO(&zh->zh_log);
	}

	mutex_exit(&zilog->zl_lock);
	}

	static int
	zil_lwb_cons(void vbuf, void unused, int kmflag)
	{
	(void) unused, (void) kmflag;
	lwb_t *lwb = vbuf;
	list_create(&lwb->lwb_itxs, sizeof (itx_t), offsetof(itx_t, itx_node));
	list_create(&lwb->lwb_waiters, sizeof (zil_commit_waiter_t),
	offsetof(zil_commit_waiter_t, zcw_node));
	avl_create(&lwb->lwb_vdev_tree, zil_lwb_vdev_compare,
	sizeof (zil_vdev_node_t), offsetof(zil_vdev_node_t, zv_node));
	mutex_init(&lwb->lwb_vdev_lock, NULL, MUTEX_DEFAULT, NULL);
	return (0);
	}

	static void
	zil_lwb_dest(void vbuf, void unused)
	{
	(void) unused;
	lwb_t *lwb = vbuf;
	mutex_destroy(&lwb->lwb_vdev_lock);
	avl_destroy(&lwb->lwb_vdev_tree);
	list_destroy(&lwb->lwb_waiters);
	list_destroy(&lwb->lwb_itxs);
	}

	void
	zil_init(void)
	{
	zil_lwb_cache = kmem_cache_create("zil_lwb_cache",
	sizeof (lwb_t), 0, zil_lwb_cons, zil_lwb_dest, NULL, NULL, NULL, 0);

	zil_zcw_cache = kmem_cache_create("zil_zcw_cache",
	sizeof (zil_commit_waiter_t), 0, NULL, NULL, NULL, NULL, NULL, 0);

	zil_sums_init(&zil_sums_global);
	zil_kstats_global = kstat_create("zfs", 0, "zil", "misc",
	KSTAT_TYPE_NAMED, sizeof (zil_stats) / sizeof (kstat_named_t),
	KSTAT_FLAG_VIRTUAL);

	if (zil_kstats_global != NULL) {
	zil_kstats_global->ks_data = &zil_stats;
	zil_kstats_global->ks_update = zil_kstats_global_update;
	zil_kstats_global->ks_private = NULL;
	kstat_install(zil_kstats_global);
	}
	}

	void
	zil_fini(void)
	{
	kmem_cache_destroy(zil_zcw_cache);
	kmem_cache_destroy(zil_lwb_cache);

	if (zil_kstats_global != NULL) {
	kstat_delete(zil_kstats_global);
	zil_kstats_global = NULL;
	}

	zil_sums_fini(&zil_sums_global);
	}

	void
	zil_set_sync(zilog_t *zilog, uint64_t sync)
	{
	zilog->zl_sync = sync;
	}

	void
	zil_set_logbias(zilog_t *zilog, uint64_t logbias)
	{
	zilog->zl_logbias = logbias;
	}

	zilog_t *
	zil_alloc(objset_t os, zil_header_t zh_phys)
	{
	zilog_t *zilog;

	zilog = kmem_zalloc(sizeof (zilog_t), KM_SLEEP);

	zilog->zl_header = zh_phys;
	zilog->zl_os = os;
	zilog->zl_spa = dmu_objset_spa(os);
	zilog->zl_dmu_pool = dmu_objset_pool(os);
	zilog->zl_destroy_txg = TXG_INITIAL - 1;
	zilog->zl_logbias = dmu_objset_logbias(os);
	zilog->zl_sync = dmu_objset_syncprop(os);
	zilog->zl_dirty_max_txg = 0;
	zilog->zl_last_lwb_opened = NULL;
	zilog->zl_last_lwb_latency = 0;
	- zilog->zl_max_block_size = zil_maxblocksize;
	+ zilog->zl_max_block_size = MIN(MAX(P2ALIGN_TYPED(zil_maxblocksize,
	+ ZIL_MIN_BLKSZ, uint64_t), ZIL_MIN_BLKSZ),
	+ spa_maxblocksize(dmu_objset_spa(os)));

	mutex_init(&zilog->zl_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&zilog->zl_issuer_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&zilog->zl_lwb_io_lock, NULL, MUTEX_DEFAULT, NULL);

	for (int i = 0; i < TXG_SIZE; i++) {
	mutex_init(&zilog->zl_itxg[i].itxg_lock, NULL,
	MUTEX_DEFAULT, NULL);
	}

	list_create(&zilog->zl_lwb_list, sizeof (lwb_t),
	offsetof(lwb_t, lwb_node));

	list_create(&zilog->zl_itx_commit_list, sizeof (itx_t),
	offsetof(itx_t, itx_node));

	cv_init(&zilog->zl_cv_suspend, NULL, CV_DEFAULT, NULL);
	cv_init(&zilog->zl_lwb_io_cv, NULL, CV_DEFAULT, NULL);

	+ for (int i = 0; i < ZIL_BURSTS; i++) {
	+ zilog->zl_prev_opt[i] = zilog->zl_max_block_size -
	+ sizeof (zil_chain_t);
	+ }
	+
	return (zilog);
	}

	void
	zil_free(zilog_t *zilog)
	{
	int i;

	zilog->zl_stop_sync = 1;

	ASSERT0(zilog->zl_suspend);
	ASSERT0(zilog->zl_suspending);

	ASSERT(list_is_empty(&zilog->zl_lwb_list));
	list_destroy(&zilog->zl_lwb_list);

	ASSERT(list_is_empty(&zilog->zl_itx_commit_list));
	list_destroy(&zilog->zl_itx_commit_list);

	for (i = 0; i < TXG_SIZE; i++) {
	/*
	* It's possible for an itx to be generated that doesn't dirty
	* a txg (e.g. ztest TX_TRUNCATE). So there's no zil_clean()
	* callback to remove the entry. We remove those here.
	*
	* Also free up the ziltest itxs.
	*/
	if (zilog->zl_itxg[i].itxg_itxs)
	zil_itxg_clean(zilog->zl_itxg[i].itxg_itxs);
	mutex_destroy(&zilog->zl_itxg[i].itxg_lock);
	}

	mutex_destroy(&zilog->zl_issuer_lock);
	mutex_destroy(&zilog->zl_lock);
	mutex_destroy(&zilog->zl_lwb_io_lock);

	cv_destroy(&zilog->zl_cv_suspend);
	cv_destroy(&zilog->zl_lwb_io_cv);

	kmem_free(zilog, sizeof (zilog_t));
	}

	/*
	* Open an intent log.
	*/
	zilog_t *
	zil_open(objset_t os, zil_get_data_t get_data, zil_sums_t *zil_sums)
	{
	zilog_t *zilog = dmu_objset_zil(os);

	ASSERT3P(zilog->zl_get_data, ==, NULL);
	ASSERT3P(zilog->zl_last_lwb_opened, ==, NULL);
	ASSERT(list_is_empty(&zilog->zl_lwb_list));

	zilog->zl_get_data = get_data;
	zilog->zl_sums = zil_sums;

	return (zilog);
	}

	/*
	* Close an intent log.
	*/
	void
	zil_close(zilog_t *zilog)
	{
	lwb_t *lwb;
	uint64_t txg;

	if (!dmu_objset_is_snapshot(zilog->zl_os)) {
	zil_commit(zilog, 0);
	} else {
	ASSERT(list_is_empty(&zilog->zl_lwb_list));
	ASSERT0(zilog->zl_dirty_max_txg);
	ASSERT3B(zilog_is_dirty(zilog), ==, B_FALSE);
	}

	mutex_enter(&zilog->zl_lock);
	txg = zilog->zl_dirty_max_txg;
	lwb = list_tail(&zilog->zl_lwb_list);
	if (lwb != NULL) {
	txg = MAX(txg, lwb->lwb_alloc_txg);
	txg = MAX(txg, lwb->lwb_max_txg);
	}
	mutex_exit(&zilog->zl_lock);

	/*
	* zl_lwb_max_issued_txg may be larger than lwb_max_txg. It depends
	* on the time when the dmu_tx transaction is assigned in
	* zil_lwb_write_issue().
	*/
	mutex_enter(&zilog->zl_lwb_io_lock);
	txg = MAX(zilog->zl_lwb_max_issued_txg, txg);
	mutex_exit(&zilog->zl_lwb_io_lock);

	/*
	* We need to use txg_wait_synced() to wait until that txg is synced.
	* zil_sync() will guarantee all lwbs up to that txg have been
	* written out, flushed, and cleaned.
	*/
	if (txg != 0)
	txg_wait_synced(zilog->zl_dmu_pool, txg);

	if (zilog_is_dirty(zilog))
	zfs_dbgmsg("zil (%px) is dirty, txg %llu", zilog,
	(u_longlong_t)txg);
	if (txg < spa_freeze_txg(zilog->zl_spa))
	VERIFY(!zilog_is_dirty(zilog));

	zilog->zl_get_data = NULL;

	/*
	* We should have only one lwb left on the list; remove it now.
	*/
	mutex_enter(&zilog->zl_lock);
	lwb = list_remove_head(&zilog->zl_lwb_list);
	if (lwb != NULL) {
	ASSERT(list_is_empty(&zilog->zl_lwb_list));
	ASSERT3S(lwb->lwb_state, ==, LWB_STATE_NEW);
	zio_buf_free(lwb->lwb_buf, lwb->lwb_sz);
	zil_free_lwb(zilog, lwb);
	}
	mutex_exit(&zilog->zl_lock);
	}

	static const char *suspend_tag = "zil suspending";

	/*
	* Suspend an intent log. While in suspended mode, we still honor
	* synchronous semantics, but we rely on txg_wait_synced() to do it.
	* On old version pools, we suspend the log briefly when taking a
	* snapshot so that it will have an empty intent log.
	*
	* Long holds are not really intended to be used the way we do here --
	* held for such a short time. A concurrent caller of dsl_dataset_long_held()
	* could fail. Therefore we take pains to only put a long hold if it is
	* actually necessary. Fortunately, it will only be necessary if the
	* objset is currently mounted (or the ZVOL equivalent). In that case it
	* will already have a long hold, so we are not really making things any worse.
	*
	* Ideally, we would locate the existing long-holder (i.e. the zfsvfs_t or
	* zvol_state_t), and use their mechanism to prevent their hold from being
	* dropped (e.g. VFS_HOLD()). However, that would be even more pain for
	* very little gain.
	*
	* if cookiep == NULL, this does both the suspend & resume.
	* Otherwise, it returns with the dataset "long held", and the cookie
	* should be passed into zil_resume().
	*/
	int
	zil_suspend(const char osname, void *cookiep)
	{
	objset_t *os;
	zilog_t *zilog;
	const zil_header_t *zh;
	int error;

	error = dmu_objset_hold(osname, suspend_tag, &os);
	if (error != 0)
	return (error);
	zilog = dmu_objset_zil(os);

	mutex_enter(&zilog->zl_lock);
	zh = zilog->zl_header;

	if (zh->zh_flags & ZIL_REPLAY_NEEDED) { /* unplayed log */
	mutex_exit(&zilog->zl_lock);
	dmu_objset_rele(os, suspend_tag);
	return (SET_ERROR(EBUSY));
	}

	/*
	* Don't put a long hold in the cases where we can avoid it. This
	* is when there is no cookie so we are doing a suspend & resume
	* (i.e. called from zil_vdev_offline()), and there's nothing to do
	* for the suspend because it's already suspended, or there's no ZIL.
	*/
	if (cookiep == NULL && !zilog->zl_suspending &&
	(zilog->zl_suspend > 0 \|\| BP_IS_HOLE(&zh->zh_log))) {
	mutex_exit(&zilog->zl_lock);
	dmu_objset_rele(os, suspend_tag);
	return (0);
	}

	dsl_dataset_long_hold(dmu_objset_ds(os), suspend_tag);
	dsl_pool_rele(dmu_objset_pool(os), suspend_tag);

	zilog->zl_suspend++;

	if (zilog->zl_suspend > 1) {
	/*
	* Someone else is already suspending it.
	* Just wait for them to finish.
	*/

	while (zilog->zl_suspending)
	cv_wait(&zilog->zl_cv_suspend, &zilog->zl_lock);
	mutex_exit(&zilog->zl_lock);

	if (cookiep == NULL)
	zil_resume(os);
	else
	*cookiep = os;
	return (0);
	}

	/*
	* If there is no pointer to an on-disk block, this ZIL must not
	* be active (e.g. filesystem not mounted), so there's nothing
	* to clean up.
	*/
	if (BP_IS_HOLE(&zh->zh_log)) {
	ASSERT(cookiep != NULL); /* fast path already handled */

	*cookiep = os;
	mutex_exit(&zilog->zl_lock);
	return (0);
	}

	/*
	* The ZIL has work to do. Ensure that the associated encryption
	* key will remain mapped while we are committing the log by
	* grabbing a reference to it. If the key isn't loaded we have no
	* choice but to return an error until the wrapping key is loaded.
	*/
	if (os->os_encrypted &&
	dsl_dataset_create_key_mapping(dmu_objset_ds(os)) != 0) {
	zilog->zl_suspend--;
	mutex_exit(&zilog->zl_lock);
	dsl_dataset_long_rele(dmu_objset_ds(os), suspend_tag);
	dsl_dataset_rele(dmu_objset_ds(os), suspend_tag);
	return (SET_ERROR(EACCES));
	}

	zilog->zl_suspending = B_TRUE;
	mutex_exit(&zilog->zl_lock);

	/*
	* We need to use zil_commit_impl to ensure we wait for all
	* LWB_STATE_OPENED, _CLOSED and _READY lwbs to be committed
	* to disk before proceeding. If we used zil_commit instead, it
	* would just call txg_wait_synced(), because zl_suspend is set.
	* txg_wait_synced() doesn't wait for these lwb's to be
	* LWB_STATE_FLUSH_DONE before returning.
	*/
	zil_commit_impl(zilog, 0);

	/*
	* Now that we've ensured all lwb's are LWB_STATE_FLUSH_DONE, we
	* use txg_wait_synced() to ensure the data from the zilog has
	* migrated to the main pool before calling zil_destroy().
	*/
	txg_wait_synced(zilog->zl_dmu_pool, 0);

	zil_destroy(zilog, B_FALSE);

	mutex_enter(&zilog->zl_lock);
	zilog->zl_suspending = B_FALSE;
	cv_broadcast(&zilog->zl_cv_suspend);
	mutex_exit(&zilog->zl_lock);

	if (os->os_encrypted)
	dsl_dataset_remove_key_mapping(dmu_objset_ds(os));

	if (cookiep == NULL)
	zil_resume(os);
	else
	*cookiep = os;
	return (0);
	}

	void
	zil_resume(void *cookie)
	{
	objset_t *os = cookie;
	zilog_t *zilog = dmu_objset_zil(os);

	mutex_enter(&zilog->zl_lock);
	ASSERT(zilog->zl_suspend != 0);
	zilog->zl_suspend--;
	mutex_exit(&zilog->zl_lock);
	dsl_dataset_long_rele(dmu_objset_ds(os), suspend_tag);
	dsl_dataset_rele(dmu_objset_ds(os), suspend_tag);
	}

	typedef struct zil_replay_arg {
	zil_replay_func_t const zr_replay;
	void *zr_arg;
	boolean_t zr_byteswap;
	char *zr_lr;
	} zil_replay_arg_t;

	static int
	zil_replay_error(zilog_t zilog, const lr_t lr, int error)
	{
	char name[ZFS_MAX_DATASET_NAME_LEN];

	zilog->zl_replaying_seq--; /* didn't actually replay this one */

	dmu_objset_name(zilog->zl_os, name);

	cmn_err(CE_WARN, "ZFS replay transaction error %d, "
	"dataset %s, seq 0x%llx, txtype %llu %s\n", error, name,
	(u_longlong_t)lr->lrc_seq,
	(u_longlong_t)(lr->lrc_txtype & ~TX_CI),
	(lr->lrc_txtype & TX_CI) ? "CI" : "");

	return (error);
	}

	static int
	zil_replay_log_record(zilog_t zilog, const lr_t lr, void *zra,
	uint64_t claim_txg)
	{
	zil_replay_arg_t *zr = zra;
	const zil_header_t *zh = zilog->zl_header;
	uint64_t reclen = lr->lrc_reclen;
	uint64_t txtype = lr->lrc_txtype;
	int error = 0;

	zilog->zl_replaying_seq = lr->lrc_seq;

	if (lr->lrc_seq <= zh->zh_replay_seq) /* already replayed */
	return (0);

	if (lr->lrc_txg < claim_txg) /* already committed */
	return (0);

	/* Strip case-insensitive bit, still present in log record */
	txtype &= ~TX_CI;

	if (txtype == 0 \|\| txtype >= TX_MAX_TYPE)
	return (zil_replay_error(zilog, lr, EINVAL));

	/*
	* If this record type can be logged out of order, the object
	* (lr_foid) may no longer exist. That's legitimate, not an error.
	*/
	if (TX_OOO(txtype)) {
	error = dmu_object_info(zilog->zl_os,
	LR_FOID_GET_OBJ(((lr_ooo_t *)lr)->lr_foid), NULL);
	if (error == ENOENT \|\| error == EEXIST)
	return (0);
	}

	/*
	* Make a copy of the data so we can revise and extend it.
	*/
	memcpy(zr->zr_lr, lr, reclen);

	/*
	* If this is a TX_WRITE with a blkptr, suck in the data.
	*/
	if (txtype == TX_WRITE && reclen == sizeof (lr_write_t)) {
	error = zil_read_log_data(zilog, (lr_write_t *)lr,
	zr->zr_lr + reclen);
	if (error != 0)
	return (zil_replay_error(zilog, lr, error));
	}

	/*
	* The log block containing this lr may have been byteswapped
	* so that we can easily examine common fields like lrc_txtype.
	* However, the log is a mix of different record types, and only the
	* replay vectors know how to byteswap their records. Therefore, if
	* the lr was byteswapped, undo it before invoking the replay vector.
	*/
	if (zr->zr_byteswap)
	byteswap_uint64_array(zr->zr_lr, reclen);

	/*
	* We must now do two things atomically: replay this log record,
	* and update the log header sequence number to reflect the fact that
	* we did so. At the end of each replay function the sequence number
	* is updated if we are in replay mode.
	*/
	error = zr->zr_replay[txtype](zr->zr_arg, zr->zr_lr, zr->zr_byteswap);
	if (error != 0) {
	/*
	* The DMU's dnode layer doesn't see removes until the txg
	* commits, so a subsequent claim can spuriously fail with
	* EEXIST. So if we receive any error we try syncing out
	* any removes then retry the transaction. Note that we
	* specify B_FALSE for byteswap now, so we don't do it twice.
	*/
	txg_wait_synced(spa_get_dsl(zilog->zl_spa), 0);
	error = zr->zr_replay[txtype](zr->zr_arg, zr->zr_lr, B_FALSE);
	if (error != 0)
	return (zil_replay_error(zilog, lr, error));
	}
	return (0);
	}

	static int
	zil_incr_blks(zilog_t zilog, const blkptr_t bp, void *arg, uint64_t claim_txg)
	{
	(void) bp, (void) arg, (void) claim_txg;

	zilog->zl_replay_blks++;

	return (0);
	}

	/*
	* If this dataset has a non-empty intent log, replay it and destroy it.
	* Return B_TRUE if there were any entries to replay.
	*/
	boolean_t
	zil_replay(objset_t os, void arg,
	zil_replay_func_t *const replay_func[TX_MAX_TYPE])
	{
	zilog_t *zilog = dmu_objset_zil(os);
	const zil_header_t *zh = zilog->zl_header;
	zil_replay_arg_t zr;

	if ((zh->zh_flags & ZIL_REPLAY_NEEDED) == 0) {
	return (zil_destroy(zilog, B_TRUE));
	}

	zr.zr_replay = replay_func;
	zr.zr_arg = arg;
	zr.zr_byteswap = BP_SHOULD_BYTESWAP(&zh->zh_log);
	zr.zr_lr = vmem_alloc(2 * SPA_MAXBLOCKSIZE, KM_SLEEP);

	/*
	* Wait for in-progress removes to sync before starting replay.
	*/
	txg_wait_synced(zilog->zl_dmu_pool, 0);

	zilog->zl_replay = B_TRUE;
	zilog->zl_replay_time = ddi_get_lbolt();
	ASSERT(zilog->zl_replay_blks == 0);
	(void) zil_parse(zilog, zil_incr_blks, zil_replay_log_record, &zr,
	zh->zh_claim_txg, B_TRUE);
	vmem_free(zr.zr_lr, 2 * SPA_MAXBLOCKSIZE);

	zil_destroy(zilog, B_FALSE);
	txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg);
	zilog->zl_replay = B_FALSE;

	return (B_TRUE);
	}

	boolean_t
	zil_replaying(zilog_t zilog, dmu_tx_t tx)
	{
	if (zilog->zl_sync == ZFS_SYNC_DISABLED)
	return (B_TRUE);

	if (zilog->zl_replay) {
	dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx);
	zilog->zl_replayed_seq[dmu_tx_get_txg(tx) & TXG_MASK] =
	zilog->zl_replaying_seq;
	return (B_TRUE);
	}

	return (B_FALSE);
	}

	int
	zil_reset(const char osname, void arg)
	{
	(void) arg;

	int error = zil_suspend(osname, NULL);
	/* EACCES means crypto key not loaded */
	if ((error == EACCES) \|\| (error == EBUSY))
	return (SET_ERROR(error));
	if (error != 0)
	return (SET_ERROR(EEXIST));
	return (0);
	}

	EXPORT_SYMBOL(zil_alloc);
	EXPORT_SYMBOL(zil_free);
	EXPORT_SYMBOL(zil_open);
	EXPORT_SYMBOL(zil_close);
	EXPORT_SYMBOL(zil_replay);
	EXPORT_SYMBOL(zil_replaying);
	EXPORT_SYMBOL(zil_destroy);
	EXPORT_SYMBOL(zil_destroy_sync);
	EXPORT_SYMBOL(zil_itx_create);
	EXPORT_SYMBOL(zil_itx_destroy);
	EXPORT_SYMBOL(zil_itx_assign);
	EXPORT_SYMBOL(zil_commit);
	EXPORT_SYMBOL(zil_claim);
	EXPORT_SYMBOL(zil_check_log_chain);
	EXPORT_SYMBOL(zil_sync);
	EXPORT_SYMBOL(zil_clean);
	EXPORT_SYMBOL(zil_suspend);
	EXPORT_SYMBOL(zil_resume);
	EXPORT_SYMBOL(zil_lwb_add_block);
	EXPORT_SYMBOL(zil_bp_tree_add);
	EXPORT_SYMBOL(zil_set_sync);
	EXPORT_SYMBOL(zil_set_logbias);
	EXPORT_SYMBOL(zil_sums_init);
	EXPORT_SYMBOL(zil_sums_fini);
	EXPORT_SYMBOL(zil_kstat_values_update);

	ZFS_MODULE_PARAM(zfs, zfs_, commit_timeout_pct, UINT, ZMOD_RW,
	"ZIL block open timeout percentage");

	-ZFS_MODULE_PARAM(zfs_zil, zil_, min_commit_timeout, U64, ZMOD_RW,
	- "Minimum delay we care for ZIL block commit");
	-
	ZFS_MODULE_PARAM(zfs_zil, zil_, replay_disable, INT, ZMOD_RW,
	"Disable intent logging replay");

	ZFS_MODULE_PARAM(zfs_zil, zil_, nocacheflush, INT, ZMOD_RW,
	"Disable ZIL cache flushes");

	ZFS_MODULE_PARAM(zfs_zil, zil_, slog_bulk, U64, ZMOD_RW,
	"Limit in bytes slog sync writes per commit");

	ZFS_MODULE_PARAM(zfs_zil, zil_, maxblocksize, UINT, ZMOD_RW,
	"Limit in bytes of ZIL log block size");

	ZFS_MODULE_PARAM(zfs_zil, zil_, maxcopied, UINT, ZMOD_RW,
	"Limit in bytes WR_COPIED size");
	diff --git a/sys/contrib/openzfs/module/zfs/zio.c b/sys/contrib/openzfs/module/zfs/zio.c
	index d8eb075eef54..046e6d64c1a9 100644
	--- a/sys/contrib/openzfs/module/zfs/zio.c
	+++ b/sys/contrib/openzfs/module/zfs/zio.c
	@@ -1,5222 +1,5218 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2011, 2022 by Delphix. All rights reserved.
	* Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2017, Intel Corporation.
	* Copyright (c) 2019, Klara Inc.
	* Copyright (c) 2019, Allan Jude
	* Copyright (c) 2021, Datto, Inc.
	*/

	#include <sys/sysmacros.h>
	#include <sys/zfs_context.h>
	#include <sys/fm/fs/zfs.h>
	#include <sys/spa.h>
	#include <sys/txg.h>
	#include <sys/spa_impl.h>
	#include <sys/vdev_impl.h>
	#include <sys/vdev_trim.h>
	#include <sys/zio_impl.h>
	#include <sys/zio_compress.h>
	#include <sys/zio_checksum.h>
	#include <sys/dmu_objset.h>
	#include <sys/arc.h>
	#include <sys/brt.h>
	#include <sys/ddt.h>
	#include <sys/blkptr.h>
	#include <sys/zfeature.h>
	#include <sys/dsl_scan.h>
	#include <sys/metaslab_impl.h>
	#include <sys/time.h>
	#include <sys/trace_zfs.h>
	#include <sys/abd.h>
	#include <sys/dsl_crypt.h>
	#include <cityhash.h>

	/*
	* ==========================================================================
	* I/O type descriptions
	* ==========================================================================
	*/
	const char *const zio_type_name[ZIO_TYPES] = {
	/*
	* Note: Linux kernel thread name length is limited
	* so these names will differ from upstream open zfs.
	*/
	"z_null", "z_rd", "z_wr", "z_fr", "z_cl", "z_ioctl", "z_trim"
	};

	int zio_dva_throttle_enabled = B_TRUE;
	static int zio_deadman_log_all = B_FALSE;

	/*
	* ==========================================================================
	* I/O kmem caches
	* ==========================================================================
	*/
	static kmem_cache_t *zio_cache;
	static kmem_cache_t *zio_link_cache;
	kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT];
	kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT];
	#if defined(ZFS_DEBUG) && !defined(_KERNEL)
	static uint64_t zio_buf_cache_allocs[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT];
	static uint64_t zio_buf_cache_frees[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT];
	#endif

	/* Mark IOs as "slow" if they take longer than 30 seconds */
	static uint_t zio_slow_io_ms = (30 * MILLISEC);

	#define BP_SPANB(indblkshift, level) \
	(((uint64_t)1) << ((level) * ((indblkshift) - SPA_BLKPTRSHIFT)))
	#define COMPARE_META_LEVEL 0x80000000ul
	/*
	* The following actions directly effect the spa's sync-to-convergence logic.
	* The values below define the sync pass when we start performing the action.
	* Care should be taken when changing these values as they directly impact
	* spa_sync() performance. Tuning these values may introduce subtle performance
	* pathologies and should only be done in the context of performance analysis.
	* These tunables will eventually be removed and replaced with #defines once
	* enough analysis has been done to determine optimal values.
	*
	* The 'zfs_sync_pass_deferred_free' pass must be greater than 1 to ensure that
	* regular blocks are not deferred.
	*
	* Starting in sync pass 8 (zfs_sync_pass_dont_compress), we disable
	* compression (including of metadata). In practice, we don't have this
	* many sync passes, so this has no effect.
	*
	* The original intent was that disabling compression would help the sync
	* passes to converge. However, in practice disabling compression increases
	* the average number of sync passes, because when we turn compression off, a
	* lot of block's size will change and thus we have to re-allocate (not
	* overwrite) them. It also increases the number of 128KB allocations (e.g.
	* for indirect blocks and spacemaps) because these will not be compressed.
	* The 128K allocations are especially detrimental to performance on highly
	* fragmented systems, which may have very few free segments of this size,
	* and may need to load new metaslabs to satisfy 128K allocations.
	*/

	/* defer frees starting in this pass */
	uint_t zfs_sync_pass_deferred_free = 2;

	/* don't compress starting in this pass */
	static uint_t zfs_sync_pass_dont_compress = 8;

	/* rewrite new bps starting in this pass */
	static uint_t zfs_sync_pass_rewrite = 2;

	/*
	* An allocating zio is one that either currently has the DVA allocate
	* stage set or will have it later in its lifetime.
	*/
	#define IO_IS_ALLOCATING(zio) ((zio)->io_orig_pipeline & ZIO_STAGE_DVA_ALLOCATE)

	/*
	* Enable smaller cores by excluding metadata
	* allocations as well.
	*/
	int zio_exclude_metadata = 0;
	static int zio_requeue_io_start_cut_in_line = 1;

	#ifdef ZFS_DEBUG
	static const int zio_buf_debug_limit = 16384;
	#else
	static const int zio_buf_debug_limit = 0;
	#endif

	static inline void __zio_execute(zio_t *zio);

	static void zio_taskq_dispatch(zio_t *, zio_taskq_type_t, boolean_t);

	void
	zio_init(void)
	{
	size_t c;

	zio_cache = kmem_cache_create("zio_cache",
	sizeof (zio_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
	zio_link_cache = kmem_cache_create("zio_link_cache",
	sizeof (zio_link_t), 0, NULL, NULL, NULL, NULL, NULL, 0);

	/*
	* For small buffers, we want a cache for each multiple of
	* SPA_MINBLOCKSIZE. For larger buffers, we want a cache
	* for each quarter-power of 2.
	*/
	for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) {
	size_t size = (c + 1) << SPA_MINBLOCKSHIFT;
	size_t p2 = size;
	size_t align = 0;
	size_t data_cflags, cflags;

	data_cflags = KMC_NODEBUG;
	cflags = (zio_exclude_metadata \|\| size > zio_buf_debug_limit) ?
	KMC_NODEBUG : 0;

	while (!ISP2(p2))
	p2 &= p2 - 1;

	#ifndef _KERNEL
	/*
	* If we are using watchpoints, put each buffer on its own page,
	* to eliminate the performance overhead of trapping to the
	* kernel when modifying a non-watched buffer that shares the
	* page with a watched buffer.
	*/
	if (arc_watch && !IS_P2ALIGNED(size, PAGESIZE))
	continue;
	/*
	* Here's the problem - on 4K native devices in userland on
	* Linux using O_DIRECT, buffers must be 4K aligned or I/O
	* will fail with EINVAL, causing zdb (and others) to coredump.
	* Since userland probably doesn't need optimized buffer caches,
	* we just force 4K alignment on everything.
	*/
	align = 8 * SPA_MINBLOCKSIZE;
	#else
	if (size < PAGESIZE) {
	align = SPA_MINBLOCKSIZE;
	} else if (IS_P2ALIGNED(size, p2 >> 2)) {
	align = PAGESIZE;
	}
	#endif

	if (align != 0) {
	char name[36];
	if (cflags == data_cflags) {
	/*
	* Resulting kmem caches would be identical.
	* Save memory by creating only one.
	*/
	(void) snprintf(name, sizeof (name),
	"zio_buf_comb_%lu", (ulong_t)size);
	zio_buf_cache[c] = kmem_cache_create(name,
	size, align, NULL, NULL, NULL, NULL, NULL,
	cflags);
	zio_data_buf_cache[c] = zio_buf_cache[c];
	continue;
	}
	(void) snprintf(name, sizeof (name), "zio_buf_%lu",
	(ulong_t)size);
	zio_buf_cache[c] = kmem_cache_create(name, size,
	align, NULL, NULL, NULL, NULL, NULL, cflags);

	(void) snprintf(name, sizeof (name), "zio_data_buf_%lu",
	(ulong_t)size);
	zio_data_buf_cache[c] = kmem_cache_create(name, size,
	align, NULL, NULL, NULL, NULL, NULL, data_cflags);
	}
	}

	while (--c != 0) {
	ASSERT(zio_buf_cache[c] != NULL);
	if (zio_buf_cache[c - 1] == NULL)
	zio_buf_cache[c - 1] = zio_buf_cache[c];

	ASSERT(zio_data_buf_cache[c] != NULL);
	if (zio_data_buf_cache[c - 1] == NULL)
	zio_data_buf_cache[c - 1] = zio_data_buf_cache[c];
	}

	zio_inject_init();

	lz4_init();
	}

	void
	zio_fini(void)
	{
	size_t n = SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT;

	#if defined(ZFS_DEBUG) && !defined(_KERNEL)
	for (size_t i = 0; i < n; i++) {
	if (zio_buf_cache_allocs[i] != zio_buf_cache_frees[i])
	(void) printf("zio_fini: [%d] %llu != %llu\n",
	(int)((i + 1) << SPA_MINBLOCKSHIFT),
	(long long unsigned)zio_buf_cache_allocs[i],
	(long long unsigned)zio_buf_cache_frees[i]);
	}
	#endif

	/*
	* The same kmem cache can show up multiple times in both zio_buf_cache
	* and zio_data_buf_cache. Do a wasteful but trivially correct scan to
	* sort it out.
	*/
	for (size_t i = 0; i < n; i++) {
	kmem_cache_t *cache = zio_buf_cache[i];
	if (cache == NULL)
	continue;
	for (size_t j = i; j < n; j++) {
	if (cache == zio_buf_cache[j])
	zio_buf_cache[j] = NULL;
	if (cache == zio_data_buf_cache[j])
	zio_data_buf_cache[j] = NULL;
	}
	kmem_cache_destroy(cache);
	}

	for (size_t i = 0; i < n; i++) {
	kmem_cache_t *cache = zio_data_buf_cache[i];
	if (cache == NULL)
	continue;
	for (size_t j = i; j < n; j++) {
	if (cache == zio_data_buf_cache[j])
	zio_data_buf_cache[j] = NULL;
	}
	kmem_cache_destroy(cache);
	}

	for (size_t i = 0; i < n; i++) {
	VERIFY3P(zio_buf_cache[i], ==, NULL);
	VERIFY3P(zio_data_buf_cache[i], ==, NULL);
	}

	kmem_cache_destroy(zio_link_cache);
	kmem_cache_destroy(zio_cache);

	zio_inject_fini();

	lz4_fini();
	}

	/*
	* ==========================================================================
	* Allocate and free I/O buffers
	* ==========================================================================
	*/

	#ifdef ZFS_DEBUG
	static const ulong_t zio_buf_canary = (ulong_t)0xdeadc0dedead210b;
	#endif

	/*
	* Use empty space after the buffer to detect overflows.
	*
	* Since zio_init() creates kmem caches only for certain set of buffer sizes,
	* allocations of different sizes may have some unused space after the data.
	* Filling part of that space with a known pattern on allocation and checking
	* it on free should allow us to detect some buffer overflows.
	*/
	static void
	zio_buf_put_canary(ulong_t p, size_t size, kmem_cache_t *cache, size_t c)
	{
	#ifdef ZFS_DEBUG
	size_t off = P2ROUNDUP(size, sizeof (ulong_t));
	ulong_t *canary = p + off / sizeof (ulong_t);
	size_t asize = (c + 1) << SPA_MINBLOCKSHIFT;
	if (c + 1 < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT &&
	cache[c] == cache[c + 1])
	asize = (c + 2) << SPA_MINBLOCKSHIFT;
	for (; off < asize; canary++, off += sizeof (ulong_t))
	*canary = zio_buf_canary;
	#endif
	}

	static void
	zio_buf_check_canary(ulong_t p, size_t size, kmem_cache_t *cache, size_t c)
	{
	#ifdef ZFS_DEBUG
	size_t off = P2ROUNDUP(size, sizeof (ulong_t));
	ulong_t *canary = p + off / sizeof (ulong_t);
	size_t asize = (c + 1) << SPA_MINBLOCKSHIFT;
	if (c + 1 < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT &&
	cache[c] == cache[c + 1])
	asize = (c + 2) << SPA_MINBLOCKSHIFT;
	for (; off < asize; canary++, off += sizeof (ulong_t)) {
	if (unlikely(*canary != zio_buf_canary)) {
	PANIC("ZIO buffer overflow %p (%zu) + %zu %#lx != %#lx",
	p, size, (canary - p) * sizeof (ulong_t),
	*canary, zio_buf_canary);
	}
	}
	#endif
	}

	/*
	* Use zio_buf_alloc to allocate ZFS metadata. This data will appear in a
	* crashdump if the kernel panics, so use it judiciously. Obviously, it's
	* useful to inspect ZFS metadata, but if possible, we should avoid keeping
	* excess / transient data in-core during a crashdump.
	*/
	void *
	zio_buf_alloc(size_t size)
	{
	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;

	VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
	#if defined(ZFS_DEBUG) && !defined(_KERNEL)
	atomic_add_64(&zio_buf_cache_allocs[c], 1);
	#endif

	void *p = kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE);
	zio_buf_put_canary(p, size, zio_buf_cache, c);
	return (p);
	}

	/*
	* Use zio_data_buf_alloc to allocate data. The data will not appear in a
	* crashdump if the kernel panics. This exists so that we will limit the amount
	* of ZFS data that shows up in a kernel crashdump. (Thus reducing the amount
	* of kernel heap dumped to disk when the kernel panics)
	*/
	void *
	zio_data_buf_alloc(size_t size)
	{
	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;

	VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);

	void *p = kmem_cache_alloc(zio_data_buf_cache[c], KM_PUSHPAGE);
	zio_buf_put_canary(p, size, zio_data_buf_cache, c);
	return (p);
	}

	void
	zio_buf_free(void *buf, size_t size)
	{
	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;

	VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
	#if defined(ZFS_DEBUG) && !defined(_KERNEL)
	atomic_add_64(&zio_buf_cache_frees[c], 1);
	#endif

	zio_buf_check_canary(buf, size, zio_buf_cache, c);
	kmem_cache_free(zio_buf_cache[c], buf);
	}

	void
	zio_data_buf_free(void *buf, size_t size)
	{
	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;

	VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);

	zio_buf_check_canary(buf, size, zio_data_buf_cache, c);
	kmem_cache_free(zio_data_buf_cache[c], buf);
	}

	static void
	zio_abd_free(void *abd, size_t size)
	{
	(void) size;
	abd_free((abd_t *)abd);
	}

	/*
	* ==========================================================================
	* Push and pop I/O transform buffers
	* ==========================================================================
	*/
	void
	zio_push_transform(zio_t zio, abd_t data, uint64_t size, uint64_t bufsize,
	zio_transform_func_t *transform)
	{
	zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP);

	zt->zt_orig_abd = zio->io_abd;
	zt->zt_orig_size = zio->io_size;
	zt->zt_bufsize = bufsize;
	zt->zt_transform = transform;

	zt->zt_next = zio->io_transform_stack;
	zio->io_transform_stack = zt;

	zio->io_abd = data;
	zio->io_size = size;
	}

	void
	zio_pop_transforms(zio_t *zio)
	{
	zio_transform_t *zt;

	while ((zt = zio->io_transform_stack) != NULL) {
	if (zt->zt_transform != NULL)
	zt->zt_transform(zio,
	zt->zt_orig_abd, zt->zt_orig_size);

	if (zt->zt_bufsize != 0)
	abd_free(zio->io_abd);

	zio->io_abd = zt->zt_orig_abd;
	zio->io_size = zt->zt_orig_size;
	zio->io_transform_stack = zt->zt_next;

	kmem_free(zt, sizeof (zio_transform_t));
	}
	}

	/*
	* ==========================================================================
	* I/O transform callbacks for subblocks, decompression, and decryption
	* ==========================================================================
	*/
	static void
	zio_subblock(zio_t zio, abd_t data, uint64_t size)
	{
	ASSERT(zio->io_size > size);

	if (zio->io_type == ZIO_TYPE_READ)
	abd_copy(data, zio->io_abd, size);
	}

	static void
	zio_decompress(zio_t zio, abd_t data, uint64_t size)
	{
	if (zio->io_error == 0) {
	void *tmp = abd_borrow_buf(data, size);
	int ret = zio_decompress_data(BP_GET_COMPRESS(zio->io_bp),
	zio->io_abd, tmp, zio->io_size, size,
	&zio->io_prop.zp_complevel);
	abd_return_buf_copy(data, tmp, size);

	if (zio_injection_enabled && ret == 0)
	ret = zio_handle_fault_injection(zio, EINVAL);

	if (ret != 0)
	zio->io_error = SET_ERROR(EIO);
	}
	}

	static void
	zio_decrypt(zio_t zio, abd_t data, uint64_t size)
	{
	int ret;
	void *tmp;
	blkptr_t *bp = zio->io_bp;
	spa_t *spa = zio->io_spa;
	uint64_t dsobj = zio->io_bookmark.zb_objset;
	uint64_t lsize = BP_GET_LSIZE(bp);
	dmu_object_type_t ot = BP_GET_TYPE(bp);
	uint8_t salt[ZIO_DATA_SALT_LEN];
	uint8_t iv[ZIO_DATA_IV_LEN];
	uint8_t mac[ZIO_DATA_MAC_LEN];
	boolean_t no_crypt = B_FALSE;

	ASSERT(BP_USES_CRYPT(bp));
	ASSERT3U(size, !=, 0);

	if (zio->io_error != 0)
	return;

	/*
	* Verify the cksum of MACs stored in an indirect bp. It will always
	* be possible to verify this since it does not require an encryption
	* key.
	*/
	if (BP_HAS_INDIRECT_MAC_CKSUM(bp)) {
	zio_crypt_decode_mac_bp(bp, mac);

	if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) {
	/*
	* We haven't decompressed the data yet, but
	* zio_crypt_do_indirect_mac_checksum() requires
	* decompressed data to be able to parse out the MACs
	* from the indirect block. We decompress it now and
	* throw away the result after we are finished.
	*/
	tmp = zio_buf_alloc(lsize);
	ret = zio_decompress_data(BP_GET_COMPRESS(bp),
	zio->io_abd, tmp, zio->io_size, lsize,
	&zio->io_prop.zp_complevel);
	if (ret != 0) {
	ret = SET_ERROR(EIO);
	goto error;
	}
	ret = zio_crypt_do_indirect_mac_checksum(B_FALSE,
	tmp, lsize, BP_SHOULD_BYTESWAP(bp), mac);
	zio_buf_free(tmp, lsize);
	} else {
	ret = zio_crypt_do_indirect_mac_checksum_abd(B_FALSE,
	zio->io_abd, size, BP_SHOULD_BYTESWAP(bp), mac);
	}
	abd_copy(data, zio->io_abd, size);

	if (zio_injection_enabled && ot != DMU_OT_DNODE && ret == 0) {
	ret = zio_handle_decrypt_injection(spa,
	&zio->io_bookmark, ot, ECKSUM);
	}
	if (ret != 0)
	goto error;

	return;
	}

	/*
	* If this is an authenticated block, just check the MAC. It would be
	* nice to separate this out into its own flag, but when this was done,
	* we had run out of bits in what is now zio_flag_t. Future cleanup
	* could make this a flag bit.
	*/
	if (BP_IS_AUTHENTICATED(bp)) {
	if (ot == DMU_OT_OBJSET) {
	ret = spa_do_crypt_objset_mac_abd(B_FALSE, spa,
	dsobj, zio->io_abd, size, BP_SHOULD_BYTESWAP(bp));
	} else {
	zio_crypt_decode_mac_bp(bp, mac);
	ret = spa_do_crypt_mac_abd(B_FALSE, spa, dsobj,
	zio->io_abd, size, mac);
	if (zio_injection_enabled && ret == 0) {
	ret = zio_handle_decrypt_injection(spa,
	&zio->io_bookmark, ot, ECKSUM);
	}
	}
	abd_copy(data, zio->io_abd, size);

	if (ret != 0)
	goto error;

	return;
	}

	zio_crypt_decode_params_bp(bp, salt, iv);

	if (ot == DMU_OT_INTENT_LOG) {
	tmp = abd_borrow_buf_copy(zio->io_abd, sizeof (zil_chain_t));
	zio_crypt_decode_mac_zil(tmp, mac);
	abd_return_buf(zio->io_abd, tmp, sizeof (zil_chain_t));
	} else {
	zio_crypt_decode_mac_bp(bp, mac);
	}

	ret = spa_do_crypt_abd(B_FALSE, spa, &zio->io_bookmark, BP_GET_TYPE(bp),
	BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp), salt, iv, mac, size, data,
	zio->io_abd, &no_crypt);
	if (no_crypt)
	abd_copy(data, zio->io_abd, size);

	if (ret != 0)
	goto error;

	return;

	error:
	/* assert that the key was found unless this was speculative */
	ASSERT(ret != EACCES \|\| (zio->io_flags & ZIO_FLAG_SPECULATIVE));

	/*
	* If there was a decryption / authentication error return EIO as
	* the io_error. If this was not a speculative zio, create an ereport.
	*/
	if (ret == ECKSUM) {
	zio->io_error = SET_ERROR(EIO);
	if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) {
	spa_log_error(spa, &zio->io_bookmark,
	&zio->io_bp->blk_birth);
	(void) zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION,
	spa, NULL, &zio->io_bookmark, zio, 0);
	}
	} else {
	zio->io_error = ret;
	}
	}

	/*
	* ==========================================================================
	* I/O parent/child relationships and pipeline interlocks
	* ==========================================================================
	*/
	zio_t *
	zio_walk_parents(zio_t cio, zio_link_t *zl)
	{
	list_t *pl = &cio->io_parent_list;

	zl = (zl == NULL) ? list_head(pl) : list_next(pl, *zl);
	if (*zl == NULL)
	return (NULL);

	ASSERT((*zl)->zl_child == cio);
	return ((*zl)->zl_parent);
	}

	zio_t *
	zio_walk_children(zio_t pio, zio_link_t *zl)
	{
	list_t *cl = &pio->io_child_list;

	ASSERT(MUTEX_HELD(&pio->io_lock));

	zl = (zl == NULL) ? list_head(cl) : list_next(cl, *zl);
	if (*zl == NULL)
	return (NULL);

	ASSERT((*zl)->zl_parent == pio);
	return ((*zl)->zl_child);
	}

	zio_t *
	zio_unique_parent(zio_t *cio)
	{
	zio_link_t *zl = NULL;
	zio_t *pio = zio_walk_parents(cio, &zl);

	VERIFY3P(zio_walk_parents(cio, &zl), ==, NULL);
	return (pio);
	}

	void
	zio_add_child(zio_t pio, zio_t cio)
	{
	/*
	* Logical I/Os can have logical, gang, or vdev children.
	* Gang I/Os can have gang or vdev children.
	* Vdev I/Os can only have vdev children.
	* The following ASSERT captures all of these constraints.
	*/
	ASSERT3S(cio->io_child_type, <=, pio->io_child_type);

	zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP);
	zl->zl_parent = pio;
	zl->zl_child = cio;

	mutex_enter(&pio->io_lock);
	mutex_enter(&cio->io_lock);

	ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0);

	uint64_t *countp = pio->io_children[cio->io_child_type];
	for (int w = 0; w < ZIO_WAIT_TYPES; w++)
	countp[w] += !cio->io_state[w];

	list_insert_head(&pio->io_child_list, zl);
	list_insert_head(&cio->io_parent_list, zl);

	mutex_exit(&cio->io_lock);
	mutex_exit(&pio->io_lock);
	}

	void
	zio_add_child_first(zio_t pio, zio_t cio)
	{
	/*
	* Logical I/Os can have logical, gang, or vdev children.
	* Gang I/Os can have gang or vdev children.
	* Vdev I/Os can only have vdev children.
	* The following ASSERT captures all of these constraints.
	*/
	ASSERT3S(cio->io_child_type, <=, pio->io_child_type);

	zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP);
	zl->zl_parent = pio;
	zl->zl_child = cio;

	ASSERT(list_is_empty(&cio->io_parent_list));
	list_insert_head(&cio->io_parent_list, zl);

	mutex_enter(&pio->io_lock);

	ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0);

	uint64_t *countp = pio->io_children[cio->io_child_type];
	for (int w = 0; w < ZIO_WAIT_TYPES; w++)
	countp[w] += !cio->io_state[w];

	list_insert_head(&pio->io_child_list, zl);

	mutex_exit(&pio->io_lock);
	}

	static void
	zio_remove_child(zio_t pio, zio_t cio, zio_link_t *zl)
	{
	ASSERT(zl->zl_parent == pio);
	ASSERT(zl->zl_child == cio);

	mutex_enter(&pio->io_lock);
	mutex_enter(&cio->io_lock);

	list_remove(&pio->io_child_list, zl);
	list_remove(&cio->io_parent_list, zl);

	mutex_exit(&cio->io_lock);
	mutex_exit(&pio->io_lock);
	kmem_cache_free(zio_link_cache, zl);
	}

	static boolean_t
	zio_wait_for_children(zio_t *zio, uint8_t childbits, enum zio_wait_type wait)
	{
	boolean_t waiting = B_FALSE;

	mutex_enter(&zio->io_lock);
	ASSERT(zio->io_stall == NULL);
	for (int c = 0; c < ZIO_CHILD_TYPES; c++) {
	if (!(ZIO_CHILD_BIT_IS_SET(childbits, c)))
	continue;

	uint64_t *countp = &zio->io_children[c][wait];
	if (*countp != 0) {
	zio->io_stage >>= 1;
	ASSERT3U(zio->io_stage, !=, ZIO_STAGE_OPEN);
	zio->io_stall = countp;
	waiting = B_TRUE;
	break;
	}
	}
	mutex_exit(&zio->io_lock);
	return (waiting);
	}

	__attribute__((always_inline))
	static inline void
	zio_notify_parent(zio_t pio, zio_t zio, enum zio_wait_type wait,
	zio_t **next_to_executep)
	{
	uint64_t *countp = &pio->io_children[zio->io_child_type][wait];
	int *errorp = &pio->io_child_error[zio->io_child_type];

	mutex_enter(&pio->io_lock);
	if (zio->io_error && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE))
	errorp = zio_worst_error(errorp, zio->io_error);
	pio->io_reexecute \|= zio->io_reexecute;
	ASSERT3U(*countp, >, 0);

	(*countp)--;

	if (*countp == 0 && pio->io_stall == countp) {
	zio_taskq_type_t type =
	pio->io_stage < ZIO_STAGE_VDEV_IO_START ? ZIO_TASKQ_ISSUE :
	ZIO_TASKQ_INTERRUPT;
	pio->io_stall = NULL;
	mutex_exit(&pio->io_lock);

	/*
	* If we can tell the caller to execute this parent next, do
	* so. We only do this if the parent's zio type matches the
	* child's type. Otherwise dispatch the parent zio in its
	* own taskq.
	*
	* Having the caller execute the parent when possible reduces
	* locking on the zio taskq's, reduces context switch
	* overhead, and has no recursion penalty. Note that one
	* read from disk typically causes at least 3 zio's: a
	* zio_null(), the logical zio_read(), and then a physical
	* zio. When the physical ZIO completes, we are able to call
	* zio_done() on all 3 of these zio's from one invocation of
	* zio_execute() by returning the parent back to
	* zio_execute(). Since the parent isn't executed until this
	* thread returns back to zio_execute(), the caller should do
	* so promptly.
	*
	* In other cases, dispatching the parent prevents
	* overflowing the stack when we have deeply nested
	* parent-child relationships, as we do with the "mega zio"
	* of writes for spa_sync(), and the chain of ZIL blocks.
	*/
	if (next_to_executep != NULL && *next_to_executep == NULL &&
	pio->io_type == zio->io_type) {
	*next_to_executep = pio;
	} else {
	zio_taskq_dispatch(pio, type, B_FALSE);
	}
	} else {
	mutex_exit(&pio->io_lock);
	}
	}

	static void
	zio_inherit_child_errors(zio_t *zio, enum zio_child c)
	{
	if (zio->io_child_error[c] != 0 && zio->io_error == 0)
	zio->io_error = zio->io_child_error[c];
	}

	int
	zio_bookmark_compare(const void x1, const void x2)
	{
	const zio_t *z1 = x1;
	const zio_t *z2 = x2;

	if (z1->io_bookmark.zb_objset < z2->io_bookmark.zb_objset)
	return (-1);
	if (z1->io_bookmark.zb_objset > z2->io_bookmark.zb_objset)
	return (1);

	if (z1->io_bookmark.zb_object < z2->io_bookmark.zb_object)
	return (-1);
	if (z1->io_bookmark.zb_object > z2->io_bookmark.zb_object)
	return (1);

	if (z1->io_bookmark.zb_level < z2->io_bookmark.zb_level)
	return (-1);
	if (z1->io_bookmark.zb_level > z2->io_bookmark.zb_level)
	return (1);

	if (z1->io_bookmark.zb_blkid < z2->io_bookmark.zb_blkid)
	return (-1);
	if (z1->io_bookmark.zb_blkid > z2->io_bookmark.zb_blkid)
	return (1);

	if (z1 < z2)
	return (-1);
	if (z1 > z2)
	return (1);

	return (0);
	}

	/*
	* ==========================================================================
	* Create the various types of I/O (read, write, free, etc)
	* ==========================================================================
	*/
	static zio_t *
	zio_create(zio_t pio, spa_t spa, uint64_t txg, const blkptr_t *bp,
	abd_t data, uint64_t lsize, uint64_t psize, zio_done_func_t done,
	void *private, zio_type_t type, zio_priority_t priority,
	zio_flag_t flags, vdev_t *vd, uint64_t offset,
	const zbookmark_phys_t *zb, enum zio_stage stage,
	enum zio_stage pipeline)
	{
	zio_t *zio;

	IMPLY(type != ZIO_TYPE_TRIM, psize <= SPA_MAXBLOCKSIZE);
	ASSERT(P2PHASE(psize, SPA_MINBLOCKSIZE) == 0);
	ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0);

	ASSERT(!vd \|\| spa_config_held(spa, SCL_STATE_ALL, RW_READER));
	ASSERT(!bp \|\| !(flags & ZIO_FLAG_CONFIG_WRITER));
	ASSERT(vd \|\| stage == ZIO_STAGE_OPEN);

	IMPLY(lsize != psize, (flags & ZIO_FLAG_RAW_COMPRESS) != 0);

	zio = kmem_cache_alloc(zio_cache, KM_SLEEP);
	memset(zio, 0, sizeof (zio_t));

	mutex_init(&zio->io_lock, NULL, MUTEX_NOLOCKDEP, NULL);
	cv_init(&zio->io_cv, NULL, CV_DEFAULT, NULL);

	list_create(&zio->io_parent_list, sizeof (zio_link_t),
	offsetof(zio_link_t, zl_parent_node));
	list_create(&zio->io_child_list, sizeof (zio_link_t),
	offsetof(zio_link_t, zl_child_node));
	metaslab_trace_init(&zio->io_alloc_list);

	if (vd != NULL)
	zio->io_child_type = ZIO_CHILD_VDEV;
	else if (flags & ZIO_FLAG_GANG_CHILD)
	zio->io_child_type = ZIO_CHILD_GANG;
	else if (flags & ZIO_FLAG_DDT_CHILD)
	zio->io_child_type = ZIO_CHILD_DDT;
	else
	zio->io_child_type = ZIO_CHILD_LOGICAL;

	if (bp != NULL) {
	if (type != ZIO_TYPE_WRITE \|\|
	zio->io_child_type == ZIO_CHILD_DDT) {
	zio->io_bp_copy = *bp;
	zio->io_bp = &zio->io_bp_copy; /* so caller can free */
	} else {
	zio->io_bp = (blkptr_t *)bp;
	}
	zio->io_bp_orig = *bp;
	if (zio->io_child_type == ZIO_CHILD_LOGICAL)
	zio->io_logical = zio;
	if (zio->io_child_type > ZIO_CHILD_GANG && BP_IS_GANG(bp))
	pipeline \|= ZIO_GANG_STAGES;
	}

	zio->io_spa = spa;
	zio->io_txg = txg;
	zio->io_done = done;
	zio->io_private = private;
	zio->io_type = type;
	zio->io_priority = priority;
	zio->io_vd = vd;
	zio->io_offset = offset;
	zio->io_orig_abd = zio->io_abd = data;
	zio->io_orig_size = zio->io_size = psize;
	zio->io_lsize = lsize;
	zio->io_orig_flags = zio->io_flags = flags;
	zio->io_orig_stage = zio->io_stage = stage;
	zio->io_orig_pipeline = zio->io_pipeline = pipeline;
	zio->io_pipeline_trace = ZIO_STAGE_OPEN;

	zio->io_state[ZIO_WAIT_READY] = (stage >= ZIO_STAGE_READY);
	zio->io_state[ZIO_WAIT_DONE] = (stage >= ZIO_STAGE_DONE);

	if (zb != NULL)
	zio->io_bookmark = *zb;

	if (pio != NULL) {
	zio->io_metaslab_class = pio->io_metaslab_class;
	if (zio->io_logical == NULL)
	zio->io_logical = pio->io_logical;
	if (zio->io_child_type == ZIO_CHILD_GANG)
	zio->io_gang_leader = pio->io_gang_leader;
	zio_add_child_first(pio, zio);
	}

	taskq_init_ent(&zio->io_tqent);

	return (zio);
	}

	void
	zio_destroy(zio_t *zio)
	{
	metaslab_trace_fini(&zio->io_alloc_list);
	list_destroy(&zio->io_parent_list);
	list_destroy(&zio->io_child_list);
	mutex_destroy(&zio->io_lock);
	cv_destroy(&zio->io_cv);
	kmem_cache_free(zio_cache, zio);
	}

	zio_t *
	zio_null(zio_t pio, spa_t spa, vdev_t vd, zio_done_func_t done,
	void *private, zio_flag_t flags)
	{
	zio_t *zio;

	zio = zio_create(pio, spa, 0, NULL, NULL, 0, 0, done, private,
	ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL,
	ZIO_STAGE_OPEN, ZIO_INTERLOCK_PIPELINE);

	return (zio);
	}

	zio_t *
	zio_root(spa_t spa, zio_done_func_t done, void *private, zio_flag_t flags)
	{
	return (zio_null(NULL, spa, NULL, done, private, flags));
	}

	static int
	zfs_blkptr_verify_log(spa_t spa, const blkptr_t bp,
	enum blk_verify_flag blk_verify, const char *fmt, ...)
	{
	va_list adx;
	char buf[256];

	va_start(adx, fmt);
	(void) vsnprintf(buf, sizeof (buf), fmt, adx);
	va_end(adx);

	zfs_dbgmsg("bad blkptr at %px: "
	"DVA[0]=%#llx/%#llx "
	"DVA[1]=%#llx/%#llx "
	"DVA[2]=%#llx/%#llx "
	"prop=%#llx "
	"pad=%#llx,%#llx "
	"phys_birth=%#llx "
	"birth=%#llx "
	"fill=%#llx "
	"cksum=%#llx/%#llx/%#llx/%#llx",
	bp,
	(long long)bp->blk_dva[0].dva_word[0],
	(long long)bp->blk_dva[0].dva_word[1],
	(long long)bp->blk_dva[1].dva_word[0],
	(long long)bp->blk_dva[1].dva_word[1],
	(long long)bp->blk_dva[2].dva_word[0],
	(long long)bp->blk_dva[2].dva_word[1],
	(long long)bp->blk_prop,
	(long long)bp->blk_pad[0],
	(long long)bp->blk_pad[1],
	(long long)bp->blk_phys_birth,
	(long long)bp->blk_birth,
	(long long)bp->blk_fill,
	(long long)bp->blk_cksum.zc_word[0],
	(long long)bp->blk_cksum.zc_word[1],
	(long long)bp->blk_cksum.zc_word[2],
	(long long)bp->blk_cksum.zc_word[3]);
	switch (blk_verify) {
	case BLK_VERIFY_HALT:
	zfs_panic_recover("%s: %s", spa_name(spa), buf);
	break;
	case BLK_VERIFY_LOG:
	zfs_dbgmsg("%s: %s", spa_name(spa), buf);
	break;
	case BLK_VERIFY_ONLY:
	break;
	}

	return (1);
	}

	/*
	* Verify the block pointer fields contain reasonable values. This means
	* it only contains known object types, checksum/compression identifiers,
	* block sizes within the maximum allowed limits, valid DVAs, etc.
	*
	* If everything checks out B_TRUE is returned. The zfs_blkptr_verify
	* argument controls the behavior when an invalid field is detected.
	*
	* Values for blk_verify_flag:
	* BLK_VERIFY_ONLY: evaluate the block
	* BLK_VERIFY_LOG: evaluate the block and log problems
	* BLK_VERIFY_HALT: call zfs_panic_recover on error
	*
	* Values for blk_config_flag:
	* BLK_CONFIG_HELD: caller holds SCL_VDEV for writer
	* BLK_CONFIG_NEEDED: caller holds no config lock, SCL_VDEV will be
	* obtained for reader
	* BLK_CONFIG_SKIP: skip checks which require SCL_VDEV, for better
	* performance
	*/
	boolean_t
	zfs_blkptr_verify(spa_t spa, const blkptr_t bp,
	enum blk_config_flag blk_config, enum blk_verify_flag blk_verify)
	{
	int errors = 0;

	if (!DMU_OT_IS_VALID(BP_GET_TYPE(bp))) {
	errors += zfs_blkptr_verify_log(spa, bp, blk_verify,
	"blkptr at %px has invalid TYPE %llu",
	bp, (longlong_t)BP_GET_TYPE(bp));
	}
	if (BP_GET_CHECKSUM(bp) >= ZIO_CHECKSUM_FUNCTIONS) {
	errors += zfs_blkptr_verify_log(spa, bp, blk_verify,
	"blkptr at %px has invalid CHECKSUM %llu",
	bp, (longlong_t)BP_GET_CHECKSUM(bp));
	}
	if (BP_GET_COMPRESS(bp) >= ZIO_COMPRESS_FUNCTIONS) {
	errors += zfs_blkptr_verify_log(spa, bp, blk_verify,
	"blkptr at %px has invalid COMPRESS %llu",
	bp, (longlong_t)BP_GET_COMPRESS(bp));
	}
	if (BP_GET_LSIZE(bp) > SPA_MAXBLOCKSIZE) {
	errors += zfs_blkptr_verify_log(spa, bp, blk_verify,
	"blkptr at %px has invalid LSIZE %llu",
	bp, (longlong_t)BP_GET_LSIZE(bp));
	}
	if (BP_GET_PSIZE(bp) > SPA_MAXBLOCKSIZE) {
	errors += zfs_blkptr_verify_log(spa, bp, blk_verify,
	"blkptr at %px has invalid PSIZE %llu",
	bp, (longlong_t)BP_GET_PSIZE(bp));
	}

	if (BP_IS_EMBEDDED(bp)) {
	if (BPE_GET_ETYPE(bp) >= NUM_BP_EMBEDDED_TYPES) {
	errors += zfs_blkptr_verify_log(spa, bp, blk_verify,
	"blkptr at %px has invalid ETYPE %llu",
	bp, (longlong_t)BPE_GET_ETYPE(bp));
	}
	}

	/*
	* Do not verify individual DVAs if the config is not trusted. This
	* will be done once the zio is executed in vdev_mirror_map_alloc.
	*/
	if (!spa->spa_trust_config)
	return (errors == 0);

	switch (blk_config) {
	case BLK_CONFIG_HELD:
	ASSERT(spa_config_held(spa, SCL_VDEV, RW_WRITER));
	break;
	case BLK_CONFIG_NEEDED:
	spa_config_enter(spa, SCL_VDEV, bp, RW_READER);
	break;
	case BLK_CONFIG_SKIP:
	return (errors == 0);
	default:
	panic("invalid blk_config %u", blk_config);
	}

	/*
	* Pool-specific checks.
	*
	* Note: it would be nice to verify that the blk_birth and
	* BP_PHYSICAL_BIRTH() are not too large. However, spa_freeze()
	* allows the birth time of log blocks (and dmu_sync()-ed blocks
	* that are in the log) to be arbitrarily large.
	*/
	for (int i = 0; i < BP_GET_NDVAS(bp); i++) {
	const dva_t *dva = &bp->blk_dva[i];
	uint64_t vdevid = DVA_GET_VDEV(dva);

	if (vdevid >= spa->spa_root_vdev->vdev_children) {
	errors += zfs_blkptr_verify_log(spa, bp, blk_verify,
	"blkptr at %px DVA %u has invalid VDEV %llu",
	bp, i, (longlong_t)vdevid);
	continue;
	}
	vdev_t *vd = spa->spa_root_vdev->vdev_child[vdevid];
	if (vd == NULL) {
	errors += zfs_blkptr_verify_log(spa, bp, blk_verify,
	"blkptr at %px DVA %u has invalid VDEV %llu",
	bp, i, (longlong_t)vdevid);
	continue;
	}
	if (vd->vdev_ops == &vdev_hole_ops) {
	errors += zfs_blkptr_verify_log(spa, bp, blk_verify,
	"blkptr at %px DVA %u has hole VDEV %llu",
	bp, i, (longlong_t)vdevid);
	continue;
	}
	if (vd->vdev_ops == &vdev_missing_ops) {
	/*
	* "missing" vdevs are valid during import, but we
	* don't have their detailed info (e.g. asize), so
	* we can't perform any more checks on them.
	*/
	continue;
	}
	uint64_t offset = DVA_GET_OFFSET(dva);
	uint64_t asize = DVA_GET_ASIZE(dva);
	if (DVA_GET_GANG(dva))
	asize = vdev_gang_header_asize(vd);
	if (offset + asize > vd->vdev_asize) {
	errors += zfs_blkptr_verify_log(spa, bp, blk_verify,
	"blkptr at %px DVA %u has invalid OFFSET %llu",
	bp, i, (longlong_t)offset);
	}
	}
	if (blk_config == BLK_CONFIG_NEEDED)
	spa_config_exit(spa, SCL_VDEV, bp);

	return (errors == 0);
	}

	boolean_t
	zfs_dva_valid(spa_t spa, const dva_t dva, const blkptr_t *bp)
	{
	(void) bp;
	uint64_t vdevid = DVA_GET_VDEV(dva);

	if (vdevid >= spa->spa_root_vdev->vdev_children)
	return (B_FALSE);

	vdev_t *vd = spa->spa_root_vdev->vdev_child[vdevid];
	if (vd == NULL)
	return (B_FALSE);

	if (vd->vdev_ops == &vdev_hole_ops)
	return (B_FALSE);

	if (vd->vdev_ops == &vdev_missing_ops) {
	return (B_FALSE);
	}

	uint64_t offset = DVA_GET_OFFSET(dva);
	uint64_t asize = DVA_GET_ASIZE(dva);

	if (DVA_GET_GANG(dva))
	asize = vdev_gang_header_asize(vd);
	if (offset + asize > vd->vdev_asize)
	return (B_FALSE);

	return (B_TRUE);
	}

	zio_t *
	zio_read(zio_t pio, spa_t spa, const blkptr_t *bp,
	abd_t data, uint64_t size, zio_done_func_t done, void *private,
	zio_priority_t priority, zio_flag_t flags, const zbookmark_phys_t *zb)
	{
	zio_t *zio;

	zio = zio_create(pio, spa, BP_PHYSICAL_BIRTH(bp), bp,
	data, size, size, done, private,
	ZIO_TYPE_READ, priority, flags, NULL, 0, zb,
	ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ?
	ZIO_DDT_CHILD_READ_PIPELINE : ZIO_READ_PIPELINE);

	return (zio);
	}

	zio_t *
	zio_write(zio_t pio, spa_t spa, uint64_t txg, blkptr_t *bp,
	abd_t data, uint64_t lsize, uint64_t psize, const zio_prop_t zp,
	zio_done_func_t ready, zio_done_func_t children_ready,
	zio_done_func_t done, void private, zio_priority_t priority,
	zio_flag_t flags, const zbookmark_phys_t *zb)
	{
	zio_t *zio;

	ASSERT(zp->zp_checksum >= ZIO_CHECKSUM_OFF &&
	zp->zp_checksum < ZIO_CHECKSUM_FUNCTIONS &&
	zp->zp_compress >= ZIO_COMPRESS_OFF &&
	zp->zp_compress < ZIO_COMPRESS_FUNCTIONS &&
	DMU_OT_IS_VALID(zp->zp_type) &&
	zp->zp_level < 32 &&
	zp->zp_copies > 0 &&
	zp->zp_copies <= spa_max_replication(spa));

	zio = zio_create(pio, spa, txg, bp, data, lsize, psize, done, private,
	ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb,
	ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ?
	ZIO_DDT_CHILD_WRITE_PIPELINE : ZIO_WRITE_PIPELINE);

	zio->io_ready = ready;
	zio->io_children_ready = children_ready;
	zio->io_prop = *zp;

	/*
	* Data can be NULL if we are going to call zio_write_override() to
	* provide the already-allocated BP. But we may need the data to
	* verify a dedup hit (if requested). In this case, don't try to
	* dedup (just take the already-allocated BP verbatim). Encrypted
	* dedup blocks need data as well so we also disable dedup in this
	* case.
	*/
	if (data == NULL &&
	(zio->io_prop.zp_dedup_verify \|\| zio->io_prop.zp_encrypt)) {
	zio->io_prop.zp_dedup = zio->io_prop.zp_dedup_verify = B_FALSE;
	}

	return (zio);
	}

	zio_t *
	zio_rewrite(zio_t pio, spa_t spa, uint64_t txg, blkptr_t bp, abd_t data,
	uint64_t size, zio_done_func_t done, void private,
	zio_priority_t priority, zio_flag_t flags, zbookmark_phys_t *zb)
	{
	zio_t *zio;

	zio = zio_create(pio, spa, txg, bp, data, size, size, done, private,
	ZIO_TYPE_WRITE, priority, flags \| ZIO_FLAG_IO_REWRITE, NULL, 0, zb,
	ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE);

	return (zio);
	}

	void
	zio_write_override(zio_t zio, blkptr_t bp, int copies, boolean_t nopwrite,
	boolean_t brtwrite)
	{
	ASSERT(zio->io_type == ZIO_TYPE_WRITE);
	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
	ASSERT(zio->io_stage == ZIO_STAGE_OPEN);
	ASSERT(zio->io_txg == spa_syncing_txg(zio->io_spa));
	ASSERT(!brtwrite \|\| !nopwrite);

	/*
	* We must reset the io_prop to match the values that existed
	* when the bp was first written by dmu_sync() keeping in mind
	* that nopwrite and dedup are mutually exclusive.
	*/
	zio->io_prop.zp_dedup = nopwrite ? B_FALSE : zio->io_prop.zp_dedup;
	zio->io_prop.zp_nopwrite = nopwrite;
	zio->io_prop.zp_brtwrite = brtwrite;
	zio->io_prop.zp_copies = copies;
	zio->io_bp_override = bp;
	}

	void
	zio_free(spa_t spa, uint64_t txg, const blkptr_t bp)
	{

	(void) zfs_blkptr_verify(spa, bp, BLK_CONFIG_NEEDED, BLK_VERIFY_HALT);

	/*
	* The check for EMBEDDED is a performance optimization. We
	* process the free here (by ignoring it) rather than
	* putting it on the list and then processing it in zio_free_sync().
	*/
	if (BP_IS_EMBEDDED(bp))
	return;

	/*
	* Frees that are for the currently-syncing txg, are not going to be
	* deferred, and which will not need to do a read (i.e. not GANG or
	* DEDUP), can be processed immediately. Otherwise, put them on the
	* in-memory list for later processing.
	*
	* Note that we only defer frees after zfs_sync_pass_deferred_free
	* when the log space map feature is disabled. [see relevant comment
	* in spa_sync_iterate_to_convergence()]
	*/
	if (BP_IS_GANG(bp) \|\|
	BP_GET_DEDUP(bp) \|\|
	txg != spa->spa_syncing_txg \|\|
	(spa_sync_pass(spa) >= zfs_sync_pass_deferred_free &&
	!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) \|\|
	brt_maybe_exists(spa, bp)) {
	metaslab_check_free(spa, bp);
	bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp);
	} else {
	VERIFY3P(zio_free_sync(NULL, spa, txg, bp, 0), ==, NULL);
	}
	}

	/*
	* To improve performance, this function may return NULL if we were able
	* to do the free immediately. This avoids the cost of creating a zio
	* (and linking it to the parent, etc).
	*/
	zio_t *
	zio_free_sync(zio_t pio, spa_t spa, uint64_t txg, const blkptr_t *bp,
	zio_flag_t flags)
	{
	ASSERT(!BP_IS_HOLE(bp));
	ASSERT(spa_syncing_txg(spa) == txg);

	if (BP_IS_EMBEDDED(bp))
	return (NULL);

	metaslab_check_free(spa, bp);
	arc_freed(spa, bp);
	dsl_scan_freed(spa, bp);

	if (BP_IS_GANG(bp) \|\|
	BP_GET_DEDUP(bp) \|\|
	brt_maybe_exists(spa, bp)) {
	/*
	* GANG, DEDUP and BRT blocks can induce a read (for the gang
	* block header, the DDT or the BRT), so issue them
	* asynchronously so that this thread is not tied up.
	*/
	enum zio_stage stage =
	ZIO_FREE_PIPELINE \| ZIO_STAGE_ISSUE_ASYNC;

	return (zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp),
	BP_GET_PSIZE(bp), NULL, NULL,
	ZIO_TYPE_FREE, ZIO_PRIORITY_NOW,
	flags, NULL, 0, NULL, ZIO_STAGE_OPEN, stage));
	} else {
	metaslab_free(spa, bp, txg, B_FALSE);
	return (NULL);
	}
	}

	zio_t *
	zio_claim(zio_t pio, spa_t spa, uint64_t txg, const blkptr_t *bp,
	zio_done_func_t done, void private, zio_flag_t flags)
	{
	zio_t *zio;

	(void) zfs_blkptr_verify(spa, bp, (flags & ZIO_FLAG_CONFIG_WRITER) ?
	BLK_CONFIG_HELD : BLK_CONFIG_NEEDED, BLK_VERIFY_HALT);

	if (BP_IS_EMBEDDED(bp))
	return (zio_null(pio, spa, NULL, NULL, NULL, 0));

	/*
	* A claim is an allocation of a specific block. Claims are needed
	* to support immediate writes in the intent log. The issue is that
	* immediate writes contain committed data, but in a txg that was
	* not committed. Upon opening the pool after an unclean shutdown,
	* the intent log claims all blocks that contain immediate write data
	* so that the SPA knows they're in use.
	*
	* All claims must be resolved in the first txg -- before the SPA
	* starts allocating blocks -- so that nothing is allocated twice.
	* If txg == 0 we just verify that the block is claimable.
	*/
	ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <,
	spa_min_claim_txg(spa));
	ASSERT(txg == spa_min_claim_txg(spa) \|\| txg == 0);
	ASSERT(!BP_GET_DEDUP(bp) \|\| !spa_writeable(spa)); /* zdb(8) */

	zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp),
	BP_GET_PSIZE(bp), done, private, ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW,
	flags, NULL, 0, NULL, ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE);
	ASSERT0(zio->io_queued_timestamp);

	return (zio);
	}

	zio_t *
	zio_ioctl(zio_t pio, spa_t spa, vdev_t *vd, int cmd,
	zio_done_func_t done, void private, zio_flag_t flags)
	{
	- zio_t *zio;
	- int c;
	-
	- if (vd->vdev_children == 0) {
	- zio = zio_create(pio, spa, 0, NULL, NULL, 0, 0, done, private,
	- ZIO_TYPE_IOCTL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL,
	- ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE);
	-
	- zio->io_cmd = cmd;
	- } else {
	- zio = zio_null(pio, spa, NULL, NULL, NULL, flags);
	-
	- for (c = 0; c < vd->vdev_children; c++)
	- zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd,
	- done, private, flags));
	- }
	-
	+ zio_t *zio = zio_create(pio, spa, 0, NULL, NULL, 0, 0, done, private,
	+ ZIO_TYPE_IOCTL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL,
	+ ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE);
	+ zio->io_cmd = cmd;
	return (zio);
	}

	zio_t *
	zio_trim(zio_t pio, vdev_t vd, uint64_t offset, uint64_t size,
	zio_done_func_t done, void private, zio_priority_t priority,
	zio_flag_t flags, enum trim_flag trim_flags)
	{
	zio_t *zio;

	ASSERT0(vd->vdev_children);
	ASSERT0(P2PHASE(offset, 1ULL << vd->vdev_ashift));
	ASSERT0(P2PHASE(size, 1ULL << vd->vdev_ashift));
	ASSERT3U(size, !=, 0);

	zio = zio_create(pio, vd->vdev_spa, 0, NULL, NULL, size, size, done,
	private, ZIO_TYPE_TRIM, priority, flags \| ZIO_FLAG_PHYSICAL,
	vd, offset, NULL, ZIO_STAGE_OPEN, ZIO_TRIM_PIPELINE);
	zio->io_trim_flags = trim_flags;

	return (zio);
	}

	zio_t *
	zio_read_phys(zio_t pio, vdev_t vd, uint64_t offset, uint64_t size,
	abd_t data, int checksum, zio_done_func_t done, void *private,
	zio_priority_t priority, zio_flag_t flags, boolean_t labels)
	{
	zio_t *zio;

	ASSERT(vd->vdev_children == 0);
	ASSERT(!labels \|\| offset + size <= VDEV_LABEL_START_SIZE \|\|
	offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE);
	ASSERT3U(offset + size, <=, vd->vdev_psize);

	zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, size, done,
	private, ZIO_TYPE_READ, priority, flags \| ZIO_FLAG_PHYSICAL, vd,
	offset, NULL, ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE);

	zio->io_prop.zp_checksum = checksum;

	return (zio);
	}

	zio_t *
	zio_write_phys(zio_t pio, vdev_t vd, uint64_t offset, uint64_t size,
	abd_t data, int checksum, zio_done_func_t done, void *private,
	zio_priority_t priority, zio_flag_t flags, boolean_t labels)
	{
	zio_t *zio;

	ASSERT(vd->vdev_children == 0);
	ASSERT(!labels \|\| offset + size <= VDEV_LABEL_START_SIZE \|\|
	offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE);
	ASSERT3U(offset + size, <=, vd->vdev_psize);

	zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, size, done,
	private, ZIO_TYPE_WRITE, priority, flags \| ZIO_FLAG_PHYSICAL, vd,
	offset, NULL, ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE);

	zio->io_prop.zp_checksum = checksum;

	if (zio_checksum_table[checksum].ci_flags & ZCHECKSUM_FLAG_EMBEDDED) {
	/*
	* zec checksums are necessarily destructive -- they modify
	* the end of the write buffer to hold the verifier/checksum.
	* Therefore, we must make a local copy in case the data is
	* being written to multiple places in parallel.
	*/
	abd_t *wbuf = abd_alloc_sametype(data, size);
	abd_copy(wbuf, data, size);

	zio_push_transform(zio, wbuf, size, size, NULL);
	}

	return (zio);
	}

	/*
	* Create a child I/O to do some work for us.
	*/
	zio_t *
	zio_vdev_child_io(zio_t pio, blkptr_t bp, vdev_t *vd, uint64_t offset,
	abd_t *data, uint64_t size, int type, zio_priority_t priority,
	zio_flag_t flags, zio_done_func_t done, void private)
	{
	enum zio_stage pipeline = ZIO_VDEV_CHILD_PIPELINE;
	zio_t *zio;

	/*
	* vdev child I/Os do not propagate their error to the parent.
	* Therefore, for correct operation the caller must check for
	* and handle the error in the child i/o's done callback.
	* The only exceptions are i/os that we don't care about
	* (OPTIONAL or REPAIR).
	*/
	ASSERT((flags & ZIO_FLAG_OPTIONAL) \|\| (flags & ZIO_FLAG_IO_REPAIR) \|\|
	done != NULL);

	if (type == ZIO_TYPE_READ && bp != NULL) {
	/*
	* If we have the bp, then the child should perform the
	* checksum and the parent need not. This pushes error
	* detection as close to the leaves as possible and
	* eliminates redundant checksums in the interior nodes.
	*/
	pipeline \|= ZIO_STAGE_CHECKSUM_VERIFY;
	pio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY;
	}

	if (vd->vdev_ops->vdev_op_leaf) {
	ASSERT0(vd->vdev_children);
	offset += VDEV_LABEL_START_SIZE;
	}

	flags \|= ZIO_VDEV_CHILD_FLAGS(pio);

	/*
	* If we've decided to do a repair, the write is not speculative --
	* even if the original read was.
	*/
	if (flags & ZIO_FLAG_IO_REPAIR)
	flags &= ~ZIO_FLAG_SPECULATIVE;

	/*
	* If we're creating a child I/O that is not associated with a
	* top-level vdev, then the child zio is not an allocating I/O.
	* If this is a retried I/O then we ignore it since we will
	* have already processed the original allocating I/O.
	*/
	if (flags & ZIO_FLAG_IO_ALLOCATING &&
	(vd != vd->vdev_top \|\| (flags & ZIO_FLAG_IO_RETRY))) {
	ASSERT(pio->io_metaslab_class != NULL);
	ASSERT(pio->io_metaslab_class->mc_alloc_throttle_enabled);
	ASSERT(type == ZIO_TYPE_WRITE);
	ASSERT(priority == ZIO_PRIORITY_ASYNC_WRITE);
	ASSERT(!(flags & ZIO_FLAG_IO_REPAIR));
	ASSERT(!(pio->io_flags & ZIO_FLAG_IO_REWRITE) \|\|
	pio->io_child_type == ZIO_CHILD_GANG);

	flags &= ~ZIO_FLAG_IO_ALLOCATING;
	}

	zio = zio_create(pio, pio->io_spa, pio->io_txg, bp, data, size, size,
	done, private, type, priority, flags, vd, offset, &pio->io_bookmark,
	ZIO_STAGE_VDEV_IO_START >> 1, pipeline);
	ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV);

	return (zio);
	}

	zio_t *
	zio_vdev_delegated_io(vdev_t vd, uint64_t offset, abd_t data, uint64_t size,
	zio_type_t type, zio_priority_t priority, zio_flag_t flags,
	zio_done_func_t done, void private)
	{
	zio_t *zio;

	ASSERT(vd->vdev_ops->vdev_op_leaf);

	zio = zio_create(NULL, vd->vdev_spa, 0, NULL,
	data, size, size, done, private, type, priority,
	flags \| ZIO_FLAG_CANFAIL \| ZIO_FLAG_DONT_RETRY \| ZIO_FLAG_DELEGATED,
	vd, offset, NULL,
	ZIO_STAGE_VDEV_IO_START >> 1, ZIO_VDEV_CHILD_PIPELINE);

	return (zio);
	}

	void
	-zio_flush(zio_t zio, vdev_t vd)
	+zio_flush(zio_t pio, vdev_t vd)
	{
	- zio_nowait(zio_ioctl(zio, zio->io_spa, vd, DKIOCFLUSHWRITECACHE,
	- NULL, NULL,
	- ZIO_FLAG_CANFAIL \| ZIO_FLAG_DONT_PROPAGATE \| ZIO_FLAG_DONT_RETRY));
	+ if (vd->vdev_nowritecache)
	+ return;
	+ if (vd->vdev_children == 0) {
	+ zio_nowait(zio_ioctl(pio, vd->vdev_spa, vd,
	+ DKIOCFLUSHWRITECACHE, NULL, NULL, ZIO_FLAG_CANFAIL \|
	+ ZIO_FLAG_DONT_PROPAGATE \| ZIO_FLAG_DONT_RETRY));
	+ } else {
	+ for (uint64_t c = 0; c < vd->vdev_children; c++)
	+ zio_flush(pio, vd->vdev_child[c]);
	+ }
	}

	void
	zio_shrink(zio_t *zio, uint64_t size)
	{
	ASSERT3P(zio->io_executor, ==, NULL);
	ASSERT3U(zio->io_orig_size, ==, zio->io_size);
	ASSERT3U(size, <=, zio->io_size);

	/*
	* We don't shrink for raidz because of problems with the
	* reconstruction when reading back less than the block size.
	* Note, BP_IS_RAIDZ() assumes no compression.
	*/
	ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF);
	if (!BP_IS_RAIDZ(zio->io_bp)) {
	/* we are not doing a raw write */
	ASSERT3U(zio->io_size, ==, zio->io_lsize);
	zio->io_orig_size = zio->io_size = zio->io_lsize = size;
	}
	}

	/*
	* Round provided allocation size up to a value that can be allocated
	* by at least some vdev(s) in the pool with minimum or no additional
	* padding and without extra space usage on others
	*/
	static uint64_t
	zio_roundup_alloc_size(spa_t *spa, uint64_t size)
	{
	if (size > spa->spa_min_alloc)
	return (roundup(size, spa->spa_gcd_alloc));
	return (spa->spa_min_alloc);
	}

	/*
	* ==========================================================================
	* Prepare to read and write logical blocks
	* ==========================================================================
	*/

	static zio_t *
	zio_read_bp_init(zio_t *zio)
	{
	blkptr_t *bp = zio->io_bp;
	uint64_t psize =
	BP_IS_EMBEDDED(bp) ? BPE_GET_PSIZE(bp) : BP_GET_PSIZE(bp);

	ASSERT3P(zio->io_bp, ==, &zio->io_bp_copy);

	if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF &&
	zio->io_child_type == ZIO_CHILD_LOGICAL &&
	!(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) {
	zio_push_transform(zio, abd_alloc_sametype(zio->io_abd, psize),
	psize, psize, zio_decompress);
	}

	if (((BP_IS_PROTECTED(bp) && !(zio->io_flags & ZIO_FLAG_RAW_ENCRYPT)) \|\|
	BP_HAS_INDIRECT_MAC_CKSUM(bp)) &&
	zio->io_child_type == ZIO_CHILD_LOGICAL) {
	zio_push_transform(zio, abd_alloc_sametype(zio->io_abd, psize),
	psize, psize, zio_decrypt);
	}

	if (BP_IS_EMBEDDED(bp) && BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA) {
	int psize = BPE_GET_PSIZE(bp);
	void *data = abd_borrow_buf(zio->io_abd, psize);

	zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
	decode_embedded_bp_compressed(bp, data);
	abd_return_buf_copy(zio->io_abd, data, psize);
	} else {
	ASSERT(!BP_IS_EMBEDDED(bp));
	}

	if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL)
	zio->io_pipeline = ZIO_DDT_READ_PIPELINE;

	return (zio);
	}

	static zio_t *
	zio_write_bp_init(zio_t *zio)
	{
	if (!IO_IS_ALLOCATING(zio))
	return (zio);

	ASSERT(zio->io_child_type != ZIO_CHILD_DDT);

	if (zio->io_bp_override) {
	blkptr_t *bp = zio->io_bp;
	zio_prop_t *zp = &zio->io_prop;

	ASSERT(bp->blk_birth != zio->io_txg);

	bp = zio->io_bp_override;
	zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;

	if (zp->zp_brtwrite)
	return (zio);

	ASSERT(!BP_GET_DEDUP(zio->io_bp_override));

	if (BP_IS_EMBEDDED(bp))
	return (zio);

	/*
	* If we've been overridden and nopwrite is set then
	* set the flag accordingly to indicate that a nopwrite
	* has already occurred.
	*/
	if (!BP_IS_HOLE(bp) && zp->zp_nopwrite) {
	ASSERT(!zp->zp_dedup);
	ASSERT3U(BP_GET_CHECKSUM(bp), ==, zp->zp_checksum);
	zio->io_flags \|= ZIO_FLAG_NOPWRITE;
	return (zio);
	}

	ASSERT(!zp->zp_nopwrite);

	if (BP_IS_HOLE(bp) \|\| !zp->zp_dedup)
	return (zio);

	ASSERT((zio_checksum_table[zp->zp_checksum].ci_flags &
	ZCHECKSUM_FLAG_DEDUP) \|\| zp->zp_dedup_verify);

	if (BP_GET_CHECKSUM(bp) == zp->zp_checksum &&
	!zp->zp_encrypt) {
	BP_SET_DEDUP(bp, 1);
	zio->io_pipeline \|= ZIO_STAGE_DDT_WRITE;
	return (zio);
	}

	/*
	* We were unable to handle this as an override bp, treat
	* it as a regular write I/O.
	*/
	zio->io_bp_override = NULL;
	*bp = zio->io_bp_orig;
	zio->io_pipeline = zio->io_orig_pipeline;
	}

	return (zio);
	}

	static zio_t *
	zio_write_compress(zio_t *zio)
	{
	spa_t *spa = zio->io_spa;
	zio_prop_t *zp = &zio->io_prop;
	enum zio_compress compress = zp->zp_compress;
	blkptr_t *bp = zio->io_bp;
	uint64_t lsize = zio->io_lsize;
	uint64_t psize = zio->io_size;
	uint32_t pass = 1;

	/*
	* If our children haven't all reached the ready stage,
	* wait for them and then repeat this pipeline stage.
	*/
	if (zio_wait_for_children(zio, ZIO_CHILD_LOGICAL_BIT \|
	ZIO_CHILD_GANG_BIT, ZIO_WAIT_READY)) {
	return (NULL);
	}

	if (!IO_IS_ALLOCATING(zio))
	return (zio);

	if (zio->io_children_ready != NULL) {
	/*
	* Now that all our children are ready, run the callback
	* associated with this zio in case it wants to modify the
	* data to be written.
	*/
	ASSERT3U(zp->zp_level, >, 0);
	zio->io_children_ready(zio);
	}

	ASSERT(zio->io_child_type != ZIO_CHILD_DDT);
	ASSERT(zio->io_bp_override == NULL);

	if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg) {
	/*
	* We're rewriting an existing block, which means we're
	* working on behalf of spa_sync(). For spa_sync() to
	* converge, it must eventually be the case that we don't
	* have to allocate new blocks. But compression changes
	* the blocksize, which forces a reallocate, and makes
	* convergence take longer. Therefore, after the first
	* few passes, stop compressing to ensure convergence.
	*/
	pass = spa_sync_pass(spa);

	ASSERT(zio->io_txg == spa_syncing_txg(spa));
	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
	ASSERT(!BP_GET_DEDUP(bp));

	if (pass >= zfs_sync_pass_dont_compress)
	compress = ZIO_COMPRESS_OFF;

	/* Make sure someone doesn't change their mind on overwrites */
	ASSERT(BP_IS_EMBEDDED(bp) \|\| BP_IS_GANG(bp) \|\|
	MIN(zp->zp_copies, spa_max_replication(spa))
	== BP_GET_NDVAS(bp));
	}

	/* If it's a compressed write that is not raw, compress the buffer. */
	if (compress != ZIO_COMPRESS_OFF &&
	!(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) {
	void *cbuf = NULL;
	psize = zio_compress_data(compress, zio->io_abd, &cbuf, lsize,
	zp->zp_complevel);
	if (psize == 0) {
	compress = ZIO_COMPRESS_OFF;
	} else if (psize >= lsize) {
	compress = ZIO_COMPRESS_OFF;
	if (cbuf != NULL)
	zio_buf_free(cbuf, lsize);
	} else if (!zp->zp_dedup && !zp->zp_encrypt &&
	psize <= BPE_PAYLOAD_SIZE &&
	zp->zp_level == 0 && !DMU_OT_HAS_FILL(zp->zp_type) &&
	spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA)) {
	encode_embedded_bp_compressed(bp,
	cbuf, compress, lsize, psize);
	BPE_SET_ETYPE(bp, BP_EMBEDDED_TYPE_DATA);
	BP_SET_TYPE(bp, zio->io_prop.zp_type);
	BP_SET_LEVEL(bp, zio->io_prop.zp_level);
	zio_buf_free(cbuf, lsize);
	bp->blk_birth = zio->io_txg;
	zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
	ASSERT(spa_feature_is_active(spa,
	SPA_FEATURE_EMBEDDED_DATA));
	return (zio);
	} else {
	/*
	* Round compressed size up to the minimum allocation
	* size of the smallest-ashift device, and zero the
	* tail. This ensures that the compressed size of the
	* BP (and thus compressratio property) are correct,
	* in that we charge for the padding used to fill out
	* the last sector.
	*/
	size_t rounded = (size_t)zio_roundup_alloc_size(spa,
	psize);
	if (rounded >= lsize) {
	compress = ZIO_COMPRESS_OFF;
	zio_buf_free(cbuf, lsize);
	psize = lsize;
	} else {
	abd_t *cdata = abd_get_from_buf(cbuf, lsize);
	abd_take_ownership_of_buf(cdata, B_TRUE);
	abd_zero_off(cdata, psize, rounded - psize);
	psize = rounded;
	zio_push_transform(zio, cdata,
	psize, lsize, NULL);
	}
	}

	/*
	* We were unable to handle this as an override bp, treat
	* it as a regular write I/O.
	*/
	zio->io_bp_override = NULL;
	*bp = zio->io_bp_orig;
	zio->io_pipeline = zio->io_orig_pipeline;

	} else if ((zio->io_flags & ZIO_FLAG_RAW_ENCRYPT) != 0 &&
	zp->zp_type == DMU_OT_DNODE) {
	/*
	* The DMU actually relies on the zio layer's compression
	* to free metadnode blocks that have had all contained
	* dnodes freed. As a result, even when doing a raw
	* receive, we must check whether the block can be compressed
	* to a hole.
	*/
	psize = zio_compress_data(ZIO_COMPRESS_EMPTY,
	zio->io_abd, NULL, lsize, zp->zp_complevel);
	if (psize == 0 \|\| psize >= lsize)
	compress = ZIO_COMPRESS_OFF;
	} else if (zio->io_flags & ZIO_FLAG_RAW_COMPRESS &&
	!(zio->io_flags & ZIO_FLAG_RAW_ENCRYPT)) {
	/*
	* If we are raw receiving an encrypted dataset we should not
	* take this codepath because it will change the on-disk block
	* and decryption will fail.
	*/
	size_t rounded = MIN((size_t)zio_roundup_alloc_size(spa, psize),
	lsize);

	if (rounded != psize) {
	abd_t *cdata = abd_alloc_linear(rounded, B_TRUE);
	abd_zero_off(cdata, psize, rounded - psize);
	abd_copy_off(cdata, zio->io_abd, 0, 0, psize);
	psize = rounded;
	zio_push_transform(zio, cdata,
	psize, rounded, NULL);
	}
	} else {
	ASSERT3U(psize, !=, 0);
	}

	/*
	* The final pass of spa_sync() must be all rewrites, but the first
	* few passes offer a trade-off: allocating blocks defers convergence,
	* but newly allocated blocks are sequential, so they can be written
	* to disk faster. Therefore, we allow the first few passes of
	* spa_sync() to allocate new blocks, but force rewrites after that.
	* There should only be a handful of blocks after pass 1 in any case.
	*/
	if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg &&
	BP_GET_PSIZE(bp) == psize &&
	pass >= zfs_sync_pass_rewrite) {
	VERIFY3U(psize, !=, 0);
	enum zio_stage gang_stages = zio->io_pipeline & ZIO_GANG_STAGES;

	zio->io_pipeline = ZIO_REWRITE_PIPELINE \| gang_stages;
	zio->io_flags \|= ZIO_FLAG_IO_REWRITE;
	} else {
	BP_ZERO(bp);
	zio->io_pipeline = ZIO_WRITE_PIPELINE;
	}

	if (psize == 0) {
	if (zio->io_bp_orig.blk_birth != 0 &&
	spa_feature_is_active(spa, SPA_FEATURE_HOLE_BIRTH)) {
	BP_SET_LSIZE(bp, lsize);
	BP_SET_TYPE(bp, zp->zp_type);
	BP_SET_LEVEL(bp, zp->zp_level);
	BP_SET_BIRTH(bp, zio->io_txg, 0);
	}
	zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
	} else {
	ASSERT(zp->zp_checksum != ZIO_CHECKSUM_GANG_HEADER);
	BP_SET_LSIZE(bp, lsize);
	BP_SET_TYPE(bp, zp->zp_type);
	BP_SET_LEVEL(bp, zp->zp_level);
	BP_SET_PSIZE(bp, psize);
	BP_SET_COMPRESS(bp, compress);
	BP_SET_CHECKSUM(bp, zp->zp_checksum);
	BP_SET_DEDUP(bp, zp->zp_dedup);
	BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER);
	if (zp->zp_dedup) {
	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
	ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));
	ASSERT(!zp->zp_encrypt \|\|
	DMU_OT_IS_ENCRYPTED(zp->zp_type));
	zio->io_pipeline = ZIO_DDT_WRITE_PIPELINE;
	}
	if (zp->zp_nopwrite) {
	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
	ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));
	zio->io_pipeline \|= ZIO_STAGE_NOP_WRITE;
	}
	}
	return (zio);
	}

	static zio_t *
	zio_free_bp_init(zio_t *zio)
	{
	blkptr_t *bp = zio->io_bp;

	if (zio->io_child_type == ZIO_CHILD_LOGICAL) {
	if (BP_GET_DEDUP(bp))
	zio->io_pipeline = ZIO_DDT_FREE_PIPELINE;
	}

	ASSERT3P(zio->io_bp, ==, &zio->io_bp_copy);

	return (zio);
	}

	/*
	* ==========================================================================
	* Execute the I/O pipeline
	* ==========================================================================
	*/

	static void
	zio_taskq_dispatch(zio_t *zio, zio_taskq_type_t q, boolean_t cutinline)
	{
	spa_t *spa = zio->io_spa;
	zio_type_t t = zio->io_type;
	int flags = (cutinline ? TQ_FRONT : 0);

	/*
	* If we're a config writer or a probe, the normal issue and
	* interrupt threads may all be blocked waiting for the config lock.
	* In this case, select the otherwise-unused taskq for ZIO_TYPE_NULL.
	*/
	if (zio->io_flags & (ZIO_FLAG_CONFIG_WRITER \| ZIO_FLAG_PROBE))
	t = ZIO_TYPE_NULL;

	/*
	* A similar issue exists for the L2ARC write thread until L2ARC 2.0.
	*/
	if (t == ZIO_TYPE_WRITE && zio->io_vd && zio->io_vd->vdev_aux)
	t = ZIO_TYPE_NULL;

	/*
	* If this is a high priority I/O, then use the high priority taskq if
	* available.
	*/
	if ((zio->io_priority == ZIO_PRIORITY_NOW \|\|
	zio->io_priority == ZIO_PRIORITY_SYNC_WRITE) &&
	spa->spa_zio_taskq[t][q + 1].stqs_count != 0)
	q++;

	ASSERT3U(q, <, ZIO_TASKQ_TYPES);

	/*
	* NB: We are assuming that the zio can only be dispatched
	* to a single taskq at a time. It would be a grievous error
	* to dispatch the zio to another taskq at the same time.
	*/
	ASSERT(taskq_empty_ent(&zio->io_tqent));
	spa_taskq_dispatch_ent(spa, t, q, zio_execute, zio, flags,
	&zio->io_tqent);
	}

	static boolean_t
	zio_taskq_member(zio_t *zio, zio_taskq_type_t q)
	{
	spa_t *spa = zio->io_spa;

	taskq_t *tq = taskq_of_curthread();

	for (zio_type_t t = 0; t < ZIO_TYPES; t++) {
	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
	uint_t i;
	for (i = 0; i < tqs->stqs_count; i++) {
	if (tqs->stqs_taskq[i] == tq)
	return (B_TRUE);
	}
	}

	return (B_FALSE);
	}

	static zio_t *
	zio_issue_async(zio_t *zio)
	{
	zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE);

	return (NULL);
	}

	void
	zio_interrupt(void *zio)
	{
	zio_taskq_dispatch(zio, ZIO_TASKQ_INTERRUPT, B_FALSE);
	}

	void
	zio_delay_interrupt(zio_t *zio)
	{
	/*
	* The timeout_generic() function isn't defined in userspace, so
	* rather than trying to implement the function, the zio delay
	* functionality has been disabled for userspace builds.
	*/

	#ifdef _KERNEL
	/*
	* If io_target_timestamp is zero, then no delay has been registered
	* for this IO, thus jump to the end of this function and "skip" the
	* delay; issuing it directly to the zio layer.
	*/
	if (zio->io_target_timestamp != 0) {
	hrtime_t now = gethrtime();

	if (now >= zio->io_target_timestamp) {
	/*
	* This IO has already taken longer than the target
	* delay to complete, so we don't want to delay it
	* any longer; we "miss" the delay and issue it
	* directly to the zio layer. This is likely due to
	* the target latency being set to a value less than
	* the underlying hardware can satisfy (e.g. delay
	* set to 1ms, but the disks take 10ms to complete an
	* IO request).
	*/

	DTRACE_PROBE2(zio__delay__miss, zio_t *, zio,
	hrtime_t, now);

	zio_interrupt(zio);
	} else {
	taskqid_t tid;
	hrtime_t diff = zio->io_target_timestamp - now;
	clock_t expire_at_tick = ddi_get_lbolt() +
	NSEC_TO_TICK(diff);

	DTRACE_PROBE3(zio__delay__hit, zio_t *, zio,
	hrtime_t, now, hrtime_t, diff);

	if (NSEC_TO_TICK(diff) == 0) {
	/* Our delay is less than a jiffy - just spin */
	zfs_sleep_until(zio->io_target_timestamp);
	zio_interrupt(zio);
	} else {
	/*
	* Use taskq_dispatch_delay() in the place of
	* OpenZFS's timeout_generic().
	*/
	tid = taskq_dispatch_delay(system_taskq,
	zio_interrupt, zio, TQ_NOSLEEP,
	expire_at_tick);
	if (tid == TASKQID_INVALID) {
	/*
	* Couldn't allocate a task. Just
	* finish the zio without a delay.
	*/
	zio_interrupt(zio);
	}
	}
	}
	return;
	}
	#endif
	DTRACE_PROBE1(zio__delay__skip, zio_t *, zio);
	zio_interrupt(zio);
	}

	static void
	zio_deadman_impl(zio_t *pio, int ziodepth)
	{
	zio_t cio, cio_next;
	zio_link_t *zl = NULL;
	vdev_t *vd = pio->io_vd;

	if (zio_deadman_log_all \|\| (vd != NULL && vd->vdev_ops->vdev_op_leaf)) {
	vdev_queue_t *vq = vd ? &vd->vdev_queue : NULL;
	zbookmark_phys_t *zb = &pio->io_bookmark;
	uint64_t delta = gethrtime() - pio->io_timestamp;
	uint64_t failmode = spa_get_deadman_failmode(pio->io_spa);

	zfs_dbgmsg("slow zio[%d]: zio=%px timestamp=%llu "
	"delta=%llu queued=%llu io=%llu "
	"path=%s "
	"last=%llu type=%d "
	"priority=%d flags=0x%llx stage=0x%x "
	"pipeline=0x%x pipeline-trace=0x%x "
	"objset=%llu object=%llu "
	"level=%llu blkid=%llu "
	"offset=%llu size=%llu "
	"error=%d",
	ziodepth, pio, pio->io_timestamp,
	(u_longlong_t)delta, pio->io_delta, pio->io_delay,
	vd ? vd->vdev_path : "NULL",
	vq ? vq->vq_io_complete_ts : 0, pio->io_type,
	pio->io_priority, (u_longlong_t)pio->io_flags,
	pio->io_stage, pio->io_pipeline, pio->io_pipeline_trace,
	(u_longlong_t)zb->zb_objset, (u_longlong_t)zb->zb_object,
	(u_longlong_t)zb->zb_level, (u_longlong_t)zb->zb_blkid,
	(u_longlong_t)pio->io_offset, (u_longlong_t)pio->io_size,
	pio->io_error);
	(void) zfs_ereport_post(FM_EREPORT_ZFS_DEADMAN,
	pio->io_spa, vd, zb, pio, 0);

	if (failmode == ZIO_FAILURE_MODE_CONTINUE &&
	taskq_empty_ent(&pio->io_tqent)) {
	zio_interrupt(pio);
	}
	}

	mutex_enter(&pio->io_lock);
	for (cio = zio_walk_children(pio, &zl); cio != NULL; cio = cio_next) {
	cio_next = zio_walk_children(pio, &zl);
	zio_deadman_impl(cio, ziodepth + 1);
	}
	mutex_exit(&pio->io_lock);
	}

	/*
	* Log the critical information describing this zio and all of its children
	* using the zfs_dbgmsg() interface then post deadman event for the ZED.
	*/
	void
	zio_deadman(zio_t pio, const char tag)
	{
	spa_t *spa = pio->io_spa;
	char *name = spa_name(spa);

	if (!zfs_deadman_enabled \|\| spa_suspended(spa))
	return;

	zio_deadman_impl(pio, 0);

	switch (spa_get_deadman_failmode(spa)) {
	case ZIO_FAILURE_MODE_WAIT:
	zfs_dbgmsg("%s waiting for hung I/O to pool '%s'", tag, name);
	break;

	case ZIO_FAILURE_MODE_CONTINUE:
	zfs_dbgmsg("%s restarting hung I/O for pool '%s'", tag, name);
	break;

	case ZIO_FAILURE_MODE_PANIC:
	fm_panic("%s determined I/O to pool '%s' is hung.", tag, name);
	break;
	}
	}

	/*
	* Execute the I/O pipeline until one of the following occurs:
	* (1) the I/O completes; (2) the pipeline stalls waiting for
	* dependent child I/Os; (3) the I/O issues, so we're waiting
	* for an I/O completion interrupt; (4) the I/O is delegated by
	* vdev-level caching or aggregation; (5) the I/O is deferred
	* due to vdev-level queueing; (6) the I/O is handed off to
	* another thread. In all cases, the pipeline stops whenever
	* there's no CPU work; it never burns a thread in cv_wait_io().
	*
	* There's no locking on io_stage because there's no legitimate way
	* for multiple threads to be attempting to process the same I/O.
	*/
	static zio_pipe_stage_t *zio_pipeline[];

	/*
	* zio_execute() is a wrapper around the static function
	* __zio_execute() so that we can force __zio_execute() to be
	* inlined. This reduces stack overhead which is important
	* because __zio_execute() is called recursively in several zio
	* code paths. zio_execute() itself cannot be inlined because
	* it is externally visible.
	*/
	void
	zio_execute(void *zio)
	{
	fstrans_cookie_t cookie;

	cookie = spl_fstrans_mark();
	__zio_execute(zio);
	spl_fstrans_unmark(cookie);
	}

	/*
	* Used to determine if in the current context the stack is sized large
	* enough to allow zio_execute() to be called recursively. A minimum
	* stack size of 16K is required to avoid needing to re-dispatch the zio.
	*/
	static boolean_t
	zio_execute_stack_check(zio_t *zio)
	{
	#if !defined(HAVE_LARGE_STACKS)
	dsl_pool_t *dp = spa_get_dsl(zio->io_spa);

	/* Executing in txg_sync_thread() context. */
	if (dp && curthread == dp->dp_tx.tx_sync_thread)
	return (B_TRUE);

	/* Pool initialization outside of zio_taskq context. */
	if (dp && spa_is_initializing(dp->dp_spa) &&
	!zio_taskq_member(zio, ZIO_TASKQ_ISSUE) &&
	!zio_taskq_member(zio, ZIO_TASKQ_ISSUE_HIGH))
	return (B_TRUE);
	#else
	(void) zio;
	#endif /* HAVE_LARGE_STACKS */

	return (B_FALSE);
	}

	__attribute__((always_inline))
	static inline void
	__zio_execute(zio_t *zio)
	{
	ASSERT3U(zio->io_queued_timestamp, >, 0);

	while (zio->io_stage < ZIO_STAGE_DONE) {
	enum zio_stage pipeline = zio->io_pipeline;
	enum zio_stage stage = zio->io_stage;

	zio->io_executor = curthread;

	ASSERT(!MUTEX_HELD(&zio->io_lock));
	ASSERT(ISP2(stage));
	ASSERT(zio->io_stall == NULL);

	do {
	stage <<= 1;
	} while ((stage & pipeline) == 0);

	ASSERT(stage <= ZIO_STAGE_DONE);

	/*
	* If we are in interrupt context and this pipeline stage
	* will grab a config lock that is held across I/O,
	* or may wait for an I/O that needs an interrupt thread
	* to complete, issue async to avoid deadlock.
	*
	* For VDEV_IO_START, we cut in line so that the io will
	* be sent to disk promptly.
	*/
	if ((stage & ZIO_BLOCKING_STAGES) && zio->io_vd == NULL &&
	zio_taskq_member(zio, ZIO_TASKQ_INTERRUPT)) {
	boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ?
	zio_requeue_io_start_cut_in_line : B_FALSE;
	zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut);
	return;
	}

	/*
	* If the current context doesn't have large enough stacks
	* the zio must be issued asynchronously to prevent overflow.
	*/
	if (zio_execute_stack_check(zio)) {
	boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ?
	zio_requeue_io_start_cut_in_line : B_FALSE;
	zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut);
	return;
	}

	zio->io_stage = stage;
	zio->io_pipeline_trace \|= zio->io_stage;

	/*
	* The zio pipeline stage returns the next zio to execute
	* (typically the same as this one), or NULL if we should
	* stop.
	*/
	zio = zio_pipeline[highbit64(stage) - 1](zio);

	if (zio == NULL)
	return;
	}
	}


	/*
	* ==========================================================================
	* Initiate I/O, either sync or async
	* ==========================================================================
	*/
	int
	zio_wait(zio_t *zio)
	{
	/*
	* Some routines, like zio_free_sync(), may return a NULL zio
	* to avoid the performance overhead of creating and then destroying
	* an unneeded zio. For the callers' simplicity, we accept a NULL
	* zio and ignore it.
	*/
	if (zio == NULL)
	return (0);

	long timeout = MSEC_TO_TICK(zfs_deadman_ziotime_ms);
	int error;

	ASSERT3S(zio->io_stage, ==, ZIO_STAGE_OPEN);
	ASSERT3P(zio->io_executor, ==, NULL);

	zio->io_waiter = curthread;
	ASSERT0(zio->io_queued_timestamp);
	zio->io_queued_timestamp = gethrtime();

	__zio_execute(zio);

	mutex_enter(&zio->io_lock);
	while (zio->io_executor != NULL) {
	error = cv_timedwait_io(&zio->io_cv, &zio->io_lock,
	ddi_get_lbolt() + timeout);

	if (zfs_deadman_enabled && error == -1 &&
	gethrtime() - zio->io_queued_timestamp >
	spa_deadman_ziotime(zio->io_spa)) {
	mutex_exit(&zio->io_lock);
	timeout = MSEC_TO_TICK(zfs_deadman_checktime_ms);
	zio_deadman(zio, FTAG);
	mutex_enter(&zio->io_lock);
	}
	}
	mutex_exit(&zio->io_lock);

	error = zio->io_error;
	zio_destroy(zio);

	return (error);
	}

	void
	zio_nowait(zio_t *zio)
	{
	/*
	* See comment in zio_wait().
	*/
	if (zio == NULL)
	return;

	ASSERT3P(zio->io_executor, ==, NULL);

	if (zio->io_child_type == ZIO_CHILD_LOGICAL &&
	list_is_empty(&zio->io_parent_list)) {
	zio_t *pio;

	/*
	* This is a logical async I/O with no parent to wait for it.
	* We add it to the spa_async_root_zio "Godfather" I/O which
	* will ensure they complete prior to unloading the pool.
	*/
	spa_t *spa = zio->io_spa;
	pio = spa->spa_async_zio_root[CPU_SEQID_UNSTABLE];

	zio_add_child(pio, zio);
	}

	ASSERT0(zio->io_queued_timestamp);
	zio->io_queued_timestamp = gethrtime();
	__zio_execute(zio);
	}

	/*
	* ==========================================================================
	* Reexecute, cancel, or suspend/resume failed I/O
	* ==========================================================================
	*/

	static void
	zio_reexecute(void *arg)
	{
	zio_t *pio = arg;
	zio_t cio, cio_next;

	ASSERT(pio->io_child_type == ZIO_CHILD_LOGICAL);
	ASSERT(pio->io_orig_stage == ZIO_STAGE_OPEN);
	ASSERT(pio->io_gang_leader == NULL);
	ASSERT(pio->io_gang_tree == NULL);

	pio->io_flags = pio->io_orig_flags;
	pio->io_stage = pio->io_orig_stage;
	pio->io_pipeline = pio->io_orig_pipeline;
	pio->io_reexecute = 0;
	pio->io_flags \|= ZIO_FLAG_REEXECUTED;
	pio->io_pipeline_trace = 0;
	pio->io_error = 0;
	for (int w = 0; w < ZIO_WAIT_TYPES; w++)
	pio->io_state[w] = 0;
	for (int c = 0; c < ZIO_CHILD_TYPES; c++)
	pio->io_child_error[c] = 0;

	if (IO_IS_ALLOCATING(pio))
	BP_ZERO(pio->io_bp);

	/*
	* As we reexecute pio's children, new children could be created.
	* New children go to the head of pio's io_child_list, however,
	* so we will (correctly) not reexecute them. The key is that
	* the remainder of pio's io_child_list, from 'cio_next' onward,
	* cannot be affected by any side effects of reexecuting 'cio'.
	*/
	zio_link_t *zl = NULL;
	mutex_enter(&pio->io_lock);
	for (cio = zio_walk_children(pio, &zl); cio != NULL; cio = cio_next) {
	cio_next = zio_walk_children(pio, &zl);
	for (int w = 0; w < ZIO_WAIT_TYPES; w++)
	pio->io_children[cio->io_child_type][w]++;
	mutex_exit(&pio->io_lock);
	zio_reexecute(cio);
	mutex_enter(&pio->io_lock);
	}
	mutex_exit(&pio->io_lock);

	/*
	* Now that all children have been reexecuted, execute the parent.
	* We don't reexecute "The Godfather" I/O here as it's the
	* responsibility of the caller to wait on it.
	*/
	if (!(pio->io_flags & ZIO_FLAG_GODFATHER)) {
	pio->io_queued_timestamp = gethrtime();
	__zio_execute(pio);
	}
	}

	void
	zio_suspend(spa_t spa, zio_t zio, zio_suspend_reason_t reason)
	{
	if (spa_get_failmode(spa) == ZIO_FAILURE_MODE_PANIC)
	fm_panic("Pool '%s' has encountered an uncorrectable I/O "
	"failure and the failure mode property for this pool "
	"is set to panic.", spa_name(spa));

	- cmn_err(CE_WARN, "Pool '%s' has encountered an uncorrectable I/O "
	- "failure and has been suspended.\n", spa_name(spa));
	+ if (reason != ZIO_SUSPEND_MMP) {
	+ cmn_err(CE_WARN, "Pool '%s' has encountered an uncorrectable "
	+ "I/O failure and has been suspended.\n", spa_name(spa));
	+ }

	(void) zfs_ereport_post(FM_EREPORT_ZFS_IO_FAILURE, spa, NULL,
	NULL, NULL, 0);

	mutex_enter(&spa->spa_suspend_lock);

	if (spa->spa_suspend_zio_root == NULL)
	spa->spa_suspend_zio_root = zio_root(spa, NULL, NULL,
	ZIO_FLAG_CANFAIL \| ZIO_FLAG_SPECULATIVE \|
	ZIO_FLAG_GODFATHER);

	spa->spa_suspended = reason;

	if (zio != NULL) {
	ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER));
	ASSERT(zio != spa->spa_suspend_zio_root);
	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
	ASSERT(zio_unique_parent(zio) == NULL);
	ASSERT(zio->io_stage == ZIO_STAGE_DONE);
	zio_add_child(spa->spa_suspend_zio_root, zio);
	}

	mutex_exit(&spa->spa_suspend_lock);
	}

	int
	zio_resume(spa_t *spa)
	{
	zio_t *pio;

	/*
	* Reexecute all previously suspended i/o.
	*/
	mutex_enter(&spa->spa_suspend_lock);
	spa->spa_suspended = ZIO_SUSPEND_NONE;
	cv_broadcast(&spa->spa_suspend_cv);
	pio = spa->spa_suspend_zio_root;
	spa->spa_suspend_zio_root = NULL;
	mutex_exit(&spa->spa_suspend_lock);

	if (pio == NULL)
	return (0);

	zio_reexecute(pio);
	return (zio_wait(pio));
	}

	void
	zio_resume_wait(spa_t *spa)
	{
	mutex_enter(&spa->spa_suspend_lock);
	while (spa_suspended(spa))
	cv_wait(&spa->spa_suspend_cv, &spa->spa_suspend_lock);
	mutex_exit(&spa->spa_suspend_lock);
	}

	/*
	* ==========================================================================
	* Gang blocks.
	*
	* A gang block is a collection of small blocks that looks to the DMU
	* like one large block. When zio_dva_allocate() cannot find a block
	* of the requested size, due to either severe fragmentation or the pool
	* being nearly full, it calls zio_write_gang_block() to construct the
	* block from smaller fragments.
	*
	* A gang block consists of a gang header (zio_gbh_phys_t) and up to
	* three (SPA_GBH_NBLKPTRS) gang members. The gang header is just like
	* an indirect block: it's an array of block pointers. It consumes
	* only one sector and hence is allocatable regardless of fragmentation.
	* The gang header's bps point to its gang members, which hold the data.
	*
	* Gang blocks are self-checksumming, using the bp's <vdev, offset, txg>
	* as the verifier to ensure uniqueness of the SHA256 checksum.
	* Critically, the gang block bp's blk_cksum is the checksum of the data,
	* not the gang header. This ensures that data block signatures (needed for
	* deduplication) are independent of how the block is physically stored.
	*
	* Gang blocks can be nested: a gang member may itself be a gang block.
	* Thus every gang block is a tree in which root and all interior nodes are
	* gang headers, and the leaves are normal blocks that contain user data.
	* The root of the gang tree is called the gang leader.
	*
	* To perform any operation (read, rewrite, free, claim) on a gang block,
	* zio_gang_assemble() first assembles the gang tree (minus data leaves)
	* in the io_gang_tree field of the original logical i/o by recursively
	* reading the gang leader and all gang headers below it. This yields
	* an in-core tree containing the contents of every gang header and the
	* bps for every constituent of the gang block.
	*
	* With the gang tree now assembled, zio_gang_issue() just walks the gang tree
	* and invokes a callback on each bp. To free a gang block, zio_gang_issue()
	* calls zio_free_gang() -- a trivial wrapper around zio_free() -- for each bp.
	* zio_claim_gang() provides a similarly trivial wrapper for zio_claim().
	* zio_read_gang() is a wrapper around zio_read() that omits reading gang
	* headers, since we already have those in io_gang_tree. zio_rewrite_gang()
	* performs a zio_rewrite() of the data or, for gang headers, a zio_rewrite()
	* of the gang header plus zio_checksum_compute() of the data to update the
	* gang header's blk_cksum as described above.
	*
	* The two-phase assemble/issue model solves the problem of partial failure --
	* what if you'd freed part of a gang block but then couldn't read the
	* gang header for another part? Assembling the entire gang tree first
	* ensures that all the necessary gang header I/O has succeeded before
	* starting the actual work of free, claim, or write. Once the gang tree
	* is assembled, free and claim are in-memory operations that cannot fail.
	*
	* In the event that a gang write fails, zio_dva_unallocate() walks the
	* gang tree to immediately free (i.e. insert back into the space map)
	* everything we've allocated. This ensures that we don't get ENOSPC
	* errors during repeated suspend/resume cycles due to a flaky device.
	*
	* Gang rewrites only happen during sync-to-convergence. If we can't assemble
	* the gang tree, we won't modify the block, so we can safely defer the free
	* (knowing that the block is still intact). If we can assemble the gang
	* tree, then even if some of the rewrites fail, zio_dva_unallocate() will free
	* each constituent bp and we can allocate a new block on the next sync pass.
	*
	* In all cases, the gang tree allows complete recovery from partial failure.
	* ==========================================================================
	*/

	static void
	zio_gang_issue_func_done(zio_t *zio)
	{
	abd_free(zio->io_abd);
	}

	static zio_t *
	zio_read_gang(zio_t pio, blkptr_t bp, zio_gang_node_t gn, abd_t data,
	uint64_t offset)
	{
	if (gn != NULL)
	return (pio);

	return (zio_read(pio, pio->io_spa, bp, abd_get_offset(data, offset),
	BP_GET_PSIZE(bp), zio_gang_issue_func_done,
	NULL, pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio),
	&pio->io_bookmark));
	}

	static zio_t *
	zio_rewrite_gang(zio_t pio, blkptr_t bp, zio_gang_node_t gn, abd_t data,
	uint64_t offset)
	{
	zio_t *zio;

	if (gn != NULL) {
	abd_t *gbh_abd =
	abd_get_from_buf(gn->gn_gbh, SPA_GANGBLOCKSIZE);
	zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp,
	gbh_abd, SPA_GANGBLOCKSIZE, zio_gang_issue_func_done, NULL,
	pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio),
	&pio->io_bookmark);
	/*
	* As we rewrite each gang header, the pipeline will compute
	* a new gang block header checksum for it; but no one will
	* compute a new data checksum, so we do that here. The one
	* exception is the gang leader: the pipeline already computed
	* its data checksum because that stage precedes gang assembly.
	* (Presently, nothing actually uses interior data checksums;
	* this is just good hygiene.)
	*/
	if (gn != pio->io_gang_leader->io_gang_tree) {
	abd_t *buf = abd_get_offset(data, offset);

	zio_checksum_compute(zio, BP_GET_CHECKSUM(bp),
	buf, BP_GET_PSIZE(bp));

	abd_free(buf);
	}
	/*
	* If we are here to damage data for testing purposes,
	* leave the GBH alone so that we can detect the damage.
	*/
	if (pio->io_gang_leader->io_flags & ZIO_FLAG_INDUCE_DAMAGE)
	zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES;
	} else {
	zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp,
	abd_get_offset(data, offset), BP_GET_PSIZE(bp),
	zio_gang_issue_func_done, NULL, pio->io_priority,
	ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark);
	}

	return (zio);
	}

	static zio_t *
	zio_free_gang(zio_t pio, blkptr_t bp, zio_gang_node_t gn, abd_t data,
	uint64_t offset)
	{
	(void) gn, (void) data, (void) offset;

	zio_t *zio = zio_free_sync(pio, pio->io_spa, pio->io_txg, bp,
	ZIO_GANG_CHILD_FLAGS(pio));
	if (zio == NULL) {
	zio = zio_null(pio, pio->io_spa,
	NULL, NULL, NULL, ZIO_GANG_CHILD_FLAGS(pio));
	}
	return (zio);
	}

	static zio_t *
	zio_claim_gang(zio_t pio, blkptr_t bp, zio_gang_node_t gn, abd_t data,
	uint64_t offset)
	{
	(void) gn, (void) data, (void) offset;
	return (zio_claim(pio, pio->io_spa, pio->io_txg, bp,
	NULL, NULL, ZIO_GANG_CHILD_FLAGS(pio)));
	}

	static zio_gang_issue_func_t *zio_gang_issue_func[ZIO_TYPES] = {
	NULL,
	zio_read_gang,
	zio_rewrite_gang,
	zio_free_gang,
	zio_claim_gang,
	NULL
	};

	static void zio_gang_tree_assemble_done(zio_t *zio);

	static zio_gang_node_t *
	zio_gang_node_alloc(zio_gang_node_t **gnpp)
	{
	zio_gang_node_t *gn;

	ASSERT(*gnpp == NULL);

	gn = kmem_zalloc(sizeof (*gn), KM_SLEEP);
	gn->gn_gbh = zio_buf_alloc(SPA_GANGBLOCKSIZE);
	*gnpp = gn;

	return (gn);
	}

	static void
	zio_gang_node_free(zio_gang_node_t **gnpp)
	{
	zio_gang_node_t gn = gnpp;

	for (int g = 0; g < SPA_GBH_NBLKPTRS; g++)
	ASSERT(gn->gn_child[g] == NULL);

	zio_buf_free(gn->gn_gbh, SPA_GANGBLOCKSIZE);
	kmem_free(gn, sizeof (*gn));
	*gnpp = NULL;
	}

	static void
	zio_gang_tree_free(zio_gang_node_t **gnpp)
	{
	zio_gang_node_t gn = gnpp;

	if (gn == NULL)
	return;

	for (int g = 0; g < SPA_GBH_NBLKPTRS; g++)
	zio_gang_tree_free(&gn->gn_child[g]);

	zio_gang_node_free(gnpp);
	}

	static void
	zio_gang_tree_assemble(zio_t gio, blkptr_t bp, zio_gang_node_t **gnpp)
	{
	zio_gang_node_t *gn = zio_gang_node_alloc(gnpp);
	abd_t *gbh_abd = abd_get_from_buf(gn->gn_gbh, SPA_GANGBLOCKSIZE);

	ASSERT(gio->io_gang_leader == gio);
	ASSERT(BP_IS_GANG(bp));

	zio_nowait(zio_read(gio, gio->io_spa, bp, gbh_abd, SPA_GANGBLOCKSIZE,
	zio_gang_tree_assemble_done, gn, gio->io_priority,
	ZIO_GANG_CHILD_FLAGS(gio), &gio->io_bookmark));
	}

	static void
	zio_gang_tree_assemble_done(zio_t *zio)
	{
	zio_t *gio = zio->io_gang_leader;
	zio_gang_node_t *gn = zio->io_private;
	blkptr_t *bp = zio->io_bp;

	ASSERT(gio == zio_unique_parent(zio));
	ASSERT(list_is_empty(&zio->io_child_list));

	if (zio->io_error)
	return;

	/* this ABD was created from a linear buf in zio_gang_tree_assemble */
	if (BP_SHOULD_BYTESWAP(bp))
	byteswap_uint64_array(abd_to_buf(zio->io_abd), zio->io_size);

	ASSERT3P(abd_to_buf(zio->io_abd), ==, gn->gn_gbh);
	ASSERT(zio->io_size == SPA_GANGBLOCKSIZE);
	ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC);

	abd_free(zio->io_abd);

	for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) {
	blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g];
	if (!BP_IS_GANG(gbp))
	continue;
	zio_gang_tree_assemble(gio, gbp, &gn->gn_child[g]);
	}
	}

	static void
	zio_gang_tree_issue(zio_t pio, zio_gang_node_t gn, blkptr_t bp, abd_t data,
	uint64_t offset)
	{
	zio_t *gio = pio->io_gang_leader;
	zio_t *zio;

	ASSERT(BP_IS_GANG(bp) == !!gn);
	ASSERT(BP_GET_CHECKSUM(bp) == BP_GET_CHECKSUM(gio->io_bp));
	ASSERT(BP_GET_LSIZE(bp) == BP_GET_PSIZE(bp) \|\| gn == gio->io_gang_tree);

	/*
	* If you're a gang header, your data is in gn->gn_gbh.
	* If you're a gang member, your data is in 'data' and gn == NULL.
	*/
	zio = zio_gang_issue_func[gio->io_type](pio, bp, gn, data, offset);

	if (gn != NULL) {
	ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC);

	for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) {
	blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g];
	if (BP_IS_HOLE(gbp))
	continue;
	zio_gang_tree_issue(zio, gn->gn_child[g], gbp, data,
	offset);
	offset += BP_GET_PSIZE(gbp);
	}
	}

	if (gn == gio->io_gang_tree)
	ASSERT3U(gio->io_size, ==, offset);

	if (zio != pio)
	zio_nowait(zio);
	}

	static zio_t *
	zio_gang_assemble(zio_t *zio)
	{
	blkptr_t *bp = zio->io_bp;

	ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == NULL);
	ASSERT(zio->io_child_type > ZIO_CHILD_GANG);

	zio->io_gang_leader = zio;

	zio_gang_tree_assemble(zio, bp, &zio->io_gang_tree);

	return (zio);
	}

	static zio_t *
	zio_gang_issue(zio_t *zio)
	{
	blkptr_t *bp = zio->io_bp;

	if (zio_wait_for_children(zio, ZIO_CHILD_GANG_BIT, ZIO_WAIT_DONE)) {
	return (NULL);
	}

	ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio);
	ASSERT(zio->io_child_type > ZIO_CHILD_GANG);

	if (zio->io_child_error[ZIO_CHILD_GANG] == 0)
	zio_gang_tree_issue(zio, zio->io_gang_tree, bp, zio->io_abd,
	0);
	else
	zio_gang_tree_free(&zio->io_gang_tree);

	zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;

	return (zio);
	}

	static void
	zio_write_gang_member_ready(zio_t *zio)
	{
	zio_t *pio = zio_unique_parent(zio);
	dva_t *cdva = zio->io_bp->blk_dva;
	dva_t *pdva = pio->io_bp->blk_dva;
	uint64_t asize;
	zio_t *gio __maybe_unused = zio->io_gang_leader;

	if (BP_IS_HOLE(zio->io_bp))
	return;

	ASSERT(BP_IS_HOLE(&zio->io_bp_orig));

	ASSERT(zio->io_child_type == ZIO_CHILD_GANG);
	ASSERT3U(zio->io_prop.zp_copies, ==, gio->io_prop.zp_copies);
	ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(zio->io_bp));
	ASSERT3U(pio->io_prop.zp_copies, <=, BP_GET_NDVAS(pio->io_bp));
	VERIFY3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp));

	mutex_enter(&pio->io_lock);
	for (int d = 0; d < BP_GET_NDVAS(zio->io_bp); d++) {
	ASSERT(DVA_GET_GANG(&pdva[d]));
	asize = DVA_GET_ASIZE(&pdva[d]);
	asize += DVA_GET_ASIZE(&cdva[d]);
	DVA_SET_ASIZE(&pdva[d], asize);
	}
	mutex_exit(&pio->io_lock);
	}

	static void
	zio_write_gang_done(zio_t *zio)
	{
	/*
	* The io_abd field will be NULL for a zio with no data. The io_flags
	* will initially have the ZIO_FLAG_NODATA bit flag set, but we can't
	* check for it here as it is cleared in zio_ready.
	*/
	if (zio->io_abd != NULL)
	abd_free(zio->io_abd);
	}

	static zio_t *
	zio_write_gang_block(zio_t pio, metaslab_class_t mc)
	{
	spa_t *spa = pio->io_spa;
	blkptr_t *bp = pio->io_bp;
	zio_t *gio = pio->io_gang_leader;
	zio_t *zio;
	zio_gang_node_t gn, *gnpp;
	zio_gbh_phys_t *gbh;
	abd_t *gbh_abd;
	uint64_t txg = pio->io_txg;
	uint64_t resid = pio->io_size;
	uint64_t lsize;
	int copies = gio->io_prop.zp_copies;
	zio_prop_t zp;
	int error;
	boolean_t has_data = !(pio->io_flags & ZIO_FLAG_NODATA);

	/*
	* If one copy was requested, store 2 copies of the GBH, so that we
	* can still traverse all the data (e.g. to free or scrub) even if a
	* block is damaged. Note that we can't store 3 copies of the GBH in
	* all cases, e.g. with encryption, which uses DVA[2] for the IV+salt.
	*/
	int gbh_copies = copies;
	if (gbh_copies == 1) {
	gbh_copies = MIN(2, spa_max_replication(spa));
	}

	int flags = METASLAB_HINTBP_FAVOR \| METASLAB_GANG_HEADER;
	if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
	ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
	ASSERT(has_data);

	flags \|= METASLAB_ASYNC_ALLOC;
	VERIFY(zfs_refcount_held(&mc->mc_allocator[pio->io_allocator].
	mca_alloc_slots, pio));

	/*
	* The logical zio has already placed a reservation for
	* 'copies' allocation slots but gang blocks may require
	* additional copies. These additional copies
	* (i.e. gbh_copies - copies) are guaranteed to succeed
	* since metaslab_class_throttle_reserve() always allows
	* additional reservations for gang blocks.
	*/
	VERIFY(metaslab_class_throttle_reserve(mc, gbh_copies - copies,
	pio->io_allocator, pio, flags));
	}

	error = metaslab_alloc(spa, mc, SPA_GANGBLOCKSIZE,
	bp, gbh_copies, txg, pio == gio ? NULL : gio->io_bp, flags,
	&pio->io_alloc_list, pio, pio->io_allocator);
	if (error) {
	if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
	ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
	ASSERT(has_data);

	/*
	* If we failed to allocate the gang block header then
	* we remove any additional allocation reservations that
	* we placed here. The original reservation will
	* be removed when the logical I/O goes to the ready
	* stage.
	*/
	metaslab_class_throttle_unreserve(mc,
	gbh_copies - copies, pio->io_allocator, pio);
	}

	pio->io_error = error;
	return (pio);
	}

	if (pio == gio) {
	gnpp = &gio->io_gang_tree;
	} else {
	gnpp = pio->io_private;
	ASSERT(pio->io_ready == zio_write_gang_member_ready);
	}

	gn = zio_gang_node_alloc(gnpp);
	gbh = gn->gn_gbh;
	memset(gbh, 0, SPA_GANGBLOCKSIZE);
	gbh_abd = abd_get_from_buf(gbh, SPA_GANGBLOCKSIZE);

	/*
	* Create the gang header.
	*/
	zio = zio_rewrite(pio, spa, txg, bp, gbh_abd, SPA_GANGBLOCKSIZE,
	zio_write_gang_done, NULL, pio->io_priority,
	ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark);

	/*
	* Create and nowait the gang children.
	*/
	for (int g = 0; resid != 0; resid -= lsize, g++) {
	lsize = P2ROUNDUP(resid / (SPA_GBH_NBLKPTRS - g),
	SPA_MINBLOCKSIZE);
	ASSERT(lsize >= SPA_MINBLOCKSIZE && lsize <= resid);

	zp.zp_checksum = gio->io_prop.zp_checksum;
	zp.zp_compress = ZIO_COMPRESS_OFF;
	zp.zp_complevel = gio->io_prop.zp_complevel;
	zp.zp_type = DMU_OT_NONE;
	zp.zp_level = 0;
	zp.zp_copies = gio->io_prop.zp_copies;
	zp.zp_dedup = B_FALSE;
	zp.zp_dedup_verify = B_FALSE;
	zp.zp_nopwrite = B_FALSE;
	zp.zp_encrypt = gio->io_prop.zp_encrypt;
	zp.zp_byteorder = gio->io_prop.zp_byteorder;
	memset(zp.zp_salt, 0, ZIO_DATA_SALT_LEN);
	memset(zp.zp_iv, 0, ZIO_DATA_IV_LEN);
	memset(zp.zp_mac, 0, ZIO_DATA_MAC_LEN);

	zio_t *cio = zio_write(zio, spa, txg, &gbh->zg_blkptr[g],
	has_data ? abd_get_offset(pio->io_abd, pio->io_size -
	resid) : NULL, lsize, lsize, &zp,
	zio_write_gang_member_ready, NULL,
	zio_write_gang_done, &gn->gn_child[g], pio->io_priority,
	ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark);

	if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
	ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
	ASSERT(has_data);

	/*
	* Gang children won't throttle but we should
	* account for their work, so reserve an allocation
	* slot for them here.
	*/
	VERIFY(metaslab_class_throttle_reserve(mc,
	zp.zp_copies, cio->io_allocator, cio, flags));
	}
	zio_nowait(cio);
	}

	/*
	* Set pio's pipeline to just wait for zio to finish.
	*/
	pio->io_pipeline = ZIO_INTERLOCK_PIPELINE;

	zio_nowait(zio);

	return (pio);
	}

	/*
	* The zio_nop_write stage in the pipeline determines if allocating a
	* new bp is necessary. The nopwrite feature can handle writes in
	* either syncing or open context (i.e. zil writes) and as a result is
	* mutually exclusive with dedup.
	*
	* By leveraging a cryptographically secure checksum, such as SHA256, we
	* can compare the checksums of the new data and the old to determine if
	* allocating a new block is required. Note that our requirements for
	* cryptographic strength are fairly weak: there can't be any accidental
	* hash collisions, but we don't need to be secure against intentional
	* (malicious) collisions. To trigger a nopwrite, you have to be able
	* to write the file to begin with, and triggering an incorrect (hash
	* collision) nopwrite is no worse than simply writing to the file.
	* That said, there are no known attacks against the checksum algorithms
	* used for nopwrite, assuming that the salt and the checksums
	* themselves remain secret.
	*/
	static zio_t *
	zio_nop_write(zio_t *zio)
	{
	blkptr_t *bp = zio->io_bp;
	blkptr_t *bp_orig = &zio->io_bp_orig;
	zio_prop_t *zp = &zio->io_prop;

	ASSERT(BP_IS_HOLE(bp));
	ASSERT(BP_GET_LEVEL(bp) == 0);
	ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));
	ASSERT(zp->zp_nopwrite);
	ASSERT(!zp->zp_dedup);
	ASSERT(zio->io_bp_override == NULL);
	ASSERT(IO_IS_ALLOCATING(zio));

	/*
	* Check to see if the original bp and the new bp have matching
	* characteristics (i.e. same checksum, compression algorithms, etc).
	* If they don't then just continue with the pipeline which will
	* allocate a new bp.
	*/
	if (BP_IS_HOLE(bp_orig) \|\|
	!(zio_checksum_table[BP_GET_CHECKSUM(bp)].ci_flags &
	ZCHECKSUM_FLAG_NOPWRITE) \|\|
	BP_IS_ENCRYPTED(bp) \|\| BP_IS_ENCRYPTED(bp_orig) \|\|
	BP_GET_CHECKSUM(bp) != BP_GET_CHECKSUM(bp_orig) \|\|
	BP_GET_COMPRESS(bp) != BP_GET_COMPRESS(bp_orig) \|\|
	BP_GET_DEDUP(bp) != BP_GET_DEDUP(bp_orig) \|\|
	zp->zp_copies != BP_GET_NDVAS(bp_orig))
	return (zio);

	/*
	* If the checksums match then reset the pipeline so that we
	* avoid allocating a new bp and issuing any I/O.
	*/
	if (ZIO_CHECKSUM_EQUAL(bp->blk_cksum, bp_orig->blk_cksum)) {
	ASSERT(zio_checksum_table[zp->zp_checksum].ci_flags &
	ZCHECKSUM_FLAG_NOPWRITE);
	ASSERT3U(BP_GET_PSIZE(bp), ==, BP_GET_PSIZE(bp_orig));
	ASSERT3U(BP_GET_LSIZE(bp), ==, BP_GET_LSIZE(bp_orig));
	ASSERT(zp->zp_compress != ZIO_COMPRESS_OFF);
	ASSERT3U(bp->blk_prop, ==, bp_orig->blk_prop);

	/*
	* If we're overwriting a block that is currently on an
	* indirect vdev, then ignore the nopwrite request and
	* allow a new block to be allocated on a concrete vdev.
	*/
	spa_config_enter(zio->io_spa, SCL_VDEV, FTAG, RW_READER);
	for (int d = 0; d < BP_GET_NDVAS(bp_orig); d++) {
	vdev_t *tvd = vdev_lookup_top(zio->io_spa,
	DVA_GET_VDEV(&bp_orig->blk_dva[d]));
	if (tvd->vdev_ops == &vdev_indirect_ops) {
	spa_config_exit(zio->io_spa, SCL_VDEV, FTAG);
	return (zio);
	}
	}
	spa_config_exit(zio->io_spa, SCL_VDEV, FTAG);

	bp = bp_orig;
	zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
	zio->io_flags \|= ZIO_FLAG_NOPWRITE;
	}

	return (zio);
	}

	/*
	* ==========================================================================
	* Block Reference Table
	* ==========================================================================
	*/
	static zio_t *
	zio_brt_free(zio_t *zio)
	{
	blkptr_t *bp;

	bp = zio->io_bp;

	if (BP_GET_LEVEL(bp) > 0 \|\|
	BP_IS_METADATA(bp) \|\|
	!brt_maybe_exists(zio->io_spa, bp)) {
	return (zio);
	}

	if (!brt_entry_decref(zio->io_spa, bp)) {
	/*
	* This isn't the last reference, so we cannot free
	* the data yet.
	*/
	zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
	}

	return (zio);
	}

	/*
	* ==========================================================================
	* Dedup
	* ==========================================================================
	*/
	static void
	zio_ddt_child_read_done(zio_t *zio)
	{
	blkptr_t *bp = zio->io_bp;
	ddt_entry_t *dde = zio->io_private;
	ddt_phys_t *ddp;
	zio_t *pio = zio_unique_parent(zio);

	mutex_enter(&pio->io_lock);
	ddp = ddt_phys_select(dde, bp);
	if (zio->io_error == 0)
	ddt_phys_clear(ddp); /* this ddp doesn't need repair */

	if (zio->io_error == 0 && dde->dde_repair_abd == NULL)
	dde->dde_repair_abd = zio->io_abd;
	else
	abd_free(zio->io_abd);
	mutex_exit(&pio->io_lock);
	}

	static zio_t *
	zio_ddt_read_start(zio_t *zio)
	{
	blkptr_t *bp = zio->io_bp;

	ASSERT(BP_GET_DEDUP(bp));
	ASSERT(BP_GET_PSIZE(bp) == zio->io_size);
	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);

	if (zio->io_child_error[ZIO_CHILD_DDT]) {
	ddt_t *ddt = ddt_select(zio->io_spa, bp);
	ddt_entry_t *dde = ddt_repair_start(ddt, bp);
	ddt_phys_t *ddp = dde->dde_phys;
	ddt_phys_t *ddp_self = ddt_phys_select(dde, bp);
	blkptr_t blk;

	ASSERT(zio->io_vsd == NULL);
	zio->io_vsd = dde;

	if (ddp_self == NULL)
	return (zio);

	for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
	if (ddp->ddp_phys_birth == 0 \|\| ddp == ddp_self)
	continue;
	ddt_bp_create(ddt->ddt_checksum, &dde->dde_key, ddp,
	&blk);
	zio_nowait(zio_read(zio, zio->io_spa, &blk,
	abd_alloc_for_io(zio->io_size, B_TRUE),
	zio->io_size, zio_ddt_child_read_done, dde,
	zio->io_priority, ZIO_DDT_CHILD_FLAGS(zio) \|
	ZIO_FLAG_DONT_PROPAGATE, &zio->io_bookmark));
	}
	return (zio);
	}

	zio_nowait(zio_read(zio, zio->io_spa, bp,
	zio->io_abd, zio->io_size, NULL, NULL, zio->io_priority,
	ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark));

	return (zio);
	}

	static zio_t *
	zio_ddt_read_done(zio_t *zio)
	{
	blkptr_t *bp = zio->io_bp;

	if (zio_wait_for_children(zio, ZIO_CHILD_DDT_BIT, ZIO_WAIT_DONE)) {
	return (NULL);
	}

	ASSERT(BP_GET_DEDUP(bp));
	ASSERT(BP_GET_PSIZE(bp) == zio->io_size);
	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);

	if (zio->io_child_error[ZIO_CHILD_DDT]) {
	ddt_t *ddt = ddt_select(zio->io_spa, bp);
	ddt_entry_t *dde = zio->io_vsd;
	if (ddt == NULL) {
	ASSERT(spa_load_state(zio->io_spa) != SPA_LOAD_NONE);
	return (zio);
	}
	if (dde == NULL) {
	zio->io_stage = ZIO_STAGE_DDT_READ_START >> 1;
	zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE);
	return (NULL);
	}
	if (dde->dde_repair_abd != NULL) {
	abd_copy(zio->io_abd, dde->dde_repair_abd,
	zio->io_size);
	zio->io_child_error[ZIO_CHILD_DDT] = 0;
	}
	ddt_repair_done(ddt, dde);
	zio->io_vsd = NULL;
	}

	ASSERT(zio->io_vsd == NULL);

	return (zio);
	}

	static boolean_t
	zio_ddt_collision(zio_t zio, ddt_t ddt, ddt_entry_t *dde)
	{
	spa_t *spa = zio->io_spa;
	boolean_t do_raw = !!(zio->io_flags & ZIO_FLAG_RAW);

	ASSERT(!(zio->io_bp_override && do_raw));

	/*
	* Note: we compare the original data, not the transformed data,
	* because when zio->io_bp is an override bp, we will not have
	* pushed the I/O transforms. That's an important optimization
	* because otherwise we'd compress/encrypt all dmu_sync() data twice.
	* However, we should never get a raw, override zio so in these
	* cases we can compare the io_abd directly. This is useful because
	* it allows us to do dedup verification even if we don't have access
	* to the original data (for instance, if the encryption keys aren't
	* loaded).
	*/

	for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) {
	zio_t *lio = dde->dde_lead_zio[p];

	if (lio != NULL && do_raw) {
	return (lio->io_size != zio->io_size \|\|
	abd_cmp(zio->io_abd, lio->io_abd) != 0);
	} else if (lio != NULL) {
	return (lio->io_orig_size != zio->io_orig_size \|\|
	abd_cmp(zio->io_orig_abd, lio->io_orig_abd) != 0);
	}
	}

	for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) {
	ddt_phys_t *ddp = &dde->dde_phys[p];

	if (ddp->ddp_phys_birth != 0 && do_raw) {
	blkptr_t blk = *zio->io_bp;
	uint64_t psize;
	abd_t *tmpabd;
	int error;

	ddt_bp_fill(ddp, &blk, ddp->ddp_phys_birth);
	psize = BP_GET_PSIZE(&blk);

	if (psize != zio->io_size)
	return (B_TRUE);

	ddt_exit(ddt);

	tmpabd = abd_alloc_for_io(psize, B_TRUE);

	error = zio_wait(zio_read(NULL, spa, &blk, tmpabd,
	psize, NULL, NULL, ZIO_PRIORITY_SYNC_READ,
	ZIO_FLAG_CANFAIL \| ZIO_FLAG_SPECULATIVE \|
	ZIO_FLAG_RAW, &zio->io_bookmark));

	if (error == 0) {
	if (abd_cmp(tmpabd, zio->io_abd) != 0)
	error = SET_ERROR(ENOENT);
	}

	abd_free(tmpabd);
	ddt_enter(ddt);
	return (error != 0);
	} else if (ddp->ddp_phys_birth != 0) {
	arc_buf_t *abuf = NULL;
	arc_flags_t aflags = ARC_FLAG_WAIT;
	blkptr_t blk = *zio->io_bp;
	int error;

	ddt_bp_fill(ddp, &blk, ddp->ddp_phys_birth);

	if (BP_GET_LSIZE(&blk) != zio->io_orig_size)
	return (B_TRUE);

	ddt_exit(ddt);

	error = arc_read(NULL, spa, &blk,
	arc_getbuf_func, &abuf, ZIO_PRIORITY_SYNC_READ,
	ZIO_FLAG_CANFAIL \| ZIO_FLAG_SPECULATIVE,
	&aflags, &zio->io_bookmark);

	if (error == 0) {
	if (abd_cmp_buf(zio->io_orig_abd, abuf->b_data,
	zio->io_orig_size) != 0)
	error = SET_ERROR(ENOENT);
	arc_buf_destroy(abuf, &abuf);
	}

	ddt_enter(ddt);
	return (error != 0);
	}
	}

	return (B_FALSE);
	}

	static void
	zio_ddt_child_write_ready(zio_t *zio)
	{
	int p = zio->io_prop.zp_copies;
	ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp);
	ddt_entry_t *dde = zio->io_private;
	ddt_phys_t *ddp = &dde->dde_phys[p];
	zio_t *pio;

	if (zio->io_error)
	return;

	ddt_enter(ddt);

	ASSERT(dde->dde_lead_zio[p] == zio);

	ddt_phys_fill(ddp, zio->io_bp);

	zio_link_t *zl = NULL;
	while ((pio = zio_walk_parents(zio, &zl)) != NULL)
	ddt_bp_fill(ddp, pio->io_bp, zio->io_txg);

	ddt_exit(ddt);
	}

	static void
	zio_ddt_child_write_done(zio_t *zio)
	{
	int p = zio->io_prop.zp_copies;
	ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp);
	ddt_entry_t *dde = zio->io_private;
	ddt_phys_t *ddp = &dde->dde_phys[p];

	ddt_enter(ddt);

	ASSERT(ddp->ddp_refcnt == 0);
	ASSERT(dde->dde_lead_zio[p] == zio);
	dde->dde_lead_zio[p] = NULL;

	if (zio->io_error == 0) {
	zio_link_t *zl = NULL;
	while (zio_walk_parents(zio, &zl) != NULL)
	ddt_phys_addref(ddp);
	} else {
	ddt_phys_clear(ddp);
	}

	ddt_exit(ddt);
	}

	static zio_t *
	zio_ddt_write(zio_t *zio)
	{
	spa_t *spa = zio->io_spa;
	blkptr_t *bp = zio->io_bp;
	uint64_t txg = zio->io_txg;
	zio_prop_t *zp = &zio->io_prop;
	int p = zp->zp_copies;
	zio_t *cio = NULL;
	ddt_t *ddt = ddt_select(spa, bp);
	ddt_entry_t *dde;
	ddt_phys_t *ddp;

	ASSERT(BP_GET_DEDUP(bp));
	ASSERT(BP_GET_CHECKSUM(bp) == zp->zp_checksum);
	ASSERT(BP_IS_HOLE(bp) \|\| zio->io_bp_override);
	ASSERT(!(zio->io_bp_override && (zio->io_flags & ZIO_FLAG_RAW)));

	ddt_enter(ddt);
	dde = ddt_lookup(ddt, bp, B_TRUE);
	ddp = &dde->dde_phys[p];

	if (zp->zp_dedup_verify && zio_ddt_collision(zio, ddt, dde)) {
	/*
	* If we're using a weak checksum, upgrade to a strong checksum
	* and try again. If we're already using a strong checksum,
	* we can't resolve it, so just convert to an ordinary write.
	* (And automatically e-mail a paper to Nature?)
	*/
	if (!(zio_checksum_table[zp->zp_checksum].ci_flags &
	ZCHECKSUM_FLAG_DEDUP)) {
	zp->zp_checksum = spa_dedup_checksum(spa);
	zio_pop_transforms(zio);
	zio->io_stage = ZIO_STAGE_OPEN;
	BP_ZERO(bp);
	} else {
	zp->zp_dedup = B_FALSE;
	BP_SET_DEDUP(bp, B_FALSE);
	}
	ASSERT(!BP_GET_DEDUP(bp));
	zio->io_pipeline = ZIO_WRITE_PIPELINE;
	ddt_exit(ddt);
	return (zio);
	}

	if (ddp->ddp_phys_birth != 0 \|\| dde->dde_lead_zio[p] != NULL) {
	if (ddp->ddp_phys_birth != 0)
	ddt_bp_fill(ddp, bp, txg);
	if (dde->dde_lead_zio[p] != NULL)
	zio_add_child(zio, dde->dde_lead_zio[p]);
	else
	ddt_phys_addref(ddp);
	} else if (zio->io_bp_override) {
	ASSERT(bp->blk_birth == txg);
	ASSERT(BP_EQUAL(bp, zio->io_bp_override));
	ddt_phys_fill(ddp, bp);
	ddt_phys_addref(ddp);
	} else {
	cio = zio_write(zio, spa, txg, bp, zio->io_orig_abd,
	zio->io_orig_size, zio->io_orig_size, zp,
	zio_ddt_child_write_ready, NULL,
	zio_ddt_child_write_done, dde, zio->io_priority,
	ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark);

	zio_push_transform(cio, zio->io_abd, zio->io_size, 0, NULL);
	dde->dde_lead_zio[p] = cio;
	}

	ddt_exit(ddt);

	zio_nowait(cio);

	return (zio);
	}

	static ddt_entry_t freedde; / for debugging */

	static zio_t *
	zio_ddt_free(zio_t *zio)
	{
	spa_t *spa = zio->io_spa;
	blkptr_t *bp = zio->io_bp;
	ddt_t *ddt = ddt_select(spa, bp);
	ddt_entry_t *dde;
	ddt_phys_t *ddp;

	ASSERT(BP_GET_DEDUP(bp));
	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);

	ddt_enter(ddt);
	freedde = dde = ddt_lookup(ddt, bp, B_TRUE);
	if (dde) {
	ddp = ddt_phys_select(dde, bp);
	if (ddp)
	ddt_phys_decref(ddp);
	}
	ddt_exit(ddt);

	return (zio);
	}

	/*
	* ==========================================================================
	* Allocate and free blocks
	* ==========================================================================
	*/

	static zio_t *
	zio_io_to_allocate(spa_t *spa, int allocator)
	{
	zio_t *zio;

	ASSERT(MUTEX_HELD(&spa->spa_allocs[allocator].spaa_lock));

	zio = avl_first(&spa->spa_allocs[allocator].spaa_tree);
	if (zio == NULL)
	return (NULL);

	ASSERT(IO_IS_ALLOCATING(zio));

	/*
	* Try to place a reservation for this zio. If we're unable to
	* reserve then we throttle.
	*/
	ASSERT3U(zio->io_allocator, ==, allocator);
	if (!metaslab_class_throttle_reserve(zio->io_metaslab_class,
	zio->io_prop.zp_copies, allocator, zio, 0)) {
	return (NULL);
	}

	avl_remove(&spa->spa_allocs[allocator].spaa_tree, zio);
	ASSERT3U(zio->io_stage, <, ZIO_STAGE_DVA_ALLOCATE);

	return (zio);
	}

	static zio_t *
	zio_dva_throttle(zio_t *zio)
	{
	spa_t *spa = zio->io_spa;
	zio_t *nio;
	metaslab_class_t *mc;

	/* locate an appropriate allocation class */
	mc = spa_preferred_class(spa, zio->io_size, zio->io_prop.zp_type,
	zio->io_prop.zp_level, zio->io_prop.zp_zpl_smallblk);

	if (zio->io_priority == ZIO_PRIORITY_SYNC_WRITE \|\|
	!mc->mc_alloc_throttle_enabled \|\|
	zio->io_child_type == ZIO_CHILD_GANG \|\|
	zio->io_flags & ZIO_FLAG_NODATA) {
	return (zio);
	}

	ASSERT(zio->io_type == ZIO_TYPE_WRITE);
	ASSERT(zio->io_child_type > ZIO_CHILD_GANG);
	ASSERT3U(zio->io_queued_timestamp, >, 0);
	ASSERT(zio->io_stage == ZIO_STAGE_DVA_THROTTLE);

	zbookmark_phys_t *bm = &zio->io_bookmark;
	/*
	* We want to try to use as many allocators as possible to help improve
	* performance, but we also want logically adjacent IOs to be physically
	* adjacent to improve sequential read performance. We chunk each object
	* into 2^20 block regions, and then hash based on the objset, object,
	* level, and region to accomplish both of these goals.
	*/
	int allocator = (uint_t)cityhash4(bm->zb_objset, bm->zb_object,
	bm->zb_level, bm->zb_blkid >> 20) % spa->spa_alloc_count;
	zio->io_allocator = allocator;
	zio->io_metaslab_class = mc;
	mutex_enter(&spa->spa_allocs[allocator].spaa_lock);
	avl_add(&spa->spa_allocs[allocator].spaa_tree, zio);
	nio = zio_io_to_allocate(spa, allocator);
	mutex_exit(&spa->spa_allocs[allocator].spaa_lock);
	return (nio);
	}

	static void
	zio_allocate_dispatch(spa_t *spa, int allocator)
	{
	zio_t *zio;

	mutex_enter(&spa->spa_allocs[allocator].spaa_lock);
	zio = zio_io_to_allocate(spa, allocator);
	mutex_exit(&spa->spa_allocs[allocator].spaa_lock);
	if (zio == NULL)
	return;

	ASSERT3U(zio->io_stage, ==, ZIO_STAGE_DVA_THROTTLE);
	ASSERT0(zio->io_error);
	zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_TRUE);
	}

	static zio_t *
	zio_dva_allocate(zio_t *zio)
	{
	spa_t *spa = zio->io_spa;
	metaslab_class_t *mc;
	blkptr_t *bp = zio->io_bp;
	int error;
	int flags = 0;

	if (zio->io_gang_leader == NULL) {
	ASSERT(zio->io_child_type > ZIO_CHILD_GANG);
	zio->io_gang_leader = zio;
	}

	ASSERT(BP_IS_HOLE(bp));
	ASSERT0(BP_GET_NDVAS(bp));
	ASSERT3U(zio->io_prop.zp_copies, >, 0);
	ASSERT3U(zio->io_prop.zp_copies, <=, spa_max_replication(spa));
	ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp));

	if (zio->io_flags & ZIO_FLAG_NODATA)
	flags \|= METASLAB_DONT_THROTTLE;
	if (zio->io_flags & ZIO_FLAG_GANG_CHILD)
	flags \|= METASLAB_GANG_CHILD;
	if (zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE)
	flags \|= METASLAB_ASYNC_ALLOC;

	/*
	* if not already chosen, locate an appropriate allocation class
	*/
	mc = zio->io_metaslab_class;
	if (mc == NULL) {
	mc = spa_preferred_class(spa, zio->io_size,
	zio->io_prop.zp_type, zio->io_prop.zp_level,
	zio->io_prop.zp_zpl_smallblk);
	zio->io_metaslab_class = mc;
	}

	/*
	* Try allocating the block in the usual metaslab class.
	* If that's full, allocate it in the normal class.
	* If that's full, allocate as a gang block,
	* and if all are full, the allocation fails (which shouldn't happen).
	*
	* Note that we do not fall back on embedded slog (ZIL) space, to
	* preserve unfragmented slog space, which is critical for decent
	* sync write performance. If a log allocation fails, we will fall
	* back to spa_sync() which is abysmal for performance.
	*/
	error = metaslab_alloc(spa, mc, zio->io_size, bp,
	zio->io_prop.zp_copies, zio->io_txg, NULL, flags,
	&zio->io_alloc_list, zio, zio->io_allocator);

	/*
	* Fallback to normal class when an alloc class is full
	*/
	if (error == ENOSPC && mc != spa_normal_class(spa)) {
	/*
	* If throttling, transfer reservation over to normal class.
	* The io_allocator slot can remain the same even though we
	* are switching classes.
	*/
	if (mc->mc_alloc_throttle_enabled &&
	(zio->io_flags & ZIO_FLAG_IO_ALLOCATING)) {
	metaslab_class_throttle_unreserve(mc,
	zio->io_prop.zp_copies, zio->io_allocator, zio);
	zio->io_flags &= ~ZIO_FLAG_IO_ALLOCATING;

	VERIFY(metaslab_class_throttle_reserve(
	spa_normal_class(spa),
	zio->io_prop.zp_copies, zio->io_allocator, zio,
	flags \| METASLAB_MUST_RESERVE));
	}
	zio->io_metaslab_class = mc = spa_normal_class(spa);
	if (zfs_flags & ZFS_DEBUG_METASLAB_ALLOC) {
	zfs_dbgmsg("%s: metaslab allocation failure, "
	"trying normal class: zio %px, size %llu, error %d",
	spa_name(spa), zio, (u_longlong_t)zio->io_size,
	error);
	}

	error = metaslab_alloc(spa, mc, zio->io_size, bp,
	zio->io_prop.zp_copies, zio->io_txg, NULL, flags,
	&zio->io_alloc_list, zio, zio->io_allocator);
	}

	if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE) {
	if (zfs_flags & ZFS_DEBUG_METASLAB_ALLOC) {
	zfs_dbgmsg("%s: metaslab allocation failure, "
	"trying ganging: zio %px, size %llu, error %d",
	spa_name(spa), zio, (u_longlong_t)zio->io_size,
	error);
	}
	return (zio_write_gang_block(zio, mc));
	}
	if (error != 0) {
	if (error != ENOSPC \|\|
	(zfs_flags & ZFS_DEBUG_METASLAB_ALLOC)) {
	zfs_dbgmsg("%s: metaslab allocation failure: zio %px, "
	"size %llu, error %d",
	spa_name(spa), zio, (u_longlong_t)zio->io_size,
	error);
	}
	zio->io_error = error;
	}

	return (zio);
	}

	static zio_t *
	zio_dva_free(zio_t *zio)
	{
	metaslab_free(zio->io_spa, zio->io_bp, zio->io_txg, B_FALSE);

	return (zio);
	}

	static zio_t *
	zio_dva_claim(zio_t *zio)
	{
	int error;

	error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg);
	if (error)
	zio->io_error = error;

	return (zio);
	}

	/*
	* Undo an allocation. This is used by zio_done() when an I/O fails
	* and we want to give back the block we just allocated.
	* This handles both normal blocks and gang blocks.
	*/
	static void
	zio_dva_unallocate(zio_t zio, zio_gang_node_t gn, blkptr_t *bp)
	{
	ASSERT(bp->blk_birth == zio->io_txg \|\| BP_IS_HOLE(bp));
	ASSERT(zio->io_bp_override == NULL);

	if (!BP_IS_HOLE(bp))
	metaslab_free(zio->io_spa, bp, bp->blk_birth, B_TRUE);

	if (gn != NULL) {
	for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) {
	zio_dva_unallocate(zio, gn->gn_child[g],
	&gn->gn_gbh->zg_blkptr[g]);
	}
	}
	}

	/*
	* Try to allocate an intent log block. Return 0 on success, errno on failure.
	*/
	int
	zio_alloc_zil(spa_t spa, objset_t os, uint64_t txg, blkptr_t *new_bp,
	uint64_t size, boolean_t *slog)
	{
	int error = 1;
	zio_alloc_list_t io_alloc_list;

	ASSERT(txg > spa_syncing_txg(spa));

	metaslab_trace_init(&io_alloc_list);

	/*
	* Block pointer fields are useful to metaslabs for stats and debugging.
	* Fill in the obvious ones before calling into metaslab_alloc().
	*/
	BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG);
	BP_SET_PSIZE(new_bp, size);
	BP_SET_LEVEL(new_bp, 0);

	/*
	* When allocating a zil block, we don't have information about
	* the final destination of the block except the objset it's part
	* of, so we just hash the objset ID to pick the allocator to get
	* some parallelism.
	*/
	int flags = METASLAB_ZIL;
	int allocator = (uint_t)cityhash4(0, 0, 0,
	os->os_dsl_dataset->ds_object) % spa->spa_alloc_count;
	error = metaslab_alloc(spa, spa_log_class(spa), size, new_bp, 1,
	txg, NULL, flags, &io_alloc_list, NULL, allocator);
	*slog = (error == 0);
	if (error != 0) {
	error = metaslab_alloc(spa, spa_embedded_log_class(spa), size,
	new_bp, 1, txg, NULL, flags,
	&io_alloc_list, NULL, allocator);
	}
	if (error != 0) {
	error = metaslab_alloc(spa, spa_normal_class(spa), size,
	new_bp, 1, txg, NULL, flags,
	&io_alloc_list, NULL, allocator);
	}
	metaslab_trace_fini(&io_alloc_list);

	if (error == 0) {
	BP_SET_LSIZE(new_bp, size);
	BP_SET_PSIZE(new_bp, size);
	BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF);
	BP_SET_CHECKSUM(new_bp,
	spa_version(spa) >= SPA_VERSION_SLIM_ZIL
	? ZIO_CHECKSUM_ZILOG2 : ZIO_CHECKSUM_ZILOG);
	BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG);
	BP_SET_LEVEL(new_bp, 0);
	BP_SET_DEDUP(new_bp, 0);
	BP_SET_BYTEORDER(new_bp, ZFS_HOST_BYTEORDER);

	/*
	* encrypted blocks will require an IV and salt. We generate
	* these now since we will not be rewriting the bp at
	* rewrite time.
	*/
	if (os->os_encrypted) {
	uint8_t iv[ZIO_DATA_IV_LEN];
	uint8_t salt[ZIO_DATA_SALT_LEN];

	BP_SET_CRYPT(new_bp, B_TRUE);
	VERIFY0(spa_crypt_get_salt(spa,
	dmu_objset_id(os), salt));
	VERIFY0(zio_crypt_generate_iv(iv));

	zio_crypt_encode_params_bp(new_bp, salt, iv);
	}
	} else {
	zfs_dbgmsg("%s: zil block allocation failure: "
	"size %llu, error %d", spa_name(spa), (u_longlong_t)size,
	error);
	}

	return (error);
	}

	/*
	* ==========================================================================
	* Read and write to physical devices
	* ==========================================================================
	*/

	/*
	* Issue an I/O to the underlying vdev. Typically the issue pipeline
	* stops after this stage and will resume upon I/O completion.
	* However, there are instances where the vdev layer may need to
	* continue the pipeline when an I/O was not issued. Since the I/O
	* that was sent to the vdev layer might be different than the one
	* currently active in the pipeline (see vdev_queue_io()), we explicitly
	* force the underlying vdev layers to call either zio_execute() or
	* zio_interrupt() to ensure that the pipeline continues with the correct I/O.
	*/
	static zio_t *
	zio_vdev_io_start(zio_t *zio)
	{
	vdev_t *vd = zio->io_vd;
	uint64_t align;
	spa_t *spa = zio->io_spa;

	zio->io_delay = 0;

	ASSERT(zio->io_error == 0);
	ASSERT(zio->io_child_error[ZIO_CHILD_VDEV] == 0);

	if (vd == NULL) {
	if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER))
	spa_config_enter(spa, SCL_ZIO, zio, RW_READER);

	/*
	* The mirror_ops handle multiple DVAs in a single BP.
	*/
	vdev_mirror_ops.vdev_op_io_start(zio);
	return (NULL);
	}

	ASSERT3P(zio->io_logical, !=, zio);
	if (zio->io_type == ZIO_TYPE_WRITE) {
	ASSERT(spa->spa_trust_config);

	/*
	* Note: the code can handle other kinds of writes,
	* but we don't expect them.
	*/
	if (zio->io_vd->vdev_noalloc) {
	ASSERT(zio->io_flags &
	(ZIO_FLAG_PHYSICAL \| ZIO_FLAG_SELF_HEAL \|
	ZIO_FLAG_RESILVER \| ZIO_FLAG_INDUCE_DAMAGE));
	}
	}

	align = 1ULL << vd->vdev_top->vdev_ashift;

	if (!(zio->io_flags & ZIO_FLAG_PHYSICAL) &&
	P2PHASE(zio->io_size, align) != 0) {
	/* Transform logical writes to be a full physical block size. */
	uint64_t asize = P2ROUNDUP(zio->io_size, align);
	abd_t *abuf = abd_alloc_sametype(zio->io_abd, asize);
	ASSERT(vd == vd->vdev_top);
	if (zio->io_type == ZIO_TYPE_WRITE) {
	abd_copy(abuf, zio->io_abd, zio->io_size);
	abd_zero_off(abuf, zio->io_size, asize - zio->io_size);
	}
	zio_push_transform(zio, abuf, asize, asize, zio_subblock);
	}

	/*
	* If this is not a physical io, make sure that it is properly aligned
	* before proceeding.
	*/
	if (!(zio->io_flags & ZIO_FLAG_PHYSICAL)) {
	ASSERT0(P2PHASE(zio->io_offset, align));
	ASSERT0(P2PHASE(zio->io_size, align));
	} else {
	/*
	* For physical writes, we allow 512b aligned writes and assume
	* the device will perform a read-modify-write as necessary.
	*/
	ASSERT0(P2PHASE(zio->io_offset, SPA_MINBLOCKSIZE));
	ASSERT0(P2PHASE(zio->io_size, SPA_MINBLOCKSIZE));
	}

	VERIFY(zio->io_type != ZIO_TYPE_WRITE \|\| spa_writeable(spa));

	/*
	* If this is a repair I/O, and there's no self-healing involved --
	* that is, we're just resilvering what we expect to resilver --
	* then don't do the I/O unless zio's txg is actually in vd's DTL.
	* This prevents spurious resilvering.
	*
	* There are a few ways that we can end up creating these spurious
	* resilver i/os:
	*
	* 1. A resilver i/o will be issued if any DVA in the BP has a
	* dirty DTL. The mirror code will issue resilver writes to
	* each DVA, including the one(s) that are not on vdevs with dirty
	* DTLs.
	*
	* 2. With nested replication, which happens when we have a
	* "replacing" or "spare" vdev that's a child of a mirror or raidz.
	* For example, given mirror(replacing(A+B), C), it's likely that
	* only A is out of date (it's the new device). In this case, we'll
	* read from C, then use the data to resilver A+B -- but we don't
	* actually want to resilver B, just A. The top-level mirror has no
	* way to know this, so instead we just discard unnecessary repairs
	* as we work our way down the vdev tree.
	*
	* 3. ZTEST also creates mirrors of mirrors, mirrors of raidz, etc.
	* The same logic applies to any form of nested replication: ditto
	* + mirror, RAID-Z + replacing, etc.
	*
	* However, indirect vdevs point off to other vdevs which may have
	* DTL's, so we never bypass them. The child i/os on concrete vdevs
	* will be properly bypassed instead.
	*
	* Leaf DTL_PARTIAL can be empty when a legitimate write comes from
	* a dRAID spare vdev. For example, when a dRAID spare is first
	* used, its spare blocks need to be written to but the leaf vdev's
	* of such blocks can have empty DTL_PARTIAL.
	*
	* There seemed no clean way to allow such writes while bypassing
	* spurious ones. At this point, just avoid all bypassing for dRAID
	* for correctness.
	*/
	if ((zio->io_flags & ZIO_FLAG_IO_REPAIR) &&
	!(zio->io_flags & ZIO_FLAG_SELF_HEAL) &&
	zio->io_txg != 0 && /* not a delegated i/o */
	vd->vdev_ops != &vdev_indirect_ops &&
	vd->vdev_top->vdev_ops != &vdev_draid_ops &&
	!vdev_dtl_contains(vd, DTL_PARTIAL, zio->io_txg, 1)) {
	ASSERT(zio->io_type == ZIO_TYPE_WRITE);
	zio_vdev_io_bypass(zio);
	return (zio);
	}

	/*
	* Select the next best leaf I/O to process. Distributed spares are
	* excluded since they dispatch the I/O directly to a leaf vdev after
	* applying the dRAID mapping.
	*/
	if (vd->vdev_ops->vdev_op_leaf &&
	vd->vdev_ops != &vdev_draid_spare_ops &&
	(zio->io_type == ZIO_TYPE_READ \|\|
	zio->io_type == ZIO_TYPE_WRITE \|\|
	zio->io_type == ZIO_TYPE_TRIM)) {

	if ((zio = vdev_queue_io(zio)) == NULL)
	return (NULL);

	if (!vdev_accessible(vd, zio)) {
	zio->io_error = SET_ERROR(ENXIO);
	zio_interrupt(zio);
	return (NULL);
	}
	zio->io_delay = gethrtime();
	}

	vd->vdev_ops->vdev_op_io_start(zio);
	return (NULL);
	}

	static zio_t *
	zio_vdev_io_done(zio_t *zio)
	{
	vdev_t *vd = zio->io_vd;
	vdev_ops_t *ops = vd ? vd->vdev_ops : &vdev_mirror_ops;
	boolean_t unexpected_error = B_FALSE;

	if (zio_wait_for_children(zio, ZIO_CHILD_VDEV_BIT, ZIO_WAIT_DONE)) {
	return (NULL);
	}

	ASSERT(zio->io_type == ZIO_TYPE_READ \|\|
	zio->io_type == ZIO_TYPE_WRITE \|\| zio->io_type == ZIO_TYPE_TRIM);

	if (zio->io_delay)
	zio->io_delay = gethrtime() - zio->io_delay;

	if (vd != NULL && vd->vdev_ops->vdev_op_leaf &&
	vd->vdev_ops != &vdev_draid_spare_ops) {
	vdev_queue_io_done(zio);

	if (zio_injection_enabled && zio->io_error == 0)
	zio->io_error = zio_handle_device_injections(vd, zio,
	EIO, EILSEQ);

	if (zio_injection_enabled && zio->io_error == 0)
	zio->io_error = zio_handle_label_injection(zio, EIO);

	if (zio->io_error && zio->io_type != ZIO_TYPE_TRIM) {
	if (!vdev_accessible(vd, zio)) {
	zio->io_error = SET_ERROR(ENXIO);
	} else {
	unexpected_error = B_TRUE;
	}
	}
	}

	ops->vdev_op_io_done(zio);

	if (unexpected_error && vd->vdev_remove_wanted == B_FALSE)
	VERIFY(vdev_probe(vd, zio) == NULL);

	return (zio);
	}

	/*
	* This function is used to change the priority of an existing zio that is
	* currently in-flight. This is used by the arc to upgrade priority in the
	* event that a demand read is made for a block that is currently queued
	* as a scrub or async read IO. Otherwise, the high priority read request
	* would end up having to wait for the lower priority IO.
	*/
	void
	zio_change_priority(zio_t *pio, zio_priority_t priority)
	{
	zio_t cio, cio_next;
	zio_link_t *zl = NULL;

	ASSERT3U(priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);

	if (pio->io_vd != NULL && pio->io_vd->vdev_ops->vdev_op_leaf) {
	vdev_queue_change_io_priority(pio, priority);
	} else {
	pio->io_priority = priority;
	}

	mutex_enter(&pio->io_lock);
	for (cio = zio_walk_children(pio, &zl); cio != NULL; cio = cio_next) {
	cio_next = zio_walk_children(pio, &zl);
	zio_change_priority(cio, priority);
	}
	mutex_exit(&pio->io_lock);
	}

	/*
	* For non-raidz ZIOs, we can just copy aside the bad data read from the
	* disk, and use that to finish the checksum ereport later.
	*/
	static void
	zio_vsd_default_cksum_finish(zio_cksum_report_t *zcr,
	const abd_t *good_buf)
	{
	/* no processing needed */
	zfs_ereport_finish_checksum(zcr, good_buf, zcr->zcr_cbdata, B_FALSE);
	}

	void
	zio_vsd_default_cksum_report(zio_t zio, zio_cksum_report_t zcr)
	{
	void *abd = abd_alloc_sametype(zio->io_abd, zio->io_size);

	abd_copy(abd, zio->io_abd, zio->io_size);

	zcr->zcr_cbinfo = zio->io_size;
	zcr->zcr_cbdata = abd;
	zcr->zcr_finish = zio_vsd_default_cksum_finish;
	zcr->zcr_free = zio_abd_free;
	}

	static zio_t *
	zio_vdev_io_assess(zio_t *zio)
	{
	vdev_t *vd = zio->io_vd;

	if (zio_wait_for_children(zio, ZIO_CHILD_VDEV_BIT, ZIO_WAIT_DONE)) {
	return (NULL);
	}

	if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER))
	spa_config_exit(zio->io_spa, SCL_ZIO, zio);

	if (zio->io_vsd != NULL) {
	zio->io_vsd_ops->vsd_free(zio);
	zio->io_vsd = NULL;
	}

	if (zio_injection_enabled && zio->io_error == 0)
	zio->io_error = zio_handle_fault_injection(zio, EIO);

	/*
	* If the I/O failed, determine whether we should attempt to retry it.
	*
	* On retry, we cut in line in the issue queue, since we don't want
	* compression/checksumming/etc. work to prevent our (cheap) IO reissue.
	*/
	if (zio->io_error && vd == NULL &&
	!(zio->io_flags & (ZIO_FLAG_DONT_RETRY \| ZIO_FLAG_IO_RETRY))) {
	ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_QUEUE)); /* not a leaf */
	ASSERT(!(zio->io_flags & ZIO_FLAG_IO_BYPASS)); /* not a leaf */
	zio->io_error = 0;
	zio->io_flags \|= ZIO_FLAG_IO_RETRY \| ZIO_FLAG_DONT_AGGREGATE;
	zio->io_stage = ZIO_STAGE_VDEV_IO_START >> 1;
	zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE,
	zio_requeue_io_start_cut_in_line);
	return (NULL);
	}

	/*
	* If we got an error on a leaf device, convert it to ENXIO
	* if the device is not accessible at all.
	*/
	if (zio->io_error && vd != NULL && vd->vdev_ops->vdev_op_leaf &&
	!vdev_accessible(vd, zio))
	zio->io_error = SET_ERROR(ENXIO);

	/*
	* If we can't write to an interior vdev (mirror or RAID-Z),
	* set vdev_cant_write so that we stop trying to allocate from it.
	*/
	if (zio->io_error == ENXIO && zio->io_type == ZIO_TYPE_WRITE &&
	vd != NULL && !vd->vdev_ops->vdev_op_leaf) {
	vdev_dbgmsg(vd, "zio_vdev_io_assess(zio=%px) setting "
	"cant_write=TRUE due to write failure with ENXIO",
	zio);
	vd->vdev_cant_write = B_TRUE;
	}

	/*
	* If a cache flush returns ENOTSUP or ENOTTY, we know that no future
	* attempts will ever succeed. In this case we set a persistent
	* boolean flag so that we don't bother with it in the future.
	*/
	if ((zio->io_error == ENOTSUP \|\| zio->io_error == ENOTTY) &&
	zio->io_type == ZIO_TYPE_IOCTL &&
	zio->io_cmd == DKIOCFLUSHWRITECACHE && vd != NULL)
	vd->vdev_nowritecache = B_TRUE;

	if (zio->io_error)
	zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;

	return (zio);
	}

	void
	zio_vdev_io_reissue(zio_t *zio)
	{
	ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START);
	ASSERT(zio->io_error == 0);

	zio->io_stage >>= 1;
	}

	void
	zio_vdev_io_redone(zio_t *zio)
	{
	ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE);

	zio->io_stage >>= 1;
	}

	void
	zio_vdev_io_bypass(zio_t *zio)
	{
	ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START);
	ASSERT(zio->io_error == 0);

	zio->io_flags \|= ZIO_FLAG_IO_BYPASS;
	zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS >> 1;
	}

	/*
	* ==========================================================================
	* Encrypt and store encryption parameters
	* ==========================================================================
	*/


	/*
	* This function is used for ZIO_STAGE_ENCRYPT. It is responsible for
	* managing the storage of encryption parameters and passing them to the
	* lower-level encryption functions.
	*/
	static zio_t *
	zio_encrypt(zio_t *zio)
	{
	zio_prop_t *zp = &zio->io_prop;
	spa_t *spa = zio->io_spa;
	blkptr_t *bp = zio->io_bp;
	uint64_t psize = BP_GET_PSIZE(bp);
	uint64_t dsobj = zio->io_bookmark.zb_objset;
	dmu_object_type_t ot = BP_GET_TYPE(bp);
	void *enc_buf = NULL;
	abd_t *eabd = NULL;
	uint8_t salt[ZIO_DATA_SALT_LEN];
	uint8_t iv[ZIO_DATA_IV_LEN];
	uint8_t mac[ZIO_DATA_MAC_LEN];
	boolean_t no_crypt = B_FALSE;

	/* the root zio already encrypted the data */
	if (zio->io_child_type == ZIO_CHILD_GANG)
	return (zio);

	/* only ZIL blocks are re-encrypted on rewrite */
	if (!IO_IS_ALLOCATING(zio) && ot != DMU_OT_INTENT_LOG)
	return (zio);

	if (!(zp->zp_encrypt \|\| BP_IS_ENCRYPTED(bp))) {
	BP_SET_CRYPT(bp, B_FALSE);
	return (zio);
	}

	/* if we are doing raw encryption set the provided encryption params */
	if (zio->io_flags & ZIO_FLAG_RAW_ENCRYPT) {
	ASSERT0(BP_GET_LEVEL(bp));
	BP_SET_CRYPT(bp, B_TRUE);
	BP_SET_BYTEORDER(bp, zp->zp_byteorder);
	if (ot != DMU_OT_OBJSET)
	zio_crypt_encode_mac_bp(bp, zp->zp_mac);

	/* dnode blocks must be written out in the provided byteorder */
	if (zp->zp_byteorder != ZFS_HOST_BYTEORDER &&
	ot == DMU_OT_DNODE) {
	void *bswap_buf = zio_buf_alloc(psize);
	abd_t *babd = abd_get_from_buf(bswap_buf, psize);

	ASSERT3U(BP_GET_COMPRESS(bp), ==, ZIO_COMPRESS_OFF);
	abd_copy_to_buf(bswap_buf, zio->io_abd, psize);
	dmu_ot_byteswap[DMU_OT_BYTESWAP(ot)].ob_func(bswap_buf,
	psize);

	abd_take_ownership_of_buf(babd, B_TRUE);
	zio_push_transform(zio, babd, psize, psize, NULL);
	}

	if (DMU_OT_IS_ENCRYPTED(ot))
	zio_crypt_encode_params_bp(bp, zp->zp_salt, zp->zp_iv);
	return (zio);
	}

	/* indirect blocks only maintain a cksum of the lower level MACs */
	if (BP_GET_LEVEL(bp) > 0) {
	BP_SET_CRYPT(bp, B_TRUE);
	VERIFY0(zio_crypt_do_indirect_mac_checksum_abd(B_TRUE,
	zio->io_orig_abd, BP_GET_LSIZE(bp), BP_SHOULD_BYTESWAP(bp),
	mac));
	zio_crypt_encode_mac_bp(bp, mac);
	return (zio);
	}

	/*
	* Objset blocks are a special case since they have 2 256-bit MACs
	* embedded within them.
	*/
	if (ot == DMU_OT_OBJSET) {
	ASSERT0(DMU_OT_IS_ENCRYPTED(ot));
	ASSERT3U(BP_GET_COMPRESS(bp), ==, ZIO_COMPRESS_OFF);
	BP_SET_CRYPT(bp, B_TRUE);
	VERIFY0(spa_do_crypt_objset_mac_abd(B_TRUE, spa, dsobj,
	zio->io_abd, psize, BP_SHOULD_BYTESWAP(bp)));
	return (zio);
	}

	/* unencrypted object types are only authenticated with a MAC */
	if (!DMU_OT_IS_ENCRYPTED(ot)) {
	BP_SET_CRYPT(bp, B_TRUE);
	VERIFY0(spa_do_crypt_mac_abd(B_TRUE, spa, dsobj,
	zio->io_abd, psize, mac));
	zio_crypt_encode_mac_bp(bp, mac);
	return (zio);
	}

	/*
	* Later passes of sync-to-convergence may decide to rewrite data
	* in place to avoid more disk reallocations. This presents a problem
	* for encryption because this constitutes rewriting the new data with
	* the same encryption key and IV. However, this only applies to blocks
	* in the MOS (particularly the spacemaps) and we do not encrypt the
	* MOS. We assert that the zio is allocating or an intent log write
	* to enforce this.
	*/
	ASSERT(IO_IS_ALLOCATING(zio) \|\| ot == DMU_OT_INTENT_LOG);
	ASSERT(BP_GET_LEVEL(bp) == 0 \|\| ot == DMU_OT_INTENT_LOG);
	ASSERT(spa_feature_is_active(spa, SPA_FEATURE_ENCRYPTION));
	ASSERT3U(psize, !=, 0);

	enc_buf = zio_buf_alloc(psize);
	eabd = abd_get_from_buf(enc_buf, psize);
	abd_take_ownership_of_buf(eabd, B_TRUE);

	/*
	* For an explanation of what encryption parameters are stored
	* where, see the block comment in zio_crypt.c.
	*/
	if (ot == DMU_OT_INTENT_LOG) {
	zio_crypt_decode_params_bp(bp, salt, iv);
	} else {
	BP_SET_CRYPT(bp, B_TRUE);
	}

	/* Perform the encryption. This should not fail */
	VERIFY0(spa_do_crypt_abd(B_TRUE, spa, &zio->io_bookmark,
	BP_GET_TYPE(bp), BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp),
	salt, iv, mac, psize, zio->io_abd, eabd, &no_crypt));

	/* encode encryption metadata into the bp */
	if (ot == DMU_OT_INTENT_LOG) {
	/*
	* ZIL blocks store the MAC in the embedded checksum, so the
	* transform must always be applied.
	*/
	zio_crypt_encode_mac_zil(enc_buf, mac);
	zio_push_transform(zio, eabd, psize, psize, NULL);
	} else {
	BP_SET_CRYPT(bp, B_TRUE);
	zio_crypt_encode_params_bp(bp, salt, iv);
	zio_crypt_encode_mac_bp(bp, mac);

	if (no_crypt) {
	ASSERT3U(ot, ==, DMU_OT_DNODE);
	abd_free(eabd);
	} else {
	zio_push_transform(zio, eabd, psize, psize, NULL);
	}
	}

	return (zio);
	}

	/*
	* ==========================================================================
	* Generate and verify checksums
	* ==========================================================================
	*/
	static zio_t *
	zio_checksum_generate(zio_t *zio)
	{
	blkptr_t *bp = zio->io_bp;
	enum zio_checksum checksum;

	if (bp == NULL) {
	/*
	* This is zio_write_phys().
	* We're either generating a label checksum, or none at all.
	*/
	checksum = zio->io_prop.zp_checksum;

	if (checksum == ZIO_CHECKSUM_OFF)
	return (zio);

	ASSERT(checksum == ZIO_CHECKSUM_LABEL);
	} else {
	if (BP_IS_GANG(bp) && zio->io_child_type == ZIO_CHILD_GANG) {
	ASSERT(!IO_IS_ALLOCATING(zio));
	checksum = ZIO_CHECKSUM_GANG_HEADER;
	} else {
	checksum = BP_GET_CHECKSUM(bp);
	}
	}

	zio_checksum_compute(zio, checksum, zio->io_abd, zio->io_size);

	return (zio);
	}

	static zio_t *
	zio_checksum_verify(zio_t *zio)
	{
	zio_bad_cksum_t info;
	blkptr_t *bp = zio->io_bp;
	int error;

	ASSERT(zio->io_vd != NULL);

	if (bp == NULL) {
	/*
	* This is zio_read_phys().
	* We're either verifying a label checksum, or nothing at all.
	*/
	if (zio->io_prop.zp_checksum == ZIO_CHECKSUM_OFF)
	return (zio);

	ASSERT3U(zio->io_prop.zp_checksum, ==, ZIO_CHECKSUM_LABEL);
	}

	if ((error = zio_checksum_error(zio, &info)) != 0) {
	zio->io_error = error;
	if (error == ECKSUM &&
	!(zio->io_flags & ZIO_FLAG_SPECULATIVE)) {
	mutex_enter(&zio->io_vd->vdev_stat_lock);
	zio->io_vd->vdev_stat.vs_checksum_errors++;
	mutex_exit(&zio->io_vd->vdev_stat_lock);
	(void) zfs_ereport_start_checksum(zio->io_spa,
	zio->io_vd, &zio->io_bookmark, zio,
	zio->io_offset, zio->io_size, &info);
	}
	}

	return (zio);
	}

	/*
	* Called by RAID-Z to ensure we don't compute the checksum twice.
	*/
	void
	zio_checksum_verified(zio_t *zio)
	{
	zio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY;
	}

	/*
	* ==========================================================================
	* Error rank. Error are ranked in the order 0, ENXIO, ECKSUM, EIO, other.
	* An error of 0 indicates success. ENXIO indicates whole-device failure,
	* which may be transient (e.g. unplugged) or permanent. ECKSUM and EIO
	* indicate errors that are specific to one I/O, and most likely permanent.
	* Any other error is presumed to be worse because we weren't expecting it.
	* ==========================================================================
	*/
	int
	zio_worst_error(int e1, int e2)
	{
	static int zio_error_rank[] = { 0, ENXIO, ECKSUM, EIO };
	int r1, r2;

	for (r1 = 0; r1 < sizeof (zio_error_rank) / sizeof (int); r1++)
	if (e1 == zio_error_rank[r1])
	break;

	for (r2 = 0; r2 < sizeof (zio_error_rank) / sizeof (int); r2++)
	if (e2 == zio_error_rank[r2])
	break;

	return (r1 > r2 ? e1 : e2);
	}

	/*
	* ==========================================================================
	* I/O completion
	* ==========================================================================
	*/
	static zio_t *
	zio_ready(zio_t *zio)
	{
	blkptr_t *bp = zio->io_bp;
	zio_t pio, pio_next;
	zio_link_t *zl = NULL;

	if (zio_wait_for_children(zio, ZIO_CHILD_LOGICAL_BIT \|
	ZIO_CHILD_GANG_BIT \| ZIO_CHILD_DDT_BIT, ZIO_WAIT_READY)) {
	return (NULL);
	}

	if (zio->io_ready) {
	ASSERT(IO_IS_ALLOCATING(zio));
	ASSERT(bp->blk_birth == zio->io_txg \|\| BP_IS_HOLE(bp) \|\|
	(zio->io_flags & ZIO_FLAG_NOPWRITE));
	ASSERT(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY] == 0);

	zio->io_ready(zio);
	}

	#ifdef ZFS_DEBUG
	if (bp != NULL && bp != &zio->io_bp_copy)
	zio->io_bp_copy = *bp;
	#endif

	if (zio->io_error != 0) {
	zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;

	if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
	ASSERT(IO_IS_ALLOCATING(zio));
	ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
	ASSERT(zio->io_metaslab_class != NULL);

	/*
	* We were unable to allocate anything, unreserve and
	* issue the next I/O to allocate.
	*/
	metaslab_class_throttle_unreserve(
	zio->io_metaslab_class, zio->io_prop.zp_copies,
	zio->io_allocator, zio);
	zio_allocate_dispatch(zio->io_spa, zio->io_allocator);
	}
	}

	mutex_enter(&zio->io_lock);
	zio->io_state[ZIO_WAIT_READY] = 1;
	pio = zio_walk_parents(zio, &zl);
	mutex_exit(&zio->io_lock);

	/*
	* As we notify zio's parents, new parents could be added.
	* New parents go to the head of zio's io_parent_list, however,
	* so we will (correctly) not notify them. The remainder of zio's
	* io_parent_list, from 'pio_next' onward, cannot change because
	* all parents must wait for us to be done before they can be done.
	*/
	for (; pio != NULL; pio = pio_next) {
	pio_next = zio_walk_parents(zio, &zl);
	zio_notify_parent(pio, zio, ZIO_WAIT_READY, NULL);
	}

	if (zio->io_flags & ZIO_FLAG_NODATA) {
	if (bp != NULL && BP_IS_GANG(bp)) {
	zio->io_flags &= ~ZIO_FLAG_NODATA;
	} else {
	ASSERT((uintptr_t)zio->io_abd < SPA_MAXBLOCKSIZE);
	zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES;
	}
	}

	if (zio_injection_enabled &&
	zio->io_spa->spa_syncing_txg == zio->io_txg)
	zio_handle_ignored_writes(zio);

	return (zio);
	}

	/*
	* Update the allocation throttle accounting.
	*/
	static void
	zio_dva_throttle_done(zio_t *zio)
	{
	zio_t *lio __maybe_unused = zio->io_logical;
	zio_t *pio = zio_unique_parent(zio);
	vdev_t *vd = zio->io_vd;
	int flags = METASLAB_ASYNC_ALLOC;

	ASSERT3P(zio->io_bp, !=, NULL);
	ASSERT3U(zio->io_type, ==, ZIO_TYPE_WRITE);
	ASSERT3U(zio->io_priority, ==, ZIO_PRIORITY_ASYNC_WRITE);
	ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV);
	ASSERT(vd != NULL);
	ASSERT3P(vd, ==, vd->vdev_top);
	ASSERT(zio_injection_enabled \|\| !(zio->io_flags & ZIO_FLAG_IO_RETRY));
	ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REPAIR));
	ASSERT(zio->io_flags & ZIO_FLAG_IO_ALLOCATING);
	ASSERT(!(lio->io_flags & ZIO_FLAG_IO_REWRITE));
	ASSERT(!(lio->io_orig_flags & ZIO_FLAG_NODATA));

	/*
	* Parents of gang children can have two flavors -- ones that
	* allocated the gang header (will have ZIO_FLAG_IO_REWRITE set)
	* and ones that allocated the constituent blocks. The allocation
	* throttle needs to know the allocating parent zio so we must find
	* it here.
	*/
	if (pio->io_child_type == ZIO_CHILD_GANG) {
	/*
	* If our parent is a rewrite gang child then our grandparent
	* would have been the one that performed the allocation.
	*/
	if (pio->io_flags & ZIO_FLAG_IO_REWRITE)
	pio = zio_unique_parent(pio);
	flags \|= METASLAB_GANG_CHILD;
	}

	ASSERT(IO_IS_ALLOCATING(pio));
	ASSERT3P(zio, !=, zio->io_logical);
	ASSERT(zio->io_logical != NULL);
	ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REPAIR));
	ASSERT0(zio->io_flags & ZIO_FLAG_NOPWRITE);
	ASSERT(zio->io_metaslab_class != NULL);

	mutex_enter(&pio->io_lock);
	metaslab_group_alloc_decrement(zio->io_spa, vd->vdev_id, pio, flags,
	pio->io_allocator, B_TRUE);
	mutex_exit(&pio->io_lock);

	metaslab_class_throttle_unreserve(zio->io_metaslab_class, 1,
	pio->io_allocator, pio);

	/*
	* Call into the pipeline to see if there is more work that
	* needs to be done. If there is work to be done it will be
	* dispatched to another taskq thread.
	*/
	zio_allocate_dispatch(zio->io_spa, pio->io_allocator);
	}

	static zio_t *
	zio_done(zio_t *zio)
	{
	/*
	* Always attempt to keep stack usage minimal here since
	* we can be called recursively up to 19 levels deep.
	*/
	const uint64_t psize = zio->io_size;
	zio_t pio, pio_next;
	zio_link_t *zl = NULL;

	/*
	* If our children haven't all completed,
	* wait for them and then repeat this pipeline stage.
	*/
	if (zio_wait_for_children(zio, ZIO_CHILD_ALL_BITS, ZIO_WAIT_DONE)) {
	return (NULL);
	}

	/*
	* If the allocation throttle is enabled, then update the accounting.
	* We only track child I/Os that are part of an allocating async
	* write. We must do this since the allocation is performed
	* by the logical I/O but the actual write is done by child I/Os.
	*/
	if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING &&
	zio->io_child_type == ZIO_CHILD_VDEV) {
	ASSERT(zio->io_metaslab_class != NULL);
	ASSERT(zio->io_metaslab_class->mc_alloc_throttle_enabled);
	zio_dva_throttle_done(zio);
	}

	/*
	* If the allocation throttle is enabled, verify that
	* we have decremented the refcounts for every I/O that was throttled.
	*/
	if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
	ASSERT(zio->io_type == ZIO_TYPE_WRITE);
	ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
	ASSERT(zio->io_bp != NULL);

	metaslab_group_alloc_verify(zio->io_spa, zio->io_bp, zio,
	zio->io_allocator);
	VERIFY(zfs_refcount_not_held(&zio->io_metaslab_class->
	mc_allocator[zio->io_allocator].mca_alloc_slots, zio));
	}


	for (int c = 0; c < ZIO_CHILD_TYPES; c++)
	for (int w = 0; w < ZIO_WAIT_TYPES; w++)
	ASSERT(zio->io_children[c][w] == 0);

	if (zio->io_bp != NULL && !BP_IS_EMBEDDED(zio->io_bp)) {
	ASSERT(zio->io_bp->blk_pad[0] == 0);
	ASSERT(zio->io_bp->blk_pad[1] == 0);
	ASSERT(memcmp(zio->io_bp, &zio->io_bp_copy,
	sizeof (blkptr_t)) == 0 \|\|
	(zio->io_bp == zio_unique_parent(zio)->io_bp));
	if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(zio->io_bp) &&
	zio->io_bp_override == NULL &&
	!(zio->io_flags & ZIO_FLAG_IO_REPAIR)) {
	ASSERT3U(zio->io_prop.zp_copies, <=,
	BP_GET_NDVAS(zio->io_bp));
	ASSERT(BP_COUNT_GANG(zio->io_bp) == 0 \|\|
	(BP_COUNT_GANG(zio->io_bp) ==
	BP_GET_NDVAS(zio->io_bp)));
	}
	if (zio->io_flags & ZIO_FLAG_NOPWRITE)
	VERIFY(BP_EQUAL(zio->io_bp, &zio->io_bp_orig));
	}

	/*
	* If there were child vdev/gang/ddt errors, they apply to us now.
	*/
	zio_inherit_child_errors(zio, ZIO_CHILD_VDEV);
	zio_inherit_child_errors(zio, ZIO_CHILD_GANG);
	zio_inherit_child_errors(zio, ZIO_CHILD_DDT);

	/*
	* If the I/O on the transformed data was successful, generate any
	* checksum reports now while we still have the transformed data.
	*/
	if (zio->io_error == 0) {
	while (zio->io_cksum_report != NULL) {
	zio_cksum_report_t *zcr = zio->io_cksum_report;
	uint64_t align = zcr->zcr_align;
	uint64_t asize = P2ROUNDUP(psize, align);
	abd_t *adata = zio->io_abd;

	if (adata != NULL && asize != psize) {
	adata = abd_alloc(asize, B_TRUE);
	abd_copy(adata, zio->io_abd, psize);
	abd_zero_off(adata, psize, asize - psize);
	}

	zio->io_cksum_report = zcr->zcr_next;
	zcr->zcr_next = NULL;
	zcr->zcr_finish(zcr, adata);
	zfs_ereport_free_checksum(zcr);

	if (adata != NULL && asize != psize)
	abd_free(adata);
	}
	}

	zio_pop_transforms(zio); /* note: may set zio->io_error */

	vdev_stat_update(zio, psize);

	/*
	* If this I/O is attached to a particular vdev is slow, exceeding
	* 30 seconds to complete, post an error described the I/O delay.
	* We ignore these errors if the device is currently unavailable.
	*/
	if (zio->io_delay >= MSEC2NSEC(zio_slow_io_ms)) {
	if (zio->io_vd != NULL && !vdev_is_dead(zio->io_vd)) {
	/*
	* We want to only increment our slow IO counters if
	* the IO is valid (i.e. not if the drive is removed).
	*
	* zfs_ereport_post() will also do these checks, but
	* it can also ratelimit and have other failures, so we
	* need to increment the slow_io counters independent
	* of it.
	*/
	if (zfs_ereport_is_valid(FM_EREPORT_ZFS_DELAY,
	zio->io_spa, zio->io_vd, zio)) {
	mutex_enter(&zio->io_vd->vdev_stat_lock);
	zio->io_vd->vdev_stat.vs_slow_ios++;
	mutex_exit(&zio->io_vd->vdev_stat_lock);

	(void) zfs_ereport_post(FM_EREPORT_ZFS_DELAY,
	zio->io_spa, zio->io_vd, &zio->io_bookmark,
	zio, 0);
	}
	}
	}

	if (zio->io_error) {
	/*
	* If this I/O is attached to a particular vdev,
	* generate an error message describing the I/O failure
	* at the block level. We ignore these errors if the
	* device is currently unavailable.
	*/
	if (zio->io_error != ECKSUM && zio->io_vd != NULL &&
	!vdev_is_dead(zio->io_vd)) {
	int ret = zfs_ereport_post(FM_EREPORT_ZFS_IO,
	zio->io_spa, zio->io_vd, &zio->io_bookmark, zio, 0);
	if (ret != EALREADY) {
	mutex_enter(&zio->io_vd->vdev_stat_lock);
	if (zio->io_type == ZIO_TYPE_READ)
	zio->io_vd->vdev_stat.vs_read_errors++;
	else if (zio->io_type == ZIO_TYPE_WRITE)
	zio->io_vd->vdev_stat.vs_write_errors++;
	mutex_exit(&zio->io_vd->vdev_stat_lock);
	}
	}

	if ((zio->io_error == EIO \|\| !(zio->io_flags &
	(ZIO_FLAG_SPECULATIVE \| ZIO_FLAG_DONT_PROPAGATE))) &&
	zio == zio->io_logical) {
	/*
	* For logical I/O requests, tell the SPA to log the
	* error and generate a logical data ereport.
	*/
	spa_log_error(zio->io_spa, &zio->io_bookmark,
	&zio->io_bp->blk_birth);
	(void) zfs_ereport_post(FM_EREPORT_ZFS_DATA,
	zio->io_spa, NULL, &zio->io_bookmark, zio, 0);
	}
	}

	if (zio->io_error && zio == zio->io_logical) {
	/*
	* Determine whether zio should be reexecuted. This will
	* propagate all the way to the root via zio_notify_parent().
	*/
	ASSERT(zio->io_vd == NULL && zio->io_bp != NULL);
	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);

	if (IO_IS_ALLOCATING(zio) &&
	!(zio->io_flags & ZIO_FLAG_CANFAIL)) {
	if (zio->io_error != ENOSPC)
	zio->io_reexecute \|= ZIO_REEXECUTE_NOW;
	else
	zio->io_reexecute \|= ZIO_REEXECUTE_SUSPEND;
	}

	if ((zio->io_type == ZIO_TYPE_READ \|\|
	zio->io_type == ZIO_TYPE_FREE) &&
	!(zio->io_flags & ZIO_FLAG_SCAN_THREAD) &&
	zio->io_error == ENXIO &&
	spa_load_state(zio->io_spa) == SPA_LOAD_NONE &&
	spa_get_failmode(zio->io_spa) != ZIO_FAILURE_MODE_CONTINUE)
	zio->io_reexecute \|= ZIO_REEXECUTE_SUSPEND;

	if (!(zio->io_flags & ZIO_FLAG_CANFAIL) && !zio->io_reexecute)
	zio->io_reexecute \|= ZIO_REEXECUTE_SUSPEND;

	/*
	* Here is a possibly good place to attempt to do
	* either combinatorial reconstruction or error correction
	* based on checksums. It also might be a good place
	* to send out preliminary ereports before we suspend
	* processing.
	*/
	}

	/*
	* If there were logical child errors, they apply to us now.
	* We defer this until now to avoid conflating logical child
	* errors with errors that happened to the zio itself when
	* updating vdev stats and reporting FMA events above.
	*/
	zio_inherit_child_errors(zio, ZIO_CHILD_LOGICAL);

	if ((zio->io_error \|\| zio->io_reexecute) &&
	IO_IS_ALLOCATING(zio) && zio->io_gang_leader == zio &&
	!(zio->io_flags & (ZIO_FLAG_IO_REWRITE \| ZIO_FLAG_NOPWRITE)))
	zio_dva_unallocate(zio, zio->io_gang_tree, zio->io_bp);

	zio_gang_tree_free(&zio->io_gang_tree);

	/*
	* Godfather I/Os should never suspend.
	*/
	if ((zio->io_flags & ZIO_FLAG_GODFATHER) &&
	(zio->io_reexecute & ZIO_REEXECUTE_SUSPEND))
	zio->io_reexecute &= ~ZIO_REEXECUTE_SUSPEND;

	if (zio->io_reexecute) {
	/*
	* This is a logical I/O that wants to reexecute.
	*
	* Reexecute is top-down. When an i/o fails, if it's not
	* the root, it simply notifies its parent and sticks around.
	* The parent, seeing that it still has children in zio_done(),
	* does the same. This percolates all the way up to the root.
	* The root i/o will reexecute or suspend the entire tree.
	*
	* This approach ensures that zio_reexecute() honors
	* all the original i/o dependency relationships, e.g.
	* parents not executing until children are ready.
	*/
	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);

	zio->io_gang_leader = NULL;

	mutex_enter(&zio->io_lock);
	zio->io_state[ZIO_WAIT_DONE] = 1;
	mutex_exit(&zio->io_lock);

	/*
	* "The Godfather" I/O monitors its children but is
	* not a true parent to them. It will track them through
	* the pipeline but severs its ties whenever they get into
	* trouble (e.g. suspended). This allows "The Godfather"
	* I/O to return status without blocking.
	*/
	zl = NULL;
	for (pio = zio_walk_parents(zio, &zl); pio != NULL;
	pio = pio_next) {
	zio_link_t *remove_zl = zl;
	pio_next = zio_walk_parents(zio, &zl);

	if ((pio->io_flags & ZIO_FLAG_GODFATHER) &&
	(zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) {
	zio_remove_child(pio, zio, remove_zl);
	/*
	* This is a rare code path, so we don't
	* bother with "next_to_execute".
	*/
	zio_notify_parent(pio, zio, ZIO_WAIT_DONE,
	NULL);
	}
	}

	if ((pio = zio_unique_parent(zio)) != NULL) {
	/*
	* We're not a root i/o, so there's nothing to do
	* but notify our parent. Don't propagate errors
	* upward since we haven't permanently failed yet.
	*/
	ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER));
	zio->io_flags \|= ZIO_FLAG_DONT_PROPAGATE;
	/*
	* This is a rare code path, so we don't bother with
	* "next_to_execute".
	*/
	zio_notify_parent(pio, zio, ZIO_WAIT_DONE, NULL);
	} else if (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND) {
	/*
	* We'd fail again if we reexecuted now, so suspend
	* until conditions improve (e.g. device comes online).
	*/
	zio_suspend(zio->io_spa, zio, ZIO_SUSPEND_IOERR);
	} else {
	/*
	* Reexecution is potentially a huge amount of work.
	* Hand it off to the otherwise-unused claim taskq.
	*/
	ASSERT(taskq_empty_ent(&zio->io_tqent));
	spa_taskq_dispatch_ent(zio->io_spa,
	ZIO_TYPE_CLAIM, ZIO_TASKQ_ISSUE,
	zio_reexecute, zio, 0, &zio->io_tqent);
	}
	return (NULL);
	}

	ASSERT(list_is_empty(&zio->io_child_list));
	ASSERT(zio->io_reexecute == 0);
	ASSERT(zio->io_error == 0 \|\| (zio->io_flags & ZIO_FLAG_CANFAIL));

	/*
	* Report any checksum errors, since the I/O is complete.
	*/
	while (zio->io_cksum_report != NULL) {
	zio_cksum_report_t *zcr = zio->io_cksum_report;
	zio->io_cksum_report = zcr->zcr_next;
	zcr->zcr_next = NULL;
	zcr->zcr_finish(zcr, NULL);
	zfs_ereport_free_checksum(zcr);
	}

	/*
	* It is the responsibility of the done callback to ensure that this
	* particular zio is no longer discoverable for adoption, and as
	* such, cannot acquire any new parents.
	*/
	if (zio->io_done)
	zio->io_done(zio);

	mutex_enter(&zio->io_lock);
	zio->io_state[ZIO_WAIT_DONE] = 1;
	mutex_exit(&zio->io_lock);

	/*
	* We are done executing this zio. We may want to execute a parent
	* next. See the comment in zio_notify_parent().
	*/
	zio_t *next_to_execute = NULL;
	zl = NULL;
	for (pio = zio_walk_parents(zio, &zl); pio != NULL; pio = pio_next) {
	zio_link_t *remove_zl = zl;
	pio_next = zio_walk_parents(zio, &zl);
	zio_remove_child(pio, zio, remove_zl);
	zio_notify_parent(pio, zio, ZIO_WAIT_DONE, &next_to_execute);
	}

	if (zio->io_waiter != NULL) {
	mutex_enter(&zio->io_lock);
	zio->io_executor = NULL;
	cv_broadcast(&zio->io_cv);
	mutex_exit(&zio->io_lock);
	} else {
	zio_destroy(zio);
	}

	return (next_to_execute);
	}

	/*
	* ==========================================================================
	* I/O pipeline definition
	* ==========================================================================
	*/
	static zio_pipe_stage_t *zio_pipeline[] = {
	NULL,
	zio_read_bp_init,
	zio_write_bp_init,
	zio_free_bp_init,
	zio_issue_async,
	zio_write_compress,
	zio_encrypt,
	zio_checksum_generate,
	zio_nop_write,
	zio_brt_free,
	zio_ddt_read_start,
	zio_ddt_read_done,
	zio_ddt_write,
	zio_ddt_free,
	zio_gang_assemble,
	zio_gang_issue,
	zio_dva_throttle,
	zio_dva_allocate,
	zio_dva_free,
	zio_dva_claim,
	zio_ready,
	zio_vdev_io_start,
	zio_vdev_io_done,
	zio_vdev_io_assess,
	zio_checksum_verify,
	zio_done
	};




	/*
	* Compare two zbookmark_phys_t's to see which we would reach first in a
	* pre-order traversal of the object tree.
	*
	* This is simple in every case aside from the meta-dnode object. For all other
	* objects, we traverse them in order (object 1 before object 2, and so on).
	* However, all of these objects are traversed while traversing object 0, since
	* the data it points to is the list of objects. Thus, we need to convert to a
	* canonical representation so we can compare meta-dnode bookmarks to
	* non-meta-dnode bookmarks.
	*
	* We do this by calculating "equivalents" for each field of the zbookmark.
	* zbookmarks outside of the meta-dnode use their own object and level, and
	* calculate the level 0 equivalent (the first L0 blkid that is contained in the
	* blocks this bookmark refers to) by multiplying their blkid by their span
	* (the number of L0 blocks contained within one block at their level).
	* zbookmarks inside the meta-dnode calculate their object equivalent
	* (which is L0equiv * dnodes per data block), use 0 for their L0equiv, and use
	* level + 1<<31 (any value larger than a level could ever be) for their level.
	* This causes them to always compare before a bookmark in their object
	* equivalent, compare appropriately to bookmarks in other objects, and to
	* compare appropriately to other bookmarks in the meta-dnode.
	*/
	int
	zbookmark_compare(uint16_t dbss1, uint8_t ibs1, uint16_t dbss2, uint8_t ibs2,
	const zbookmark_phys_t zb1, const zbookmark_phys_t zb2)
	{
	/*
	* These variables represent the "equivalent" values for the zbookmark,
	* after converting zbookmarks inside the meta dnode to their
	* normal-object equivalents.
	*/
	uint64_t zb1obj, zb2obj;
	uint64_t zb1L0, zb2L0;
	uint64_t zb1level, zb2level;

	if (zb1->zb_object == zb2->zb_object &&
	zb1->zb_level == zb2->zb_level &&
	zb1->zb_blkid == zb2->zb_blkid)
	return (0);

	IMPLY(zb1->zb_level > 0, ibs1 >= SPA_MINBLOCKSHIFT);
	IMPLY(zb2->zb_level > 0, ibs2 >= SPA_MINBLOCKSHIFT);

	/*
	* BP_SPANB calculates the span in blocks.
	*/
	zb1L0 = (zb1->zb_blkid) * BP_SPANB(ibs1, zb1->zb_level);
	zb2L0 = (zb2->zb_blkid) * BP_SPANB(ibs2, zb2->zb_level);

	if (zb1->zb_object == DMU_META_DNODE_OBJECT) {
	zb1obj = zb1L0 * (dbss1 << (SPA_MINBLOCKSHIFT - DNODE_SHIFT));
	zb1L0 = 0;
	zb1level = zb1->zb_level + COMPARE_META_LEVEL;
	} else {
	zb1obj = zb1->zb_object;
	zb1level = zb1->zb_level;
	}

	if (zb2->zb_object == DMU_META_DNODE_OBJECT) {
	zb2obj = zb2L0 * (dbss2 << (SPA_MINBLOCKSHIFT - DNODE_SHIFT));
	zb2L0 = 0;
	zb2level = zb2->zb_level + COMPARE_META_LEVEL;
	} else {
	zb2obj = zb2->zb_object;
	zb2level = zb2->zb_level;
	}

	/* Now that we have a canonical representation, do the comparison. */
	if (zb1obj != zb2obj)
	return (zb1obj < zb2obj ? -1 : 1);
	else if (zb1L0 != zb2L0)
	return (zb1L0 < zb2L0 ? -1 : 1);
	else if (zb1level != zb2level)
	return (zb1level > zb2level ? -1 : 1);
	/*
	* This can (theoretically) happen if the bookmarks have the same object
	* and level, but different blkids, if the block sizes are not the same.
	* There is presently no way to change the indirect block sizes
	*/
	return (0);
	}

	/*
	* This function checks the following: given that last_block is the place that
	* our traversal stopped last time, does that guarantee that we've visited
	* every node under subtree_root? Therefore, we can't just use the raw output
	* of zbookmark_compare. We have to pass in a modified version of
	* subtree_root; by incrementing the block id, and then checking whether
	* last_block is before or equal to that, we can tell whether or not having
	* visited last_block implies that all of subtree_root's children have been
	* visited.
	*/
	boolean_t
	zbookmark_subtree_completed(const dnode_phys_t *dnp,
	const zbookmark_phys_t subtree_root, const zbookmark_phys_t last_block)
	{
	zbookmark_phys_t mod_zb = *subtree_root;
	mod_zb.zb_blkid++;
	ASSERT0(last_block->zb_level);

	/* The objset_phys_t isn't before anything. */
	if (dnp == NULL)
	return (B_FALSE);

	/*
	* We pass in 1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT) for the
	* data block size in sectors, because that variable is only used if
	* the bookmark refers to a block in the meta-dnode. Since we don't
	* know without examining it what object it refers to, and there's no
	* harm in passing in this value in other cases, we always pass it in.
	*
	* We pass in 0 for the indirect block size shift because zb2 must be
	* level 0. The indirect block size is only used to calculate the span
	* of the bookmark, but since the bookmark must be level 0, the span is
	* always 1, so the math works out.
	*
	* If you make changes to how the zbookmark_compare code works, be sure
	* to make sure that this code still works afterwards.
	*/
	return (zbookmark_compare(dnp->dn_datablkszsec, dnp->dn_indblkshift,
	1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT), 0, &mod_zb,
	last_block) <= 0);
	}

	/*
	* This function is similar to zbookmark_subtree_completed(), but returns true
	* if subtree_root is equal or ahead of last_block, i.e. still to be done.
	*/
	boolean_t
	zbookmark_subtree_tbd(const dnode_phys_t *dnp,
	const zbookmark_phys_t subtree_root, const zbookmark_phys_t last_block)
	{
	ASSERT0(last_block->zb_level);
	if (dnp == NULL)
	return (B_FALSE);
	return (zbookmark_compare(dnp->dn_datablkszsec, dnp->dn_indblkshift,
	1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT), 0, subtree_root,
	last_block) >= 0);
	}

	EXPORT_SYMBOL(zio_type_name);
	EXPORT_SYMBOL(zio_buf_alloc);
	EXPORT_SYMBOL(zio_data_buf_alloc);
	EXPORT_SYMBOL(zio_buf_free);
	EXPORT_SYMBOL(zio_data_buf_free);

	ZFS_MODULE_PARAM(zfs_zio, zio_, slow_io_ms, INT, ZMOD_RW,
	"Max I/O completion time (milliseconds) before marking it as slow");

	ZFS_MODULE_PARAM(zfs_zio, zio_, requeue_io_start_cut_in_line, INT, ZMOD_RW,
	"Prioritize requeued I/O");

	ZFS_MODULE_PARAM(zfs, zfs_, sync_pass_deferred_free, UINT, ZMOD_RW,
	"Defer frees starting in this pass");

	ZFS_MODULE_PARAM(zfs, zfs_, sync_pass_dont_compress, UINT, ZMOD_RW,
	"Don't compress starting in this pass");

	ZFS_MODULE_PARAM(zfs, zfs_, sync_pass_rewrite, UINT, ZMOD_RW,
	"Rewrite new bps starting in this pass");

	ZFS_MODULE_PARAM(zfs_zio, zio_, dva_throttle_enabled, INT, ZMOD_RW,
	"Throttle block allocations in the ZIO pipeline");

	ZFS_MODULE_PARAM(zfs_zio, zio_, deadman_log_all, INT, ZMOD_RW,
	"Log all slow ZIOs, not just those with vdevs");
	diff --git a/sys/contrib/openzfs/module/zfs/zio_inject.c b/sys/contrib/openzfs/module/zfs/zio_inject.c
	index 3598351c499d..66ad72fb88e9 100644
	--- a/sys/contrib/openzfs/module/zfs/zio_inject.c
	+++ b/sys/contrib/openzfs/module/zfs/zio_inject.c
	@@ -1,972 +1,978 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
	* Copyright (c) 2012, 2015 by Delphix. All rights reserved.
	* Copyright (c) 2017, Intel Corporation.
	*/

	/*
	* ZFS fault injection
	*
	* To handle fault injection, we keep track of a series of zinject_record_t
	* structures which describe which logical block(s) should be injected with a
	* fault. These are kept in a global list. Each record corresponds to a given
	* spa_t and maintains a special hold on the spa_t so that it cannot be deleted
	* or exported while the injection record exists.
	*
	* Device level injection is done using the 'zi_guid' field. If this is set, it
	* means that the error is destined for a particular device, not a piece of
	* data.
	*
	* This is a rather poor data structure and algorithm, but we don't expect more
	* than a few faults at any one time, so it should be sufficient for our needs.
	*/

	#include <sys/arc.h>
	#include <sys/zio.h>
	#include <sys/zfs_ioctl.h>
	#include <sys/vdev_impl.h>
	#include <sys/dmu_objset.h>
	#include <sys/dsl_dataset.h>
	#include <sys/fs/zfs.h>

	uint32_t zio_injection_enabled = 0;

	/*
	* Data describing each zinject handler registered on the system, and
	* contains the list node linking the handler in the global zinject
	* handler list.
	*/
	typedef struct inject_handler {
	int zi_id;
	spa_t *zi_spa;
	zinject_record_t zi_record;
	uint64_t *zi_lanes;
	int zi_next_lane;
	list_node_t zi_link;
	} inject_handler_t;

	/*
	* List of all zinject handlers registered on the system, protected by
	* the inject_lock defined below.
	*/
	static list_t inject_handlers;

	/*
	* This protects insertion into, and traversal of, the inject handler
	* list defined above; as well as the inject_delay_count. Any time a
	* handler is inserted or removed from the list, this lock should be
	* taken as a RW_WRITER; and any time traversal is done over the list
	* (without modification to it) this lock should be taken as a RW_READER.
	*/
	static krwlock_t inject_lock;

	/*
	* This holds the number of zinject delay handlers that have been
	* registered on the system. It is protected by the inject_lock defined
	* above. Thus modifications to this count must be a RW_WRITER of the
	* inject_lock, and reads of this count must be (at least) a RW_READER
	* of the lock.
	*/
	static int inject_delay_count = 0;

	/*
	* This lock is used only in zio_handle_io_delay(), refer to the comment
	* in that function for more details.
	*/
	static kmutex_t inject_delay_mtx;

	/*
	* Used to assign unique identifying numbers to each new zinject handler.
	*/
	static int inject_next_id = 1;

	/*
	* Test if the requested frequency was triggered
	*/
	static boolean_t
	freq_triggered(uint32_t frequency)
	{
	/*
	* zero implies always (100%)
	*/
	if (frequency == 0)
	return (B_TRUE);

	/*
	* Note: we still handle legacy (unscaled) frequency values
	*/
	uint32_t maximum = (frequency <= 100) ? 100 : ZI_PERCENTAGE_MAX;

	return (random_in_range(maximum) < frequency);
	}

	/*
	* Returns true if the given record matches the I/O in progress.
	*/
	static boolean_t
	zio_match_handler(const zbookmark_phys_t *zb, uint64_t type, int dva,
	zinject_record_t *record, int error)
	{
	/*
	* Check for a match against the MOS, which is based on type
	*/
	if (zb->zb_objset == DMU_META_OBJSET &&
	record->zi_objset == DMU_META_OBJSET &&
	record->zi_object == DMU_META_DNODE_OBJECT) {
	if (record->zi_type == DMU_OT_NONE \|\|
	type == record->zi_type)
	return (freq_triggered(record->zi_freq));
	else
	return (B_FALSE);
	}

	/*
	* Check for an exact match.
	*/
	if (zb->zb_objset == record->zi_objset &&
	zb->zb_object == record->zi_object &&
	zb->zb_level == record->zi_level &&
	zb->zb_blkid >= record->zi_start &&
	zb->zb_blkid <= record->zi_end &&
	(record->zi_dvas == 0 \|\|
	(dva != ZI_NO_DVA && (record->zi_dvas & (1ULL << dva)))) &&
	error == record->zi_error) {
	return (freq_triggered(record->zi_freq));
	}

	return (B_FALSE);
	}

	/*
	* Panic the system when a config change happens in the function
	* specified by tag.
	*/
	void
	zio_handle_panic_injection(spa_t spa, const char tag, uint64_t type)
	{
	inject_handler_t *handler;

	rw_enter(&inject_lock, RW_READER);

	for (handler = list_head(&inject_handlers); handler != NULL;
	handler = list_next(&inject_handlers, handler)) {

	if (spa != handler->zi_spa)
	continue;

	if (handler->zi_record.zi_type == type &&
	strcmp(tag, handler->zi_record.zi_func) == 0)
	panic("Panic requested in function %s\n", tag);
	}

	rw_exit(&inject_lock);
	}

	/*
	* Inject a decryption failure. Decryption failures can occur in
	* both the ARC and the ZIO layers.
	*/
	int
	zio_handle_decrypt_injection(spa_t spa, const zbookmark_phys_t zb,
	uint64_t type, int error)
	{
	int ret = 0;
	inject_handler_t *handler;

	rw_enter(&inject_lock, RW_READER);

	for (handler = list_head(&inject_handlers); handler != NULL;
	handler = list_next(&inject_handlers, handler)) {

	if (spa != handler->zi_spa \|\|
	handler->zi_record.zi_cmd != ZINJECT_DECRYPT_FAULT)
	continue;

	if (zio_match_handler(zb, type, ZI_NO_DVA,
	&handler->zi_record, error)) {
	ret = error;
	break;
	}
	}

	rw_exit(&inject_lock);
	return (ret);
	}

	/*
	* If this is a physical I/O for a vdev child determine which DVA it is
	* for. We iterate backwards through the DVAs matching on the offset so
	* that we end up with ZI_NO_DVA (-1) if we don't find a match.
	*/
	static int
	zio_match_dva(zio_t *zio)
	{
	int i = ZI_NO_DVA;

	if (zio->io_bp != NULL && zio->io_vd != NULL &&
	zio->io_child_type == ZIO_CHILD_VDEV) {
	for (i = BP_GET_NDVAS(zio->io_bp) - 1; i >= 0; i--) {
	dva_t *dva = &zio->io_bp->blk_dva[i];
	uint64_t off = DVA_GET_OFFSET(dva);
	vdev_t *vd = vdev_lookup_top(zio->io_spa,
	DVA_GET_VDEV(dva));

	/* Compensate for vdev label added to leaves */
	if (zio->io_vd->vdev_ops->vdev_op_leaf)
	off += VDEV_LABEL_START_SIZE;

	if (zio->io_vd == vd && zio->io_offset == off)
	break;
	}
	}

	return (i);
	}


	/*
	* Determine if the I/O in question should return failure. Returns the errno
	* to be returned to the caller.
	*/
	int
	zio_handle_fault_injection(zio_t *zio, int error)
	{
	int ret = 0;
	inject_handler_t *handler;

	/*
	* Ignore I/O not associated with any logical data.
	*/
	if (zio->io_logical == NULL)
	return (0);

	/*
	* Currently, we only support fault injection on reads.
	*/
	if (zio->io_type != ZIO_TYPE_READ)
	return (0);

	/*
	* A rebuild I/O has no checksum to verify.
	*/
	if (zio->io_priority == ZIO_PRIORITY_REBUILD && error == ECKSUM)
	return (0);

	rw_enter(&inject_lock, RW_READER);

	for (handler = list_head(&inject_handlers); handler != NULL;
	handler = list_next(&inject_handlers, handler)) {
	if (zio->io_spa != handler->zi_spa \|\|
	handler->zi_record.zi_cmd != ZINJECT_DATA_FAULT)
	continue;

	/* If this handler matches, return the specified error */
	if (zio_match_handler(&zio->io_logical->io_bookmark,
	zio->io_bp ? BP_GET_TYPE(zio->io_bp) : DMU_OT_NONE,
	zio_match_dva(zio), &handler->zi_record, error)) {
	ret = error;
	break;
	}
	}

	rw_exit(&inject_lock);

	return (ret);
	}

	/*
	* Determine if the zio is part of a label update and has an injection
	* handler associated with that portion of the label. Currently, we
	* allow error injection in either the nvlist or the uberblock region of
	* of the vdev label.
	*/
	int
	zio_handle_label_injection(zio_t *zio, int error)
	{
	inject_handler_t *handler;
	vdev_t *vd = zio->io_vd;
	uint64_t offset = zio->io_offset;
	int label;
	int ret = 0;

	if (offset >= VDEV_LABEL_START_SIZE &&
	offset < vd->vdev_psize - VDEV_LABEL_END_SIZE)
	return (0);

	rw_enter(&inject_lock, RW_READER);

	for (handler = list_head(&inject_handlers); handler != NULL;
	handler = list_next(&inject_handlers, handler)) {
	uint64_t start = handler->zi_record.zi_start;
	uint64_t end = handler->zi_record.zi_end;

	if (handler->zi_record.zi_cmd != ZINJECT_LABEL_FAULT)
	continue;

	/*
	* The injection region is the relative offsets within a
	* vdev label. We must determine the label which is being
	* updated and adjust our region accordingly.
	*/
	label = vdev_label_number(vd->vdev_psize, offset);
	start = vdev_label_offset(vd->vdev_psize, label, start);
	end = vdev_label_offset(vd->vdev_psize, label, end);

	if (zio->io_vd->vdev_guid == handler->zi_record.zi_guid &&
	(offset >= start && offset <= end)) {
	ret = error;
	break;
	}
	}
	rw_exit(&inject_lock);
	return (ret);
	}

	static int
	zio_inject_bitflip_cb(void data, size_t len, void private)
	{
	zio_t *zio = private;
	uint8_t *buffer = data;
	uint_t byte = random_in_range(len);

	ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ);

	/* flip a single random bit in an abd data buffer */
	buffer[byte] ^= 1 << random_in_range(8);

	return (1); /* stop after first flip */
	}

	static int
	zio_handle_device_injection_impl(vdev_t vd, zio_t zio, int err1, int err2)
	{
	inject_handler_t *handler;
	int ret = 0;

	/*
	* We skip over faults in the labels unless it's during
	* device open (i.e. zio == NULL).
	*/
	if (zio != NULL) {
	uint64_t offset = zio->io_offset;

	if (offset < VDEV_LABEL_START_SIZE \|\|
	offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE)
	return (0);
	}

	rw_enter(&inject_lock, RW_READER);

	for (handler = list_head(&inject_handlers); handler != NULL;
	handler = list_next(&inject_handlers, handler)) {

	if (handler->zi_record.zi_cmd != ZINJECT_DEVICE_FAULT)
	continue;

	if (vd->vdev_guid == handler->zi_record.zi_guid) {
	if (handler->zi_record.zi_failfast &&
	(zio == NULL \|\| (zio->io_flags &
	(ZIO_FLAG_IO_RETRY \| ZIO_FLAG_TRYHARD)))) {
	continue;
	}

	/* Handle type specific I/O failures */
	if (zio != NULL &&
	handler->zi_record.zi_iotype != ZIO_TYPES &&
	handler->zi_record.zi_iotype != zio->io_type)
	continue;

	if (handler->zi_record.zi_error == err1 \|\|
	handler->zi_record.zi_error == err2) {
	/*
	* limit error injection if requested
	*/
	if (!freq_triggered(handler->zi_record.zi_freq))
	continue;

	/*
	* For a failed open, pretend like the device
	* has gone away.
	*/
	if (err1 == ENXIO)
	vd->vdev_stat.vs_aux =
	VDEV_AUX_OPEN_FAILED;

	/*
	* Treat these errors as if they had been
	* retried so that all the appropriate stats
	* and FMA events are generated.
	*/
	if (!handler->zi_record.zi_failfast &&
	zio != NULL)
	zio->io_flags \|= ZIO_FLAG_IO_RETRY;

	/*
	* EILSEQ means flip a bit after a read
	*/
	if (handler->zi_record.zi_error == EILSEQ) {
	if (zio == NULL)
	break;

	/* locate buffer data and flip a bit */
	(void) abd_iterate_func(zio->io_abd, 0,
	zio->io_size, zio_inject_bitflip_cb,
	zio);
	break;
	}

	ret = handler->zi_record.zi_error;
	break;
	}
	if (handler->zi_record.zi_error == ENXIO) {
	ret = SET_ERROR(EIO);
	break;
	}
	}
	}

	rw_exit(&inject_lock);

	return (ret);
	}

	int
	zio_handle_device_injection(vdev_t vd, zio_t zio, int error)
	{
	return (zio_handle_device_injection_impl(vd, zio, error, INT_MAX));
	}

	int
	zio_handle_device_injections(vdev_t vd, zio_t zio, int err1, int err2)
	{
	return (zio_handle_device_injection_impl(vd, zio, err1, err2));
	}

	/*
	* Simulate hardware that ignores cache flushes. For requested number
	* of seconds nix the actual writing to disk.
	*/
	void
	zio_handle_ignored_writes(zio_t *zio)
	{
	inject_handler_t *handler;

	rw_enter(&inject_lock, RW_READER);

	for (handler = list_head(&inject_handlers); handler != NULL;
	handler = list_next(&inject_handlers, handler)) {

	/* Ignore errors not destined for this pool */
	if (zio->io_spa != handler->zi_spa \|\|
	handler->zi_record.zi_cmd != ZINJECT_IGNORED_WRITES)
	continue;

	/*
	* Positive duration implies # of seconds, negative
	* a number of txgs
	*/
	if (handler->zi_record.zi_timer == 0) {
	if (handler->zi_record.zi_duration > 0)
	handler->zi_record.zi_timer = ddi_get_lbolt64();
	else
	handler->zi_record.zi_timer = zio->io_txg;
	}

	/* Have a "problem" writing 60% of the time */
	if (random_in_range(100) < 60)
	zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES;
	break;
	}

	rw_exit(&inject_lock);
	}

	void
	spa_handle_ignored_writes(spa_t *spa)
	{
	inject_handler_t *handler;

	if (zio_injection_enabled == 0)
	return;

	rw_enter(&inject_lock, RW_READER);

	for (handler = list_head(&inject_handlers); handler != NULL;
	handler = list_next(&inject_handlers, handler)) {

	if (spa != handler->zi_spa \|\|
	handler->zi_record.zi_cmd != ZINJECT_IGNORED_WRITES)
	continue;

	if (handler->zi_record.zi_duration > 0) {
	VERIFY(handler->zi_record.zi_timer == 0 \|\|
	ddi_time_after64(
	(int64_t)handler->zi_record.zi_timer +
	handler->zi_record.zi_duration * hz,
	ddi_get_lbolt64()));
	} else {
	/* duration is negative so the subtraction here adds */
	VERIFY(handler->zi_record.zi_timer == 0 \|\|
	handler->zi_record.zi_timer -
	handler->zi_record.zi_duration >=
	spa_syncing_txg(spa));
	}
	}

	rw_exit(&inject_lock);
	}

	hrtime_t
	zio_handle_io_delay(zio_t *zio)
	{
	vdev_t *vd = zio->io_vd;
	inject_handler_t *min_handler = NULL;
	hrtime_t min_target = 0;

	rw_enter(&inject_lock, RW_READER);

	/*
	* inject_delay_count is a subset of zio_injection_enabled that
	* is only incremented for delay handlers. These checks are
	* mainly added to remind the reader why we're not explicitly
	* checking zio_injection_enabled like the other functions.
	*/
	IMPLY(inject_delay_count > 0, zio_injection_enabled > 0);
	IMPLY(zio_injection_enabled == 0, inject_delay_count == 0);

	/*
	* If there aren't any inject delay handlers registered, then we
	* can short circuit and simply return 0 here. A value of zero
	* informs zio_delay_interrupt() that this request should not be
	* delayed. This short circuit keeps us from acquiring the
	* inject_delay_mutex unnecessarily.
	*/
	if (inject_delay_count == 0) {
	rw_exit(&inject_lock);
	return (0);
	}

	/*
	* Each inject handler has a number of "lanes" associated with
	* it. Each lane is able to handle requests independently of one
	* another, and at a latency defined by the inject handler
	* record's zi_timer field. Thus if a handler in configured with
	* a single lane with a 10ms latency, it will delay requests
	* such that only a single request is completed every 10ms. So,
	* if more than one request is attempted per each 10ms interval,
	* the average latency of the requests will be greater than
	* 10ms; but if only a single request is submitted each 10ms
	* interval the average latency will be 10ms.
	*
	* We need to acquire this mutex to prevent multiple concurrent
	* threads being assigned to the same lane of a given inject
	* handler. The mutex allows us to perform the following two
	* operations atomically:
	*
	* 1. determine the minimum handler and minimum target
	* value of all the possible handlers
	* 2. update that minimum handler's lane array
	*
	* Without atomicity, two (or more) threads could pick the same
	* lane in step (1), and then conflict with each other in step
	* (2). This could allow a single lane handler to process
	* multiple requests simultaneously, which shouldn't be possible.
	*/
	mutex_enter(&inject_delay_mtx);

	for (inject_handler_t *handler = list_head(&inject_handlers);
	handler != NULL; handler = list_next(&inject_handlers, handler)) {
	if (handler->zi_record.zi_cmd != ZINJECT_DELAY_IO)
	continue;

	if (!freq_triggered(handler->zi_record.zi_freq))
	continue;

	if (vd->vdev_guid != handler->zi_record.zi_guid)
	continue;

	+ /* also match on I/O type (e.g., -T read) */
	+ if (handler->zi_record.zi_iotype != ZIO_TYPES &&
	+ handler->zi_record.zi_iotype != zio->io_type) {
	+ continue;
	+ }
	+
	/*
	* Defensive; should never happen as the array allocation
	* occurs prior to inserting this handler on the list.
	*/
	ASSERT3P(handler->zi_lanes, !=, NULL);

	/*
	* This should never happen, the zinject command should
	* prevent a user from setting an IO delay with zero lanes.
	*/
	ASSERT3U(handler->zi_record.zi_nlanes, !=, 0);

	ASSERT3U(handler->zi_record.zi_nlanes, >,
	handler->zi_next_lane);

	/*
	* We want to issue this IO to the lane that will become
	* idle the soonest, so we compare the soonest this
	* specific handler can complete the IO with all other
	* handlers, to find the lowest value of all possible
	* lanes. We then use this lane to submit the request.
	*
	* Since each handler has a constant value for its
	* delay, we can just use the "next" lane for that
	* handler; as it will always be the lane with the
	* lowest value for that particular handler (i.e. the
	* lane that will become idle the soonest). This saves a
	* scan of each handler's lanes array.
	*
	* There's two cases to consider when determining when
	* this specific IO request should complete. If this
	* lane is idle, we want to "submit" the request now so
	* it will complete after zi_timer milliseconds. Thus,
	* we set the target to now + zi_timer.
	*
	* If the lane is busy, we want this request to complete
	* zi_timer milliseconds after the lane becomes idle.
	* Since the 'zi_lanes' array holds the time at which
	* each lane will become idle, we use that value to
	* determine when this request should complete.
	*/
	hrtime_t idle = handler->zi_record.zi_timer + gethrtime();
	hrtime_t busy = handler->zi_record.zi_timer +
	handler->zi_lanes[handler->zi_next_lane];
	hrtime_t target = MAX(idle, busy);

	if (min_handler == NULL) {
	min_handler = handler;
	min_target = target;
	continue;
	}

	ASSERT3P(min_handler, !=, NULL);
	ASSERT3U(min_target, !=, 0);

	/*
	* We don't yet increment the "next lane" variable since
	* we still might find a lower value lane in another
	* handler during any remaining iterations. Once we're
	* sure we've selected the absolute minimum, we'll claim
	* the lane and increment the handler's "next lane"
	* field below.
	*/

	if (target < min_target) {
	min_handler = handler;
	min_target = target;
	}
	}

	/*
	* 'min_handler' will be NULL if no IO delays are registered for
	* this vdev, otherwise it will point to the handler containing
	* the lane that will become idle the soonest.
	*/
	if (min_handler != NULL) {
	ASSERT3U(min_target, !=, 0);
	min_handler->zi_lanes[min_handler->zi_next_lane] = min_target;

	/*
	* If we've used all possible lanes for this handler,
	* loop back and start using the first lane again;
	* otherwise, just increment the lane index.
	*/
	min_handler->zi_next_lane = (min_handler->zi_next_lane + 1) %
	min_handler->zi_record.zi_nlanes;
	}

	mutex_exit(&inject_delay_mtx);
	rw_exit(&inject_lock);

	return (min_target);
	}

	static int
	zio_calculate_range(const char pool, zinject_record_t record)
	{
	dsl_pool_t *dp;
	dsl_dataset_t *ds;
	objset_t *os = NULL;
	dnode_t *dn = NULL;
	int error;

	/*
	* Obtain the dnode for object using pool, objset, and object
	*/
	error = dsl_pool_hold(pool, FTAG, &dp);
	if (error)
	return (error);

	error = dsl_dataset_hold_obj(dp, record->zi_objset, FTAG, &ds);
	dsl_pool_rele(dp, FTAG);
	if (error)
	return (error);

	error = dmu_objset_from_ds(ds, &os);
	dsl_dataset_rele(ds, FTAG);
	if (error)
	return (error);

	error = dnode_hold(os, record->zi_object, FTAG, &dn);
	if (error)
	return (error);

	/*
	* Translate the range into block IDs
	*/
	if (record->zi_start != 0 \|\| record->zi_end != -1ULL) {
	record->zi_start >>= dn->dn_datablkshift;
	record->zi_end >>= dn->dn_datablkshift;
	}
	if (record->zi_level > 0) {
	if (record->zi_level >= dn->dn_nlevels) {
	dnode_rele(dn, FTAG);
	return (SET_ERROR(EDOM));
	}

	if (record->zi_start != 0 \|\| record->zi_end != 0) {
	int shift = dn->dn_indblkshift - SPA_BLKPTRSHIFT;

	for (int level = record->zi_level; level > 0; level--) {
	record->zi_start >>= shift;
	record->zi_end >>= shift;
	}
	}
	}

	dnode_rele(dn, FTAG);
	return (0);
	}

	/*
	* Create a new handler for the given record. We add it to the list, adding
	* a reference to the spa_t in the process. We increment zio_injection_enabled,
	* which is the switch to trigger all fault injection.
	*/
	int
	zio_inject_fault(char name, int flags, int id, zinject_record_t *record)
	{
	inject_handler_t *handler;
	int error;
	spa_t *spa;

	/*
	* If this is pool-wide metadata, make sure we unload the corresponding
	* spa_t, so that the next attempt to load it will trigger the fault.
	* We call spa_reset() to unload the pool appropriately.
	*/
	if (flags & ZINJECT_UNLOAD_SPA)
	if ((error = spa_reset(name)) != 0)
	return (error);

	if (record->zi_cmd == ZINJECT_DELAY_IO) {
	/*
	* A value of zero for the number of lanes or for the
	* delay time doesn't make sense.
	*/
	if (record->zi_timer == 0 \|\| record->zi_nlanes == 0)
	return (SET_ERROR(EINVAL));

	/*
	* The number of lanes is directly mapped to the size of
	* an array used by the handler. Thus, to ensure the
	* user doesn't trigger an allocation that's "too large"
	* we cap the number of lanes here.
	*/
	if (record->zi_nlanes >= UINT16_MAX)
	return (SET_ERROR(EINVAL));
	}

	/*
	* If the supplied range was in bytes -- calculate the actual blkid
	*/
	if (flags & ZINJECT_CALC_RANGE) {
	error = zio_calculate_range(name, record);
	if (error != 0)
	return (error);
	}

	if (!(flags & ZINJECT_NULL)) {
	/*
	* spa_inject_ref() will add an injection reference, which will
	* prevent the pool from being removed from the namespace while
	* still allowing it to be unloaded.
	*/
	if ((spa = spa_inject_addref(name)) == NULL)
	return (SET_ERROR(ENOENT));

	handler = kmem_alloc(sizeof (inject_handler_t), KM_SLEEP);

	handler->zi_spa = spa;
	handler->zi_record = *record;

	if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {
	handler->zi_lanes = kmem_zalloc(
	sizeof (handler->zi_lanes)
	handler->zi_record.zi_nlanes, KM_SLEEP);
	handler->zi_next_lane = 0;
	} else {
	handler->zi_lanes = NULL;
	handler->zi_next_lane = 0;
	}

	rw_enter(&inject_lock, RW_WRITER);

	/*
	* We can't move this increment into the conditional
	* above because we need to hold the RW_WRITER lock of
	* inject_lock, and we don't want to hold that while
	* allocating the handler's zi_lanes array.
	*/
	if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {
	ASSERT3S(inject_delay_count, >=, 0);
	inject_delay_count++;
	ASSERT3S(inject_delay_count, >, 0);
	}

	*id = handler->zi_id = inject_next_id++;
	list_insert_tail(&inject_handlers, handler);
	atomic_inc_32(&zio_injection_enabled);

	rw_exit(&inject_lock);
	}

	/*
	* Flush the ARC, so that any attempts to read this data will end up
	* going to the ZIO layer. Note that this is a little overkill, but
	* we don't have the necessary ARC interfaces to do anything else, and
	* fault injection isn't a performance critical path.
	*/
	if (flags & ZINJECT_FLUSH_ARC)
	/*
	* We must use FALSE to ensure arc_flush returns, since
	* we're not preventing concurrent ARC insertions.
	*/
	arc_flush(NULL, FALSE);

	return (0);
	}

	/*
	* Returns the next record with an ID greater than that supplied to the
	* function. Used to iterate over all handlers in the system.
	*/
	int
	zio_inject_list_next(int id, char name, size_t buflen,
	zinject_record_t *record)
	{
	inject_handler_t *handler;
	int ret;

	mutex_enter(&spa_namespace_lock);
	rw_enter(&inject_lock, RW_READER);

	for (handler = list_head(&inject_handlers); handler != NULL;
	handler = list_next(&inject_handlers, handler))
	if (handler->zi_id > *id)
	break;

	if (handler) {
	*record = handler->zi_record;
	*id = handler->zi_id;
	(void) strlcpy(name, spa_name(handler->zi_spa), buflen);
	ret = 0;
	} else {
	ret = SET_ERROR(ENOENT);
	}

	rw_exit(&inject_lock);
	mutex_exit(&spa_namespace_lock);

	return (ret);
	}

	/*
	* Clear the fault handler with the given identifier, or return ENOENT if none
	* exists.
	*/
	int
	zio_clear_fault(int id)
	{
	inject_handler_t *handler;

	rw_enter(&inject_lock, RW_WRITER);

	for (handler = list_head(&inject_handlers); handler != NULL;
	handler = list_next(&inject_handlers, handler))
	if (handler->zi_id == id)
	break;

	if (handler == NULL) {
	rw_exit(&inject_lock);
	return (SET_ERROR(ENOENT));
	}

	if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {
	ASSERT3S(inject_delay_count, >, 0);
	inject_delay_count--;
	ASSERT3S(inject_delay_count, >=, 0);
	}

	list_remove(&inject_handlers, handler);
	rw_exit(&inject_lock);

	if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {
	ASSERT3P(handler->zi_lanes, !=, NULL);
	kmem_free(handler->zi_lanes, sizeof (handler->zi_lanes)
	handler->zi_record.zi_nlanes);
	} else {
	ASSERT3P(handler->zi_lanes, ==, NULL);
	}

	spa_inject_delref(handler->zi_spa);
	kmem_free(handler, sizeof (inject_handler_t));
	atomic_dec_32(&zio_injection_enabled);

	return (0);
	}

	void
	zio_inject_init(void)
	{
	rw_init(&inject_lock, NULL, RW_DEFAULT, NULL);
	mutex_init(&inject_delay_mtx, NULL, MUTEX_DEFAULT, NULL);
	list_create(&inject_handlers, sizeof (inject_handler_t),
	offsetof(inject_handler_t, zi_link));
	}

	void
	zio_inject_fini(void)
	{
	list_destroy(&inject_handlers);
	mutex_destroy(&inject_delay_mtx);
	rw_destroy(&inject_lock);
	}

	#if defined(_KERNEL)
	EXPORT_SYMBOL(zio_injection_enabled);
	EXPORT_SYMBOL(zio_inject_fault);
	EXPORT_SYMBOL(zio_inject_list_next);
	EXPORT_SYMBOL(zio_clear_fault);
	EXPORT_SYMBOL(zio_handle_fault_injection);
	EXPORT_SYMBOL(zio_handle_device_injection);
	EXPORT_SYMBOL(zio_handle_label_injection);
	#endif
	diff --git a/sys/contrib/openzfs/module/zfs/zvol.c b/sys/contrib/openzfs/module/zfs/zvol.c
	index 20ea71f23376..89b523ddd903 100644
	--- a/sys/contrib/openzfs/module/zfs/zvol.c
	+++ b/sys/contrib/openzfs/module/zfs/zvol.c
	@@ -1,1741 +1,1741 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC.
	* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
	* Rewritten for Linux by Brian Behlendorf <behlendorf1@llnl.gov>.
	* LLNL-CODE-403049.
	*
	* ZFS volume emulation driver.
	*
	* Makes a DMU object look like a volume of arbitrary size, up to 2^64 bytes.
	* Volumes are accessed through the symbolic links named:
	*
	* /dev/<pool_name>/<dataset_name>
	*
	* Volumes are persistent through reboot and module load. No user command
	* needs to be run before opening and using a device.
	*
	* Copyright 2014 Nexenta Systems, Inc. All rights reserved.
	* Copyright (c) 2016 Actifio, Inc. All rights reserved.
	* Copyright (c) 2012, 2019 by Delphix. All rights reserved.
	*/

	/*
	* Note on locking of zvol state structures.
	*
	* These structures are used to maintain internal state used to emulate block
	* devices on top of zvols. In particular, management of device minor number
	* operations - create, remove, rename, and set_snapdev - involves access to
	* these structures. The zvol_state_lock is primarily used to protect the
	* zvol_state_list. The zv->zv_state_lock is used to protect the contents
	* of the zvol_state_t structures, as well as to make sure that when the
	* time comes to remove the structure from the list, it is not in use, and
	* therefore, it can be taken off zvol_state_list and freed.
	*
	* The zv_suspend_lock was introduced to allow for suspending I/O to a zvol,
	* e.g. for the duration of receive and rollback operations. This lock can be
	* held for significant periods of time. Given that it is undesirable to hold
	* mutexes for long periods of time, the following lock ordering applies:
	* - take zvol_state_lock if necessary, to protect zvol_state_list
	* - take zv_suspend_lock if necessary, by the code path in question
	* - take zv_state_lock to protect zvol_state_t
	*
	* The minor operations are issued to spa->spa_zvol_taskq queues, that are
	* single-threaded (to preserve order of minor operations), and are executed
	* through the zvol_task_cb that dispatches the specific operations. Therefore,
	* these operations are serialized per pool. Consequently, we can be certain
	* that for a given zvol, there is only one operation at a time in progress.
	* That is why one can be sure that first, zvol_state_t for a given zvol is
	* allocated and placed on zvol_state_list, and then other minor operations
	* for this zvol are going to proceed in the order of issue.
	*
	*/

	#include <sys/dataset_kstats.h>
	#include <sys/dbuf.h>
	#include <sys/dmu_traverse.h>
	#include <sys/dsl_dataset.h>
	#include <sys/dsl_prop.h>
	#include <sys/dsl_dir.h>
	#include <sys/zap.h>
	#include <sys/zfeature.h>
	#include <sys/zil_impl.h>
	#include <sys/dmu_tx.h>
	#include <sys/zio.h>
	#include <sys/zfs_rlock.h>
	#include <sys/spa_impl.h>
	#include <sys/zvol.h>
	#include <sys/zvol_impl.h>

	unsigned int zvol_inhibit_dev = 0;
	unsigned int zvol_volmode = ZFS_VOLMODE_GEOM;

	struct hlist_head *zvol_htable;
	static list_t zvol_state_list;
	krwlock_t zvol_state_lock;

	typedef enum {
	ZVOL_ASYNC_REMOVE_MINORS,
	ZVOL_ASYNC_RENAME_MINORS,
	ZVOL_ASYNC_SET_SNAPDEV,
	ZVOL_ASYNC_SET_VOLMODE,
	ZVOL_ASYNC_MAX
	} zvol_async_op_t;

	typedef struct {
	zvol_async_op_t op;
	char name1[MAXNAMELEN];
	char name2[MAXNAMELEN];
	uint64_t value;
	} zvol_task_t;

	uint64_t
	zvol_name_hash(const char *name)
	{
	int i;
	uint64_t crc = -1ULL;
	const uint8_t p = (const uint8_t )name;
	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
	for (i = 0; i < MAXNAMELEN - 1 && *p; i++, p++) {
	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (*p)) & 0xFF];
	}
	return (crc);
	}

	/*
	* Find a zvol_state_t given the name and hash generated by zvol_name_hash.
	* If found, return with zv_suspend_lock and zv_state_lock taken, otherwise,
	* return (NULL) without the taking locks. The zv_suspend_lock is always taken
	* before zv_state_lock. The mode argument indicates the mode (including none)
	* for zv_suspend_lock to be taken.
	*/
	zvol_state_t *
	zvol_find_by_name_hash(const char *name, uint64_t hash, int mode)
	{
	zvol_state_t *zv;
	struct hlist_node *p = NULL;

	rw_enter(&zvol_state_lock, RW_READER);
	hlist_for_each(p, ZVOL_HT_HEAD(hash)) {
	zv = hlist_entry(p, zvol_state_t, zv_hlink);
	mutex_enter(&zv->zv_state_lock);
	if (zv->zv_hash == hash &&
	strncmp(zv->zv_name, name, MAXNAMELEN) == 0) {
	/*
	* this is the right zvol, take the locks in the
	* right order
	*/
	if (mode != RW_NONE &&
	!rw_tryenter(&zv->zv_suspend_lock, mode)) {
	mutex_exit(&zv->zv_state_lock);
	rw_enter(&zv->zv_suspend_lock, mode);
	mutex_enter(&zv->zv_state_lock);
	/*
	* zvol cannot be renamed as we continue
	* to hold zvol_state_lock
	*/
	ASSERT(zv->zv_hash == hash &&
	strncmp(zv->zv_name, name, MAXNAMELEN)
	== 0);
	}
	rw_exit(&zvol_state_lock);
	return (zv);
	}
	mutex_exit(&zv->zv_state_lock);
	}
	rw_exit(&zvol_state_lock);

	return (NULL);
	}

	/*
	* Find a zvol_state_t given the name.
	* If found, return with zv_suspend_lock and zv_state_lock taken, otherwise,
	* return (NULL) without the taking locks. The zv_suspend_lock is always taken
	* before zv_state_lock. The mode argument indicates the mode (including none)
	* for zv_suspend_lock to be taken.
	*/
	static zvol_state_t *
	zvol_find_by_name(const char *name, int mode)
	{
	return (zvol_find_by_name_hash(name, zvol_name_hash(name), mode));
	}

	/*
	* ZFS_IOC_CREATE callback handles dmu zvol and zap object creation.
	*/
	void
	zvol_create_cb(objset_t os, void arg, cred_t cr, dmu_tx_t tx)
	{
	zfs_creat_t *zct = arg;
	nvlist_t *nvprops = zct->zct_props;
	int error;
	uint64_t volblocksize, volsize;

	VERIFY(nvlist_lookup_uint64(nvprops,
	zfs_prop_to_name(ZFS_PROP_VOLSIZE), &volsize) == 0);
	if (nvlist_lookup_uint64(nvprops,
	zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), &volblocksize) != 0)
	volblocksize = zfs_prop_default_numeric(ZFS_PROP_VOLBLOCKSIZE);

	/*
	* These properties must be removed from the list so the generic
	* property setting step won't apply to them.
	*/
	VERIFY(nvlist_remove_all(nvprops,
	zfs_prop_to_name(ZFS_PROP_VOLSIZE)) == 0);
	(void) nvlist_remove_all(nvprops,
	zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE));

	error = dmu_object_claim(os, ZVOL_OBJ, DMU_OT_ZVOL, volblocksize,
	DMU_OT_NONE, 0, tx);
	ASSERT(error == 0);

	error = zap_create_claim(os, ZVOL_ZAP_OBJ, DMU_OT_ZVOL_PROP,
	DMU_OT_NONE, 0, tx);
	ASSERT(error == 0);

	error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize, tx);
	ASSERT(error == 0);
	}

	/*
	* ZFS_IOC_OBJSET_STATS entry point.
	*/
	int
	zvol_get_stats(objset_t os, nvlist_t nv)
	{
	int error;
	dmu_object_info_t *doi;
	uint64_t val;

	error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &val);
	if (error)
	return (SET_ERROR(error));

	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLSIZE, val);
	doi = kmem_alloc(sizeof (dmu_object_info_t), KM_SLEEP);
	error = dmu_object_info(os, ZVOL_OBJ, doi);

	if (error == 0) {
	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLBLOCKSIZE,
	doi->doi_data_block_size);
	}

	kmem_free(doi, sizeof (dmu_object_info_t));

	return (SET_ERROR(error));
	}

	/*
	* Sanity check volume size.
	*/
	int
	zvol_check_volsize(uint64_t volsize, uint64_t blocksize)
	{
	if (volsize == 0)
	return (SET_ERROR(EINVAL));

	if (volsize % blocksize != 0)
	return (SET_ERROR(EINVAL));

	#ifdef _ILP32
	if (volsize - 1 > SPEC_MAXOFFSET_T)
	return (SET_ERROR(EOVERFLOW));
	#endif
	return (0);
	}

	/*
	* Ensure the zap is flushed then inform the VFS of the capacity change.
	*/
	static int
	zvol_update_volsize(uint64_t volsize, objset_t *os)
	{
	dmu_tx_t *tx;
	int error;
	uint64_t txg;

	tx = dmu_tx_create(os);
	dmu_tx_hold_zap(tx, ZVOL_ZAP_OBJ, TRUE, NULL);
	dmu_tx_mark_netfree(tx);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error) {
	dmu_tx_abort(tx);
	return (SET_ERROR(error));
	}
	txg = dmu_tx_get_txg(tx);

	error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1,
	&volsize, tx);
	dmu_tx_commit(tx);

	txg_wait_synced(dmu_objset_pool(os), txg);

	if (error == 0)
	error = dmu_free_long_range(os,
	ZVOL_OBJ, volsize, DMU_OBJECT_END);

	return (error);
	}

	/*
	* Set ZFS_PROP_VOLSIZE set entry point. Note that modifying the volume
	* size will result in a udev "change" event being generated.
	*/
	int
	zvol_set_volsize(const char *name, uint64_t volsize)
	{
	objset_t *os = NULL;
	uint64_t readonly;
	int error;
	boolean_t owned = B_FALSE;

	error = dsl_prop_get_integer(name,
	zfs_prop_to_name(ZFS_PROP_READONLY), &readonly, NULL);
	if (error != 0)
	return (SET_ERROR(error));
	if (readonly)
	return (SET_ERROR(EROFS));

	zvol_state_t *zv = zvol_find_by_name(name, RW_READER);

	ASSERT(zv == NULL \|\| (MUTEX_HELD(&zv->zv_state_lock) &&
	RW_READ_HELD(&zv->zv_suspend_lock)));

	if (zv == NULL \|\| zv->zv_objset == NULL) {
	if (zv != NULL)
	rw_exit(&zv->zv_suspend_lock);
	if ((error = dmu_objset_own(name, DMU_OST_ZVOL, B_FALSE, B_TRUE,
	FTAG, &os)) != 0) {
	if (zv != NULL)
	mutex_exit(&zv->zv_state_lock);
	return (SET_ERROR(error));
	}
	owned = B_TRUE;
	if (zv != NULL)
	zv->zv_objset = os;
	} else {
	os = zv->zv_objset;
	}

	dmu_object_info_t doi = kmem_alloc(sizeof (doi), KM_SLEEP);

	if ((error = dmu_object_info(os, ZVOL_OBJ, doi)) \|\|
	(error = zvol_check_volsize(volsize, doi->doi_data_block_size)))
	goto out;

	error = zvol_update_volsize(volsize, os);
	if (error == 0 && zv != NULL) {
	zv->zv_volsize = volsize;
	zv->zv_changed = 1;
	}
	out:
	kmem_free(doi, sizeof (dmu_object_info_t));

	if (owned) {
	dmu_objset_disown(os, B_TRUE, FTAG);
	if (zv != NULL)
	zv->zv_objset = NULL;
	} else {
	rw_exit(&zv->zv_suspend_lock);
	}

	if (zv != NULL)
	mutex_exit(&zv->zv_state_lock);

	if (error == 0 && zv != NULL)
	zvol_os_update_volsize(zv, volsize);

	return (SET_ERROR(error));
	}

	/*
	* Sanity check volume block size.
	*/
	int
	zvol_check_volblocksize(const char *name, uint64_t volblocksize)
	{
	/* Record sizes above 128k need the feature to be enabled */
	if (volblocksize > SPA_OLD_MAXBLOCKSIZE) {
	spa_t *spa;
	int error;

	if ((error = spa_open(name, &spa, FTAG)) != 0)
	return (error);

	if (!spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) {
	spa_close(spa, FTAG);
	return (SET_ERROR(ENOTSUP));
	}

	/*
	* We don't allow setting the property above 1MB,
	* unless the tunable has been changed.
	*/
	if (volblocksize > zfs_max_recordsize)
	return (SET_ERROR(EDOM));

	spa_close(spa, FTAG);
	}

	if (volblocksize < SPA_MINBLOCKSIZE \|\|
	volblocksize > SPA_MAXBLOCKSIZE \|\|
	!ISP2(volblocksize))
	return (SET_ERROR(EDOM));

	return (0);
	}

	/*
	* Replay a TX_TRUNCATE ZIL transaction if asked. TX_TRUNCATE is how we
	* implement DKIOCFREE/free-long-range.
	*/
	static int
	zvol_replay_truncate(void arg1, void arg2, boolean_t byteswap)
	{
	zvol_state_t *zv = arg1;
	lr_truncate_t *lr = arg2;
	uint64_t offset, length;

	ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr));

	if (byteswap)
	byteswap_uint64_array(lr, sizeof (*lr));

	offset = lr->lr_offset;
	length = lr->lr_length;

	dmu_tx_t *tx = dmu_tx_create(zv->zv_objset);
	dmu_tx_mark_netfree(tx);
	int error = dmu_tx_assign(tx, TXG_WAIT);
	if (error != 0) {
	dmu_tx_abort(tx);
	} else {
	(void) zil_replaying(zv->zv_zilog, tx);
	dmu_tx_commit(tx);
	error = dmu_free_long_range(zv->zv_objset, ZVOL_OBJ, offset,
	length);
	}

	return (error);
	}

	/*
	* Replay a TX_WRITE ZIL transaction that didn't get committed
	* after a system failure
	*/
	static int
	zvol_replay_write(void arg1, void arg2, boolean_t byteswap)
	{
	zvol_state_t *zv = arg1;
	lr_write_t *lr = arg2;
	objset_t *os = zv->zv_objset;
	char data = (char )(lr + 1); /* data follows lr_write_t */
	uint64_t offset, length;
	dmu_tx_t *tx;
	int error;

	ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr));

	if (byteswap)
	byteswap_uint64_array(lr, sizeof (*lr));

	offset = lr->lr_offset;
	length = lr->lr_length;

	/* If it's a dmu_sync() block, write the whole block */
	if (lr->lr_common.lrc_reclen == sizeof (lr_write_t)) {
	uint64_t blocksize = BP_GET_LSIZE(&lr->lr_blkptr);
	if (length < blocksize) {
	offset -= offset % blocksize;
	length = blocksize;
	}
	}

	tx = dmu_tx_create(os);
	dmu_tx_hold_write(tx, ZVOL_OBJ, offset, length);
	error = dmu_tx_assign(tx, TXG_WAIT);
	if (error) {
	dmu_tx_abort(tx);
	} else {
	dmu_write(os, ZVOL_OBJ, offset, length, data, tx);
	(void) zil_replaying(zv->zv_zilog, tx);
	dmu_tx_commit(tx);
	}

	return (error);
	}

	static int
	zvol_replay_err(void arg1, void arg2, boolean_t byteswap)
	{
	(void) arg1, (void) arg2, (void) byteswap;
	return (SET_ERROR(ENOTSUP));
	}

	/*
	* Callback vectors for replaying records.
	* Only TX_WRITE and TX_TRUNCATE are needed for zvol.
	*/
	zil_replay_func_t *const zvol_replay_vector[TX_MAX_TYPE] = {
	zvol_replay_err, /* no such transaction type */
	zvol_replay_err, /* TX_CREATE */
	zvol_replay_err, /* TX_MKDIR */
	zvol_replay_err, /* TX_MKXATTR */
	zvol_replay_err, /* TX_SYMLINK */
	zvol_replay_err, /* TX_REMOVE */
	zvol_replay_err, /* TX_RMDIR */
	zvol_replay_err, /* TX_LINK */
	zvol_replay_err, /* TX_RENAME */
	zvol_replay_write, /* TX_WRITE */
	zvol_replay_truncate, /* TX_TRUNCATE */
	zvol_replay_err, /* TX_SETATTR */
	zvol_replay_err, /* TX_ACL */
	zvol_replay_err, /* TX_CREATE_ATTR */
	zvol_replay_err, /* TX_CREATE_ACL_ATTR */
	zvol_replay_err, /* TX_MKDIR_ACL */
	zvol_replay_err, /* TX_MKDIR_ATTR */
	zvol_replay_err, /* TX_MKDIR_ACL_ATTR */
	zvol_replay_err, /* TX_WRITE2 */
	zvol_replay_err, /* TX_SETSAXATTR */
	zvol_replay_err, /* TX_RENAME_EXCHANGE */
	zvol_replay_err, /* TX_RENAME_WHITEOUT */
	zvol_replay_err, /* TX_CLONE_RANGE */
	};

	/*
	* zvol_log_write() handles synchronous writes using TX_WRITE ZIL transactions.
	*
	* We store data in the log buffers if it's small enough.
	* Otherwise we will later flush the data out via dmu_sync().
	*/
	static const ssize_t zvol_immediate_write_sz = 32768;

	void
	zvol_log_write(zvol_state_t zv, dmu_tx_t tx, uint64_t offset,
	uint64_t size, int sync)
	{
	uint32_t blocksize = zv->zv_volblocksize;
	zilog_t *zilog = zv->zv_zilog;
	itx_wr_state_t write_state;
	uint64_t sz = size;

	if (zil_replaying(zilog, tx))
	return;

	if (zilog->zl_logbias == ZFS_LOGBIAS_THROUGHPUT)
	write_state = WR_INDIRECT;
	else if (!spa_has_slogs(zilog->zl_spa) &&
	size >= blocksize && blocksize > zvol_immediate_write_sz)
	write_state = WR_INDIRECT;
	else if (sync)
	write_state = WR_COPIED;
	else
	write_state = WR_NEED_COPY;

	while (size) {
	itx_t *itx;
	lr_write_t *lr;
	itx_wr_state_t wr_state = write_state;
	ssize_t len = size;

	if (wr_state == WR_COPIED && size > zil_max_copied_data(zilog))
	wr_state = WR_NEED_COPY;
	else if (wr_state == WR_INDIRECT)
	len = MIN(blocksize - P2PHASE(offset, blocksize), size);

	itx = zil_itx_create(TX_WRITE, sizeof (*lr) +
	(wr_state == WR_COPIED ? len : 0));
	lr = (lr_write_t *)&itx->itx_lr;
	if (wr_state == WR_COPIED && dmu_read_by_dnode(zv->zv_dn,
	offset, len, lr+1, DMU_READ_NO_PREFETCH) != 0) {
	zil_itx_destroy(itx);
	itx = zil_itx_create(TX_WRITE, sizeof (*lr));
	lr = (lr_write_t *)&itx->itx_lr;
	wr_state = WR_NEED_COPY;
	}

	itx->itx_wr_state = wr_state;
	lr->lr_foid = ZVOL_OBJ;
	lr->lr_offset = offset;
	lr->lr_length = len;
	lr->lr_blkoff = 0;
	BP_ZERO(&lr->lr_blkptr);

	itx->itx_private = zv;
	itx->itx_sync = sync;

	(void) zil_itx_assign(zilog, itx, tx);

	offset += len;
	size -= len;
	}

	if (write_state == WR_COPIED \|\| write_state == WR_NEED_COPY) {
	dsl_pool_wrlog_count(zilog->zl_dmu_pool, sz, tx->tx_txg);
	}
	}

	/*
	* Log a DKIOCFREE/free-long-range to the ZIL with TX_TRUNCATE.
	*/
	void
	zvol_log_truncate(zvol_state_t zv, dmu_tx_t tx, uint64_t off, uint64_t len,
	boolean_t sync)
	{
	itx_t *itx;
	lr_truncate_t *lr;
	zilog_t *zilog = zv->zv_zilog;

	if (zil_replaying(zilog, tx))
	return;

	itx = zil_itx_create(TX_TRUNCATE, sizeof (*lr));
	lr = (lr_truncate_t *)&itx->itx_lr;
	lr->lr_foid = ZVOL_OBJ;
	lr->lr_offset = off;
	lr->lr_length = len;

	itx->itx_sync = sync;
	zil_itx_assign(zilog, itx, tx);
	}


	static void
	zvol_get_done(zgd_t *zgd, int error)
	{
	(void) error;
	if (zgd->zgd_db)
	dmu_buf_rele(zgd->zgd_db, zgd);

	zfs_rangelock_exit(zgd->zgd_lr);

	kmem_free(zgd, sizeof (zgd_t));
	}

	/*
	* Get data to generate a TX_WRITE intent log record.
	*/
	int
	zvol_get_data(void arg, uint64_t arg2, lr_write_t lr, char *buf,
	struct lwb lwb, zio_t zio)
	{
	zvol_state_t *zv = arg;
	uint64_t offset = lr->lr_offset;
	uint64_t size = lr->lr_length;
	dmu_buf_t *db;
	zgd_t *zgd;
	int error;

	ASSERT3P(lwb, !=, NULL);
	ASSERT3U(size, !=, 0);

	zgd = kmem_zalloc(sizeof (zgd_t), KM_SLEEP);
	zgd->zgd_lwb = lwb;

	/*
	* Write records come in two flavors: immediate and indirect.
	* For small writes it's cheaper to store the data with the
	* log record (immediate); for large writes it's cheaper to
	* sync the data and get a pointer to it (indirect) so that
	* we don't have to write the data twice.
	*/
	if (buf != NULL) { /* immediate write */
	zgd->zgd_lr = zfs_rangelock_enter(&zv->zv_rangelock, offset,
	size, RL_READER);
	error = dmu_read_by_dnode(zv->zv_dn, offset, size, buf,
	DMU_READ_NO_PREFETCH);
	} else { /* indirect write */
	ASSERT3P(zio, !=, NULL);
	/*
	* Have to lock the whole block to ensure when it's written out
	* and its checksum is being calculated that no one can change
	* the data. Contrarily to zfs_get_data we need not re-check
	* blocksize after we get the lock because it cannot be changed.
	*/
	size = zv->zv_volblocksize;
	offset = P2ALIGN_TYPED(offset, size, uint64_t);
	zgd->zgd_lr = zfs_rangelock_enter(&zv->zv_rangelock, offset,
	size, RL_READER);
	error = dmu_buf_hold_noread_by_dnode(zv->zv_dn, offset, zgd,
	&db);
	if (error == 0) {
	blkptr_t *bp = &lr->lr_blkptr;

	zgd->zgd_db = db;
	zgd->zgd_bp = bp;

	ASSERT(db != NULL);
	ASSERT(db->db_offset == offset);
	ASSERT(db->db_size == size);

	error = dmu_sync(zio, lr->lr_common.lrc_txg,
	zvol_get_done, zgd);

	if (error == 0)
	return (0);
	}
	}

	zvol_get_done(zgd, error);

	return (SET_ERROR(error));
	}

	/*
	* The zvol_state_t's are inserted into zvol_state_list and zvol_htable.
	*/

	void
	zvol_insert(zvol_state_t *zv)
	{
	ASSERT(RW_WRITE_HELD(&zvol_state_lock));
	list_insert_head(&zvol_state_list, zv);
	hlist_add_head(&zv->zv_hlink, ZVOL_HT_HEAD(zv->zv_hash));
	}

	/*
	* Simply remove the zvol from to list of zvols.
	*/
	static void
	zvol_remove(zvol_state_t *zv)
	{
	ASSERT(RW_WRITE_HELD(&zvol_state_lock));
	list_remove(&zvol_state_list, zv);
	hlist_del(&zv->zv_hlink);
	}

	/*
	* Setup zv after we just own the zv->objset
	*/
	static int
	zvol_setup_zv(zvol_state_t *zv)
	{
	uint64_t volsize;
	int error;
	uint64_t ro;
	objset_t *os = zv->zv_objset;

	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
	ASSERT(RW_LOCK_HELD(&zv->zv_suspend_lock));

	zv->zv_zilog = NULL;
	zv->zv_flags &= ~ZVOL_WRITTEN_TO;

	error = dsl_prop_get_integer(zv->zv_name, "readonly", &ro, NULL);
	if (error)
	return (SET_ERROR(error));

	error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize);
	if (error)
	return (SET_ERROR(error));

	error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn);
	if (error)
	return (SET_ERROR(error));

	zvol_os_set_capacity(zv, volsize >> 9);
	zv->zv_volsize = volsize;

	if (ro \|\| dmu_objset_is_snapshot(os) \|\|
	!spa_writeable(dmu_objset_spa(os))) {
	zvol_os_set_disk_ro(zv, 1);
	zv->zv_flags \|= ZVOL_RDONLY;
	} else {
	zvol_os_set_disk_ro(zv, 0);
	zv->zv_flags &= ~ZVOL_RDONLY;
	}
	return (0);
	}

	/*
	* Shutdown every zv_objset related stuff except zv_objset itself.
	* The is the reverse of zvol_setup_zv.
	*/
	static void
	zvol_shutdown_zv(zvol_state_t *zv)
	{
	ASSERT(MUTEX_HELD(&zv->zv_state_lock) &&
	RW_LOCK_HELD(&zv->zv_suspend_lock));

	if (zv->zv_flags & ZVOL_WRITTEN_TO) {
	ASSERT(zv->zv_zilog != NULL);
	zil_close(zv->zv_zilog);
	}

	zv->zv_zilog = NULL;

	dnode_rele(zv->zv_dn, zv);
	zv->zv_dn = NULL;

	/*
	* Evict cached data. We must write out any dirty data before
	* disowning the dataset.
	*/
	if (zv->zv_flags & ZVOL_WRITTEN_TO)
	txg_wait_synced(dmu_objset_pool(zv->zv_objset), 0);
	(void) dmu_objset_evict_dbufs(zv->zv_objset);
	}

	/*
	* return the proper tag for rollback and recv
	*/
	void *
	zvol_tag(zvol_state_t *zv)
	{
	ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock));
	return (zv->zv_open_count > 0 ? zv : NULL);
	}

	/*
	* Suspend the zvol for recv and rollback.
	*/
	zvol_state_t *
	zvol_suspend(const char *name)
	{
	zvol_state_t *zv;

	zv = zvol_find_by_name(name, RW_WRITER);

	if (zv == NULL)
	return (NULL);

	/* block all I/O, release in zvol_resume. */
	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
	ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock));

	atomic_inc(&zv->zv_suspend_ref);

	if (zv->zv_open_count > 0)
	zvol_shutdown_zv(zv);

	/*
	* do not hold zv_state_lock across suspend/resume to
	* avoid locking up zvol lookups
	*/
	mutex_exit(&zv->zv_state_lock);

	/* zv_suspend_lock is released in zvol_resume() */
	return (zv);
	}

	int
	zvol_resume(zvol_state_t *zv)
	{
	int error = 0;

	ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock));

	mutex_enter(&zv->zv_state_lock);

	if (zv->zv_open_count > 0) {
	VERIFY0(dmu_objset_hold(zv->zv_name, zv, &zv->zv_objset));
	VERIFY3P(zv->zv_objset->os_dsl_dataset->ds_owner, ==, zv);
	VERIFY(dsl_dataset_long_held(zv->zv_objset->os_dsl_dataset));
	dmu_objset_rele(zv->zv_objset, zv);

	error = zvol_setup_zv(zv);
	}

	mutex_exit(&zv->zv_state_lock);

	rw_exit(&zv->zv_suspend_lock);
	/*
	* We need this because we don't hold zvol_state_lock while releasing
	* zv_suspend_lock. zvol_remove_minors_impl thus cannot check
	* zv_suspend_lock to determine it is safe to free because rwlock is
	* not inherent atomic.
	*/
	atomic_dec(&zv->zv_suspend_ref);

	return (SET_ERROR(error));
	}

	int
	zvol_first_open(zvol_state_t *zv, boolean_t readonly)
	{
	objset_t *os;
	int error;

	ASSERT(RW_READ_HELD(&zv->zv_suspend_lock));
	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
	ASSERT(mutex_owned(&spa_namespace_lock));

	boolean_t ro = (readonly \|\| (strchr(zv->zv_name, '@') != NULL));
	error = dmu_objset_own(zv->zv_name, DMU_OST_ZVOL, ro, B_TRUE, zv, &os);
	if (error)
	return (SET_ERROR(error));

	zv->zv_objset = os;

	error = zvol_setup_zv(zv);
	if (error) {
	dmu_objset_disown(os, 1, zv);
	zv->zv_objset = NULL;
	}

	return (error);
	}

	void
	zvol_last_close(zvol_state_t *zv)
	{
	ASSERT(RW_READ_HELD(&zv->zv_suspend_lock));
	ASSERT(MUTEX_HELD(&zv->zv_state_lock));

	zvol_shutdown_zv(zv);

	dmu_objset_disown(zv->zv_objset, 1, zv);
	zv->zv_objset = NULL;
	}

	typedef struct minors_job {
	list_t *list;
	list_node_t link;
	/* input */
	char *name;
	/* output */
	int error;
	} minors_job_t;

	/*
	* Prefetch zvol dnodes for the minors_job
	*/
	static void
	zvol_prefetch_minors_impl(void *arg)
	{
	minors_job_t *job = arg;
	char *dsname = job->name;
	objset_t *os = NULL;

	job->error = dmu_objset_own(dsname, DMU_OST_ZVOL, B_TRUE, B_TRUE,
	FTAG, &os);
	if (job->error == 0) {
	- dmu_prefetch(os, ZVOL_OBJ, 0, 0, 0, ZIO_PRIORITY_SYNC_READ);
	+ dmu_prefetch_dnode(os, ZVOL_OBJ, ZIO_PRIORITY_SYNC_READ);
	dmu_objset_disown(os, B_TRUE, FTAG);
	}
	}

	/*
	* Mask errors to continue dmu_objset_find() traversal
	*/
	static int
	zvol_create_snap_minor_cb(const char dsname, void arg)
	{
	minors_job_t *j = arg;
	list_t *minors_list = j->list;
	const char *name = j->name;

	ASSERT0(MUTEX_HELD(&spa_namespace_lock));

	/* skip the designated dataset */
	if (name && strcmp(dsname, name) == 0)
	return (0);

	/* at this point, the dsname should name a snapshot */
	if (strchr(dsname, '@') == 0) {
	dprintf("zvol_create_snap_minor_cb(): "
	"%s is not a snapshot name\n", dsname);
	} else {
	minors_job_t *job;
	char *n = kmem_strdup(dsname);
	if (n == NULL)
	return (0);

	job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP);
	job->name = n;
	job->list = minors_list;
	job->error = 0;
	list_insert_tail(minors_list, job);
	/* don't care if dispatch fails, because job->error is 0 */
	taskq_dispatch(system_taskq, zvol_prefetch_minors_impl, job,
	TQ_SLEEP);
	}

	return (0);
	}

	/*
	* If spa_keystore_load_wkey() is called for an encrypted zvol,
	* we need to look for any clones also using the key. This function
	* is "best effort" - so we just skip over it if there are failures.
	*/
	static void
	zvol_add_clones(const char dsname, list_t minors_list)
	{
	/* Also check if it has clones */
	dsl_dir_t *dd = NULL;
	dsl_pool_t *dp = NULL;

	if (dsl_pool_hold(dsname, FTAG, &dp) != 0)
	return;

	if (!spa_feature_is_enabled(dp->dp_spa,
	SPA_FEATURE_ENCRYPTION))
	goto out;

	if (dsl_dir_hold(dp, dsname, FTAG, &dd, NULL) != 0)
	goto out;

	if (dsl_dir_phys(dd)->dd_clones == 0)
	goto out;

	zap_cursor_t *zc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP);
	zap_attribute_t *za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP);
	objset_t *mos = dd->dd_pool->dp_meta_objset;

	for (zap_cursor_init(zc, mos, dsl_dir_phys(dd)->dd_clones);
	zap_cursor_retrieve(zc, za) == 0;
	zap_cursor_advance(zc)) {
	dsl_dataset_t *clone;
	minors_job_t *job;

	if (dsl_dataset_hold_obj(dd->dd_pool,
	za->za_first_integer, FTAG, &clone) == 0) {

	char name[ZFS_MAX_DATASET_NAME_LEN];
	dsl_dataset_name(clone, name);

	char *n = kmem_strdup(name);
	job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP);
	job->name = n;
	job->list = minors_list;
	job->error = 0;
	list_insert_tail(minors_list, job);

	dsl_dataset_rele(clone, FTAG);
	}
	}
	zap_cursor_fini(zc);
	kmem_free(za, sizeof (zap_attribute_t));
	kmem_free(zc, sizeof (zap_cursor_t));

	out:
	if (dd != NULL)
	dsl_dir_rele(dd, FTAG);
	dsl_pool_rele(dp, FTAG);
	}

	/*
	* Mask errors to continue dmu_objset_find() traversal
	*/
	static int
	zvol_create_minors_cb(const char dsname, void arg)
	{
	uint64_t snapdev;
	int error;
	list_t *minors_list = arg;

	ASSERT0(MUTEX_HELD(&spa_namespace_lock));

	error = dsl_prop_get_integer(dsname, "snapdev", &snapdev, NULL);
	if (error)
	return (0);

	/*
	* Given the name and the 'snapdev' property, create device minor nodes
	* with the linkages to zvols/snapshots as needed.
	* If the name represents a zvol, create a minor node for the zvol, then
	* check if its snapshots are 'visible', and if so, iterate over the
	* snapshots and create device minor nodes for those.
	*/
	if (strchr(dsname, '@') == 0) {
	minors_job_t *job;
	char *n = kmem_strdup(dsname);
	if (n == NULL)
	return (0);

	job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP);
	job->name = n;
	job->list = minors_list;
	job->error = 0;
	list_insert_tail(minors_list, job);
	/* don't care if dispatch fails, because job->error is 0 */
	taskq_dispatch(system_taskq, zvol_prefetch_minors_impl, job,
	TQ_SLEEP);

	zvol_add_clones(dsname, minors_list);

	if (snapdev == ZFS_SNAPDEV_VISIBLE) {
	/*
	* traverse snapshots only, do not traverse children,
	* and skip the 'dsname'
	*/
	(void) dmu_objset_find(dsname,
	zvol_create_snap_minor_cb, (void *)job,
	DS_FIND_SNAPSHOTS);
	}
	} else {
	dprintf("zvol_create_minors_cb(): %s is not a zvol name\n",
	dsname);
	}

	return (0);
	}

	/*
	* Create minors for the specified dataset, including children and snapshots.
	* Pay attention to the 'snapdev' property and iterate over the snapshots
	* only if they are 'visible'. This approach allows one to assure that the
	* snapshot metadata is read from disk only if it is needed.
	*
	* The name can represent a dataset to be recursively scanned for zvols and
	* their snapshots, or a single zvol snapshot. If the name represents a
	* dataset, the scan is performed in two nested stages:
	* - scan the dataset for zvols, and
	* - for each zvol, create a minor node, then check if the zvol's snapshots
	* are 'visible', and only then iterate over the snapshots if needed
	*
	* If the name represents a snapshot, a check is performed if the snapshot is
	* 'visible' (which also verifies that the parent is a zvol), and if so,
	* a minor node for that snapshot is created.
	*/
	void
	zvol_create_minors_recursive(const char *name)
	{
	list_t minors_list;
	minors_job_t *job;

	if (zvol_inhibit_dev)
	return;

	/*
	* This is the list for prefetch jobs. Whenever we found a match
	* during dmu_objset_find, we insert a minors_job to the list and do
	* taskq_dispatch to parallel prefetch zvol dnodes. Note we don't need
	* any lock because all list operation is done on the current thread.
	*
	* We will use this list to do zvol_os_create_minor after prefetch
	* so we don't have to traverse using dmu_objset_find again.
	*/
	list_create(&minors_list, sizeof (minors_job_t),
	offsetof(minors_job_t, link));


	if (strchr(name, '@') != NULL) {
	uint64_t snapdev;

	int error = dsl_prop_get_integer(name, "snapdev",
	&snapdev, NULL);

	if (error == 0 && snapdev == ZFS_SNAPDEV_VISIBLE)
	(void) zvol_os_create_minor(name);
	} else {
	fstrans_cookie_t cookie = spl_fstrans_mark();
	(void) dmu_objset_find(name, zvol_create_minors_cb,
	&minors_list, DS_FIND_CHILDREN);
	spl_fstrans_unmark(cookie);
	}

	taskq_wait_outstanding(system_taskq, 0);

	/*
	* Prefetch is completed, we can do zvol_os_create_minor
	* sequentially.
	*/
	while ((job = list_remove_head(&minors_list)) != NULL) {
	if (!job->error)
	(void) zvol_os_create_minor(job->name);
	kmem_strfree(job->name);
	kmem_free(job, sizeof (minors_job_t));
	}

	list_destroy(&minors_list);
	}

	void
	zvol_create_minor(const char *name)
	{
	/*
	* Note: the dsl_pool_config_lock must not be held.
	* Minor node creation needs to obtain the zvol_state_lock.
	* zvol_open() obtains the zvol_state_lock and then the dsl pool
	* config lock. Therefore, we can't have the config lock now if
	* we are going to wait for the zvol_state_lock, because it
	* would be a lock order inversion which could lead to deadlock.
	*/

	if (zvol_inhibit_dev)
	return;

	if (strchr(name, '@') != NULL) {
	uint64_t snapdev;

	int error = dsl_prop_get_integer(name,
	"snapdev", &snapdev, NULL);

	if (error == 0 && snapdev == ZFS_SNAPDEV_VISIBLE)
	(void) zvol_os_create_minor(name);
	} else {
	(void) zvol_os_create_minor(name);
	}
	}

	/*
	* Remove minors for specified dataset including children and snapshots.
	*/

	static void
	zvol_free_task(void *arg)
	{
	zvol_os_free(arg);
	}

	void
	zvol_remove_minors_impl(const char *name)
	{
	zvol_state_t zv, zv_next;
	int namelen = ((name) ? strlen(name) : 0);
	taskqid_t t;
	list_t free_list;

	if (zvol_inhibit_dev)
	return;

	list_create(&free_list, sizeof (zvol_state_t),
	offsetof(zvol_state_t, zv_next));

	rw_enter(&zvol_state_lock, RW_WRITER);

	for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) {
	zv_next = list_next(&zvol_state_list, zv);

	mutex_enter(&zv->zv_state_lock);
	if (name == NULL \|\| strcmp(zv->zv_name, name) == 0 \|\|
	(strncmp(zv->zv_name, name, namelen) == 0 &&
	(zv->zv_name[namelen] == '/' \|\|
	zv->zv_name[namelen] == '@'))) {
	/*
	* By holding zv_state_lock here, we guarantee that no
	* one is currently using this zv
	*/

	/* If in use, leave alone */
	if (zv->zv_open_count > 0 \|\|
	atomic_read(&zv->zv_suspend_ref)) {
	mutex_exit(&zv->zv_state_lock);
	continue;
	}

	zvol_remove(zv);

	/*
	* Cleared while holding zvol_state_lock as a writer
	* which will prevent zvol_open() from opening it.
	*/
	zvol_os_clear_private(zv);

	/* Drop zv_state_lock before zvol_free() */
	mutex_exit(&zv->zv_state_lock);

	/* Try parallel zv_free, if failed do it in place */
	t = taskq_dispatch(system_taskq, zvol_free_task, zv,
	TQ_SLEEP);
	if (t == TASKQID_INVALID)
	list_insert_head(&free_list, zv);
	} else {
	mutex_exit(&zv->zv_state_lock);
	}
	}
	rw_exit(&zvol_state_lock);

	/* Drop zvol_state_lock before calling zvol_free() */
	while ((zv = list_remove_head(&free_list)) != NULL)
	zvol_os_free(zv);
	}

	/* Remove minor for this specific volume only */
	static void
	zvol_remove_minor_impl(const char *name)
	{
	zvol_state_t zv = NULL, zv_next;

	if (zvol_inhibit_dev)
	return;

	rw_enter(&zvol_state_lock, RW_WRITER);

	for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) {
	zv_next = list_next(&zvol_state_list, zv);

	mutex_enter(&zv->zv_state_lock);
	if (strcmp(zv->zv_name, name) == 0) {
	/*
	* By holding zv_state_lock here, we guarantee that no
	* one is currently using this zv
	*/

	/* If in use, leave alone */
	if (zv->zv_open_count > 0 \|\|
	atomic_read(&zv->zv_suspend_ref)) {
	mutex_exit(&zv->zv_state_lock);
	continue;
	}
	zvol_remove(zv);

	zvol_os_clear_private(zv);
	mutex_exit(&zv->zv_state_lock);
	break;
	} else {
	mutex_exit(&zv->zv_state_lock);
	}
	}

	/* Drop zvol_state_lock before calling zvol_free() */
	rw_exit(&zvol_state_lock);

	if (zv != NULL)
	zvol_os_free(zv);
	}

	/*
	* Rename minors for specified dataset including children and snapshots.
	*/
	static void
	zvol_rename_minors_impl(const char oldname, const char newname)
	{
	zvol_state_t zv, zv_next;
	int oldnamelen;

	if (zvol_inhibit_dev)
	return;

	oldnamelen = strlen(oldname);

	rw_enter(&zvol_state_lock, RW_READER);

	for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) {
	zv_next = list_next(&zvol_state_list, zv);

	mutex_enter(&zv->zv_state_lock);

	if (strcmp(zv->zv_name, oldname) == 0) {
	zvol_os_rename_minor(zv, newname);
	} else if (strncmp(zv->zv_name, oldname, oldnamelen) == 0 &&
	(zv->zv_name[oldnamelen] == '/' \|\|
	zv->zv_name[oldnamelen] == '@')) {
	char *name = kmem_asprintf("%s%c%s", newname,
	zv->zv_name[oldnamelen],
	zv->zv_name + oldnamelen + 1);
	zvol_os_rename_minor(zv, name);
	kmem_strfree(name);
	}

	mutex_exit(&zv->zv_state_lock);
	}

	rw_exit(&zvol_state_lock);
	}

	typedef struct zvol_snapdev_cb_arg {
	uint64_t snapdev;
	} zvol_snapdev_cb_arg_t;

	static int
	zvol_set_snapdev_cb(const char dsname, void param)
	{
	zvol_snapdev_cb_arg_t *arg = param;

	if (strchr(dsname, '@') == NULL)
	return (0);

	switch (arg->snapdev) {
	case ZFS_SNAPDEV_VISIBLE:
	(void) zvol_os_create_minor(dsname);
	break;
	case ZFS_SNAPDEV_HIDDEN:
	(void) zvol_remove_minor_impl(dsname);
	break;
	}

	return (0);
	}

	static void
	zvol_set_snapdev_impl(char *name, uint64_t snapdev)
	{
	zvol_snapdev_cb_arg_t arg = {snapdev};
	fstrans_cookie_t cookie = spl_fstrans_mark();
	/*
	* The zvol_set_snapdev_sync() sets snapdev appropriately
	* in the dataset hierarchy. Here, we only scan snapshots.
	*/
	dmu_objset_find(name, zvol_set_snapdev_cb, &arg, DS_FIND_SNAPSHOTS);
	spl_fstrans_unmark(cookie);
	}

	static void
	zvol_set_volmode_impl(char *name, uint64_t volmode)
	{
	fstrans_cookie_t cookie;
	uint64_t old_volmode;
	zvol_state_t *zv;

	if (strchr(name, '@') != NULL)
	return;

	/*
	* It's unfortunate we need to remove minors before we create new ones:
	* this is necessary because our backing gendisk (zvol_state->zv_disk)
	* could be different when we set, for instance, volmode from "geom"
	* to "dev" (or vice versa).
	*/
	zv = zvol_find_by_name(name, RW_NONE);
	if (zv == NULL && volmode == ZFS_VOLMODE_NONE)
	return;
	if (zv != NULL) {
	old_volmode = zv->zv_volmode;
	mutex_exit(&zv->zv_state_lock);
	if (old_volmode == volmode)
	return;
	zvol_wait_close(zv);
	}
	cookie = spl_fstrans_mark();
	switch (volmode) {
	case ZFS_VOLMODE_NONE:
	(void) zvol_remove_minor_impl(name);
	break;
	case ZFS_VOLMODE_GEOM:
	case ZFS_VOLMODE_DEV:
	(void) zvol_remove_minor_impl(name);
	(void) zvol_os_create_minor(name);
	break;
	case ZFS_VOLMODE_DEFAULT:
	(void) zvol_remove_minor_impl(name);
	if (zvol_volmode == ZFS_VOLMODE_NONE)
	break;
	else /* if zvol_volmode is invalid defaults to "geom" */
	(void) zvol_os_create_minor(name);
	break;
	}
	spl_fstrans_unmark(cookie);
	}

	static zvol_task_t *
	zvol_task_alloc(zvol_async_op_t op, const char name1, const char name2,
	uint64_t value)
	{
	zvol_task_t *task;

	/* Never allow tasks on hidden names. */
	if (name1[0] == '$')
	return (NULL);

	task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP);
	task->op = op;
	task->value = value;

	strlcpy(task->name1, name1, MAXNAMELEN);
	if (name2 != NULL)
	strlcpy(task->name2, name2, MAXNAMELEN);

	return (task);
	}

	static void
	zvol_task_free(zvol_task_t *task)
	{
	kmem_free(task, sizeof (zvol_task_t));
	}

	/*
	* The worker thread function performed asynchronously.
	*/
	static void
	zvol_task_cb(void *arg)
	{
	zvol_task_t *task = arg;

	switch (task->op) {
	case ZVOL_ASYNC_REMOVE_MINORS:
	zvol_remove_minors_impl(task->name1);
	break;
	case ZVOL_ASYNC_RENAME_MINORS:
	zvol_rename_minors_impl(task->name1, task->name2);
	break;
	case ZVOL_ASYNC_SET_SNAPDEV:
	zvol_set_snapdev_impl(task->name1, task->value);
	break;
	case ZVOL_ASYNC_SET_VOLMODE:
	zvol_set_volmode_impl(task->name1, task->value);
	break;
	default:
	VERIFY(0);
	break;
	}

	zvol_task_free(task);
	}

	typedef struct zvol_set_prop_int_arg {
	const char *zsda_name;
	uint64_t zsda_value;
	zprop_source_t zsda_source;
	dmu_tx_t *zsda_tx;
	} zvol_set_prop_int_arg_t;

	/*
	* Sanity check the dataset for safe use by the sync task. No additional
	* conditions are imposed.
	*/
	static int
	zvol_set_snapdev_check(void arg, dmu_tx_t tx)
	{
	zvol_set_prop_int_arg_t *zsda = arg;
	dsl_pool_t *dp = dmu_tx_pool(tx);
	dsl_dir_t *dd;
	int error;

	error = dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL);
	if (error != 0)
	return (error);

	dsl_dir_rele(dd, FTAG);

	return (error);
	}

	static int
	zvol_set_snapdev_sync_cb(dsl_pool_t dp, dsl_dataset_t ds, void *arg)
	{
	(void) arg;
	char dsname[MAXNAMELEN];
	zvol_task_t *task;
	uint64_t snapdev;

	dsl_dataset_name(ds, dsname);
	if (dsl_prop_get_int_ds(ds, "snapdev", &snapdev) != 0)
	return (0);
	task = zvol_task_alloc(ZVOL_ASYNC_SET_SNAPDEV, dsname, NULL, snapdev);
	if (task == NULL)
	return (0);

	(void) taskq_dispatch(dp->dp_spa->spa_zvol_taskq, zvol_task_cb,
	task, TQ_SLEEP);
	return (0);
	}

	/*
	* Traverse all child datasets and apply snapdev appropriately.
	* We call dsl_prop_set_sync_impl() here to set the value only on the toplevel
	* dataset and read the effective "snapdev" on every child in the callback
	* function: this is because the value is not guaranteed to be the same in the
	* whole dataset hierarchy.
	*/
	static void
	zvol_set_snapdev_sync(void arg, dmu_tx_t tx)
	{
	zvol_set_prop_int_arg_t *zsda = arg;
	dsl_pool_t *dp = dmu_tx_pool(tx);
	dsl_dir_t *dd;
	dsl_dataset_t *ds;
	int error;

	VERIFY0(dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL));
	zsda->zsda_tx = tx;

	error = dsl_dataset_hold(dp, zsda->zsda_name, FTAG, &ds);
	if (error == 0) {
	dsl_prop_set_sync_impl(ds, zfs_prop_to_name(ZFS_PROP_SNAPDEV),
	zsda->zsda_source, sizeof (zsda->zsda_value), 1,
	&zsda->zsda_value, zsda->zsda_tx);
	dsl_dataset_rele(ds, FTAG);
	}
	dmu_objset_find_dp(dp, dd->dd_object, zvol_set_snapdev_sync_cb,
	zsda, DS_FIND_CHILDREN);

	dsl_dir_rele(dd, FTAG);
	}

	int
	zvol_set_snapdev(const char *ddname, zprop_source_t source, uint64_t snapdev)
	{
	zvol_set_prop_int_arg_t zsda;

	zsda.zsda_name = ddname;
	zsda.zsda_source = source;
	zsda.zsda_value = snapdev;

	return (dsl_sync_task(ddname, zvol_set_snapdev_check,
	zvol_set_snapdev_sync, &zsda, 0, ZFS_SPACE_CHECK_NONE));
	}

	/*
	* Sanity check the dataset for safe use by the sync task. No additional
	* conditions are imposed.
	*/
	static int
	zvol_set_volmode_check(void arg, dmu_tx_t tx)
	{
	zvol_set_prop_int_arg_t *zsda = arg;
	dsl_pool_t *dp = dmu_tx_pool(tx);
	dsl_dir_t *dd;
	int error;

	error = dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL);
	if (error != 0)
	return (error);

	dsl_dir_rele(dd, FTAG);

	return (error);
	}

	static int
	zvol_set_volmode_sync_cb(dsl_pool_t dp, dsl_dataset_t ds, void *arg)
	{
	(void) arg;
	char dsname[MAXNAMELEN];
	zvol_task_t *task;
	uint64_t volmode;

	dsl_dataset_name(ds, dsname);
	if (dsl_prop_get_int_ds(ds, "volmode", &volmode) != 0)
	return (0);
	task = zvol_task_alloc(ZVOL_ASYNC_SET_VOLMODE, dsname, NULL, volmode);
	if (task == NULL)
	return (0);

	(void) taskq_dispatch(dp->dp_spa->spa_zvol_taskq, zvol_task_cb,
	task, TQ_SLEEP);
	return (0);
	}

	/*
	* Traverse all child datasets and apply volmode appropriately.
	* We call dsl_prop_set_sync_impl() here to set the value only on the toplevel
	* dataset and read the effective "volmode" on every child in the callback
	* function: this is because the value is not guaranteed to be the same in the
	* whole dataset hierarchy.
	*/
	static void
	zvol_set_volmode_sync(void arg, dmu_tx_t tx)
	{
	zvol_set_prop_int_arg_t *zsda = arg;
	dsl_pool_t *dp = dmu_tx_pool(tx);
	dsl_dir_t *dd;
	dsl_dataset_t *ds;
	int error;

	VERIFY0(dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL));
	zsda->zsda_tx = tx;

	error = dsl_dataset_hold(dp, zsda->zsda_name, FTAG, &ds);
	if (error == 0) {
	dsl_prop_set_sync_impl(ds, zfs_prop_to_name(ZFS_PROP_VOLMODE),
	zsda->zsda_source, sizeof (zsda->zsda_value), 1,
	&zsda->zsda_value, zsda->zsda_tx);
	dsl_dataset_rele(ds, FTAG);
	}

	dmu_objset_find_dp(dp, dd->dd_object, zvol_set_volmode_sync_cb,
	zsda, DS_FIND_CHILDREN);

	dsl_dir_rele(dd, FTAG);
	}

	int
	zvol_set_volmode(const char *ddname, zprop_source_t source, uint64_t volmode)
	{
	zvol_set_prop_int_arg_t zsda;

	zsda.zsda_name = ddname;
	zsda.zsda_source = source;
	zsda.zsda_value = volmode;

	return (dsl_sync_task(ddname, zvol_set_volmode_check,
	zvol_set_volmode_sync, &zsda, 0, ZFS_SPACE_CHECK_NONE));
	}

	void
	zvol_remove_minors(spa_t spa, const char name, boolean_t async)
	{
	zvol_task_t *task;
	taskqid_t id;

	task = zvol_task_alloc(ZVOL_ASYNC_REMOVE_MINORS, name, NULL, ~0ULL);
	if (task == NULL)
	return;

	id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP);
	if ((async == B_FALSE) && (id != TASKQID_INVALID))
	taskq_wait_id(spa->spa_zvol_taskq, id);
	}

	void
	zvol_rename_minors(spa_t spa, const char name1, const char *name2,
	boolean_t async)
	{
	zvol_task_t *task;
	taskqid_t id;

	task = zvol_task_alloc(ZVOL_ASYNC_RENAME_MINORS, name1, name2, ~0ULL);
	if (task == NULL)
	return;

	id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP);
	if ((async == B_FALSE) && (id != TASKQID_INVALID))
	taskq_wait_id(spa->spa_zvol_taskq, id);
	}

	boolean_t
	zvol_is_zvol(const char *name)
	{

	return (zvol_os_is_zvol(name));
	}

	int
	zvol_init_impl(void)
	{
	int i;

	list_create(&zvol_state_list, sizeof (zvol_state_t),
	offsetof(zvol_state_t, zv_next));
	rw_init(&zvol_state_lock, NULL, RW_DEFAULT, NULL);

	zvol_htable = kmem_alloc(ZVOL_HT_SIZE * sizeof (struct hlist_head),
	KM_SLEEP);
	for (i = 0; i < ZVOL_HT_SIZE; i++)
	INIT_HLIST_HEAD(&zvol_htable[i]);

	return (0);
	}

	void
	zvol_fini_impl(void)
	{
	zvol_remove_minors_impl(NULL);

	/*
	* The call to "zvol_remove_minors_impl" may dispatch entries to
	* the system_taskq, but it doesn't wait for those entries to
	* complete before it returns. Thus, we must wait for all of the
	* removals to finish, before we can continue.
	*/
	taskq_wait_outstanding(system_taskq, 0);

	kmem_free(zvol_htable, ZVOL_HT_SIZE * sizeof (struct hlist_head));
	list_destroy(&zvol_state_list);
	rw_destroy(&zvol_state_lock);
	}
	diff --git a/sys/contrib/openzfs/rpm/redhat/zfs-kmod.spec.in b/sys/contrib/openzfs/rpm/redhat/zfs-kmod.spec.in
	index 9c836786baea..876c198c64de 100644
	--- a/sys/contrib/openzfs/rpm/redhat/zfs-kmod.spec.in
	+++ b/sys/contrib/openzfs/rpm/redhat/zfs-kmod.spec.in
	@@ -1,112 +1,112 @@
	%bcond_with debug
	%bcond_with debuginfo

	Name: @PACKAGE@-kmod
	Version: @VERSION@
	Release: @RELEASE@%{?dist}

	Summary: Kernel module(s)
	Group: System Environment/Kernel
	License: @ZFS_META_LICENSE@
	URL: https://github.com/openzfs/zfs
	BuildRequires: %kernel_module_package_buildreqs
	Source0: @PACKAGE@-%{version}.tar.gz
	BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)

	# Additional dependency information for the kmod sub-package must be specified
	# by generating a preamble text file which kmodtool can append to the spec file.
	%(/bin/echo -e "\
	Requires: @PACKAGE@ = %{version}\n\
	-Conflicts: @PACKAGE@-dkms)
	+Conflicts: @PACKAGE@-dkms" > %{_sourcedir}/kmod-preamble)

	# LDFLAGS are not sanitized by arch/*/Makefile for these architectures.
	%ifarch ppc ppc64 ppc64le aarch64
	%global __global_ldflags %{nil}
	%endif

	%description
	This package contains the ZFS kernel modules.

	%define kmod_name @PACKAGE@

	%kernel_module_package -n %{kmod_name} -p %{_sourcedir}/kmod-preamble

	%define ksrc %{_usrsrc}/kernels/%{kverrel}
	%define kobj %{ksrc}

	%package -n kmod-%{kmod_name}-devel
	Summary: ZFS kernel module(s) devel common
	Group: System Environment/Kernel

	%description -n kmod-%{kmod_name}-devel
	This package provides the header files and objects to build kernel modules.

	%prep
	if ! [ -d "%{ksrc}" ]; then
	echo "Kernel build directory isn't set properly, cannot continue"
	exit 1
	fi

	%if %{with debug}
	%define debug --enable-debug
	%else
	%define debug --disable-debug
	%endif

	%if %{with debuginfo}
	%define debuginfo --enable-debuginfo
	%else
	%define debuginfo --disable-debuginfo
	%endif

	%setup -n %{kmod_name}-%{version}
	%build
	%configure \
	--with-config=kernel \
	--with-linux=%{ksrc} \
	--with-linux-obj=%{kobj} \
	%{debug} \
	%{debuginfo} \
	%{?kernel_cc} \
	%{?kernel_ld} \
	%{?kernel_llvm}
	make %{?_smp_mflags}

	# Module signing (modsign)
	#
	# This must be run _after_ find-debuginfo.sh runs, otherwise that will strip
	# the signature off of the modules.
	# (Based on Fedora's kernel.spec workaround)
	%define __modsign_install_post \
	sign_pem="%{ksrc}/certs/signing_key.pem"; \
	sign_x509="%{ksrc}/certs/signing_key.x509"; \
	if [ -f "${sign_x509}" ]\
	then \
	echo "Signing kernel modules ..."; \
	for kmod in $(find %{buildroot}/lib/modules/%{kverrel}/extra/ -name \*.ko); do \
	%{ksrc}/scripts/sign-file sha256 ${sign_pem} ${sign_x509} ${kmod}; \
	done \
	fi \
	%{nil}

	# hack to ensure signing happens after find-debuginfo.sh runs
	%define __spec_install_post \
	%{?__debug_package:%{__debug_install_post}}\
	%{__arch_install_post}\
	%{__os_install_post}\
	%{__modsign_install_post}

	%install
	make install \
	DESTDIR=${RPM_BUILD_ROOT} \
	INSTALL_MOD_DIR=extra/%{kmod_name}
	%{__rm} -f %{buildroot}/lib/modules/%{kverrel}/modules.*

	# find-debuginfo.sh only considers executables
	%{__chmod} u+x %{buildroot}/lib/modules/%{kverrel}/extra//

	%clean
	rm -rf $RPM_BUILD_ROOT

	%files -n kmod-%{kmod_name}-devel
	%{_usrsrc}/%{kmod_name}-%{version}
	diff --git a/sys/contrib/openzfs/tests/runfiles/common.run b/sys/contrib/openzfs/tests/runfiles/common.run
	index 7331244515f6..3e1a3aeb6cbe 100644
	--- a/sys/contrib/openzfs/tests/runfiles/common.run
	+++ b/sys/contrib/openzfs/tests/runfiles/common.run
	@@ -1,1017 +1,1019 @@
	#
	# This file and its contents are supplied under the terms of the
	# Common Development and Distribution License ("CDDL"), version 1.0.
	# You may only use this file in accordance with the terms of version
	# 1.0 of the CDDL.
	#
	# A full copy of the text of the CDDL should have accompanied this
	# source. A copy of the CDDL is also available via the Internet at
	# http://www.illumos.org/license/CDDL.
	#
	# This run file contains all of the common functional tests. When
	# adding a new test consider also adding it to the sanity.run file
	# if the new test runs to completion in only a few seconds.
	#
	# Approximate run time: 4-5 hours
	#

	[DEFAULT]
	pre = setup
	quiet = False
	pre_user = root
	user = root
	timeout = 600
	post_user = root
	post = cleanup
	failsafe_user = root
	failsafe = callbacks/zfs_failsafe
	outputdir = /var/tmp/test_results
	tags = ['functional']

	[tests/functional/acl/off]
	tests = ['dosmode', 'posixmode']
	tags = ['functional', 'acl']

	[tests/functional/alloc_class]
	tests = ['alloc_class_001_pos', 'alloc_class_002_neg', 'alloc_class_003_pos',
	'alloc_class_004_pos', 'alloc_class_005_pos', 'alloc_class_006_pos',
	'alloc_class_007_pos', 'alloc_class_008_pos', 'alloc_class_009_pos',
	'alloc_class_010_pos', 'alloc_class_011_neg', 'alloc_class_012_pos',
	'alloc_class_013_pos', 'alloc_class_014_neg', 'alloc_class_015_pos']
	tags = ['functional', 'alloc_class']

	[tests/functional/append]
	tests = ['file_append', 'threadsappend_001_pos']
	tags = ['functional', 'append']

	[tests/functional/arc]
	tests = ['dbufstats_001_pos', 'dbufstats_002_pos', 'dbufstats_003_pos',
	'arcstats_runtime_tuning']
	tags = ['functional', 'arc']

	[tests/functional/atime]
	tests = ['atime_001_pos', 'atime_002_neg', 'root_atime_off', 'root_atime_on']
	tags = ['functional', 'atime']

	[tests/functional/bclone]
	tests = ['bclone_crossfs_corner_cases_limited',
	'bclone_crossfs_data',
	'bclone_crossfs_embedded',
	'bclone_crossfs_hole',
	'bclone_diffprops_all',
	'bclone_diffprops_checksum',
	'bclone_diffprops_compress',
	'bclone_diffprops_copies',
	'bclone_diffprops_recordsize',
	'bclone_prop_sync',
	'bclone_samefs_corner_cases_limited',
	'bclone_samefs_data',
	'bclone_samefs_embedded',
	'bclone_samefs_hole']
	tags = ['functional', 'bclone']
	timeout = 7200

	[tests/functional/block_cloning]
	tests = ['block_cloning_clone_mmap_cached',
	'block_cloning_copyfilerange',
	'block_cloning_copyfilerange_partial',
	'block_cloning_copyfilerange_fallback',
	'block_cloning_disabled_copyfilerange',
	'block_cloning_copyfilerange_cross_dataset',
	'block_cloning_cross_enc_dataset',
	'block_cloning_copyfilerange_fallback_same_txg',
	'block_cloning_replay', 'block_cloning_replay_encrypted',
	'block_cloning_lwb_buffer_overflow', 'block_cloning_clone_mmap_write']
	tags = ['functional', 'block_cloning']

	[tests/functional/bootfs]
	tests = ['bootfs_001_pos', 'bootfs_002_neg', 'bootfs_003_pos',
	'bootfs_004_neg', 'bootfs_005_neg', 'bootfs_006_pos', 'bootfs_007_pos',
	'bootfs_008_pos']
	tags = ['functional', 'bootfs']

	[tests/functional/btree]
	tests = ['btree_positive', 'btree_negative']
	tags = ['functional', 'btree']
	pre =
	post =

	[tests/functional/cache]
	tests = ['cache_001_pos', 'cache_002_pos', 'cache_003_pos', 'cache_004_neg',
	'cache_005_neg', 'cache_006_pos', 'cache_007_neg', 'cache_008_neg',
	'cache_009_pos', 'cache_010_pos', 'cache_011_pos', 'cache_012_pos']
	tags = ['functional', 'cache']

	[tests/functional/cachefile]
	tests = ['cachefile_001_pos', 'cachefile_002_pos', 'cachefile_003_pos',
	'cachefile_004_pos']
	tags = ['functional', 'cachefile']

	[tests/functional/casenorm]
	tests = ['case_all_values', 'norm_all_values', 'mixed_create_failure',
	'sensitive_none_lookup', 'sensitive_none_delete',
	'sensitive_formd_lookup', 'sensitive_formd_delete',
	'insensitive_none_lookup', 'insensitive_none_delete',
	'insensitive_formd_lookup', 'insensitive_formd_delete',
	'mixed_none_lookup', 'mixed_none_lookup_ci', 'mixed_none_delete',
	'mixed_formd_lookup', 'mixed_formd_lookup_ci', 'mixed_formd_delete']
	tags = ['functional', 'casenorm']

	[tests/functional/channel_program/lua_core]
	tests = ['tst.args_to_lua', 'tst.divide_by_zero', 'tst.exists',
	'tst.integer_illegal', 'tst.integer_overflow', 'tst.language_functions_neg',
	'tst.language_functions_pos', 'tst.large_prog', 'tst.libraries',
	'tst.memory_limit', 'tst.nested_neg', 'tst.nested_pos', 'tst.nvlist_to_lua',
	'tst.recursive_neg', 'tst.recursive_pos', 'tst.return_large',
	'tst.return_nvlist_neg', 'tst.return_nvlist_pos',
	'tst.return_recursive_table', 'tst.stack_gsub', 'tst.timeout']
	tags = ['functional', 'channel_program', 'lua_core']

	[tests/functional/channel_program/synctask_core]
	tests = ['tst.destroy_fs', 'tst.destroy_snap', 'tst.get_count_and_limit',
	'tst.get_index_props', 'tst.get_mountpoint', 'tst.get_neg',
	'tst.get_number_props', 'tst.get_string_props', 'tst.get_type',
	'tst.get_userquota', 'tst.get_written', 'tst.inherit', 'tst.list_bookmarks',
	'tst.list_children', 'tst.list_clones', 'tst.list_holds',
	'tst.list_snapshots', 'tst.list_system_props',
	'tst.list_user_props', 'tst.parse_args_neg','tst.promote_conflict',
	'tst.promote_multiple', 'tst.promote_simple', 'tst.rollback_mult',
	'tst.rollback_one', 'tst.set_props', 'tst.snapshot_destroy', 'tst.snapshot_neg',
	'tst.snapshot_recursive', 'tst.snapshot_rename', 'tst.snapshot_simple',
	'tst.bookmark.create', 'tst.bookmark.copy',
	'tst.terminate_by_signal'
	]
	tags = ['functional', 'channel_program', 'synctask_core']

	[tests/functional/checksum]
	tests = ['run_edonr_test', 'run_sha2_test', 'run_skein_test', 'run_blake3_test',
	'filetest_001_pos', 'filetest_002_pos']
	tags = ['functional', 'checksum']

	[tests/functional/clean_mirror]
	tests = [ 'clean_mirror_001_pos', 'clean_mirror_002_pos',
	'clean_mirror_003_pos', 'clean_mirror_004_pos']
	tags = ['functional', 'clean_mirror']

	[tests/functional/cli_root/zdb]
	tests = ['zdb_002_pos', 'zdb_003_pos', 'zdb_004_pos', 'zdb_005_pos',
	'zdb_006_pos', 'zdb_args_neg', 'zdb_args_pos',
	'zdb_block_size_histogram', 'zdb_checksum', 'zdb_decompress',
	'zdb_display_block', 'zdb_encrypted', 'zdb_label_checksum',
	'zdb_object_range_neg', 'zdb_object_range_pos', 'zdb_objset_id',
	'zdb_decompress_zstd', 'zdb_recover', 'zdb_recover_2', 'zdb_backup']
	pre =
	post =
	tags = ['functional', 'cli_root', 'zdb']
	timeout = 1200

	[tests/functional/cli_root/zfs]
	tests = ['zfs_001_neg', 'zfs_002_pos']
	tags = ['functional', 'cli_root', 'zfs']

	[tests/functional/cli_root/zfs_bookmark]
	tests = ['zfs_bookmark_cliargs']
	tags = ['functional', 'cli_root', 'zfs_bookmark']

	[tests/functional/cli_root/zfs_change-key]
	tests = ['zfs_change-key', 'zfs_change-key_child', 'zfs_change-key_format',
	'zfs_change-key_inherit', 'zfs_change-key_load', 'zfs_change-key_location',
	'zfs_change-key_pbkdf2iters', 'zfs_change-key_clones']
	tags = ['functional', 'cli_root', 'zfs_change-key']

	[tests/functional/cli_root/zfs_clone]
	tests = ['zfs_clone_001_neg', 'zfs_clone_002_pos', 'zfs_clone_003_pos',
	'zfs_clone_004_pos', 'zfs_clone_005_pos', 'zfs_clone_006_pos',
	'zfs_clone_007_pos', 'zfs_clone_008_neg', 'zfs_clone_009_neg',
	'zfs_clone_010_pos', 'zfs_clone_encrypted', 'zfs_clone_deeply_nested',
	'zfs_clone_rm_nested']
	tags = ['functional', 'cli_root', 'zfs_clone']

	[tests/functional/cli_root/zfs_copies]
	tests = ['zfs_copies_001_pos', 'zfs_copies_002_pos', 'zfs_copies_003_pos',
	'zfs_copies_004_neg', 'zfs_copies_005_neg', 'zfs_copies_006_pos']
	tags = ['functional', 'cli_root', 'zfs_copies']

	[tests/functional/cli_root/zfs_create]
	tests = ['zfs_create_001_pos', 'zfs_create_002_pos', 'zfs_create_003_pos',
	'zfs_create_004_pos', 'zfs_create_005_pos', 'zfs_create_006_pos',
	'zfs_create_007_pos', 'zfs_create_008_neg', 'zfs_create_009_neg',
	'zfs_create_010_neg', 'zfs_create_011_pos', 'zfs_create_012_pos',
	'zfs_create_013_pos', 'zfs_create_014_pos', 'zfs_create_encrypted',
	'zfs_create_crypt_combos', 'zfs_create_dryrun', 'zfs_create_nomount',
	'zfs_create_verbose']
	tags = ['functional', 'cli_root', 'zfs_create']

	[tests/functional/cli_root/zfs_destroy]
	tests = ['zfs_clone_livelist_condense_and_disable',
	'zfs_clone_livelist_condense_races', 'zfs_clone_livelist_dedup',
	'zfs_destroy_001_pos', 'zfs_destroy_002_pos', 'zfs_destroy_003_pos',
	'zfs_destroy_004_pos', 'zfs_destroy_005_neg', 'zfs_destroy_006_neg',
	'zfs_destroy_007_neg', 'zfs_destroy_008_pos', 'zfs_destroy_009_pos',
	'zfs_destroy_010_pos', 'zfs_destroy_011_pos', 'zfs_destroy_012_pos',
	'zfs_destroy_013_neg', 'zfs_destroy_014_pos', 'zfs_destroy_015_pos',
	'zfs_destroy_016_pos', 'zfs_destroy_clone_livelist',
	'zfs_destroy_dev_removal', 'zfs_destroy_dev_removal_condense']
	tags = ['functional', 'cli_root', 'zfs_destroy']

	[tests/functional/cli_root/zfs_diff]
	tests = ['zfs_diff_changes', 'zfs_diff_cliargs', 'zfs_diff_timestamp',
	'zfs_diff_types', 'zfs_diff_encrypted', 'zfs_diff_mangle']
	tags = ['functional', 'cli_root', 'zfs_diff']

	[tests/functional/cli_root/zfs_get]
	tests = ['zfs_get_001_pos', 'zfs_get_002_pos', 'zfs_get_003_pos',
	'zfs_get_004_pos', 'zfs_get_005_neg', 'zfs_get_006_neg', 'zfs_get_007_neg',
	'zfs_get_008_pos', 'zfs_get_009_pos', 'zfs_get_010_neg']
	tags = ['functional', 'cli_root', 'zfs_get']

	[tests/functional/cli_root/zfs_ids_to_path]
	tests = ['zfs_ids_to_path_001_pos']
	tags = ['functional', 'cli_root', 'zfs_ids_to_path']

	[tests/functional/cli_root/zfs_inherit]
	tests = ['zfs_inherit_001_neg', 'zfs_inherit_002_neg', 'zfs_inherit_003_pos',
	'zfs_inherit_mountpoint']
	tags = ['functional', 'cli_root', 'zfs_inherit']

	[tests/functional/cli_root/zfs_load-key]
	tests = ['zfs_load-key', 'zfs_load-key_all', 'zfs_load-key_file',
	'zfs_load-key_https', 'zfs_load-key_location', 'zfs_load-key_noop',
	'zfs_load-key_recursive']
	tags = ['functional', 'cli_root', 'zfs_load-key']

	[tests/functional/cli_root/zfs_mount]
	tests = ['zfs_mount_001_pos', 'zfs_mount_002_pos', 'zfs_mount_003_pos',
	'zfs_mount_004_pos', 'zfs_mount_005_pos', 'zfs_mount_007_pos',
	'zfs_mount_009_neg', 'zfs_mount_010_neg', 'zfs_mount_011_neg',
	'zfs_mount_012_pos', 'zfs_mount_all_001_pos', 'zfs_mount_encrypted',
	'zfs_mount_remount', 'zfs_mount_all_fail', 'zfs_mount_all_mountpoints',
	- 'zfs_mount_test_race']
	+ 'zfs_mount_test_race', 'zfs_mount_recursive']
	tags = ['functional', 'cli_root', 'zfs_mount']

	[tests/functional/cli_root/zfs_program]
	tests = ['zfs_program_json']
	tags = ['functional', 'cli_root', 'zfs_program']

	[tests/functional/cli_root/zfs_promote]
	tests = ['zfs_promote_001_pos', 'zfs_promote_002_pos', 'zfs_promote_003_pos',
	'zfs_promote_004_pos', 'zfs_promote_005_pos', 'zfs_promote_006_neg',
	'zfs_promote_007_neg', 'zfs_promote_008_pos', 'zfs_promote_encryptionroot']
	tags = ['functional', 'cli_root', 'zfs_promote']

	[tests/functional/cli_root/zfs_property]
	tests = ['zfs_written_property_001_pos']
	tags = ['functional', 'cli_root', 'zfs_property']

	[tests/functional/cli_root/zfs_receive]
	tests = ['zfs_receive_001_pos', 'zfs_receive_002_pos', 'zfs_receive_003_pos',
	'zfs_receive_004_neg', 'zfs_receive_005_neg', 'zfs_receive_006_pos',
	'zfs_receive_007_neg', 'zfs_receive_008_pos', 'zfs_receive_009_neg',
	'zfs_receive_010_pos', 'zfs_receive_011_pos', 'zfs_receive_012_pos',
	'zfs_receive_013_pos', 'zfs_receive_014_pos', 'zfs_receive_015_pos',
	'zfs_receive_016_pos', 'receive-o-x_props_override',
	'receive-o-x_props_aliases',
	'zfs_receive_from_encrypted', 'zfs_receive_to_encrypted',
	'zfs_receive_raw', 'zfs_receive_raw_incremental', 'zfs_receive_-e',
	'zfs_receive_raw_-d', 'zfs_receive_from_zstd', 'zfs_receive_new_props',
	'zfs_receive_-wR-encrypted-mix', 'zfs_receive_corrective',
	'zfs_receive_compressed_corrective', 'zfs_receive_large_block_corrective']
	tags = ['functional', 'cli_root', 'zfs_receive']

	[tests/functional/cli_root/zfs_rename]
	tests = ['zfs_rename_001_pos', 'zfs_rename_002_pos', 'zfs_rename_003_pos',
	'zfs_rename_004_neg', 'zfs_rename_005_neg', 'zfs_rename_006_pos',
	'zfs_rename_007_pos', 'zfs_rename_008_pos', 'zfs_rename_009_neg',
	'zfs_rename_010_neg', 'zfs_rename_011_pos', 'zfs_rename_012_neg',
	'zfs_rename_013_pos', 'zfs_rename_014_neg', 'zfs_rename_encrypted_child',
	'zfs_rename_to_encrypted', 'zfs_rename_mountpoint', 'zfs_rename_nounmount']
	tags = ['functional', 'cli_root', 'zfs_rename']

	[tests/functional/cli_root/zfs_reservation]
	tests = ['zfs_reservation_001_pos', 'zfs_reservation_002_pos']
	tags = ['functional', 'cli_root', 'zfs_reservation']

	[tests/functional/cli_root/zfs_rollback]
	tests = ['zfs_rollback_001_pos', 'zfs_rollback_002_pos',
	'zfs_rollback_003_neg', 'zfs_rollback_004_neg']
	tags = ['functional', 'cli_root', 'zfs_rollback']

	[tests/functional/cli_root/zfs_send]
	tests = ['zfs_send_001_pos', 'zfs_send_002_pos', 'zfs_send_003_pos',
	'zfs_send_004_neg', 'zfs_send_005_pos', 'zfs_send_006_pos',
	'zfs_send_007_pos', 'zfs_send_encrypted', 'zfs_send_encrypted_unloaded',
	'zfs_send_raw', 'zfs_send_sparse', 'zfs_send-b', 'zfs_send_skip_missing']
	tags = ['functional', 'cli_root', 'zfs_send']

	[tests/functional/cli_root/zfs_set]
	tests = ['cache_001_pos', 'cache_002_neg', 'canmount_001_pos',
	'canmount_002_pos', 'canmount_003_pos', 'canmount_004_pos',
	'checksum_001_pos', 'compression_001_pos', 'mountpoint_001_pos',
	'mountpoint_002_pos', 'reservation_001_neg', 'user_property_002_pos',
	'share_mount_001_neg', 'snapdir_001_pos', 'onoffs_001_pos',
	'user_property_001_pos', 'user_property_003_neg', 'readonly_001_pos',
	'user_property_004_pos', 'version_001_neg', 'zfs_set_001_neg',
	'zfs_set_002_neg', 'zfs_set_003_neg', 'property_alias_001_pos',
	'mountpoint_003_pos', 'ro_props_001_pos', 'zfs_set_keylocation',
	'zfs_set_feature_activation', 'zfs_set_nomount']
	tags = ['functional', 'cli_root', 'zfs_set']

	[tests/functional/cli_root/zfs_share]
	tests = ['zfs_share_001_pos', 'zfs_share_002_pos', 'zfs_share_003_pos',
	'zfs_share_004_pos', 'zfs_share_006_pos', 'zfs_share_008_neg',
	'zfs_share_010_neg', 'zfs_share_011_pos', 'zfs_share_concurrent_shares',
	'zfs_share_after_mount']
	tags = ['functional', 'cli_root', 'zfs_share']

	[tests/functional/cli_root/zfs_snapshot]
	tests = ['zfs_snapshot_001_neg', 'zfs_snapshot_002_neg',
	'zfs_snapshot_003_neg', 'zfs_snapshot_004_neg', 'zfs_snapshot_005_neg',
	'zfs_snapshot_006_pos', 'zfs_snapshot_007_neg', 'zfs_snapshot_008_neg',
	'zfs_snapshot_009_pos']
	tags = ['functional', 'cli_root', 'zfs_snapshot']

	[tests/functional/cli_root/zfs_unload-key]
	tests = ['zfs_unload-key', 'zfs_unload-key_all', 'zfs_unload-key_recursive']
	tags = ['functional', 'cli_root', 'zfs_unload-key']

	[tests/functional/cli_root/zfs_unmount]
	tests = ['zfs_unmount_001_pos', 'zfs_unmount_002_pos', 'zfs_unmount_003_pos',
	'zfs_unmount_004_pos', 'zfs_unmount_005_pos', 'zfs_unmount_006_pos',
	'zfs_unmount_007_neg', 'zfs_unmount_008_neg', 'zfs_unmount_009_pos',
	'zfs_unmount_all_001_pos', 'zfs_unmount_nested', 'zfs_unmount_unload_keys']
	tags = ['functional', 'cli_root', 'zfs_unmount']

	[tests/functional/cli_root/zfs_unshare]
	tests = ['zfs_unshare_001_pos', 'zfs_unshare_002_pos', 'zfs_unshare_003_pos',
	'zfs_unshare_004_neg', 'zfs_unshare_005_neg', 'zfs_unshare_006_pos',
	'zfs_unshare_007_pos']
	tags = ['functional', 'cli_root', 'zfs_unshare']

	[tests/functional/cli_root/zfs_upgrade]
	tests = ['zfs_upgrade_001_pos', 'zfs_upgrade_002_pos', 'zfs_upgrade_003_pos',
	'zfs_upgrade_004_pos', 'zfs_upgrade_005_pos', 'zfs_upgrade_006_neg',
	'zfs_upgrade_007_neg']
	tags = ['functional', 'cli_root', 'zfs_upgrade']

	[tests/functional/cli_root/zfs_wait]
	tests = ['zfs_wait_deleteq', 'zfs_wait_getsubopt']
	tags = ['functional', 'cli_root', 'zfs_wait']

	[tests/functional/cli_root/zhack]
	tests = ['zhack_label_repair_001', 'zhack_label_repair_002',
	'zhack_label_repair_003', 'zhack_label_repair_004']
	pre =
	post =
	tags = ['functional', 'cli_root', 'zhack']

	[tests/functional/cli_root/zpool]
	tests = ['zpool_001_neg', 'zpool_002_pos', 'zpool_003_pos', 'zpool_colors']
	tags = ['functional', 'cli_root', 'zpool']

	[tests/functional/cli_root/zpool_add]
	tests = ['zpool_add_001_pos', 'zpool_add_002_pos', 'zpool_add_003_pos',
	'zpool_add_004_pos', 'zpool_add_006_pos', 'zpool_add_007_neg',
	'zpool_add_008_neg', 'zpool_add_009_neg', 'zpool_add_010_pos',
	- 'add-o_ashift', 'add_prop_ashift', 'zpool_add_dryrun_output']
	+ 'add-o_ashift', 'add_prop_ashift', 'zpool_add_dryrun_output',
	+ 'zpool_add--allow-ashift-mismatch']
	tags = ['functional', 'cli_root', 'zpool_add']

	[tests/functional/cli_root/zpool_attach]
	tests = ['zpool_attach_001_neg', 'attach-o_ashift']
	tags = ['functional', 'cli_root', 'zpool_attach']

	[tests/functional/cli_root/zpool_clear]
	tests = ['zpool_clear_001_pos', 'zpool_clear_002_neg', 'zpool_clear_003_neg',
	'zpool_clear_readonly']
	tags = ['functional', 'cli_root', 'zpool_clear']

	[tests/functional/cli_root/zpool_create]
	tests = ['zpool_create_001_pos', 'zpool_create_002_pos',
	'zpool_create_003_pos', 'zpool_create_004_pos', 'zpool_create_005_pos',
	'zpool_create_006_pos', 'zpool_create_007_neg', 'zpool_create_008_pos',
	'zpool_create_009_neg', 'zpool_create_010_neg', 'zpool_create_011_neg',
	'zpool_create_012_neg', 'zpool_create_014_neg', 'zpool_create_015_neg',
	'zpool_create_017_neg', 'zpool_create_018_pos', 'zpool_create_019_pos',
	'zpool_create_020_pos', 'zpool_create_021_pos', 'zpool_create_022_pos',
	'zpool_create_023_neg', 'zpool_create_024_pos',
	'zpool_create_encrypted', 'zpool_create_crypt_combos',
	'zpool_create_draid_001_pos', 'zpool_create_draid_002_pos',
	'zpool_create_draid_003_pos', 'zpool_create_draid_004_pos',
	'zpool_create_features_001_pos', 'zpool_create_features_002_pos',
	'zpool_create_features_003_pos', 'zpool_create_features_004_neg',
	'zpool_create_features_005_pos', 'zpool_create_features_006_pos',
	'zpool_create_features_007_pos', 'zpool_create_features_008_pos',
	'zpool_create_features_009_pos', 'create-o_ashift',
	'zpool_create_tempname', 'zpool_create_dryrun_output']
	tags = ['functional', 'cli_root', 'zpool_create']

	[tests/functional/cli_root/zpool_destroy]
	tests = ['zpool_destroy_001_pos', 'zpool_destroy_002_pos',
	'zpool_destroy_003_neg']
	pre =
	post =
	tags = ['functional', 'cli_root', 'zpool_destroy']

	[tests/functional/cli_root/zpool_detach]
	tests = ['zpool_detach_001_neg']
	tags = ['functional', 'cli_root', 'zpool_detach']

	[tests/functional/cli_root/zpool_events]
	tests = ['zpool_events_clear', 'zpool_events_cliargs', 'zpool_events_follow',
	'zpool_events_poolname', 'zpool_events_errors', 'zpool_events_duplicates',
	'zpool_events_clear_retained']
	tags = ['functional', 'cli_root', 'zpool_events']

	[tests/functional/cli_root/zpool_export]
	tests = ['zpool_export_001_pos', 'zpool_export_002_pos',
	'zpool_export_003_neg', 'zpool_export_004_pos']
	tags = ['functional', 'cli_root', 'zpool_export']

	[tests/functional/cli_root/zpool_get]
	tests = ['zpool_get_001_pos', 'zpool_get_002_pos', 'zpool_get_003_pos',
	'zpool_get_004_neg', 'zpool_get_005_pos', 'vdev_get_001_pos']
	tags = ['functional', 'cli_root', 'zpool_get']

	[tests/functional/cli_root/zpool_history]
	tests = ['zpool_history_001_neg', 'zpool_history_002_pos']
	tags = ['functional', 'cli_root', 'zpool_history']

	[tests/functional/cli_root/zpool_import]
	tests = ['zpool_import_001_pos', 'zpool_import_002_pos',
	'zpool_import_003_pos', 'zpool_import_004_pos', 'zpool_import_005_pos',
	'zpool_import_006_pos', 'zpool_import_007_pos', 'zpool_import_008_pos',
	'zpool_import_009_neg', 'zpool_import_010_pos', 'zpool_import_011_neg',
	'zpool_import_012_pos', 'zpool_import_013_neg', 'zpool_import_014_pos',
	'zpool_import_015_pos', 'zpool_import_016_pos', 'zpool_import_017_pos',
	'zpool_import_features_001_pos', 'zpool_import_features_002_neg',
	'zpool_import_features_003_pos', 'zpool_import_missing_001_pos',
	'zpool_import_missing_002_pos', 'zpool_import_missing_003_pos',
	'zpool_import_rename_001_pos', 'zpool_import_all_001_pos',
	'zpool_import_encrypted', 'zpool_import_encrypted_load',
	'zpool_import_errata3', 'zpool_import_errata4',
	'import_cachefile_device_added',
	'import_cachefile_device_removed',
	'import_cachefile_device_replaced',
	'import_cachefile_mirror_attached',
	'import_cachefile_mirror_detached',
	'import_cachefile_paths_changed',
	'import_cachefile_shared_device',
	'import_devices_missing', 'import_log_missing',
	'import_paths_changed',
	'import_rewind_config_changed',
	- 'import_rewind_device_replaced']
	+ 'import_rewind_device_replaced',
	+ 'zpool_import_status']
	tags = ['functional', 'cli_root', 'zpool_import']
	timeout = 1200

	[tests/functional/cli_root/zpool_labelclear]
	tests = ['zpool_labelclear_active', 'zpool_labelclear_exported',
	'zpool_labelclear_removed', 'zpool_labelclear_valid']
	pre =
	post =
	tags = ['functional', 'cli_root', 'zpool_labelclear']

	[tests/functional/cli_root/zpool_initialize]
	tests = ['zpool_initialize_attach_detach_add_remove',
	'zpool_initialize_fault_export_import_online',
	'zpool_initialize_import_export',
	'zpool_initialize_offline_export_import_online',
	'zpool_initialize_online_offline',
	'zpool_initialize_split',
	'zpool_initialize_start_and_cancel_neg',
	'zpool_initialize_start_and_cancel_pos',
	'zpool_initialize_suspend_resume',
	'zpool_initialize_uninit',
	'zpool_initialize_unsupported_vdevs',
	'zpool_initialize_verify_checksums',
	'zpool_initialize_verify_initialized']
	pre =
	tags = ['functional', 'cli_root', 'zpool_initialize']

	[tests/functional/cli_root/zpool_offline]
	tests = ['zpool_offline_001_pos', 'zpool_offline_002_neg',
	'zpool_offline_003_pos']
	tags = ['functional', 'cli_root', 'zpool_offline']

	[tests/functional/cli_root/zpool_online]
	tests = ['zpool_online_001_pos', 'zpool_online_002_neg']
	tags = ['functional', 'cli_root', 'zpool_online']

	[tests/functional/cli_root/zpool_remove]
	tests = ['zpool_remove_001_neg', 'zpool_remove_002_pos',
	'zpool_remove_003_pos']
	tags = ['functional', 'cli_root', 'zpool_remove']

	[tests/functional/cli_root/zpool_replace]
	tests = ['zpool_replace_001_neg', 'replace-o_ashift', 'replace_prop_ashift']
	tags = ['functional', 'cli_root', 'zpool_replace']

	[tests/functional/cli_root/zpool_resilver]
	tests = ['zpool_resilver_bad_args', 'zpool_resilver_restart',
	'zpool_resilver_concurrent']
	tags = ['functional', 'cli_root', 'zpool_resilver']

	[tests/functional/cli_root/zpool_scrub]
	tests = ['zpool_scrub_001_neg', 'zpool_scrub_002_pos', 'zpool_scrub_003_pos',
	'zpool_scrub_004_pos', 'zpool_scrub_005_pos',
	'zpool_scrub_encrypted_unloaded', 'zpool_scrub_print_repairing',
	'zpool_scrub_offline_device', 'zpool_scrub_multiple_copies',
	'zpool_error_scrub_001_pos', 'zpool_error_scrub_002_pos',
	'zpool_error_scrub_003_pos', 'zpool_error_scrub_004_pos']
	tags = ['functional', 'cli_root', 'zpool_scrub']

	[tests/functional/cli_root/zpool_set]
	tests = ['zpool_set_001_pos', 'zpool_set_002_neg', 'zpool_set_003_neg',
	'zpool_set_ashift', 'zpool_set_features', 'vdev_set_001_pos',
	'user_property_001_pos', 'user_property_002_neg']
	tags = ['functional', 'cli_root', 'zpool_set']

	[tests/functional/cli_root/zpool_split]
	tests = ['zpool_split_cliargs', 'zpool_split_devices',
	'zpool_split_encryption', 'zpool_split_props', 'zpool_split_vdevs',
	'zpool_split_resilver', 'zpool_split_indirect',
	'zpool_split_dryrun_output']
	tags = ['functional', 'cli_root', 'zpool_split']

	[tests/functional/cli_root/zpool_status]
	tests = ['zpool_status_001_pos', 'zpool_status_002_pos',
	'zpool_status_003_pos', 'zpool_status_004_pos',
	'zpool_status_005_pos', 'zpool_status_006_pos',
	'zpool_status_007_pos', 'zpool_status_008_pos',
	'zpool_status_features_001_pos']
	tags = ['functional', 'cli_root', 'zpool_status']

	[tests/functional/cli_root/zpool_sync]
	tests = ['zpool_sync_001_pos', 'zpool_sync_002_neg']
	tags = ['functional', 'cli_root', 'zpool_sync']

	[tests/functional/cli_root/zpool_trim]
	tests = ['zpool_trim_attach_detach_add_remove',
	'zpool_trim_fault_export_import_online',
	'zpool_trim_import_export', 'zpool_trim_multiple', 'zpool_trim_neg',
	'zpool_trim_offline_export_import_online', 'zpool_trim_online_offline',
	'zpool_trim_partial', 'zpool_trim_rate', 'zpool_trim_rate_neg',
	'zpool_trim_secure', 'zpool_trim_split', 'zpool_trim_start_and_cancel_neg',
	'zpool_trim_start_and_cancel_pos', 'zpool_trim_suspend_resume',
	'zpool_trim_unsupported_vdevs', 'zpool_trim_verify_checksums',
	'zpool_trim_verify_trimmed']
	tags = ['functional', 'zpool_trim']

	[tests/functional/cli_root/zpool_upgrade]
	tests = ['zpool_upgrade_001_pos', 'zpool_upgrade_002_pos',
	'zpool_upgrade_003_pos', 'zpool_upgrade_004_pos',
	'zpool_upgrade_005_neg', 'zpool_upgrade_006_neg',
	'zpool_upgrade_007_pos', 'zpool_upgrade_008_pos',
	'zpool_upgrade_009_neg', 'zpool_upgrade_features_001_pos']
	tags = ['functional', 'cli_root', 'zpool_upgrade']

	[tests/functional/cli_root/zpool_wait]
	tests = ['zpool_wait_discard', 'zpool_wait_freeing',
	'zpool_wait_initialize_basic', 'zpool_wait_initialize_cancel',
	'zpool_wait_initialize_flag', 'zpool_wait_multiple',
	'zpool_wait_no_activity', 'zpool_wait_remove', 'zpool_wait_remove_cancel',
	'zpool_wait_trim_basic', 'zpool_wait_trim_cancel', 'zpool_wait_trim_flag',
	'zpool_wait_usage']
	tags = ['functional', 'cli_root', 'zpool_wait']

	[tests/functional/cli_root/zpool_wait/scan]
	tests = ['zpool_wait_replace_cancel', 'zpool_wait_rebuild',
	'zpool_wait_resilver', 'zpool_wait_scrub_cancel',
	'zpool_wait_replace', 'zpool_wait_scrub_basic', 'zpool_wait_scrub_flag']
	tags = ['functional', 'cli_root', 'zpool_wait']

	[tests/functional/cli_user/misc]
	tests = ['zdb_001_neg', 'zfs_001_neg', 'zfs_allow_001_neg',
	'zfs_clone_001_neg', 'zfs_create_001_neg', 'zfs_destroy_001_neg',
	'zfs_get_001_neg', 'zfs_inherit_001_neg', 'zfs_mount_001_neg',
	'zfs_promote_001_neg', 'zfs_receive_001_neg', 'zfs_rename_001_neg',
	'zfs_rollback_001_neg', 'zfs_send_001_neg', 'zfs_set_001_neg',
	'zfs_share_001_neg', 'zfs_snapshot_001_neg', 'zfs_unallow_001_neg',
	'zfs_unmount_001_neg', 'zfs_unshare_001_neg', 'zfs_upgrade_001_neg',
	'zpool_001_neg', 'zpool_add_001_neg', 'zpool_attach_001_neg',
	'zpool_clear_001_neg', 'zpool_create_001_neg', 'zpool_destroy_001_neg',
	'zpool_detach_001_neg', 'zpool_export_001_neg', 'zpool_get_001_neg',
	'zpool_history_001_neg', 'zpool_import_001_neg', 'zpool_import_002_neg',
	'zpool_offline_001_neg', 'zpool_online_001_neg', 'zpool_remove_001_neg',
	'zpool_replace_001_neg', 'zpool_scrub_001_neg', 'zpool_set_001_neg',
	'zpool_status_001_neg', 'zpool_upgrade_001_neg', 'arcstat_001_pos',
	'arc_summary_001_pos', 'arc_summary_002_neg', 'zpool_wait_privilege',
	'zilstat_001_pos']
	user =
	tags = ['functional', 'cli_user', 'misc']

	[tests/functional/cli_user/zfs_list]
	tests = ['zfs_list_001_pos', 'zfs_list_002_pos', 'zfs_list_003_pos',
	'zfs_list_004_neg', 'zfs_list_005_neg', 'zfs_list_007_pos',
	'zfs_list_008_neg']
	user =
	tags = ['functional', 'cli_user', 'zfs_list']

	[tests/functional/cli_user/zpool_iostat]
	tests = ['zpool_iostat_001_neg', 'zpool_iostat_002_pos',
	'zpool_iostat_003_neg', 'zpool_iostat_004_pos',
	'zpool_iostat_005_pos', 'zpool_iostat_-c_disable',
	'zpool_iostat_-c_homedir', 'zpool_iostat_-c_searchpath']
	user =
	tags = ['functional', 'cli_user', 'zpool_iostat']

	[tests/functional/cli_user/zpool_list]
	tests = ['zpool_list_001_pos', 'zpool_list_002_neg']
	user =
	tags = ['functional', 'cli_user', 'zpool_list']

	[tests/functional/cli_user/zpool_status]
	tests = ['zpool_status_003_pos', 'zpool_status_-c_disable',
	'zpool_status_-c_homedir', 'zpool_status_-c_searchpath']
	user =
	tags = ['functional', 'cli_user', 'zpool_status']

	[tests/functional/compression]
	tests = ['compress_001_pos', 'compress_002_pos', 'compress_003_pos',
	'l2arc_compressed_arc', 'l2arc_compressed_arc_disabled',
	'l2arc_encrypted', 'l2arc_encrypted_no_compressed_arc']
	tags = ['functional', 'compression']

	[tests/functional/cp_files]
	tests = ['cp_files_001_pos', 'cp_files_002_pos', 'cp_stress']
	tags = ['functional', 'cp_files']

	[tests/functional/crtime]
	tests = ['crtime_001_pos' ]
	tags = ['functional', 'crtime']

	[tests/functional/ctime]
	tests = ['ctime_001_pos' ]
	tags = ['functional', 'ctime']

	[tests/functional/deadman]
	tests = ['deadman_ratelimit', 'deadman_sync', 'deadman_zio']
	pre =
	post =
	tags = ['functional', 'deadman']

	[tests/functional/delegate]
	tests = ['zfs_allow_001_pos', 'zfs_allow_002_pos', 'zfs_allow_003_pos',
	'zfs_allow_004_pos', 'zfs_allow_005_pos', 'zfs_allow_006_pos',
	'zfs_allow_007_pos', 'zfs_allow_008_pos', 'zfs_allow_009_neg',
	'zfs_allow_010_pos', 'zfs_allow_011_neg', 'zfs_allow_012_neg',
	'zfs_unallow_001_pos', 'zfs_unallow_002_pos', 'zfs_unallow_003_pos',
	'zfs_unallow_004_pos', 'zfs_unallow_005_pos', 'zfs_unallow_006_pos',
	'zfs_unallow_007_neg', 'zfs_unallow_008_neg']
	tags = ['functional', 'delegate']

	[tests/functional/exec]
	tests = ['exec_001_pos', 'exec_002_neg']
	tags = ['functional', 'exec']

	[tests/functional/fallocate]
	tests = ['fallocate_punch-hole']
	tags = ['functional', 'fallocate']

	[tests/functional/features/async_destroy]
	tests = ['async_destroy_001_pos']
	tags = ['functional', 'features', 'async_destroy']

	[tests/functional/features/large_dnode]
	tests = ['large_dnode_001_pos', 'large_dnode_003_pos', 'large_dnode_004_neg',
	'large_dnode_005_pos', 'large_dnode_007_neg', 'large_dnode_009_pos']
	tags = ['functional', 'features', 'large_dnode']

	[tests/functional/grow]
	pre =
	post =
	tests = ['grow_pool_001_pos', 'grow_replicas_001_pos']
	tags = ['functional', 'grow']

	[tests/functional/history]
	tests = ['history_001_pos', 'history_002_pos', 'history_003_pos',
	'history_004_pos', 'history_005_neg', 'history_006_neg',
	'history_007_pos', 'history_008_pos', 'history_009_pos',
	'history_010_pos']
	tags = ['functional', 'history']

	[tests/functional/hkdf]
	pre =
	post =
	tests = ['hkdf_test']
	tags = ['functional', 'hkdf']

	[tests/functional/inheritance]
	tests = ['inherit_001_pos']
	pre =
	tags = ['functional', 'inheritance']

	[tests/functional/io]
	tests = ['sync', 'psync', 'posixaio', 'mmap']
	tags = ['functional', 'io']

	[tests/functional/inuse]
	tests = ['inuse_004_pos', 'inuse_005_pos', 'inuse_008_pos', 'inuse_009_pos']
	post =
	tags = ['functional', 'inuse']

	[tests/functional/large_files]
	tests = ['large_files_001_pos', 'large_files_002_pos']
	tags = ['functional', 'large_files']

	[tests/functional/limits]
	tests = ['filesystem_count', 'filesystem_limit', 'snapshot_count',
	'snapshot_limit']
	tags = ['functional', 'limits']

	[tests/functional/link_count]
	tests = ['link_count_001', 'link_count_root_inode']
	tags = ['functional', 'link_count']

	[tests/functional/migration]
	tests = ['migration_001_pos', 'migration_002_pos', 'migration_003_pos',
	'migration_004_pos', 'migration_005_pos', 'migration_006_pos',
	'migration_007_pos', 'migration_008_pos', 'migration_009_pos',
	'migration_010_pos', 'migration_011_pos', 'migration_012_pos']
	tags = ['functional', 'migration']

	[tests/functional/mmap]
	tests = ['mmap_mixed', 'mmap_read_001_pos', 'mmap_seek_001_pos',
	'mmap_sync_001_pos', 'mmap_write_001_pos']
	tags = ['functional', 'mmap']

	[tests/functional/mount]
	tests = ['umount_001', 'umountall_001']
	tags = ['functional', 'mount']

	[tests/functional/mv_files]
	tests = ['mv_files_001_pos', 'mv_files_002_pos', 'random_creation']
	tags = ['functional', 'mv_files']

	[tests/functional/nestedfs]
	tests = ['nestedfs_001_pos']
	tags = ['functional', 'nestedfs']

	[tests/functional/no_space]
	tests = ['enospc_001_pos', 'enospc_002_pos', 'enospc_003_pos',
	'enospc_df', 'enospc_ganging', 'enospc_rm']
	tags = ['functional', 'no_space']

	[tests/functional/nopwrite]
	tests = ['nopwrite_copies', 'nopwrite_mtime', 'nopwrite_negative',
	'nopwrite_promoted_clone', 'nopwrite_recsize', 'nopwrite_sync',
	'nopwrite_varying_compression', 'nopwrite_volume']
	tags = ['functional', 'nopwrite']

	[tests/functional/online_offline]
	tests = ['online_offline_001_pos', 'online_offline_002_neg',
	'online_offline_003_neg']
	tags = ['functional', 'online_offline']

	[tests/functional/pool_checkpoint]
	tests = ['checkpoint_after_rewind', 'checkpoint_big_rewind',
	'checkpoint_capacity', 'checkpoint_conf_change', 'checkpoint_discard',
	'checkpoint_discard_busy', 'checkpoint_discard_many',
	'checkpoint_indirect', 'checkpoint_invalid', 'checkpoint_lun_expsz',
	'checkpoint_open', 'checkpoint_removal', 'checkpoint_rewind',
	'checkpoint_ro_rewind', 'checkpoint_sm_scale', 'checkpoint_twice',
	'checkpoint_vdev_add', 'checkpoint_zdb', 'checkpoint_zhack_feat']
	tags = ['functional', 'pool_checkpoint']
	timeout = 1800

	[tests/functional/pool_names]
	tests = ['pool_names_001_pos', 'pool_names_002_neg']
	pre =
	post =
	tags = ['functional', 'pool_names']

	[tests/functional/poolversion]
	tests = ['poolversion_001_pos', 'poolversion_002_pos']
	tags = ['functional', 'poolversion']

	[tests/functional/pyzfs]
	tests = ['pyzfs_unittest']
	pre =
	post =
	tags = ['functional', 'pyzfs']

	[tests/functional/quota]
	tests = ['quota_001_pos', 'quota_002_pos', 'quota_003_pos',
	'quota_004_pos', 'quota_005_pos', 'quota_006_neg']
	tags = ['functional', 'quota']

	[tests/functional/redacted_send]
	tests = ['redacted_compressed', 'redacted_contents', 'redacted_deleted',
	'redacted_disabled_feature', 'redacted_embedded', 'redacted_holes',
	'redacted_incrementals', 'redacted_largeblocks', 'redacted_many_clones',
	'redacted_mixed_recsize', 'redacted_mounts', 'redacted_negative',
	'redacted_origin', 'redacted_panic', 'redacted_props', 'redacted_resume',
	'redacted_size', 'redacted_volume']
	tags = ['functional', 'redacted_send']

	[tests/functional/raidz]
	tests = ['raidz_001_neg', 'raidz_002_pos', 'raidz_003_pos', 'raidz_004_pos']
	tags = ['functional', 'raidz']

	[tests/functional/redundancy]
	tests = ['redundancy_draid', 'redundancy_draid1', 'redundancy_draid2',
	'redundancy_draid3', 'redundancy_draid_damaged1',
	'redundancy_draid_damaged2', 'redundancy_draid_spare1',
	'redundancy_draid_spare2', 'redundancy_draid_spare3', 'redundancy_mirror',
	'redundancy_raidz', 'redundancy_raidz1', 'redundancy_raidz2',
	'redundancy_raidz3', 'redundancy_stripe']
	tags = ['functional', 'redundancy']
	timeout = 1200

	[tests/functional/refquota]
	tests = ['refquota_001_pos', 'refquota_002_pos', 'refquota_003_pos',
	'refquota_004_pos', 'refquota_005_pos', 'refquota_006_neg',
	'refquota_007_neg', 'refquota_008_neg']
	tags = ['functional', 'refquota']

	[tests/functional/refreserv]
	tests = ['refreserv_001_pos', 'refreserv_002_pos', 'refreserv_003_pos',
	'refreserv_004_pos', 'refreserv_005_pos', 'refreserv_multi_raidz',
	'refreserv_raidz']
	tags = ['functional', 'refreserv']

	[tests/functional/removal]
	pre =
	tests = ['removal_all_vdev', 'removal_cancel', 'removal_check_space',
	'removal_condense_export', 'removal_multiple_indirection',
	'removal_nopwrite', 'removal_remap_deadlists',
	'removal_resume_export', 'removal_sanity', 'removal_with_add',
	'removal_with_create_fs', 'removal_with_dedup',
	'removal_with_errors', 'removal_with_export', 'removal_with_indirect',
	'removal_with_ganging', 'removal_with_faulted',
	'removal_with_remove', 'removal_with_scrub', 'removal_with_send',
	'removal_with_send_recv', 'removal_with_snapshot',
	'removal_with_write', 'removal_with_zdb', 'remove_expanded',
	'remove_mirror', 'remove_mirror_sanity', 'remove_raidz',
	'remove_indirect', 'remove_attach_mirror', 'removal_reservation']
	tags = ['functional', 'removal']

	[tests/functional/rename_dirs]
	tests = ['rename_dirs_001_pos']
	tags = ['functional', 'rename_dirs']

	[tests/functional/replacement]
	tests = ['attach_import', 'attach_multiple', 'attach_rebuild',
	'attach_resilver', 'detach', 'rebuild_disabled_feature',
	'rebuild_multiple', 'rebuild_raidz', 'replace_import', 'replace_rebuild',
	'replace_resilver', 'resilver_restart_001', 'resilver_restart_002',
	'scrub_cancel']
	tags = ['functional', 'replacement']

	[tests/functional/reservation]
	tests = ['reservation_001_pos', 'reservation_002_pos', 'reservation_003_pos',
	'reservation_004_pos', 'reservation_005_pos', 'reservation_006_pos',
	'reservation_007_pos', 'reservation_008_pos', 'reservation_009_pos',
	'reservation_010_pos', 'reservation_011_pos', 'reservation_012_pos',
	'reservation_013_pos', 'reservation_014_pos', 'reservation_015_pos',
	'reservation_016_pos', 'reservation_017_pos', 'reservation_018_pos',
	'reservation_019_pos', 'reservation_020_pos', 'reservation_021_neg',
	'reservation_022_pos']
	tags = ['functional', 'reservation']

	[tests/functional/rootpool]
	tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_pos']
	tags = ['functional', 'rootpool']

	[tests/functional/rsend]
	tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos',
	'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', 'rsend_005_pos',
	'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos', 'rsend_009_pos',
	'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos', 'rsend_013_pos',
	'rsend_014_pos', 'rsend_016_neg', 'rsend_019_pos', 'rsend_020_pos',
	'rsend_021_pos', 'rsend_022_pos', 'rsend_024_pos', 'rsend_025_pos',
	'rsend_026_neg', 'rsend_027_pos', 'rsend_028_neg', 'rsend_029_neg',
	'rsend_030_pos', 'rsend_031_pos', 'send-c_verify_ratio',
	'send-c_verify_contents', 'send-c_props', 'send-c_incremental',
	'send-c_volume', 'send-c_zstream_recompress', 'send-c_zstreamdump',
	'send-c_lz4_disabled', 'send-c_recv_lz4_disabled',
	'send-c_mixed_compression', 'send-c_stream_size_estimate',
	'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize',
	'send-c_recv_dedup', 'send-L_toggle', 'send_encrypted_incremental',
	'send_encrypted_freeobjects', 'send_encrypted_hierarchy',
	'send_encrypted_props', 'send_encrypted_truncated_files',
	'send_freeobjects', 'send_realloc_files', 'send_realloc_encrypted_files',
	'send_spill_block', 'send_holds', 'send_hole_birth', 'send_mixed_raw',
	'send-wR_encrypted_zvol', 'send_partial_dataset', 'send_invalid',
	'send_doall', 'send_raw_spill_block', 'send_raw_ashift',
	'send_raw_large_blocks']
	tags = ['functional', 'rsend']

	[tests/functional/scrub_mirror]
	tests = ['scrub_mirror_001_pos', 'scrub_mirror_002_pos',
	'scrub_mirror_003_pos', 'scrub_mirror_004_pos']
	tags = ['functional', 'scrub_mirror']

	[tests/functional/slog]
	tests = ['slog_001_pos', 'slog_002_pos', 'slog_003_pos', 'slog_004_pos',
	'slog_005_pos', 'slog_006_pos', 'slog_007_pos', 'slog_008_neg',
	'slog_009_neg', 'slog_010_neg', 'slog_011_neg', 'slog_012_neg',
	'slog_013_pos', 'slog_014_pos', 'slog_015_neg', 'slog_replay_fs_001',
	'slog_replay_fs_002', 'slog_replay_volume', 'slog_016_pos']
	tags = ['functional', 'slog']

	[tests/functional/snapshot]
	tests = ['clone_001_pos', 'rollback_001_pos', 'rollback_002_pos',
	'rollback_003_pos', 'snapshot_001_pos', 'snapshot_002_pos',
	'snapshot_003_pos', 'snapshot_004_pos', 'snapshot_005_pos',
	'snapshot_006_pos', 'snapshot_007_pos', 'snapshot_008_pos',
	'snapshot_009_pos', 'snapshot_010_pos', 'snapshot_011_pos',
	'snapshot_012_pos', 'snapshot_013_pos', 'snapshot_014_pos',
	'snapshot_017_pos', 'snapshot_018_pos']
	tags = ['functional', 'snapshot']

	[tests/functional/snapused]
	tests = ['snapused_001_pos', 'snapused_002_pos', 'snapused_003_pos',
	'snapused_004_pos', 'snapused_005_pos']
	tags = ['functional', 'snapused']

	[tests/functional/sparse]
	tests = ['sparse_001_pos']
	tags = ['functional', 'sparse']

	[tests/functional/stat]
	tests = ['stat_001_pos']
	tags = ['functional', 'stat']

	[tests/functional/suid]
	tests = ['suid_write_to_suid', 'suid_write_to_sgid', 'suid_write_to_suid_sgid',
	'suid_write_to_none', 'suid_write_zil_replay']
	tags = ['functional', 'suid']

	[tests/functional/trim]
	tests = ['autotrim_integrity', 'autotrim_config', 'autotrim_trim_integrity',
	'trim_integrity', 'trim_config', 'trim_l2arc']
	tags = ['functional', 'trim']

	[tests/functional/truncate]
	tests = ['truncate_001_pos', 'truncate_002_pos', 'truncate_timestamps']
	tags = ['functional', 'truncate']

	[tests/functional/upgrade]
	tests = ['upgrade_userobj_001_pos', 'upgrade_readonly_pool']
	tags = ['functional', 'upgrade']

	[tests/functional/userquota]
	tests = [
	'userquota_001_pos', 'userquota_002_pos', 'userquota_003_pos',
	'userquota_004_pos', 'userquota_005_neg', 'userquota_006_pos',
	'userquota_007_pos', 'userquota_008_pos', 'userquota_009_pos',
	'userquota_010_pos', 'userquota_011_pos', 'userquota_012_neg',
	'userspace_001_pos', 'userspace_002_pos', 'userspace_encrypted',
	'userspace_send_encrypted', 'userspace_encrypted_13709']
	tags = ['functional', 'userquota']

	[tests/functional/vdev_zaps]
	tests = ['vdev_zaps_001_pos', 'vdev_zaps_002_pos', 'vdev_zaps_003_pos',
	'vdev_zaps_004_pos', 'vdev_zaps_005_pos', 'vdev_zaps_006_pos',
	'vdev_zaps_007_pos']
	tags = ['functional', 'vdev_zaps']

	[tests/functional/write_dirs]
	tests = ['write_dirs_001_pos', 'write_dirs_002_pos']
	tags = ['functional', 'write_dirs']

	[tests/functional/xattr]
	tests = ['xattr_001_pos', 'xattr_002_neg', 'xattr_003_neg', 'xattr_004_pos',
	'xattr_005_pos', 'xattr_006_pos', 'xattr_007_neg',
	'xattr_011_pos', 'xattr_012_pos', 'xattr_013_pos', 'xattr_compat']
	tags = ['functional', 'xattr']

	[tests/functional/zvol/zvol_ENOSPC]
	tests = ['zvol_ENOSPC_001_pos']
	tags = ['functional', 'zvol', 'zvol_ENOSPC']

	[tests/functional/zvol/zvol_cli]
	tests = ['zvol_cli_001_pos', 'zvol_cli_002_pos', 'zvol_cli_003_neg']
	tags = ['functional', 'zvol', 'zvol_cli']

	[tests/functional/zvol/zvol_misc]
	tests = ['zvol_misc_002_pos', 'zvol_misc_hierarchy', 'zvol_misc_rename_inuse',
	'zvol_misc_snapdev', 'zvol_misc_trim', 'zvol_misc_volmode', 'zvol_misc_zil']
	tags = ['functional', 'zvol', 'zvol_misc']

	[tests/functional/zvol/zvol_stress]
	tests = ['zvol_stress']
	tags = ['functional', 'zvol', 'zvol_stress']

	[tests/functional/zvol/zvol_swap]
	tests = ['zvol_swap_001_pos', 'zvol_swap_002_pos', 'zvol_swap_004_pos']
	tags = ['functional', 'zvol', 'zvol_swap']

	[tests/functional/libzfs]
	tests = ['many_fds', 'libzfs_input']
	tags = ['functional', 'libzfs']

	[tests/functional/log_spacemap]
	tests = ['log_spacemap_import_logs']
	pre =
	post =
	tags = ['functional', 'log_spacemap']

	[tests/functional/l2arc]
	tests = ['l2arc_arcstats_pos', 'l2arc_mfuonly_pos', 'l2arc_l2miss_pos',
	'persist_l2arc_001_pos', 'persist_l2arc_002_pos',
	'persist_l2arc_003_neg', 'persist_l2arc_004_pos', 'persist_l2arc_005_pos']
	tags = ['functional', 'l2arc']

	[tests/functional/zpool_influxdb]
	tests = ['zpool_influxdb']
	tags = ['functional', 'zpool_influxdb']
	diff --git a/sys/contrib/openzfs/tests/runfiles/linux.run b/sys/contrib/openzfs/tests/runfiles/linux.run
	index 6a4cd3fe691c..92ce09ec6fcb 100644
	--- a/sys/contrib/openzfs/tests/runfiles/linux.run
	+++ b/sys/contrib/openzfs/tests/runfiles/linux.run
	@@ -1,223 +1,224 @@
	#
	# This file and its contents are supplied under the terms of the
	# Common Development and Distribution License ("CDDL"), version 1.0.
	# You may only use this file in accordance with the terms of version
	# 1.0 of the CDDL.
	#
	# A full copy of the text of the CDDL should have accompanied this
	# source. A copy of the CDDL is also available via the Internet at
	# http://www.illumos.org/license/CDDL.
	#

	[DEFAULT]
	pre = setup
	quiet = False
	pre_user = root
	user = root
	timeout = 600
	post_user = root
	post = cleanup
	failsafe_user = root
	failsafe = callbacks/zfs_failsafe
	outputdir = /var/tmp/test_results
	tags = ['functional']

	[tests/functional/acl/posix:Linux]
	tests = ['posix_001_pos', 'posix_002_pos', 'posix_003_pos', 'posix_004_pos']
	tags = ['functional', 'acl', 'posix']

	[tests/functional/acl/posix-sa:Linux]
	tests = ['posix_001_pos', 'posix_002_pos', 'posix_003_pos', 'posix_004_pos']
	tags = ['functional', 'acl', 'posix-sa']

	[tests/functional/atime:Linux]
	tests = ['atime_003_pos', 'root_relatime_on']
	tags = ['functional', 'atime']

	[tests/functional/block_cloning:Linux]
	tests = ['block_cloning_ficlone', 'block_cloning_ficlonerange',
	'block_cloning_ficlonerange_partial', 'block_cloning_disabled_ficlone',
	'block_cloning_disabled_ficlonerange']
	tags = ['functional', 'block_cloning']

	[tests/functional/chattr:Linux]
	tests = ['chattr_001_pos', 'chattr_002_neg']
	tags = ['functional', 'chattr']

	[tests/functional/cli_root/zfs:Linux]
	tests = ['zfs_003_neg']
	tags = ['functional', 'cli_root', 'zfs']

	[tests/functional/cli_root/zfs_mount:Linux]
	tests = ['zfs_mount_006_pos', 'zfs_mount_008_pos', 'zfs_mount_013_pos',
	'zfs_mount_014_neg', 'zfs_multi_mount']
	tags = ['functional', 'cli_root', 'zfs_mount']

	[tests/functional/cli_root/zfs_share:Linux]
	tests = ['zfs_share_005_pos', 'zfs_share_007_neg', 'zfs_share_009_neg',
	'zfs_share_012_pos', 'zfs_share_013_pos']
	tags = ['functional', 'cli_root', 'zfs_share']

	[tests/functional/cli_root/zfs_unshare:Linux]
	tests = ['zfs_unshare_008_pos']
	tags = ['functional', 'cli_root', 'zfs_unshare']

	[tests/functional/cli_root/zfs_sysfs:Linux]
	tests = ['zfeature_set_unsupported', 'zfs_get_unsupported',
	'zfs_set_unsupported', 'zfs_sysfs_live', 'zpool_get_unsupported',
	'zpool_set_unsupported']
	tags = ['functional', 'cli_root', 'zfs_sysfs']

	[tests/functional/cli_root/zpool_add:Linux]
	tests = ['add_nested_replacing_spare']
	tags = ['functional', 'cli_root', 'zpool_add']

	[tests/functional/cli_root/zpool_expand:Linux]
	tests = ['zpool_expand_001_pos', 'zpool_expand_002_pos',
	'zpool_expand_003_neg', 'zpool_expand_004_pos', 'zpool_expand_005_pos']
	tags = ['functional', 'cli_root', 'zpool_expand']

	[tests/functional/cli_root/zpool_import:Linux]
	tests = ['zpool_import_hostid_changed',
	'zpool_import_hostid_changed_unclean_export',
	'zpool_import_hostid_changed_cachefile',
	'zpool_import_hostid_changed_cachefile_unclean_export']
	tags = ['functional', 'cli_root', 'zpool_import']

	[tests/functional/cli_root/zpool_reopen:Linux]
	tests = ['zpool_reopen_001_pos', 'zpool_reopen_002_pos',
	'zpool_reopen_003_pos', 'zpool_reopen_004_pos', 'zpool_reopen_005_pos',
	'zpool_reopen_006_neg', 'zpool_reopen_007_pos']
	tags = ['functional', 'cli_root', 'zpool_reopen']

	[tests/functional/cli_root/zpool_split:Linux]
	tests = ['zpool_split_wholedisk']
	tags = ['functional', 'cli_root', 'zpool_split']

	[tests/functional/compression:Linux]
	tests = ['compress_004_pos']
	tags = ['functional', 'compression']

	[tests/functional/devices:Linux]
	tests = ['devices_001_pos', 'devices_002_neg', 'devices_003_pos']
	tags = ['functional', 'devices']

	[tests/functional/events:Linux]
	tests = ['events_001_pos', 'events_002_pos', 'zed_rc_filter', 'zed_fd_spill',
	- 'zed_cksum_reported', 'zed_cksum_config', 'zed_io_config']
	+ 'zed_cksum_reported', 'zed_cksum_config', 'zed_io_config',
	+ 'zed_slow_io', 'zed_slow_io_many_vdevs']
	tags = ['functional', 'events']

	[tests/functional/fadvise:Linux]
	tests = ['fadvise_sequential']
	tags = ['functional', 'fadvise']

	[tests/functional/fallocate:Linux]
	tests = ['fallocate_prealloc', 'fallocate_zero-range']
	tags = ['functional', 'fallocate']

	[tests/functional/fault:Linux]
	tests = ['auto_offline_001_pos', 'auto_online_001_pos', 'auto_online_002_pos',
	'auto_replace_001_pos', 'auto_replace_002_pos', 'auto_spare_001_pos',
	'auto_spare_002_pos', 'auto_spare_multiple', 'auto_spare_ashift',
	'auto_spare_shared', 'decrypt_fault', 'decompress_fault',
	'scrub_after_resilver', 'zpool_status_-s']
	tags = ['functional', 'fault']

	[tests/functional/features/large_dnode:Linux]
	tests = ['large_dnode_002_pos', 'large_dnode_006_pos', 'large_dnode_008_pos']
	tags = ['functional', 'features', 'large_dnode']

	[tests/functional/io:Linux]
	tests = ['libaio', 'io_uring']
	tags = ['functional', 'io']

	[tests/functional/largest_pool:Linux]
	tests = ['largest_pool_001_pos']
	pre =
	post =
	tags = ['functional', 'largest_pool']

	[tests/functional/mmap:Linux]
	tests = ['mmap_libaio_001_pos', 'mmap_sync_001_pos']
	tags = ['functional', 'mmap']

	[tests/functional/mmp:Linux]
	tests = ['mmp_on_thread', 'mmp_on_uberblocks', 'mmp_on_off', 'mmp_interval',
	'mmp_active_import', 'mmp_inactive_import', 'mmp_exported_import',
	'mmp_write_uberblocks', 'mmp_reset_interval', 'multihost_history',
	- 'mmp_on_zdb', 'mmp_write_distribution', 'mmp_hostid']
	+ 'mmp_on_zdb', 'mmp_write_distribution', 'mmp_hostid', 'mmp_write_slow_disk']
	tags = ['functional', 'mmp']

	[tests/functional/mount:Linux]
	tests = ['umount_unlinked_drain']
	tags = ['functional', 'mount']

	[tests/functional/pam:Linux]
	tests = ['pam_basic', 'pam_change_unmounted', 'pam_nounmount', 'pam_recursive',
	'pam_short_password']
	tags = ['functional', 'pam']

	[tests/functional/procfs:Linux]
	tests = ['procfs_list_basic', 'procfs_list_concurrent_readers',
	'procfs_list_stale_read', 'pool_state']
	tags = ['functional', 'procfs']

	[tests/functional/projectquota:Linux]
	tests = ['projectid_001_pos', 'projectid_002_pos', 'projectid_003_pos',
	'projectquota_001_pos', 'projectquota_002_pos', 'projectquota_003_pos',
	'projectquota_004_neg', 'projectquota_005_pos', 'projectquota_006_pos',
	'projectquota_007_pos', 'projectquota_008_pos', 'projectquota_009_pos',
	'projectspace_001_pos', 'projectspace_002_pos', 'projectspace_003_pos',
	'projectspace_004_pos',
	'projecttree_001_pos', 'projecttree_002_pos', 'projecttree_003_neg']
	tags = ['functional', 'projectquota']

	[tests/functional/dos_attributes:Linux]
	tests = ['read_dos_attrs_001', 'write_dos_attrs_001']
	tags = ['functional', 'dos_attributes']

	[tests/functional/renameat2:Linux]
	tests = ['renameat2_noreplace', 'renameat2_exchange', 'renameat2_whiteout']
	tags = ['functional', 'renameat2']

	[tests/functional/rsend:Linux]
	tests = ['send_realloc_dnode_size', 'send_encrypted_files']
	tags = ['functional', 'rsend']

	[tests/functional/simd:Linux]
	pre =
	post =
	tests = ['simd_supported']
	tags = ['functional', 'simd']

	[tests/functional/snapshot:Linux]
	tests = ['snapshot_015_pos', 'snapshot_016_pos']
	tags = ['functional', 'snapshot']

	[tests/functional/tmpfile:Linux]
	tests = ['tmpfile_001_pos', 'tmpfile_002_pos', 'tmpfile_003_pos',
	'tmpfile_stat_mode']
	tags = ['functional', 'tmpfile']

	[tests/functional/upgrade:Linux]
	tests = ['upgrade_projectquota_001_pos']
	tags = ['functional', 'upgrade']

	[tests/functional/user_namespace:Linux]
	tests = ['user_namespace_001', 'user_namespace_002', 'user_namespace_003',
	'user_namespace_004']
	tags = ['functional', 'user_namespace']

	[tests/functional/userquota:Linux]
	tests = ['groupspace_001_pos', 'groupspace_002_pos', 'groupspace_003_pos',
	'userquota_013_pos', 'userspace_003_pos']
	tags = ['functional', 'userquota']

	[tests/functional/zvol/zvol_misc:Linux]
	tests = ['zvol_misc_fua']
	tags = ['functional', 'zvol', 'zvol_misc']

	[tests/functional/idmap_mount:Linux]
	tests = ['idmap_mount_001', 'idmap_mount_002', 'idmap_mount_003',
	'idmap_mount_004', 'idmap_mount_005']
	tags = ['functional', 'idmap_mount']
	diff --git a/sys/contrib/openzfs/tests/runfiles/sanity.run b/sys/contrib/openzfs/tests/runfiles/sanity.run
	index ab41c05b8473..695d3697a602 100644
	--- a/sys/contrib/openzfs/tests/runfiles/sanity.run
	+++ b/sys/contrib/openzfs/tests/runfiles/sanity.run
	@@ -1,633 +1,634 @@
	#
	# This file and its contents are supplied under the terms of the
	# Common Development and Distribution License ("CDDL"), version 1.0.
	# You may only use this file in accordance with the terms of version
	# 1.0 of the CDDL.
	#
	# A full copy of the text of the CDDL should have accompanied this
	# source. A copy of the CDDL is also available via the Internet at
	# http://www.illumos.org/license/CDDL.
	#
	# This run file contains a subset of functional tests which exercise
	# as much functionality as possible while still executing relatively
	# quickly. The included tests should take no more than a few seconds
	# each to run at most. This provides a convenient way to sanity test a
	# change before committing to a full test run which takes several hours.
	#
	# Approximate run time: 15 minutes
	#

	[DEFAULT]
	pre = setup
	quiet = False
	pre_user = root
	user = root
	timeout = 180
	post_user = root
	post = cleanup
	failsafe_user = root
	failsafe = callbacks/zfs_failsafe
	outputdir = /var/tmp/test_results
	tags = ['functional']

	[tests/functional/acl/off]
	tests = ['posixmode']
	tags = ['functional', 'acl']

	[tests/functional/alloc_class]
	tests = ['alloc_class_003_pos', 'alloc_class_004_pos', 'alloc_class_005_pos',
	'alloc_class_006_pos', 'alloc_class_008_pos', 'alloc_class_010_pos',
	'alloc_class_011_neg']
	tags = ['functional', 'alloc_class']

	[tests/functional/arc]
	tests = ['dbufstats_001_pos', 'dbufstats_002_pos', 'arcstats_runtime_tuning']
	tags = ['functional', 'arc']

	[tests/functional/bootfs]
	tests = ['bootfs_004_neg', 'bootfs_007_pos']
	tags = ['functional', 'bootfs']

	[tests/functional/cache]
	tests = ['cache_004_neg', 'cache_005_neg', 'cache_007_neg', 'cache_010_pos']
	tags = ['functional', 'cache']

	[tests/functional/cachefile]
	tests = ['cachefile_001_pos', 'cachefile_002_pos', 'cachefile_003_pos',
	'cachefile_004_pos']
	tags = ['functional', 'cachefile']

	[tests/functional/casenorm]
	tests = ['case_all_values', 'norm_all_values', 'sensitive_none_lookup',
	'sensitive_none_delete', 'insensitive_none_lookup',
	'insensitive_none_delete', 'mixed_none_lookup', 'mixed_none_delete']
	tags = ['functional', 'casenorm']

	[tests/functional/channel_program/lua_core]
	tests = ['tst.args_to_lua', 'tst.divide_by_zero', 'tst.exists',
	'tst.integer_illegal', 'tst.integer_overflow', 'tst.language_functions_neg',
	'tst.language_functions_pos', 'tst.large_prog', 'tst.libraries',
	'tst.memory_limit', 'tst.nested_neg', 'tst.nested_pos', 'tst.nvlist_to_lua',
	'tst.recursive_neg', 'tst.recursive_pos', 'tst.return_large',
	'tst.return_nvlist_neg', 'tst.return_nvlist_pos',
	'tst.return_recursive_table', 'tst.stack_gsub', 'tst.timeout']
	tags = ['functional', 'channel_program', 'lua_core']

	[tests/functional/channel_program/synctask_core]
	tests = ['tst.destroy_fs', 'tst.destroy_snap', 'tst.get_count_and_limit',
	'tst.get_index_props', 'tst.get_mountpoint', 'tst.get_neg',
	'tst.get_number_props', 'tst.get_string_props', 'tst.get_type',
	'tst.get_userquota', 'tst.get_written', 'tst.inherit', 'tst.list_bookmarks',
	'tst.list_children', 'tst.list_clones', 'tst.list_holds',
	'tst.list_snapshots', 'tst.list_system_props',
	'tst.list_user_props', 'tst.parse_args_neg','tst.promote_conflict',
	'tst.promote_multiple', 'tst.promote_simple', 'tst.rollback_mult',
	'tst.rollback_one', 'tst.set_props', 'tst.snapshot_destroy',
	'tst.snapshot_neg', 'tst.snapshot_recursive', 'tst.snapshot_simple',
	'tst.bookmark.create', 'tst.bookmark.copy']
	tags = ['functional', 'channel_program', 'synctask_core']

	[tests/functional/cli_root/zdb]
	tests = ['zdb_003_pos', 'zdb_004_pos', 'zdb_005_pos']
	pre =
	post =
	tags = ['functional', 'cli_root', 'zdb']

	[tests/functional/cli_root/zfs]
	tests = ['zfs_001_neg', 'zfs_002_pos']
	tags = ['functional', 'cli_root', 'zfs']

	[tests/functional/cli_root/zfs_bookmark]
	tests = ['zfs_bookmark_cliargs']
	tags = ['functional', 'cli_root', 'zfs_bookmark']

	[tests/functional/cli_root/zfs_change-key]
	tests = ['zfs_change-key', 'zfs_change-key_child', 'zfs_change-key_format',
	'zfs_change-key_inherit', 'zfs_change-key_load', 'zfs_change-key_location',
	'zfs_change-key_pbkdf2iters', 'zfs_change-key_clones']
	tags = ['functional', 'cli_root', 'zfs_change-key']

	[tests/functional/cli_root/zfs_clone]
	tests = ['zfs_clone_001_neg', 'zfs_clone_002_pos', 'zfs_clone_003_pos',
	'zfs_clone_004_pos', 'zfs_clone_005_pos', 'zfs_clone_006_pos',
	'zfs_clone_007_pos', 'zfs_clone_008_neg', 'zfs_clone_009_neg',
	'zfs_clone_encrypted']
	tags = ['functional', 'cli_root', 'zfs_clone']

	[tests/functional/cli_root/zfs_create]
	tests = ['zfs_create_001_pos', 'zfs_create_002_pos', 'zfs_create_003_pos',
	'zfs_create_004_pos', 'zfs_create_005_pos', 'zfs_create_006_pos',
	'zfs_create_007_pos', 'zfs_create_011_pos', 'zfs_create_012_pos',
	'zfs_create_013_pos', 'zfs_create_014_pos', 'zfs_create_encrypted',
	'zfs_create_dryrun', 'zfs_create_verbose']
	tags = ['functional', 'cli_root', 'zfs_create']

	[tests/functional/cli_root/zfs_destroy]
	tests = ['zfs_destroy_002_pos', 'zfs_destroy_003_pos',
	'zfs_destroy_004_pos', 'zfs_destroy_006_neg', 'zfs_destroy_007_neg',
	'zfs_destroy_008_pos', 'zfs_destroy_009_pos', 'zfs_destroy_010_pos',
	'zfs_destroy_011_pos', 'zfs_destroy_012_pos', 'zfs_destroy_013_neg',
	'zfs_destroy_014_pos', 'zfs_destroy_dev_removal',
	'zfs_destroy_dev_removal_condense']
	tags = ['functional', 'cli_root', 'zfs_destroy']

	[tests/functional/cli_root/zfs_diff]
	tests = ['zfs_diff_cliargs', 'zfs_diff_encrypted']
	tags = ['functional', 'cli_root', 'zfs_diff']

	[tests/functional/cli_root/zfs_get]
	tests = ['zfs_get_003_pos', 'zfs_get_006_neg', 'zfs_get_007_neg',
	'zfs_get_010_neg']
	tags = ['functional', 'cli_root', 'zfs_get']

	[tests/functional/cli_root/zfs_inherit]
	tests = ['zfs_inherit_001_neg', 'zfs_inherit_003_pos', 'zfs_inherit_mountpoint']
	tags = ['functional', 'cli_root', 'zfs_inherit']

	[tests/functional/cli_root/zfs_load-key]
	tests = ['zfs_load-key', 'zfs_load-key_all', 'zfs_load-key_file',
	'zfs_load-key_https', 'zfs_load-key_location', 'zfs_load-key_noop',
	'zfs_load-key_recursive']
	tags = ['functional', 'cli_root', 'zfs_load-key']

	[tests/functional/cli_root/zfs_mount]
	tests = ['zfs_mount_001_pos', 'zfs_mount_002_pos', 'zfs_mount_003_pos',
	'zfs_mount_004_pos', 'zfs_mount_005_pos', 'zfs_mount_007_pos',
	'zfs_mount_009_neg', 'zfs_mount_010_neg', 'zfs_mount_011_neg',
	'zfs_mount_012_pos', 'zfs_mount_encrypted', 'zfs_mount_remount',
	- 'zfs_mount_all_fail', 'zfs_mount_all_mountpoints', 'zfs_mount_test_race']
	+ 'zfs_mount_all_fail', 'zfs_mount_all_mountpoints',
	+ 'zfs_mount_test_race', 'zfs_mount_recursive']
	tags = ['functional', 'cli_root', 'zfs_mount']

	[tests/functional/cli_root/zfs_program]
	tests = ['zfs_program_json']
	tags = ['functional', 'cli_root', 'zfs_program']

	[tests/functional/cli_root/zfs_promote]
	tests = ['zfs_promote_001_pos', 'zfs_promote_002_pos', 'zfs_promote_003_pos',
	'zfs_promote_004_pos', 'zfs_promote_005_pos', 'zfs_promote_006_neg',
	'zfs_promote_007_neg', 'zfs_promote_008_pos', 'zfs_promote_encryptionroot']
	tags = ['functional', 'cli_root', 'zfs_promote']

	[tests/functional/cli_root/zfs_receive]
	tests = ['zfs_receive_001_pos', 'zfs_receive_002_pos', 'zfs_receive_003_pos',
	'zfs_receive_004_neg', 'zfs_receive_005_neg', 'zfs_receive_006_pos',
	'zfs_receive_007_neg', 'zfs_receive_008_pos', 'zfs_receive_009_neg',
	'zfs_receive_010_pos', 'zfs_receive_011_pos', 'zfs_receive_012_pos',
	'zfs_receive_013_pos', 'zfs_receive_014_pos', 'zfs_receive_015_pos',
	'zfs_receive_016_pos', 'zfs_receive_from_encrypted',
	'zfs_receive_to_encrypted', 'zfs_receive_raw',
	'zfs_receive_raw_incremental', 'zfs_receive_-e',
	'zfs_receive_raw_-d', 'zfs_receive_from_zstd', 'zfs_receive_new_props']
	tags = ['functional', 'cli_root', 'zfs_receive']

	[tests/functional/cli_root/zfs_rename]
	tests = ['zfs_rename_003_pos', 'zfs_rename_004_neg',
	'zfs_rename_005_neg', 'zfs_rename_006_pos', 'zfs_rename_007_pos',
	'zfs_rename_008_pos', 'zfs_rename_009_neg', 'zfs_rename_010_neg',
	'zfs_rename_011_pos', 'zfs_rename_012_neg', 'zfs_rename_013_pos',
	'zfs_rename_encrypted_child', 'zfs_rename_to_encrypted',
	'zfs_rename_mountpoint', 'zfs_rename_nounmount']
	tags = ['functional', 'cli_root', 'zfs_rename']

	[tests/functional/cli_root/zfs_reservation]
	tests = ['zfs_reservation_001_pos', 'zfs_reservation_002_pos']
	tags = ['functional', 'cli_root', 'zfs_reservation']

	[tests/functional/cli_root/zfs_rollback]
	tests = ['zfs_rollback_003_neg', 'zfs_rollback_004_neg']
	tags = ['functional', 'cli_root', 'zfs_rollback']

	[tests/functional/cli_root/zfs_send]
	tests = ['zfs_send_001_pos', 'zfs_send_002_pos', 'zfs_send_003_pos',
	'zfs_send_004_neg', 'zfs_send_005_pos', 'zfs_send_encrypted',
	'zfs_send_raw']
	tags = ['functional', 'cli_root', 'zfs_send']

	[tests/functional/cli_root/zfs_set]
	tests = ['cache_001_pos', 'cache_002_neg', 'canmount_001_pos',
	'canmount_002_pos', 'canmount_003_pos', 'canmount_004_pos',
	'checksum_001_pos', 'compression_001_pos', 'mountpoint_001_pos',
	'mountpoint_002_pos', 'user_property_002_pos',
	'share_mount_001_neg', 'snapdir_001_pos', 'onoffs_001_pos',
	'user_property_001_pos', 'user_property_003_neg', 'readonly_001_pos',
	'user_property_004_pos', 'version_001_neg',
	'zfs_set_003_neg', 'property_alias_001_pos',
	'zfs_set_keylocation', 'zfs_set_feature_activation', 'zfs_set_nomount']
	tags = ['functional', 'cli_root', 'zfs_set']

	[tests/functional/cli_root/zfs_snapshot]
	tests = ['zfs_snapshot_001_neg', 'zfs_snapshot_002_neg',
	'zfs_snapshot_003_neg', 'zfs_snapshot_006_pos', 'zfs_snapshot_007_neg']
	tags = ['functional', 'cli_root', 'zfs_snapshot']

	[tests/functional/cli_root/zfs_unload-key]
	tests = ['zfs_unload-key', 'zfs_unload-key_all', 'zfs_unload-key_recursive']
	tags = ['functional', 'cli_root', 'zfs_unload-key']

	[tests/functional/cli_root/zfs_unmount]
	tests = ['zfs_unmount_001_pos', 'zfs_unmount_002_pos', 'zfs_unmount_003_pos',
	'zfs_unmount_004_pos', 'zfs_unmount_007_neg', 'zfs_unmount_008_neg',
	'zfs_unmount_009_pos', 'zfs_unmount_unload_keys']
	tags = ['functional', 'cli_root', 'zfs_unmount']

	[tests/functional/cli_root/zfs_upgrade]
	tests = ['zfs_upgrade_001_pos', 'zfs_upgrade_002_pos', 'zfs_upgrade_006_neg',
	'zfs_upgrade_007_neg']
	tags = ['functional', 'cli_root', 'zfs_upgrade']

	[tests/functional/cli_root/zfs_wait]
	tests = ['zfs_wait_deleteq', 'zfs_wait_getsubopt']
	tags = ['functional', 'cli_root', 'zfs_wait']

	[tests/functional/cli_root/zpool]
	tests = ['zpool_001_neg', 'zpool_003_pos', 'zpool_colors']
	tags = ['functional', 'cli_root', 'zpool']

	[tests/functional/cli_root/zpool_add]
	tests = ['zpool_add_002_pos', 'zpool_add_003_pos',
	'zpool_add_004_pos', 'zpool_add_006_pos', 'zpool_add_007_neg',
	'zpool_add_008_neg', 'zpool_add_009_neg']
	tags = ['functional', 'cli_root', 'zpool_add']

	[tests/functional/cli_root/zpool_attach]
	tests = ['zpool_attach_001_neg']
	tags = ['functional', 'cli_root', 'zpool_attach']

	[tests/functional/cli_root/zpool_clear]
	tests = ['zpool_clear_002_neg']
	tags = ['functional', 'cli_root', 'zpool_clear']

	[tests/functional/cli_root/zpool_create]
	tests = ['zpool_create_001_pos', 'zpool_create_002_pos',
	'zpool_create_003_pos', 'zpool_create_004_pos', 'zpool_create_007_neg',
	'zpool_create_008_pos', 'zpool_create_010_neg', 'zpool_create_011_neg',
	'zpool_create_012_neg', 'zpool_create_014_neg', 'zpool_create_015_neg',
	'zpool_create_017_neg', 'zpool_create_018_pos', 'zpool_create_019_pos',
	'zpool_create_020_pos', 'zpool_create_021_pos', 'zpool_create_022_pos',
	'zpool_create_encrypted',
	'zpool_create_features_001_pos', 'zpool_create_features_002_pos',
	'zpool_create_features_003_pos', 'zpool_create_features_004_neg',
	'zpool_create_features_005_pos']
	tags = ['functional', 'cli_root', 'zpool_create']

	[tests/functional/cli_root/zpool_destroy]
	tests = ['zpool_destroy_001_pos', 'zpool_destroy_002_pos',
	'zpool_destroy_003_neg']
	pre =
	post =
	tags = ['functional', 'cli_root', 'zpool_destroy']

	[tests/functional/cli_root/zpool_detach]
	tests = ['zpool_detach_001_neg']
	tags = ['functional', 'cli_root', 'zpool_detach']

	[tests/functional/cli_root/zpool_events]
	tests = ['zpool_events_clear', 'zpool_events_follow', 'zpool_events_poolname']
	tags = ['functional', 'cli_root', 'zpool_events']

	[tests/functional/cli_root/zpool_export]
	tests = ['zpool_export_001_pos', 'zpool_export_002_pos', 'zpool_export_003_neg']
	tags = ['functional', 'cli_root', 'zpool_export']

	[tests/functional/cli_root/zpool_get]
	tests = ['zpool_get_001_pos', 'zpool_get_002_pos', 'zpool_get_003_pos',
	'zpool_get_004_neg', 'zpool_get_005_pos']
	tags = ['functional', 'cli_root', 'zpool_get']

	[tests/functional/cli_root/zpool_history]
	tests = ['zpool_history_001_neg', 'zpool_history_002_pos']
	tags = ['functional', 'cli_root', 'zpool_history']

	[tests/functional/cli_root/zpool_import]
	tests = ['zpool_import_003_pos', 'zpool_import_010_pos', 'zpool_import_011_neg',
	'zpool_import_014_pos', 'zpool_import_features_001_pos',
	'zpool_import_all_001_pos', 'zpool_import_encrypted']
	tags = ['functional', 'cli_root', 'zpool_import']

	[tests/functional/cli_root/zpool_labelclear]
	tests = ['zpool_labelclear_active', 'zpool_labelclear_exported',
	'zpool_labelclear_removed', 'zpool_labelclear_valid']
	pre =
	post =
	tags = ['functional', 'cli_root', 'zpool_labelclear']

	[tests/functional/cli_root/zpool_initialize]
	tests = ['zpool_initialize_online_offline']
	pre =
	tags = ['functional', 'cli_root', 'zpool_initialize']

	[tests/functional/cli_root/zpool_offline]
	tests = ['zpool_offline_001_pos', 'zpool_offline_002_neg']
	tags = ['functional', 'cli_root', 'zpool_offline']

	[tests/functional/cli_root/zpool_online]
	tests = ['zpool_online_001_pos', 'zpool_online_002_neg']
	tags = ['functional', 'cli_root', 'zpool_online']

	[tests/functional/cli_root/zpool_remove]
	tests = ['zpool_remove_001_neg', 'zpool_remove_002_pos',
	'zpool_remove_003_pos']
	tags = ['functional', 'cli_root', 'zpool_remove']

	[tests/functional/cli_root/zpool_replace]
	tests = ['zpool_replace_001_neg']
	tags = ['functional', 'cli_root', 'zpool_replace']

	[tests/functional/cli_root/zpool_resilver]
	tests = ['zpool_resilver_bad_args']
	tags = ['functional', 'cli_root', 'zpool_resilver']

	[tests/functional/cli_root/zpool_scrub]
	tests = ['zpool_scrub_001_neg', 'zpool_scrub_003_pos',
	'zpool_scrub_encrypted_unloaded', 'zpool_scrub_print_repairing',
	'zpool_scrub_offline_device', 'zpool_scrub_multiple_copies']
	tags = ['functional', 'cli_root', 'zpool_scrub']

	[tests/functional/cli_root/zpool_set]
	tests = ['zpool_set_001_pos', 'zpool_set_002_neg', 'zpool_set_003_neg',
	'zpool_set_ashift', 'zpool_set_features']
	tags = ['functional', 'cli_root', 'zpool_set']

	[tests/functional/cli_root/zpool_split]
	tests = ['zpool_split_cliargs', 'zpool_split_devices',
	'zpool_split_props', 'zpool_split_vdevs', 'zpool_split_indirect']
	tags = ['functional', 'cli_root', 'zpool_split']

	[tests/functional/cli_root/zpool_status]
	tests = ['zpool_status_001_pos', 'zpool_status_002_pos']
	tags = ['functional', 'cli_root', 'zpool_status']

	[tests/functional/cli_root/zpool_sync]
	tests = ['zpool_sync_002_neg']
	tags = ['functional', 'cli_root', 'zpool_sync']

	[tests/functional/cli_root/zpool_trim]
	tests = ['zpool_trim_attach_detach_add_remove', 'zpool_trim_neg',
	'zpool_trim_offline_export_import_online', 'zpool_trim_online_offline',
	'zpool_trim_rate_neg', 'zpool_trim_secure', 'zpool_trim_split',
	'zpool_trim_start_and_cancel_neg', 'zpool_trim_start_and_cancel_pos']
	tags = ['functional', 'zpool_trim']

	[tests/functional/cli_root/zpool_upgrade]
	tests = ['zpool_upgrade_001_pos', 'zpool_upgrade_003_pos',
	'zpool_upgrade_005_neg', 'zpool_upgrade_006_neg',
	'zpool_upgrade_009_neg']
	tags = ['functional', 'cli_root', 'zpool_upgrade']

	[tests/functional/cli_root/zpool_wait]
	tests = ['zpool_wait_no_activity', 'zpool_wait_usage']
	tags = ['functional', 'cli_root', 'zpool_wait']

	[tests/functional/cli_root/zpool_wait/scan]
	tests = ['zpool_wait_scrub_flag']
	tags = ['functional', 'cli_root', 'zpool_wait']

	[tests/functional/cli_user/misc]
	tests = ['zdb_001_neg', 'zfs_001_neg', 'zfs_allow_001_neg',
	'zfs_clone_001_neg', 'zfs_create_001_neg', 'zfs_destroy_001_neg',
	'zfs_get_001_neg', 'zfs_inherit_001_neg', 'zfs_mount_001_neg',
	'zfs_promote_001_neg', 'zfs_receive_001_neg', 'zfs_rename_001_neg',
	'zfs_rollback_001_neg', 'zfs_send_001_neg', 'zfs_set_001_neg',
	'zfs_snapshot_001_neg', 'zfs_unallow_001_neg',
	'zfs_unmount_001_neg', 'zfs_upgrade_001_neg',
	'zpool_001_neg', 'zpool_add_001_neg', 'zpool_attach_001_neg',
	'zpool_clear_001_neg', 'zpool_create_001_neg', 'zpool_destroy_001_neg',
	'zpool_detach_001_neg', 'zpool_export_001_neg', 'zpool_get_001_neg',
	'zpool_history_001_neg', 'zpool_offline_001_neg', 'zpool_online_001_neg',
	'zpool_remove_001_neg', 'zpool_scrub_001_neg', 'zpool_set_001_neg',
	'zpool_status_001_neg', 'zpool_upgrade_001_neg', 'arcstat_001_pos',
	'arc_summary_001_pos', 'arc_summary_002_neg', 'zpool_wait_privilege',
	'zilstat_001_pos']
	user =
	tags = ['functional', 'cli_user', 'misc']

	[tests/functional/cli_user/zpool_iostat]
	tests = ['zpool_iostat_001_neg', 'zpool_iostat_002_pos',
	'zpool_iostat_003_neg', 'zpool_iostat_004_pos',
	'zpool_iostat_-c_disable',
	'zpool_iostat_-c_homedir', 'zpool_iostat_-c_searchpath']
	user =
	tags = ['functional', 'cli_user', 'zpool_iostat']

	[tests/functional/cli_user/zpool_list]
	tests = ['zpool_list_001_pos', 'zpool_list_002_neg']
	user =
	tags = ['functional', 'cli_user', 'zpool_list']

	[tests/functional/compression]
	tests = ['compress_003_pos','compress_zstd_bswap']
	tags = ['functional', 'compression']

	[tests/functional/exec]
	tests = ['exec_001_pos', 'exec_002_neg']
	tags = ['functional', 'exec']

	[tests/functional/features/large_dnode]
	tests = ['large_dnode_003_pos', 'large_dnode_004_neg',
	'large_dnode_005_pos', 'large_dnode_007_neg']
	tags = ['functional', 'features', 'large_dnode']

	[tests/functional/grow]
	pre =
	post =
	tests = ['grow_pool_001_pos', 'grow_replicas_001_pos']
	tags = ['functional', 'grow']

	[tests/functional/history]
	tests = ['history_004_pos', 'history_005_neg', 'history_007_pos',
	'history_009_pos']
	tags = ['functional', 'history']

	[tests/functional/hkdf]
	pre =
	post =
	tests = ['hkdf_test']
	tags = ['functional', 'hkdf']

	[tests/functional/inuse]
	tests = ['inuse_004_pos', 'inuse_005_pos']
	post =
	tags = ['functional', 'inuse']

	[tests/functional/large_files]
	tests = ['large_files_001_pos', 'large_files_002_pos']
	tags = ['functional', 'large_files']

	[tests/functional/libzfs]
	tests = ['many_fds', 'libzfs_input']
	tags = ['functional', 'libzfs']

	[tests/functional/limits]
	tests = ['filesystem_count', 'snapshot_count']
	tags = ['functional', 'limits']

	[tests/functional/link_count]
	tests = ['link_count_root_inode']
	tags = ['functional', 'link_count']

	[tests/functional/log_spacemap]
	tests = ['log_spacemap_import_logs']
	pre =
	post =
	tags = ['functional', 'log_spacemap']

	[tests/functional/migration]
	tests = ['migration_001_pos', 'migration_002_pos', 'migration_003_pos',
	'migration_004_pos', 'migration_005_pos', 'migration_006_pos',
	'migration_007_pos', 'migration_008_pos', 'migration_009_pos',
	'migration_010_pos', 'migration_011_pos', 'migration_012_pos']
	tags = ['functional', 'migration']

	[tests/functional/mmap]
	tests = ['mmap_read_001_pos']
	tags = ['functional', 'mmap']

	[tests/functional/nestedfs]
	tests = ['nestedfs_001_pos']
	tags = ['functional', 'nestedfs']

	[tests/functional/nopwrite]
	tests = ['nopwrite_sync', 'nopwrite_volume']
	tags = ['functional', 'nopwrite']

	[tests/functional/pool_checkpoint]
	tests = ['checkpoint_conf_change', 'checkpoint_discard_many',
	'checkpoint_removal', 'checkpoint_sm_scale', 'checkpoint_twice']
	tags = ['functional', 'pool_checkpoint']
	timeout = 1800

	[tests/functional/poolversion]
	tests = ['poolversion_001_pos', 'poolversion_002_pos']
	tags = ['functional', 'poolversion']

	[tests/functional/redacted_send]
	tests = ['redacted_compressed', 'redacted_contents', 'redacted_deleted',
	'redacted_disabled_feature', 'redacted_incrementals',
	'redacted_largeblocks', 'redacted_mixed_recsize', 'redacted_negative',
	'redacted_origin', 'redacted_props', 'redacted_resume', 'redacted_size']
	tags = ['functional', 'redacted_send']

	[tests/functional/raidz]
	tests = ['raidz_001_neg']
	tags = ['functional', 'raidz']

	[tests/functional/refquota]
	tests = ['refquota_001_pos', 'refquota_002_pos', 'refquota_003_pos',
	'refquota_004_pos', 'refquota_005_pos', 'refquota_006_neg',
	'refquota_007_neg']
	tags = ['functional', 'refquota']

	[tests/functional/refreserv]
	tests = ['refreserv_001_pos', 'refreserv_002_pos', 'refreserv_003_pos',
	'refreserv_005_pos', 'refreserv_multi_raidz']
	tags = ['functional', 'refreserv']

	[tests/functional/removal]
	pre =
	tests = ['removal_all_vdev', 'removal_sanity', 'removal_with_dedup',
	'removal_with_ganging', 'removal_with_faulted']
	tags = ['functional', 'removal']

	[tests/functional/replacement]
	tests = ['rebuild_raidz']
	tags = ['functional', 'replacement']

	[tests/functional/reservation]
	tests = ['reservation_001_pos', 'reservation_002_pos', 'reservation_003_pos',
	'reservation_004_pos', 'reservation_005_pos', 'reservation_006_pos',
	'reservation_007_pos', 'reservation_008_pos', 'reservation_009_pos',
	'reservation_010_pos', 'reservation_011_pos', 'reservation_012_pos',
	'reservation_014_pos', 'reservation_015_pos',
	'reservation_016_pos', 'reservation_017_pos', 'reservation_018_pos',
	'reservation_019_pos', 'reservation_020_pos', 'reservation_021_neg',
	'reservation_022_pos']
	tags = ['functional', 'reservation']

	[tests/functional/rsend]
	tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos',
	'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', 'rsend_005_pos',
	'rsend_006_pos', 'rsend_009_pos', 'rsend_010_pos', 'rsend_011_pos',
	'rsend_014_pos', 'rsend_016_neg', 'send-c_verify_contents',
	'send-c_volume', 'send-c_zstreamdump', 'send-c_recv_dedup',
	'send-L_toggle', 'send_encrypted_hierarchy', 'send_encrypted_props',
	'send_encrypted_freeobjects',
	'send_encrypted_truncated_files', 'send_freeobjects', 'send_holds',
	'send_mixed_raw', 'send-wR_encrypted_zvol', 'send_partial_dataset',
	'send_invalid']
	tags = ['functional', 'rsend']

	[tests/functional/scrub_mirror]
	tests = ['scrub_mirror_001_pos', 'scrub_mirror_002_pos']
	tags = ['functional', 'scrub_mirror']

	[tests/functional/slog]
	tests = ['slog_008_neg', 'slog_009_neg', 'slog_010_neg']
	tags = ['functional', 'slog']

	[tests/functional/snapshot]
	tests = ['clone_001_pos', 'rollback_001_pos', 'rollback_002_pos',
	'snapshot_001_pos', 'snapshot_002_pos', 'snapshot_003_pos',
	'snapshot_004_pos', 'snapshot_005_pos', 'snapshot_006_pos',
	'snapshot_007_pos', 'snapshot_008_pos', 'snapshot_009_pos',
	'snapshot_010_pos', 'snapshot_011_pos', 'snapshot_012_pos',
	'snapshot_013_pos', 'snapshot_014_pos', 'snapshot_017_pos',
	'snapshot_018_pos']
	tags = ['functional', 'snapshot']

	[tests/functional/snapused]
	tests = ['snapused_002_pos', 'snapused_004_pos', 'snapused_005_pos']
	tags = ['functional', 'snapused']

	[tests/functional/sparse]
	tests = ['sparse_001_pos']
	tags = ['functional', 'sparse']

	[tests/functional/suid]
	tests = ['suid_write_to_suid', 'suid_write_to_sgid', 'suid_write_to_suid_sgid',
	'suid_write_to_none']
	tags = ['functional', 'suid']

	[tests/functional/append]
	tests = ['threadsappend_001_pos']
	tags = ['functional', 'threadsappend']

	[tests/functional/truncate]
	tests = ['truncate_001_pos', 'truncate_002_pos']
	tags = ['functional', 'truncate']

	[tests/functional/upgrade]
	tests = ['upgrade_userobj_001_pos', 'upgrade_readonly_pool']
	tags = ['functional', 'upgrade']

	[tests/functional/vdev_zaps]
	tests = ['vdev_zaps_001_pos', 'vdev_zaps_003_pos', 'vdev_zaps_004_pos',
	'vdev_zaps_005_pos', 'vdev_zaps_006_pos']
	tags = ['functional', 'vdev_zaps']

	[tests/functional/xattr]
	tests = ['xattr_001_pos', 'xattr_002_neg', 'xattr_003_neg', 'xattr_004_pos',
	'xattr_005_pos', 'xattr_006_pos', 'xattr_007_neg',
	'xattr_011_pos', 'xattr_013_pos', 'xattr_compat']
	tags = ['functional', 'xattr']

	[tests/functional/zvol/zvol_ENOSPC]
	tests = ['zvol_ENOSPC_001_pos']
	tags = ['functional', 'zvol', 'zvol_ENOSPC']

	[tests/functional/zvol/zvol_cli]
	tests = ['zvol_cli_001_pos', 'zvol_cli_002_pos', 'zvol_cli_003_neg']
	tags = ['functional', 'zvol', 'zvol_cli']

	[tests/functional/zvol/zvol_swap]
	tests = ['zvol_swap_001_pos', 'zvol_swap_002_pos']
	tags = ['functional', 'zvol', 'zvol_swap']

	[tests/functional/zpool_influxdb]
	tests = ['zpool_influxdb']
	tags = ['functional', 'zpool_influxdb']

	[tests/functional/pyzfs]
	tests = ['pyzfs_unittest']
	pre =
	post =
	tags = ['functional', 'pyzfs']
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am b/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am
	index e2824ee065e8..cc66d762f3c2 100644
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am
	@@ -1,2105 +1,2110 @@
	CLEANFILES =
	dist_noinst_DATA =
	include $(top_srcdir)/config/Substfiles.am


	datadir_zfs_tests_testsdir = $(datadir)/$(PACKAGE)/zfs-tests/tests
	nobase_dist_datadir_zfs_tests_tests_DATA = \
	perf/nfs-sample.cfg \
	perf/perf.shlib \
	\
	perf/fio/mkfiles.fio \
	perf/fio/random_reads.fio \
	perf/fio/random_readwrite.fio \
	perf/fio/random_readwrite_fixed.fio \
	perf/fio/random_writes.fio \
	perf/fio/sequential_reads.fio \
	perf/fio/sequential_readwrite.fio \
	perf/fio/sequential_writes.fio

	nobase_dist_datadir_zfs_tests_tests_SCRIPTS = \
	perf/regression/random_reads.ksh \
	perf/regression/random_readwrite.ksh \
	perf/regression/random_readwrite_fixed.ksh \
	perf/regression/random_writes.ksh \
	perf/regression/random_writes_zil.ksh \
	perf/regression/sequential_reads_arc_cached_clone.ksh \
	perf/regression/sequential_reads_arc_cached.ksh \
	perf/regression/sequential_reads_dbuf_cached.ksh \
	perf/regression/sequential_reads.ksh \
	perf/regression/sequential_writes.ksh \
	perf/regression/setup.ksh \
	\
	perf/scripts/prefetch_io.sh

	# These lists can be regenerated by running make regen-tests at the root, or, on a clean source:
	# find functional/ ! -type d ! -name .gitignore ! -name .dirstamp ! -name '.Po' ! -executable -name '.in' \| sort \| sed 's/\.in$//;s/^/\t/;$!s/$/ \\/'
	# find functional/ ! -type d ! -name .gitignore ! -name .dirstamp ! -name '.Po' -executable -name '.in' \| sort \| sed 's/\.in$//;s/^/\t/;$!s/$/ \\/'
	# find functional/ ! -type d ! -name .gitignore ! -name .dirstamp ! -name '.Po' ! -name '.in' ! -name '*.c' \| grep -Fe /simd -e /tmpfile \| sort \| sed 's/^/\t/;$!s/$/ \\/'
	# find functional/ ! -type d ! -name .gitignore ! -name .dirstamp ! -name '.Po' ! -executable ! -name '.in' ! -name '*.c' \| grep -vFe /simd -e /tmpfile \| sort \| sed 's/^/\t/;$!s/$/ \\/'
	# find functional/ ! -type d ! -name .gitignore ! -name .dirstamp ! -name '.Po' -executable ! -name '.in' ! -name '*.c' \| grep -vFe /simd -e /tmpfile \| sort \| sed 's/^/\t/;$!s/$/ \\/'
	#
	# simd and tmpfile are Linux-only and not installed elsewhere
	#
	# C programs are specced in ../Makefile.am above as part of the main Makefile

	find_common := find functional/ ! -type d ! -name .gitignore ! -name .dirstamp ! -name '*.Po'
	regen:
	@$(MAKE) -C $(top_builddir) clean
	@$(MAKE) clean
	$(SED) $(ac_inplace) '/^# -- >8 --/q' Makefile.am
	echo >> Makefile.am
	echo 'nobase_nodist_datadir_zfs_tests_tests_DATA = \' >> Makefile.am
	$(find_common) ! -executable -name '*.in' \| sort \| sed 's/\.in$$//;s/^/\t/;$$!s/$$/ \\/' >> Makefile.am
	echo 'nobase_nodist_datadir_zfs_tests_tests_SCRIPTS = \' >> Makefile.am
	$(find_common) -executable -name '*.in' \| sort \| sed 's/\.in$$//;s/^/\t/;$$!s/$$/ \\/' >> Makefile.am
	echo >> Makefile.am
	echo 'SUBSTFILES += $$(nobase_nodist_datadir_zfs_tests_tests_DATA) $$(nobase_nodist_datadir_zfs_tests_tests_SCRIPTS)' >> Makefile.am
	echo >> Makefile.am
	echo 'if BUILD_LINUX' >> Makefile.am
	echo 'nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \' >> Makefile.am
	$(find_common) ! -name '.in' ! -name '.c' \| grep -Fe /simd -e /tmpfile \| sort \| sed 's/^/\t/;$$!s/$$/ \\/' >> Makefile.am
	echo 'endif' >> Makefile.am
	echo >> Makefile.am
	echo 'nobase_dist_datadir_zfs_tests_tests_DATA += \' >> Makefile.am
	$(find_common) ! -executable ! -name '.in' ! -name '.c' \| grep -vFe /simd -e /tmpfile \| sort \| sed 's/^/\t/;$$!s/$$/ \\/' >> Makefile.am
	echo >> Makefile.am
	echo 'nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \' >> Makefile.am
	$(find_common) -executable ! -name '.in' ! -name '.c' \| grep -vFe /simd -e /tmpfile \| sort \| sed 's/^/\t/;$$!s/$$/ \\/' >> Makefile.am

	# -- >8 --

	nobase_nodist_datadir_zfs_tests_tests_DATA = \
	functional/pam/utilities.kshlib
	nobase_nodist_datadir_zfs_tests_tests_SCRIPTS = \
	functional/pyzfs/pyzfs_unittest.ksh

	SUBSTFILES += $(nobase_nodist_datadir_zfs_tests_tests_DATA) $(nobase_nodist_datadir_zfs_tests_tests_SCRIPTS)

	if BUILD_LINUX
	nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
	functional/simd/simd_supported.ksh \
	functional/tmpfile/cleanup.ksh \
	functional/tmpfile/setup.ksh
	endif

	nobase_dist_datadir_zfs_tests_tests_DATA += \
	functional/acl/acl.cfg \
	functional/acl/acl_common.kshlib \
	functional/alloc_class/alloc_class.cfg \
	functional/alloc_class/alloc_class.kshlib \
	functional/atime/atime.cfg \
	functional/atime/atime_common.kshlib \
	functional/bclone/bclone.cfg \
	functional/bclone/bclone_common.kshlib \
	functional/bclone/bclone_corner_cases.kshlib \
	functional/block_cloning/block_cloning.kshlib \
	functional/cache/cache.cfg \
	functional/cache/cache.kshlib \
	functional/cachefile/cachefile.cfg \
	functional/cachefile/cachefile.kshlib \
	functional/casenorm/casenorm.cfg \
	functional/casenorm/casenorm.kshlib \
	functional/channel_program/channel_common.kshlib \
	functional/channel_program/lua_core/tst.args_to_lua.out \
	functional/channel_program/lua_core/tst.args_to_lua.zcp \
	functional/channel_program/lua_core/tst.divide_by_zero.err \
	functional/channel_program/lua_core/tst.divide_by_zero.zcp \
	functional/channel_program/lua_core/tst.exists.zcp \
	functional/channel_program/lua_core/tst.large_prog.out \
	functional/channel_program/lua_core/tst.large_prog.zcp \
	functional/channel_program/lua_core/tst.lib_base.lua \
	functional/channel_program/lua_core/tst.lib_coroutine.lua \
	functional/channel_program/lua_core/tst.lib_strings.lua \
	functional/channel_program/lua_core/tst.lib_table.lua \
	functional/channel_program/lua_core/tst.nested_neg.zcp \
	functional/channel_program/lua_core/tst.nested_pos.zcp \
	functional/channel_program/lua_core/tst.recursive.zcp \
	functional/channel_program/lua_core/tst.return_large.zcp \
	functional/channel_program/lua_core/tst.return_recursive_table.zcp \
	functional/channel_program/lua_core/tst.stack_gsub.err \
	functional/channel_program/lua_core/tst.stack_gsub.zcp \
	functional/channel_program/lua_core/tst.timeout.zcp \
	functional/channel_program/synctask_core/tst.bookmark.copy.zcp \
	functional/channel_program/synctask_core/tst.bookmark.create.zcp \
	functional/channel_program/synctask_core/tst.get_index_props.out \
	functional/channel_program/synctask_core/tst.get_index_props.zcp \
	functional/channel_program/synctask_core/tst.get_number_props.out \
	functional/channel_program/synctask_core/tst.get_number_props.zcp \
	functional/channel_program/synctask_core/tst.get_string_props.out \
	functional/channel_program/synctask_core/tst.get_string_props.zcp \
	functional/channel_program/synctask_core/tst.promote_conflict.zcp \
	functional/channel_program/synctask_core/tst.set_props.zcp \
	functional/channel_program/synctask_core/tst.snapshot_destroy.zcp \
	functional/channel_program/synctask_core/tst.snapshot_neg.zcp \
	functional/channel_program/synctask_core/tst.snapshot_recursive.zcp \
	functional/channel_program/synctask_core/tst.snapshot_rename.zcp \
	functional/channel_program/synctask_core/tst.snapshot_simple.zcp \
	functional/checksum/default.cfg \
	functional/clean_mirror/clean_mirror_common.kshlib \
	functional/clean_mirror/default.cfg \
	functional/cli_root/cli_common.kshlib \
	functional/cli_root/zfs_copies/zfs_copies.cfg \
	functional/cli_root/zfs_copies/zfs_copies.kshlib \
	functional/cli_root/zfs_create/properties.kshlib \
	functional/cli_root/zfs_create/zfs_create.cfg \
	functional/cli_root/zfs_create/zfs_create_common.kshlib \
	functional/cli_root/zfs_destroy/zfs_destroy.cfg \
	functional/cli_root/zfs_destroy/zfs_destroy_common.kshlib \
	functional/cli_root/zfs_get/zfs_get_common.kshlib \
	functional/cli_root/zfs_get/zfs_get_list_d.kshlib \
	functional/cli_root/zfs_jail/jail.conf \
	functional/cli_root/zfs_load-key/HEXKEY \
	functional/cli_root/zfs_load-key/PASSPHRASE \
	functional/cli_root/zfs_load-key/RAWKEY \
	functional/cli_root/zfs_load-key/zfs_load-key.cfg \
	functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib \
	functional/cli_root/zfs_mount/zfs_mount.cfg \
	functional/cli_root/zfs_mount/zfs_mount.kshlib \
	functional/cli_root/zfs_promote/zfs_promote.cfg \
	functional/cli_root/zfs_receive/zstd_test_data.txt \
	functional/cli_root/zfs_rename/zfs_rename.cfg \
	functional/cli_root/zfs_rename/zfs_rename.kshlib \
	functional/cli_root/zfs_rollback/zfs_rollback.cfg \
	functional/cli_root/zfs_rollback/zfs_rollback_common.kshlib \
	functional/cli_root/zfs_send/zfs_send.cfg \
	functional/cli_root/zfs_set/zfs_set_common.kshlib \
	functional/cli_root/zfs_share/zfs_share.cfg \
	functional/cli_root/zfs_snapshot/zfs_snapshot.cfg \
	functional/cli_root/zfs_unmount/zfs_unmount.cfg \
	functional/cli_root/zfs_unmount/zfs_unmount.kshlib \
	functional/cli_root/zfs_upgrade/zfs_upgrade.kshlib \
	functional/cli_root/zfs_wait/zfs_wait.kshlib \
	functional/cli_root/zpool_add/zpool_add.cfg \
	functional/cli_root/zpool_add/zpool_add.kshlib \
	functional/cli_root/zpool_clear/zpool_clear.cfg \
	functional/cli_root/zpool_create/draidcfg.gz \
	functional/cli_root/zpool_create/zpool_create.cfg \
	functional/cli_root/zpool_create/zpool_create.shlib \
	functional/cli_root/zpool_destroy/zpool_destroy.cfg \
	functional/cli_root/zpool_events/zpool_events.cfg \
	functional/cli_root/zpool_events/zpool_events.kshlib \
	functional/cli_root/zpool_expand/zpool_expand.cfg \
	functional/cli_root/zpool_export/zpool_export.cfg \
	functional/cli_root/zpool_export/zpool_export.kshlib \
	functional/cli_root/zpool_get/vdev_get.cfg \
	functional/cli_root/zpool_get/zpool_get.cfg \
	functional/cli_root/zpool_get/zpool_get_parsable.cfg \
	functional/cli_root/zpool_import/blockfiles/cryptv0.dat.bz2 \
	functional/cli_root/zpool_import/blockfiles/missing_ivset.dat.bz2 \
	functional/cli_root/zpool_import/blockfiles/unclean_export.dat.bz2 \
	functional/cli_root/zpool_import/zpool_import.cfg \
	functional/cli_root/zpool_import/zpool_import.kshlib \
	functional/cli_root/zpool_initialize/zpool_initialize.kshlib \
	functional/cli_root/zpool_labelclear/labelclear.cfg \
	functional/cli_root/zpool_remove/zpool_remove.cfg \
	functional/cli_root/zpool_reopen/zpool_reopen.cfg \
	functional/cli_root/zpool_reopen/zpool_reopen.shlib \
	functional/cli_root/zpool_resilver/zpool_resilver.cfg \
	functional/cli_root/zpool_scrub/zpool_scrub.cfg \
	functional/cli_root/zpool_split/zpool_split.cfg \
	functional/cli_root/zpool_trim/zpool_trim.kshlib \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-broken-mirror1.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-broken-mirror2.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v10.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v11.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v12.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v13.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v14.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v15.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1mirror1.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1mirror2.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1mirror3.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1raidz1.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1raidz2.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1raidz3.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1stripe1.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1stripe2.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1stripe3.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2mirror1.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2mirror2.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2mirror3.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2raidz1.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2raidz2.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2raidz3.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2stripe1.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2stripe2.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2stripe3.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3hotspare1.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3hotspare2.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3hotspare3.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3mirror1.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3mirror2.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3mirror3.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3raidz1.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3raidz21.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3raidz22.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3raidz23.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3raidz2.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3raidz3.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3stripe1.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3stripe2.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3stripe3.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v4.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v5.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v6.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v7.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v8.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v999.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v9.dat.bz2 \
	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-vBROKEN.dat.bz2 \
	functional/cli_root/zpool_upgrade/zpool_upgrade.cfg \
	functional/cli_root/zpool_upgrade/zpool_upgrade.kshlib \
	functional/cli_root/zpool_wait/zpool_wait.kshlib \
	functional/cli_root/zhack/library.kshlib \
	functional/cli_user/misc/misc.cfg \
	functional/cli_user/zfs_list/zfs_list.cfg \
	functional/cli_user/zfs_list/zfs_list.kshlib \
	functional/compression/compress.cfg \
	functional/compression/testpool_zstd.tar.gz \
	functional/deadman/deadman.cfg \
	functional/delegate/delegate.cfg \
	functional/delegate/delegate_common.kshlib \
	functional/devices/devices.cfg \
	functional/devices/devices_common.kshlib \
	functional/events/events.cfg \
	functional/events/events_common.kshlib \
	functional/fault/fault.cfg \
	functional/grow/grow.cfg \
	functional/history/history.cfg \
	functional/history/history_common.kshlib \
	functional/history/i386.migratedpool.DAT.Z \
	functional/history/i386.orig_history.txt \
	functional/history/sparc.migratedpool.DAT.Z \
	functional/history/sparc.orig_history.txt \
	functional/history/zfs-pool-v4.dat.Z \
	functional/inheritance/config001.cfg \
	functional/inheritance/config002.cfg \
	functional/inheritance/config003.cfg \
	functional/inheritance/config004.cfg \
	functional/inheritance/config005.cfg \
	functional/inheritance/config006.cfg \
	functional/inheritance/config007.cfg \
	functional/inheritance/config008.cfg \
	functional/inheritance/config009.cfg \
	functional/inheritance/config010.cfg \
	functional/inheritance/config011.cfg \
	functional/inheritance/config012.cfg \
	functional/inheritance/config013.cfg \
	functional/inheritance/config014.cfg \
	functional/inheritance/config015.cfg \
	functional/inheritance/config016.cfg \
	functional/inheritance/config017.cfg \
	functional/inheritance/config018.cfg \
	functional/inheritance/config019.cfg \
	functional/inheritance/config020.cfg \
	functional/inheritance/config021.cfg \
	functional/inheritance/config022.cfg \
	functional/inheritance/config023.cfg \
	functional/inheritance/config024.cfg \
	functional/inheritance/inherit.kshlib \
	functional/inheritance/README.config \
	functional/inheritance/README.state \
	functional/inheritance/state001.cfg \
	functional/inheritance/state002.cfg \
	functional/inheritance/state003.cfg \
	functional/inheritance/state004.cfg \
	functional/inheritance/state005.cfg \
	functional/inheritance/state006.cfg \
	functional/inheritance/state007.cfg \
	functional/inheritance/state008.cfg \
	functional/inheritance/state009.cfg \
	functional/inheritance/state010.cfg \
	functional/inheritance/state011.cfg \
	functional/inheritance/state012.cfg \
	functional/inheritance/state013.cfg \
	functional/inheritance/state014.cfg \
	functional/inheritance/state015.cfg \
	functional/inheritance/state016.cfg \
	functional/inheritance/state017.cfg \
	functional/inheritance/state018.cfg \
	functional/inheritance/state019.cfg \
	functional/inheritance/state020.cfg \
	functional/inheritance/state021.cfg \
	functional/inheritance/state022.cfg \
	functional/inheritance/state023.cfg \
	functional/inheritance/state024.cfg \
	functional/inuse/inuse.cfg \
	functional/io/io.cfg \
	functional/l2arc/l2arc.cfg \
	functional/largest_pool/largest_pool.cfg \
	functional/migration/migration.cfg \
	functional/migration/migration.kshlib \
	functional/mmap/mmap.cfg \
	functional/mmp/mmp.cfg \
	functional/mmp/mmp.kshlib \
	functional/mv_files/mv_files.cfg \
	functional/mv_files/mv_files_common.kshlib \
	functional/nopwrite/nopwrite.shlib \
	functional/no_space/enospc.cfg \
	functional/online_offline/online_offline.cfg \
	functional/pool_checkpoint/pool_checkpoint.kshlib \
	functional/projectquota/projectquota.cfg \
	functional/projectquota/projectquota_common.kshlib \
	functional/quota/quota.cfg \
	functional/quota/quota.kshlib \
	functional/redacted_send/redacted.cfg \
	functional/redacted_send/redacted.kshlib \
	functional/redundancy/redundancy.cfg \
	functional/redundancy/redundancy.kshlib \
	functional/refreserv/refreserv.cfg \
	functional/removal/removal.kshlib \
	functional/replacement/replacement.cfg \
	functional/reservation/reservation.cfg \
	functional/reservation/reservation.shlib \
	functional/rsend/dedup_encrypted_zvol.bz2 \
	functional/rsend/dedup_encrypted_zvol.zsend.bz2 \
	functional/rsend/dedup.zsend.bz2 \
	functional/rsend/fs.tar.gz \
	functional/rsend/rsend.cfg \
	functional/rsend/rsend.kshlib \
	functional/scrub_mirror/default.cfg \
	functional/scrub_mirror/scrub_mirror_common.kshlib \
	functional/slog/slog.cfg \
	functional/slog/slog.kshlib \
	functional/snapshot/snapshot.cfg \
	functional/snapused/snapused.kshlib \
	functional/sparse/sparse.cfg \
	functional/trim/trim.cfg \
	functional/trim/trim.kshlib \
	functional/truncate/truncate.cfg \
	functional/upgrade/upgrade_common.kshlib \
	functional/user_namespace/user_namespace.cfg \
	functional/user_namespace/user_namespace_common.kshlib \
	functional/userquota/13709_reproducer.bz2 \
	functional/userquota/userquota.cfg \
	functional/userquota/userquota_common.kshlib \
	functional/vdev_zaps/vdev_zaps.kshlib \
	functional/xattr/xattr.cfg \
	functional/xattr/xattr_common.kshlib \
	functional/zvol/zvol.cfg \
	functional/zvol/zvol_cli/zvol_cli.cfg \
	functional/zvol/zvol_common.shlib \
	functional/zvol/zvol_ENOSPC/zvol_ENOSPC.cfg \
	functional/zvol/zvol_misc/zvol_misc_common.kshlib \
	functional/zvol/zvol_swap/zvol_swap.cfg \
	functional/idmap_mount/idmap_mount.cfg \
	functional/idmap_mount/idmap_mount_common.kshlib

	nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
	functional/acl/off/cleanup.ksh \
	functional/acl/off/dosmode.ksh \
	functional/acl/off/posixmode.ksh \
	functional/acl/off/setup.ksh \
	functional/acl/posix/cleanup.ksh \
	functional/acl/posix/posix_001_pos.ksh \
	functional/acl/posix/posix_002_pos.ksh \
	functional/acl/posix/posix_003_pos.ksh \
	functional/acl/posix/posix_004_pos.ksh \
	functional/acl/posix-sa/cleanup.ksh \
	functional/acl/posix-sa/posix_001_pos.ksh \
	functional/acl/posix-sa/posix_002_pos.ksh \
	functional/acl/posix-sa/posix_003_pos.ksh \
	functional/acl/posix-sa/posix_004_pos.ksh \
	functional/acl/posix-sa/setup.ksh \
	functional/acl/posix/setup.ksh \
	functional/alloc_class/alloc_class_001_pos.ksh \
	functional/alloc_class/alloc_class_002_neg.ksh \
	functional/alloc_class/alloc_class_003_pos.ksh \
	functional/alloc_class/alloc_class_004_pos.ksh \
	functional/alloc_class/alloc_class_005_pos.ksh \
	functional/alloc_class/alloc_class_006_pos.ksh \
	functional/alloc_class/alloc_class_007_pos.ksh \
	functional/alloc_class/alloc_class_008_pos.ksh \
	functional/alloc_class/alloc_class_009_pos.ksh \
	functional/alloc_class/alloc_class_010_pos.ksh \
	functional/alloc_class/alloc_class_011_neg.ksh \
	functional/alloc_class/alloc_class_012_pos.ksh \
	functional/alloc_class/alloc_class_013_pos.ksh \
	functional/alloc_class/alloc_class_014_neg.ksh \
	functional/alloc_class/alloc_class_015_pos.ksh \
	functional/alloc_class/cleanup.ksh \
	functional/alloc_class/setup.ksh \
	functional/append/file_append.ksh \
	functional/append/threadsappend_001_pos.ksh \
	functional/append/cleanup.ksh \
	functional/append/setup.ksh \
	functional/arc/arcstats_runtime_tuning.ksh \
	functional/arc/cleanup.ksh \
	functional/arc/dbufstats_001_pos.ksh \
	functional/arc/dbufstats_002_pos.ksh \
	functional/arc/dbufstats_003_pos.ksh \
	functional/arc/setup.ksh \
	functional/atime/atime_001_pos.ksh \
	functional/atime/atime_002_neg.ksh \
	functional/atime/atime_003_pos.ksh \
	functional/atime/cleanup.ksh \
	functional/atime/root_atime_off.ksh \
	functional/atime/root_atime_on.ksh \
	functional/atime/root_relatime_on.ksh \
	functional/atime/setup.ksh \
	functional/bclone/bclone_crossfs_corner_cases.ksh \
	functional/bclone/bclone_crossfs_corner_cases_limited.ksh \
	functional/bclone/bclone_crossfs_data.ksh \
	functional/bclone/bclone_crossfs_embedded.ksh \
	functional/bclone/bclone_crossfs_hole.ksh \
	functional/bclone/bclone_diffprops_all.ksh \
	functional/bclone/bclone_diffprops_checksum.ksh \
	functional/bclone/bclone_diffprops_compress.ksh \
	functional/bclone/bclone_diffprops_copies.ksh \
	functional/bclone/bclone_diffprops_recordsize.ksh \
	functional/bclone/bclone_prop_sync.ksh \
	functional/bclone/bclone_samefs_corner_cases.ksh \
	functional/bclone/bclone_samefs_corner_cases_limited.ksh \
	functional/bclone/bclone_samefs_data.ksh \
	functional/bclone/bclone_samefs_embedded.ksh \
	functional/bclone/bclone_samefs_hole.ksh \
	functional/bclone/cleanup.ksh \
	functional/bclone/setup.ksh \
	functional/block_cloning/cleanup.ksh \
	functional/block_cloning/setup.ksh \
	functional/block_cloning/block_cloning_clone_mmap_cached.ksh \
	functional/block_cloning/block_cloning_clone_mmap_write.ksh \
	functional/block_cloning/block_cloning_copyfilerange_cross_dataset.ksh \
	functional/block_cloning/block_cloning_copyfilerange_fallback.ksh \
	functional/block_cloning/block_cloning_copyfilerange_fallback_same_txg.ksh \
	functional/block_cloning/block_cloning_copyfilerange.ksh \
	functional/block_cloning/block_cloning_copyfilerange_partial.ksh \
	functional/block_cloning/block_cloning_disabled_copyfilerange.ksh \
	functional/block_cloning/block_cloning_disabled_ficlone.ksh \
	functional/block_cloning/block_cloning_disabled_ficlonerange.ksh \
	functional/block_cloning/block_cloning_ficlone.ksh \
	functional/block_cloning/block_cloning_ficlonerange.ksh \
	functional/block_cloning/block_cloning_ficlonerange_partial.ksh \
	functional/block_cloning/block_cloning_cross_enc_dataset.ksh \
	functional/block_cloning/block_cloning_replay.ksh \
	functional/block_cloning/block_cloning_replay_encrypted.ksh \
	functional/block_cloning/block_cloning_lwb_buffer_overflow.ksh \
	functional/bootfs/bootfs_001_pos.ksh \
	functional/bootfs/bootfs_002_neg.ksh \
	functional/bootfs/bootfs_003_pos.ksh \
	functional/bootfs/bootfs_004_neg.ksh \
	functional/bootfs/bootfs_005_neg.ksh \
	functional/bootfs/bootfs_006_pos.ksh \
	functional/bootfs/bootfs_007_pos.ksh \
	functional/bootfs/bootfs_008_pos.ksh \
	functional/bootfs/cleanup.ksh \
	functional/bootfs/setup.ksh \
	functional/btree/btree_negative.ksh \
	functional/btree/btree_positive.ksh \
	functional/cache/cache_001_pos.ksh \
	functional/cache/cache_002_pos.ksh \
	functional/cache/cache_003_pos.ksh \
	functional/cache/cache_004_neg.ksh \
	functional/cache/cache_005_neg.ksh \
	functional/cache/cache_006_pos.ksh \
	functional/cache/cache_007_neg.ksh \
	functional/cache/cache_008_neg.ksh \
	functional/cache/cache_009_pos.ksh \
	functional/cache/cache_010_pos.ksh \
	functional/cache/cache_011_pos.ksh \
	functional/cache/cache_012_pos.ksh \
	functional/cache/cleanup.ksh \
	functional/cachefile/cachefile_001_pos.ksh \
	functional/cachefile/cachefile_002_pos.ksh \
	functional/cachefile/cachefile_003_pos.ksh \
	functional/cachefile/cachefile_004_pos.ksh \
	functional/cachefile/cleanup.ksh \
	functional/cachefile/setup.ksh \
	functional/cache/setup.ksh \
	functional/casenorm/case_all_values.ksh \
	functional/casenorm/cleanup.ksh \
	functional/casenorm/insensitive_formd_delete.ksh \
	functional/casenorm/insensitive_formd_lookup.ksh \
	functional/casenorm/insensitive_none_delete.ksh \
	functional/casenorm/insensitive_none_lookup.ksh \
	functional/casenorm/mixed_create_failure.ksh \
	functional/casenorm/mixed_formd_delete.ksh \
	functional/casenorm/mixed_formd_lookup_ci.ksh \
	functional/casenorm/mixed_formd_lookup.ksh \
	functional/casenorm/mixed_none_delete.ksh \
	functional/casenorm/mixed_none_lookup_ci.ksh \
	functional/casenorm/mixed_none_lookup.ksh \
	functional/casenorm/norm_all_values.ksh \
	functional/casenorm/sensitive_formd_delete.ksh \
	functional/casenorm/sensitive_formd_lookup.ksh \
	functional/casenorm/sensitive_none_delete.ksh \
	functional/casenorm/sensitive_none_lookup.ksh \
	functional/casenorm/setup.ksh \
	functional/channel_program/lua_core/cleanup.ksh \
	functional/channel_program/lua_core/setup.ksh \
	functional/channel_program/lua_core/tst.args_to_lua.ksh \
	functional/channel_program/lua_core/tst.divide_by_zero.ksh \
	functional/channel_program/lua_core/tst.exists.ksh \
	functional/channel_program/lua_core/tst.integer_illegal.ksh \
	functional/channel_program/lua_core/tst.integer_overflow.ksh \
	functional/channel_program/lua_core/tst.language_functions_neg.ksh \
	functional/channel_program/lua_core/tst.language_functions_pos.ksh \
	functional/channel_program/lua_core/tst.large_prog.ksh \
	functional/channel_program/lua_core/tst.libraries.ksh \
	functional/channel_program/lua_core/tst.memory_limit.ksh \
	functional/channel_program/lua_core/tst.nested_neg.ksh \
	functional/channel_program/lua_core/tst.nested_pos.ksh \
	functional/channel_program/lua_core/tst.nvlist_to_lua.ksh \
	functional/channel_program/lua_core/tst.recursive_neg.ksh \
	functional/channel_program/lua_core/tst.recursive_pos.ksh \
	functional/channel_program/lua_core/tst.return_large.ksh \
	functional/channel_program/lua_core/tst.return_nvlist_neg.ksh \
	functional/channel_program/lua_core/tst.return_nvlist_pos.ksh \
	functional/channel_program/lua_core/tst.return_recursive_table.ksh \
	functional/channel_program/lua_core/tst.stack_gsub.ksh \
	functional/channel_program/lua_core/tst.timeout.ksh \
	functional/channel_program/synctask_core/cleanup.ksh \
	functional/channel_program/synctask_core/setup.ksh \
	functional/channel_program/synctask_core/tst.bookmark.copy.ksh \
	functional/channel_program/synctask_core/tst.bookmark.create.ksh \
	functional/channel_program/synctask_core/tst.destroy_fs.ksh \
	functional/channel_program/synctask_core/tst.destroy_snap.ksh \
	functional/channel_program/synctask_core/tst.get_count_and_limit.ksh \
	functional/channel_program/synctask_core/tst.get_index_props.ksh \
	functional/channel_program/synctask_core/tst.get_mountpoint.ksh \
	functional/channel_program/synctask_core/tst.get_neg.ksh \
	functional/channel_program/synctask_core/tst.get_number_props.ksh \
	functional/channel_program/synctask_core/tst.get_string_props.ksh \
	functional/channel_program/synctask_core/tst.get_type.ksh \
	functional/channel_program/synctask_core/tst.get_userquota.ksh \
	functional/channel_program/synctask_core/tst.get_written.ksh \
	functional/channel_program/synctask_core/tst.inherit.ksh \
	functional/channel_program/synctask_core/tst.list_bookmarks.ksh \
	functional/channel_program/synctask_core/tst.list_children.ksh \
	functional/channel_program/synctask_core/tst.list_clones.ksh \
	functional/channel_program/synctask_core/tst.list_holds.ksh \
	functional/channel_program/synctask_core/tst.list_snapshots.ksh \
	functional/channel_program/synctask_core/tst.list_system_props.ksh \
	functional/channel_program/synctask_core/tst.list_user_props.ksh \
	functional/channel_program/synctask_core/tst.parse_args_neg.ksh \
	functional/channel_program/synctask_core/tst.promote_conflict.ksh \
	functional/channel_program/synctask_core/tst.promote_multiple.ksh \
	functional/channel_program/synctask_core/tst.promote_simple.ksh \
	functional/channel_program/synctask_core/tst.rollback_mult.ksh \
	functional/channel_program/synctask_core/tst.rollback_one.ksh \
	functional/channel_program/synctask_core/tst.set_props.ksh \
	functional/channel_program/synctask_core/tst.snapshot_destroy.ksh \
	functional/channel_program/synctask_core/tst.snapshot_neg.ksh \
	functional/channel_program/synctask_core/tst.snapshot_recursive.ksh \
	functional/channel_program/synctask_core/tst.snapshot_rename.ksh \
	functional/channel_program/synctask_core/tst.snapshot_simple.ksh \
	functional/channel_program/synctask_core/tst.terminate_by_signal.ksh \
	functional/chattr/chattr_001_pos.ksh \
	functional/chattr/chattr_002_neg.ksh \
	functional/chattr/cleanup.ksh \
	functional/chattr/setup.ksh \
	functional/checksum/cleanup.ksh \
	functional/checksum/filetest_001_pos.ksh \
	functional/checksum/filetest_002_pos.ksh \
	functional/checksum/run_blake3_test.ksh \
	functional/checksum/run_edonr_test.ksh \
	functional/checksum/run_sha2_test.ksh \
	functional/checksum/run_skein_test.ksh \
	functional/checksum/setup.ksh \
	functional/clean_mirror/clean_mirror_001_pos.ksh \
	functional/clean_mirror/clean_mirror_002_pos.ksh \
	functional/clean_mirror/clean_mirror_003_pos.ksh \
	functional/clean_mirror/clean_mirror_004_pos.ksh \
	functional/clean_mirror/cleanup.ksh \
	functional/clean_mirror/setup.ksh \
	functional/cli_root/zdb/zdb_002_pos.ksh \
	functional/cli_root/zdb/zdb_003_pos.ksh \
	functional/cli_root/zdb/zdb_004_pos.ksh \
	functional/cli_root/zdb/zdb_005_pos.ksh \
	functional/cli_root/zdb/zdb_006_pos.ksh \
	functional/cli_root/zdb/zdb_args_neg.ksh \
	functional/cli_root/zdb/zdb_args_pos.ksh \
	functional/cli_root/zdb/zdb_backup.ksh \
	functional/cli_root/zdb/zdb_block_size_histogram.ksh \
	functional/cli_root/zdb/zdb_checksum.ksh \
	functional/cli_root/zdb/zdb_decompress.ksh \
	functional/cli_root/zdb/zdb_decompress_zstd.ksh \
	functional/cli_root/zdb/zdb_display_block.ksh \
	functional/cli_root/zdb/zdb_encrypted.ksh \
	functional/cli_root/zdb/zdb_label_checksum.ksh \
	functional/cli_root/zdb/zdb_object_range_neg.ksh \
	functional/cli_root/zdb/zdb_object_range_pos.ksh \
	functional/cli_root/zdb/zdb_objset_id.ksh \
	functional/cli_root/zdb/zdb_recover_2.ksh \
	functional/cli_root/zdb/zdb_recover.ksh \
	functional/cli_root/zfs_bookmark/cleanup.ksh \
	functional/cli_root/zfs_bookmark/setup.ksh \
	functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh \
	functional/cli_root/zfs_change-key/cleanup.ksh \
	functional/cli_root/zfs_change-key/setup.ksh \
	functional/cli_root/zfs_change-key/zfs_change-key_child.ksh \
	functional/cli_root/zfs_change-key/zfs_change-key_clones.ksh \
	functional/cli_root/zfs_change-key/zfs_change-key_format.ksh \
	functional/cli_root/zfs_change-key/zfs_change-key_inherit.ksh \
	functional/cli_root/zfs_change-key/zfs_change-key.ksh \
	functional/cli_root/zfs_change-key/zfs_change-key_load.ksh \
	functional/cli_root/zfs_change-key/zfs_change-key_location.ksh \
	functional/cli_root/zfs_change-key/zfs_change-key_pbkdf2iters.ksh \
	functional/cli_root/zfs/cleanup.ksh \
	functional/cli_root/zfs_clone/cleanup.ksh \
	functional/cli_root/zfs_clone/setup.ksh \
	functional/cli_root/zfs_clone/zfs_clone_001_neg.ksh \
	functional/cli_root/zfs_clone/zfs_clone_002_pos.ksh \
	functional/cli_root/zfs_clone/zfs_clone_003_pos.ksh \
	functional/cli_root/zfs_clone/zfs_clone_004_pos.ksh \
	functional/cli_root/zfs_clone/zfs_clone_005_pos.ksh \
	functional/cli_root/zfs_clone/zfs_clone_006_pos.ksh \
	functional/cli_root/zfs_clone/zfs_clone_007_pos.ksh \
	functional/cli_root/zfs_clone/zfs_clone_008_neg.ksh \
	functional/cli_root/zfs_clone/zfs_clone_009_neg.ksh \
	functional/cli_root/zfs_clone/zfs_clone_010_pos.ksh \
	functional/cli_root/zfs_clone/zfs_clone_deeply_nested.ksh \
	functional/cli_root/zfs_clone/zfs_clone_encrypted.ksh \
	functional/cli_root/zfs_clone/zfs_clone_rm_nested.ksh \
	functional/cli_root/zfs_copies/cleanup.ksh \
	functional/cli_root/zfs_copies/setup.ksh \
	functional/cli_root/zfs_copies/zfs_copies_001_pos.ksh \
	functional/cli_root/zfs_copies/zfs_copies_002_pos.ksh \
	functional/cli_root/zfs_copies/zfs_copies_003_pos.ksh \
	functional/cli_root/zfs_copies/zfs_copies_004_neg.ksh \
	functional/cli_root/zfs_copies/zfs_copies_005_neg.ksh \
	functional/cli_root/zfs_copies/zfs_copies_006_pos.ksh \
	functional/cli_root/zfs_create/cleanup.ksh \
	functional/cli_root/zfs_create/setup.ksh \
	functional/cli_root/zfs_create/zfs_create_001_pos.ksh \
	functional/cli_root/zfs_create/zfs_create_002_pos.ksh \
	functional/cli_root/zfs_create/zfs_create_003_pos.ksh \
	functional/cli_root/zfs_create/zfs_create_004_pos.ksh \
	functional/cli_root/zfs_create/zfs_create_005_pos.ksh \
	functional/cli_root/zfs_create/zfs_create_006_pos.ksh \
	functional/cli_root/zfs_create/zfs_create_007_pos.ksh \
	functional/cli_root/zfs_create/zfs_create_008_neg.ksh \
	functional/cli_root/zfs_create/zfs_create_009_neg.ksh \
	functional/cli_root/zfs_create/zfs_create_010_neg.ksh \
	functional/cli_root/zfs_create/zfs_create_011_pos.ksh \
	functional/cli_root/zfs_create/zfs_create_012_pos.ksh \
	functional/cli_root/zfs_create/zfs_create_013_pos.ksh \
	functional/cli_root/zfs_create/zfs_create_014_pos.ksh \
	functional/cli_root/zfs_create/zfs_create_crypt_combos.ksh \
	functional/cli_root/zfs_create/zfs_create_dryrun.ksh \
	functional/cli_root/zfs_create/zfs_create_encrypted.ksh \
	functional/cli_root/zfs_create/zfs_create_nomount.ksh \
	functional/cli_root/zfs_create/zfs_create_verbose.ksh \
	functional/cli_root/zfs_destroy/cleanup.ksh \
	functional/cli_root/zfs_destroy/setup.ksh \
	functional/cli_root/zfs_destroy/zfs_clone_livelist_condense_and_disable.ksh \
	functional/cli_root/zfs_destroy/zfs_clone_livelist_condense_races.ksh \
	functional/cli_root/zfs_destroy/zfs_clone_livelist_dedup.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_001_pos.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_002_pos.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_003_pos.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_004_pos.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_005_neg.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_006_neg.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_007_neg.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_008_pos.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_009_pos.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_010_pos.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_011_pos.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_012_pos.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_013_neg.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_014_pos.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_015_pos.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_016_pos.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_clone_livelist.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_dev_removal_condense.ksh \
	functional/cli_root/zfs_destroy/zfs_destroy_dev_removal.ksh \
	functional/cli_root/zfs_diff/cleanup.ksh \
	functional/cli_root/zfs_diff/setup.ksh \
	functional/cli_root/zfs_diff/zfs_diff_changes.ksh \
	functional/cli_root/zfs_diff/zfs_diff_cliargs.ksh \
	functional/cli_root/zfs_diff/zfs_diff_encrypted.ksh \
	functional/cli_root/zfs_diff/zfs_diff_mangle.ksh \
	functional/cli_root/zfs_diff/zfs_diff_timestamp.ksh \
	functional/cli_root/zfs_diff/zfs_diff_types.ksh \
	functional/cli_root/zfs_get/cleanup.ksh \
	functional/cli_root/zfs_get/setup.ksh \
	functional/cli_root/zfs_get/zfs_get_001_pos.ksh \
	functional/cli_root/zfs_get/zfs_get_002_pos.ksh \
	functional/cli_root/zfs_get/zfs_get_003_pos.ksh \
	functional/cli_root/zfs_get/zfs_get_004_pos.ksh \
	functional/cli_root/zfs_get/zfs_get_005_neg.ksh \
	functional/cli_root/zfs_get/zfs_get_006_neg.ksh \
	functional/cli_root/zfs_get/zfs_get_007_neg.ksh \
	functional/cli_root/zfs_get/zfs_get_008_pos.ksh \
	functional/cli_root/zfs_get/zfs_get_009_pos.ksh \
	functional/cli_root/zfs_get/zfs_get_010_neg.ksh \
	functional/cli_root/zfs_ids_to_path/cleanup.ksh \
	functional/cli_root/zfs_ids_to_path/setup.ksh \
	functional/cli_root/zfs_ids_to_path/zfs_ids_to_path_001_pos.ksh \
	functional/cli_root/zfs_inherit/cleanup.ksh \
	functional/cli_root/zfs_inherit/setup.ksh \
	functional/cli_root/zfs_inherit/zfs_inherit_001_neg.ksh \
	functional/cli_root/zfs_inherit/zfs_inherit_002_neg.ksh \
	functional/cli_root/zfs_inherit/zfs_inherit_003_pos.ksh \
	functional/cli_root/zfs_inherit/zfs_inherit_mountpoint.ksh \
	functional/cli_root/zfs_jail/cleanup.ksh \
	functional/cli_root/zfs_jail/setup.ksh \
	functional/cli_root/zfs_jail/zfs_jail_001_pos.ksh \
	functional/cli_root/zfs_load-key/cleanup.ksh \
	functional/cli_root/zfs_load-key/setup.ksh \
	functional/cli_root/zfs_load-key/zfs_load-key_all.ksh \
	functional/cli_root/zfs_load-key/zfs_load-key_file.ksh \
	functional/cli_root/zfs_load-key/zfs_load-key_https.ksh \
	functional/cli_root/zfs_load-key/zfs_load-key.ksh \
	functional/cli_root/zfs_load-key/zfs_load-key_location.ksh \
	functional/cli_root/zfs_load-key/zfs_load-key_noop.ksh \
	functional/cli_root/zfs_load-key/zfs_load-key_recursive.ksh \
	functional/cli_root/zfs_mount/cleanup.ksh \
	functional/cli_root/zfs_mount/setup.ksh \
	functional/cli_root/zfs_mount/zfs_mount_001_pos.ksh \
	functional/cli_root/zfs_mount/zfs_mount_002_pos.ksh \
	functional/cli_root/zfs_mount/zfs_mount_003_pos.ksh \
	functional/cli_root/zfs_mount/zfs_mount_004_pos.ksh \
	functional/cli_root/zfs_mount/zfs_mount_005_pos.ksh \
	functional/cli_root/zfs_mount/zfs_mount_006_pos.ksh \
	functional/cli_root/zfs_mount/zfs_mount_007_pos.ksh \
	functional/cli_root/zfs_mount/zfs_mount_008_pos.ksh \
	functional/cli_root/zfs_mount/zfs_mount_009_neg.ksh \
	functional/cli_root/zfs_mount/zfs_mount_010_neg.ksh \
	functional/cli_root/zfs_mount/zfs_mount_011_neg.ksh \
	functional/cli_root/zfs_mount/zfs_mount_012_pos.ksh \
	functional/cli_root/zfs_mount/zfs_mount_013_pos.ksh \
	functional/cli_root/zfs_mount/zfs_mount_014_neg.ksh \
	functional/cli_root/zfs_mount/zfs_mount_all_001_pos.ksh \
	functional/cli_root/zfs_mount/zfs_mount_all_fail.ksh \
	functional/cli_root/zfs_mount/zfs_mount_all_mountpoints.ksh \
	functional/cli_root/zfs_mount/zfs_mount_encrypted.ksh \
	+ functional/cli_root/zfs_mount/zfs_mount_recursive.ksh \
	functional/cli_root/zfs_mount/zfs_mount_remount.ksh \
	functional/cli_root/zfs_mount/zfs_mount_test_race.ksh \
	functional/cli_root/zfs_mount/zfs_multi_mount.ksh \
	functional/cli_root/zfs_program/cleanup.ksh \
	functional/cli_root/zfs_program/setup.ksh \
	functional/cli_root/zfs_program/zfs_program_json.ksh \
	functional/cli_root/zfs_promote/cleanup.ksh \
	functional/cli_root/zfs_promote/setup.ksh \
	functional/cli_root/zfs_promote/zfs_promote_001_pos.ksh \
	functional/cli_root/zfs_promote/zfs_promote_002_pos.ksh \
	functional/cli_root/zfs_promote/zfs_promote_003_pos.ksh \
	functional/cli_root/zfs_promote/zfs_promote_004_pos.ksh \
	functional/cli_root/zfs_promote/zfs_promote_005_pos.ksh \
	functional/cli_root/zfs_promote/zfs_promote_006_neg.ksh \
	functional/cli_root/zfs_promote/zfs_promote_007_neg.ksh \
	functional/cli_root/zfs_promote/zfs_promote_008_pos.ksh \
	functional/cli_root/zfs_promote/zfs_promote_encryptionroot.ksh \
	functional/cli_root/zfs_property/cleanup.ksh \
	functional/cli_root/zfs_property/setup.ksh \
	functional/cli_root/zfs_property/zfs_written_property_001_pos.ksh \
	functional/cli_root/zfs_receive/cleanup.ksh \
	functional/cli_root/zfs_receive/receive-o-x_props_aliases.ksh \
	functional/cli_root/zfs_receive/receive-o-x_props_override.ksh \
	functional/cli_root/zfs_receive/setup.ksh \
	functional/cli_root/zfs_receive/zfs_receive_001_pos.ksh \
	functional/cli_root/zfs_receive/zfs_receive_002_pos.ksh \
	functional/cli_root/zfs_receive/zfs_receive_003_pos.ksh \
	functional/cli_root/zfs_receive/zfs_receive_004_neg.ksh \
	functional/cli_root/zfs_receive/zfs_receive_005_neg.ksh \
	functional/cli_root/zfs_receive/zfs_receive_006_pos.ksh \
	functional/cli_root/zfs_receive/zfs_receive_007_neg.ksh \
	functional/cli_root/zfs_receive/zfs_receive_008_pos.ksh \
	functional/cli_root/zfs_receive/zfs_receive_009_neg.ksh \
	functional/cli_root/zfs_receive/zfs_receive_010_pos.ksh \
	functional/cli_root/zfs_receive/zfs_receive_011_pos.ksh \
	functional/cli_root/zfs_receive/zfs_receive_012_pos.ksh \
	functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh \
	functional/cli_root/zfs_receive/zfs_receive_014_pos.ksh \
	functional/cli_root/zfs_receive/zfs_receive_015_pos.ksh \
	functional/cli_root/zfs_receive/zfs_receive_016_pos.ksh \
	functional/cli_root/zfs_receive/zfs_receive_-e.ksh \
	functional/cli_root/zfs_receive/zfs_receive_from_encrypted.ksh \
	functional/cli_root/zfs_receive/zfs_receive_from_zstd.ksh \
	functional/cli_root/zfs_receive/zfs_receive_new_props.ksh \
	functional/cli_root/zfs_receive/zfs_receive_raw_-d.ksh \
	functional/cli_root/zfs_receive/zfs_receive_raw_incremental.ksh \
	functional/cli_root/zfs_receive/zfs_receive_raw.ksh \
	functional/cli_root/zfs_receive/zfs_receive_to_encrypted.ksh \
	functional/cli_root/zfs_receive/zfs_receive_-wR-encrypted-mix.ksh \
	functional/cli_root/zfs_receive/zfs_receive_corrective.ksh \
	functional/cli_root/zfs_receive/zfs_receive_compressed_corrective.ksh \
	functional/cli_root/zfs_receive/zfs_receive_large_block_corrective.ksh \
	functional/cli_root/zfs_rename/cleanup.ksh \
	functional/cli_root/zfs_rename/setup.ksh \
	functional/cli_root/zfs_rename/zfs_rename_001_pos.ksh \
	functional/cli_root/zfs_rename/zfs_rename_002_pos.ksh \
	functional/cli_root/zfs_rename/zfs_rename_003_pos.ksh \
	functional/cli_root/zfs_rename/zfs_rename_004_neg.ksh \
	functional/cli_root/zfs_rename/zfs_rename_005_neg.ksh \
	functional/cli_root/zfs_rename/zfs_rename_006_pos.ksh \
	functional/cli_root/zfs_rename/zfs_rename_007_pos.ksh \
	functional/cli_root/zfs_rename/zfs_rename_008_pos.ksh \
	functional/cli_root/zfs_rename/zfs_rename_009_neg.ksh \
	functional/cli_root/zfs_rename/zfs_rename_010_neg.ksh \
	functional/cli_root/zfs_rename/zfs_rename_011_pos.ksh \
	functional/cli_root/zfs_rename/zfs_rename_012_neg.ksh \
	functional/cli_root/zfs_rename/zfs_rename_013_pos.ksh \
	functional/cli_root/zfs_rename/zfs_rename_014_neg.ksh \
	functional/cli_root/zfs_rename/zfs_rename_encrypted_child.ksh \
	functional/cli_root/zfs_rename/zfs_rename_mountpoint.ksh \
	functional/cli_root/zfs_rename/zfs_rename_nounmount.ksh \
	functional/cli_root/zfs_rename/zfs_rename_to_encrypted.ksh \
	functional/cli_root/zfs_reservation/cleanup.ksh \
	functional/cli_root/zfs_reservation/setup.ksh \
	functional/cli_root/zfs_reservation/zfs_reservation_001_pos.ksh \
	functional/cli_root/zfs_reservation/zfs_reservation_002_pos.ksh \
	functional/cli_root/zfs_rollback/cleanup.ksh \
	functional/cli_root/zfs_rollback/setup.ksh \
	functional/cli_root/zfs_rollback/zfs_rollback_001_pos.ksh \
	functional/cli_root/zfs_rollback/zfs_rollback_002_pos.ksh \
	functional/cli_root/zfs_rollback/zfs_rollback_003_neg.ksh \
	functional/cli_root/zfs_rollback/zfs_rollback_004_neg.ksh \
	functional/cli_root/zfs_send/cleanup.ksh \
	functional/cli_root/zfs_send/setup.ksh \
	functional/cli_root/zfs_send/zfs_send_001_pos.ksh \
	functional/cli_root/zfs_send/zfs_send_002_pos.ksh \
	functional/cli_root/zfs_send/zfs_send_003_pos.ksh \
	functional/cli_root/zfs_send/zfs_send_004_neg.ksh \
	functional/cli_root/zfs_send/zfs_send_005_pos.ksh \
	functional/cli_root/zfs_send/zfs_send_006_pos.ksh \
	functional/cli_root/zfs_send/zfs_send_007_pos.ksh \
	functional/cli_root/zfs_send/zfs_send-b.ksh \
	functional/cli_root/zfs_send/zfs_send_encrypted.ksh \
	functional/cli_root/zfs_send/zfs_send_encrypted_unloaded.ksh \
	functional/cli_root/zfs_send/zfs_send_raw.ksh \
	functional/cli_root/zfs_send/zfs_send_skip_missing.ksh \
	functional/cli_root/zfs_send/zfs_send_sparse.ksh \
	functional/cli_root/zfs_set/cache_001_pos.ksh \
	functional/cli_root/zfs_set/cache_002_neg.ksh \
	functional/cli_root/zfs_set/canmount_001_pos.ksh \
	functional/cli_root/zfs_set/canmount_002_pos.ksh \
	functional/cli_root/zfs_set/canmount_003_pos.ksh \
	functional/cli_root/zfs_set/canmount_004_pos.ksh \
	functional/cli_root/zfs_set/checksum_001_pos.ksh \
	functional/cli_root/zfs_set/cleanup.ksh \
	functional/cli_root/zfs_set/compression_001_pos.ksh \
	functional/cli_root/zfs_set/mountpoint_001_pos.ksh \
	functional/cli_root/zfs_set/mountpoint_002_pos.ksh \
	functional/cli_root/zfs_set/mountpoint_003_pos.ksh \
	functional/cli_root/zfs_set/onoffs_001_pos.ksh \
	functional/cli_root/zfs_set/property_alias_001_pos.ksh \
	functional/cli_root/zfs_set/readonly_001_pos.ksh \
	functional/cli_root/zfs_set/reservation_001_neg.ksh \
	functional/cli_root/zfs_set/ro_props_001_pos.ksh \
	functional/cli_root/zfs_set/setup.ksh \
	functional/cli_root/zfs_set/share_mount_001_neg.ksh \
	functional/cli_root/zfs_set/snapdir_001_pos.ksh \
	functional/cli_root/zfs/setup.ksh \
	functional/cli_root/zfs_set/user_property_001_pos.ksh \
	functional/cli_root/zfs_set/user_property_002_pos.ksh \
	functional/cli_root/zfs_set/user_property_003_neg.ksh \
	functional/cli_root/zfs_set/user_property_004_pos.ksh \
	functional/cli_root/zfs_set/version_001_neg.ksh \
	functional/cli_root/zfs_set/zfs_set_001_neg.ksh \
	functional/cli_root/zfs_set/zfs_set_002_neg.ksh \
	functional/cli_root/zfs_set/zfs_set_003_neg.ksh \
	functional/cli_root/zfs_set/zfs_set_feature_activation.ksh \
	functional/cli_root/zfs_set/zfs_set_keylocation.ksh \
	functional/cli_root/zfs_set/zfs_set_nomount.ksh \
	functional/cli_root/zfs_share/cleanup.ksh \
	functional/cli_root/zfs_share/setup.ksh \
	functional/cli_root/zfs_share/zfs_share_001_pos.ksh \
	functional/cli_root/zfs_share/zfs_share_002_pos.ksh \
	functional/cli_root/zfs_share/zfs_share_003_pos.ksh \
	functional/cli_root/zfs_share/zfs_share_004_pos.ksh \
	functional/cli_root/zfs_share/zfs_share_005_pos.ksh \
	functional/cli_root/zfs_share/zfs_share_006_pos.ksh \
	functional/cli_root/zfs_share/zfs_share_007_neg.ksh \
	functional/cli_root/zfs_share/zfs_share_008_neg.ksh \
	functional/cli_root/zfs_share/zfs_share_009_neg.ksh \
	functional/cli_root/zfs_share/zfs_share_010_neg.ksh \
	functional/cli_root/zfs_share/zfs_share_011_pos.ksh \
	functional/cli_root/zfs_share/zfs_share_012_pos.ksh \
	functional/cli_root/zfs_share/zfs_share_013_pos.ksh \
	functional/cli_root/zfs_share/zfs_share_concurrent_shares.ksh \
	functional/cli_root/zfs_share/zfs_share_after_mount.ksh \
	functional/cli_root/zfs_snapshot/cleanup.ksh \
	functional/cli_root/zfs_snapshot/setup.ksh \
	functional/cli_root/zfs_snapshot/zfs_snapshot_001_neg.ksh \
	functional/cli_root/zfs_snapshot/zfs_snapshot_002_neg.ksh \
	functional/cli_root/zfs_snapshot/zfs_snapshot_003_neg.ksh \
	functional/cli_root/zfs_snapshot/zfs_snapshot_004_neg.ksh \
	functional/cli_root/zfs_snapshot/zfs_snapshot_005_neg.ksh \
	functional/cli_root/zfs_snapshot/zfs_snapshot_006_pos.ksh \
	functional/cli_root/zfs_snapshot/zfs_snapshot_007_neg.ksh \
	functional/cli_root/zfs_snapshot/zfs_snapshot_008_neg.ksh \
	functional/cli_root/zfs_snapshot/zfs_snapshot_009_pos.ksh \
	functional/cli_root/zfs_sysfs/cleanup.ksh \
	functional/cli_root/zfs_sysfs/setup.ksh \
	functional/cli_root/zfs_sysfs/zfeature_set_unsupported.ksh \
	functional/cli_root/zfs_sysfs/zfs_get_unsupported.ksh \
	functional/cli_root/zfs_sysfs/zfs_set_unsupported.ksh \
	functional/cli_root/zfs_sysfs/zfs_sysfs_live.ksh \
	functional/cli_root/zfs_sysfs/zpool_get_unsupported.ksh \
	functional/cli_root/zfs_sysfs/zpool_set_unsupported.ksh \
	functional/cli_root/zfs_unload-key/cleanup.ksh \
	functional/cli_root/zfs_unload-key/setup.ksh \
	functional/cli_root/zfs_unload-key/zfs_unload-key_all.ksh \
	functional/cli_root/zfs_unload-key/zfs_unload-key.ksh \
	functional/cli_root/zfs_unload-key/zfs_unload-key_recursive.ksh \
	functional/cli_root/zfs_unmount/cleanup.ksh \
	functional/cli_root/zfs_unmount/setup.ksh \
	functional/cli_root/zfs_unmount/zfs_unmount_001_pos.ksh \
	functional/cli_root/zfs_unmount/zfs_unmount_002_pos.ksh \
	functional/cli_root/zfs_unmount/zfs_unmount_003_pos.ksh \
	functional/cli_root/zfs_unmount/zfs_unmount_004_pos.ksh \
	functional/cli_root/zfs_unmount/zfs_unmount_005_pos.ksh \
	functional/cli_root/zfs_unmount/zfs_unmount_006_pos.ksh \
	functional/cli_root/zfs_unmount/zfs_unmount_007_neg.ksh \
	functional/cli_root/zfs_unmount/zfs_unmount_008_neg.ksh \
	functional/cli_root/zfs_unmount/zfs_unmount_009_pos.ksh \
	functional/cli_root/zfs_unmount/zfs_unmount_all_001_pos.ksh \
	functional/cli_root/zfs_unmount/zfs_unmount_nested.ksh \
	functional/cli_root/zfs_unmount/zfs_unmount_unload_keys.ksh \
	functional/cli_root/zfs_unshare/cleanup.ksh \
	functional/cli_root/zfs_unshare/setup.ksh \
	functional/cli_root/zfs_unshare/zfs_unshare_001_pos.ksh \
	functional/cli_root/zfs_unshare/zfs_unshare_002_pos.ksh \
	functional/cli_root/zfs_unshare/zfs_unshare_003_pos.ksh \
	functional/cli_root/zfs_unshare/zfs_unshare_004_neg.ksh \
	functional/cli_root/zfs_unshare/zfs_unshare_005_neg.ksh \
	functional/cli_root/zfs_unshare/zfs_unshare_006_pos.ksh \
	functional/cli_root/zfs_unshare/zfs_unshare_007_pos.ksh \
	functional/cli_root/zfs_unshare/zfs_unshare_008_pos.ksh \
	functional/cli_root/zfs_upgrade/cleanup.ksh \
	functional/cli_root/zfs_upgrade/setup.ksh \
	functional/cli_root/zfs_upgrade/zfs_upgrade_001_pos.ksh \
	functional/cli_root/zfs_upgrade/zfs_upgrade_002_pos.ksh \
	functional/cli_root/zfs_upgrade/zfs_upgrade_003_pos.ksh \
	functional/cli_root/zfs_upgrade/zfs_upgrade_004_pos.ksh \
	functional/cli_root/zfs_upgrade/zfs_upgrade_005_pos.ksh \
	functional/cli_root/zfs_upgrade/zfs_upgrade_006_neg.ksh \
	functional/cli_root/zfs_upgrade/zfs_upgrade_007_neg.ksh \
	functional/cli_root/zfs_wait/cleanup.ksh \
	functional/cli_root/zfs_wait/setup.ksh \
	functional/cli_root/zfs_wait/zfs_wait_deleteq.ksh \
	functional/cli_root/zfs_wait/zfs_wait_getsubopt.ksh \
	functional/cli_root/zfs/zfs_001_neg.ksh \
	functional/cli_root/zfs/zfs_002_pos.ksh \
	functional/cli_root/zfs/zfs_003_neg.ksh \
	functional/cli_root/zhack/zhack_label_repair_001.ksh \
	functional/cli_root/zhack/zhack_label_repair_002.ksh \
	functional/cli_root/zhack/zhack_label_repair_003.ksh \
	functional/cli_root/zhack/zhack_label_repair_004.ksh \
	functional/cli_root/zpool_add/add_nested_replacing_spare.ksh \
	functional/cli_root/zpool_add/add-o_ashift.ksh \
	functional/cli_root/zpool_add/add_prop_ashift.ksh \
	functional/cli_root/zpool_add/cleanup.ksh \
	functional/cli_root/zpool_add/setup.ksh \
	+ functional/cli_root/zpool_add/zpool_add--allow-ashift-mismatch.ksh \
	functional/cli_root/zpool_add/zpool_add_001_pos.ksh \
	functional/cli_root/zpool_add/zpool_add_002_pos.ksh \
	functional/cli_root/zpool_add/zpool_add_003_pos.ksh \
	functional/cli_root/zpool_add/zpool_add_004_pos.ksh \
	functional/cli_root/zpool_add/zpool_add_005_pos.ksh \
	functional/cli_root/zpool_add/zpool_add_006_pos.ksh \
	functional/cli_root/zpool_add/zpool_add_007_neg.ksh \
	functional/cli_root/zpool_add/zpool_add_008_neg.ksh \
	functional/cli_root/zpool_add/zpool_add_009_neg.ksh \
	functional/cli_root/zpool_add/zpool_add_010_pos.ksh \
	functional/cli_root/zpool_add/zpool_add_dryrun_output.ksh \
	functional/cli_root/zpool_attach/attach-o_ashift.ksh \
	functional/cli_root/zpool_attach/cleanup.ksh \
	functional/cli_root/zpool_attach/setup.ksh \
	functional/cli_root/zpool_attach/zpool_attach_001_neg.ksh \
	functional/cli_root/zpool/cleanup.ksh \
	functional/cli_root/zpool_clear/cleanup.ksh \
	functional/cli_root/zpool_clear/setup.ksh \
	functional/cli_root/zpool_clear/zpool_clear_001_pos.ksh \
	functional/cli_root/zpool_clear/zpool_clear_002_neg.ksh \
	functional/cli_root/zpool_clear/zpool_clear_003_neg.ksh \
	functional/cli_root/zpool_clear/zpool_clear_readonly.ksh \
	functional/cli_root/zpool_create/cleanup.ksh \
	functional/cli_root/zpool_create/create-o_ashift.ksh \
	functional/cli_root/zpool_create/setup.ksh \
	functional/cli_root/zpool_create/zpool_create_001_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_002_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_003_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_004_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_005_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_006_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_007_neg.ksh \
	functional/cli_root/zpool_create/zpool_create_008_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_009_neg.ksh \
	functional/cli_root/zpool_create/zpool_create_010_neg.ksh \
	functional/cli_root/zpool_create/zpool_create_011_neg.ksh \
	functional/cli_root/zpool_create/zpool_create_012_neg.ksh \
	functional/cli_root/zpool_create/zpool_create_014_neg.ksh \
	functional/cli_root/zpool_create/zpool_create_015_neg.ksh \
	functional/cli_root/zpool_create/zpool_create_016_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_017_neg.ksh \
	functional/cli_root/zpool_create/zpool_create_018_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_019_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_020_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_021_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_022_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_023_neg.ksh \
	functional/cli_root/zpool_create/zpool_create_024_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_crypt_combos.ksh \
	functional/cli_root/zpool_create/zpool_create_draid_001_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_draid_002_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_draid_003_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_draid_004_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_dryrun_output.ksh \
	functional/cli_root/zpool_create/zpool_create_encrypted.ksh \
	functional/cli_root/zpool_create/zpool_create_features_001_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_features_002_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_features_003_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_features_004_neg.ksh \
	functional/cli_root/zpool_create/zpool_create_features_005_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_features_006_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_features_007_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_features_008_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_features_009_pos.ksh \
	functional/cli_root/zpool_create/zpool_create_tempname.ksh \
	functional/cli_root/zpool_destroy/zpool_destroy_001_pos.ksh \
	functional/cli_root/zpool_destroy/zpool_destroy_002_pos.ksh \
	functional/cli_root/zpool_destroy/zpool_destroy_003_neg.ksh \
	functional/cli_root/zpool_detach/cleanup.ksh \
	functional/cli_root/zpool_detach/setup.ksh \
	functional/cli_root/zpool_detach/zpool_detach_001_neg.ksh \
	functional/cli_root/zpool_events/cleanup.ksh \
	functional/cli_root/zpool_events/setup.ksh \
	functional/cli_root/zpool_events/zpool_events_clear.ksh \
	functional/cli_root/zpool_events/zpool_events_clear_retained.ksh \
	functional/cli_root/zpool_events/zpool_events_cliargs.ksh \
	functional/cli_root/zpool_events/zpool_events_duplicates.ksh \
	functional/cli_root/zpool_events/zpool_events_errors.ksh \
	functional/cli_root/zpool_events/zpool_events_follow.ksh \
	functional/cli_root/zpool_events/zpool_events_poolname.ksh \
	functional/cli_root/zpool_expand/cleanup.ksh \
	functional/cli_root/zpool_expand/setup.ksh \
	functional/cli_root/zpool_expand/zpool_expand_001_pos.ksh \
	functional/cli_root/zpool_expand/zpool_expand_002_pos.ksh \
	functional/cli_root/zpool_expand/zpool_expand_003_neg.ksh \
	functional/cli_root/zpool_expand/zpool_expand_004_pos.ksh \
	functional/cli_root/zpool_expand/zpool_expand_005_pos.ksh \
	functional/cli_root/zpool_export/cleanup.ksh \
	functional/cli_root/zpool_export/setup.ksh \
	functional/cli_root/zpool_export/zpool_export_001_pos.ksh \
	functional/cli_root/zpool_export/zpool_export_002_pos.ksh \
	functional/cli_root/zpool_export/zpool_export_003_neg.ksh \
	functional/cli_root/zpool_export/zpool_export_004_pos.ksh \
	functional/cli_root/zpool_get/cleanup.ksh \
	functional/cli_root/zpool_get/setup.ksh \
	functional/cli_root/zpool_get/vdev_get_001_pos.ksh \
	functional/cli_root/zpool_get/zpool_get_001_pos.ksh \
	functional/cli_root/zpool_get/zpool_get_002_pos.ksh \
	functional/cli_root/zpool_get/zpool_get_003_pos.ksh \
	functional/cli_root/zpool_get/zpool_get_004_neg.ksh \
	functional/cli_root/zpool_get/zpool_get_005_pos.ksh \
	functional/cli_root/zpool_history/cleanup.ksh \
	functional/cli_root/zpool_history/setup.ksh \
	functional/cli_root/zpool_history/zpool_history_001_neg.ksh \
	functional/cli_root/zpool_history/zpool_history_002_pos.ksh \
	functional/cli_root/zpool_import/cleanup.ksh \
	functional/cli_root/zpool_import/import_cachefile_device_added.ksh \
	functional/cli_root/zpool_import/import_cachefile_device_removed.ksh \
	functional/cli_root/zpool_import/import_cachefile_device_replaced.ksh \
	functional/cli_root/zpool_import/import_cachefile_mirror_attached.ksh \
	functional/cli_root/zpool_import/import_cachefile_mirror_detached.ksh \
	functional/cli_root/zpool_import/import_cachefile_paths_changed.ksh \
	functional/cli_root/zpool_import/import_cachefile_shared_device.ksh \
	functional/cli_root/zpool_import/import_devices_missing.ksh \
	functional/cli_root/zpool_import/import_log_missing.ksh \
	functional/cli_root/zpool_import/import_paths_changed.ksh \
	functional/cli_root/zpool_import/import_rewind_config_changed.ksh \
	functional/cli_root/zpool_import/import_rewind_device_replaced.ksh \
	functional/cli_root/zpool_import/setup.ksh \
	functional/cli_root/zpool_import/zpool_import_001_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_002_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_003_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_004_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_005_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_006_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_007_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_008_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_009_neg.ksh \
	functional/cli_root/zpool_import/zpool_import_010_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_011_neg.ksh \
	functional/cli_root/zpool_import/zpool_import_012_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_013_neg.ksh \
	functional/cli_root/zpool_import/zpool_import_014_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_015_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_016_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_017_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_all_001_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_encrypted.ksh \
	functional/cli_root/zpool_import/zpool_import_encrypted_load.ksh \
	functional/cli_root/zpool_import/zpool_import_errata3.ksh \
	functional/cli_root/zpool_import/zpool_import_errata4.ksh \
	functional/cli_root/zpool_import/zpool_import_features_001_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_features_002_neg.ksh \
	functional/cli_root/zpool_import/zpool_import_features_003_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_hostid_changed.ksh \
	functional/cli_root/zpool_import/zpool_import_hostid_changed_unclean_export.ksh \
	functional/cli_root/zpool_import/zpool_import_hostid_changed_cachefile.ksh \
	functional/cli_root/zpool_import/zpool_import_hostid_changed_cachefile_unclean_export.ksh \
	functional/cli_root/zpool_import/zpool_import_missing_001_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_missing_002_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_missing_003_pos.ksh \
	functional/cli_root/zpool_import/zpool_import_rename_001_pos.ksh \
	functional/cli_root/zpool_initialize/cleanup.ksh \
	functional/cli_root/zpool_initialize/zpool_initialize_attach_detach_add_remove.ksh \
	functional/cli_root/zpool_initialize/zpool_initialize_fault_export_import_online.ksh \
	functional/cli_root/zpool_initialize/zpool_initialize_import_export.ksh \
	functional/cli_root/zpool_initialize/zpool_initialize_offline_export_import_online.ksh \
	functional/cli_root/zpool_initialize/zpool_initialize_online_offline.ksh \
	functional/cli_root/zpool_initialize/zpool_initialize_split.ksh \
	functional/cli_root/zpool_initialize/zpool_initialize_start_and_cancel_neg.ksh \
	functional/cli_root/zpool_initialize/zpool_initialize_start_and_cancel_pos.ksh \
	functional/cli_root/zpool_initialize/zpool_initialize_suspend_resume.ksh \
	functional/cli_root/zpool_initialize/zpool_initialize_uninit.ksh \
	functional/cli_root/zpool_initialize/zpool_initialize_unsupported_vdevs.ksh \
	functional/cli_root/zpool_initialize/zpool_initialize_verify_checksums.ksh \
	functional/cli_root/zpool_initialize/zpool_initialize_verify_initialized.ksh \
	functional/cli_root/zpool_labelclear/zpool_labelclear_active.ksh \
	functional/cli_root/zpool_labelclear/zpool_labelclear_exported.ksh \
	functional/cli_root/zpool_labelclear/zpool_labelclear_removed.ksh \
	functional/cli_root/zpool_labelclear/zpool_labelclear_valid.ksh \
	functional/cli_root/zpool_offline/cleanup.ksh \
	functional/cli_root/zpool_offline/setup.ksh \
	functional/cli_root/zpool_offline/zpool_offline_001_pos.ksh \
	functional/cli_root/zpool_offline/zpool_offline_002_neg.ksh \
	functional/cli_root/zpool_offline/zpool_offline_003_pos.ksh \
	functional/cli_root/zpool_online/cleanup.ksh \
	functional/cli_root/zpool_online/setup.ksh \
	functional/cli_root/zpool_online/zpool_online_001_pos.ksh \
	functional/cli_root/zpool_online/zpool_online_002_neg.ksh \
	functional/cli_root/zpool_remove/cleanup.ksh \
	functional/cli_root/zpool_remove/setup.ksh \
	functional/cli_root/zpool_remove/zpool_remove_001_neg.ksh \
	functional/cli_root/zpool_remove/zpool_remove_002_pos.ksh \
	functional/cli_root/zpool_remove/zpool_remove_003_pos.ksh \
	functional/cli_root/zpool_reopen/cleanup.ksh \
	functional/cli_root/zpool_reopen/setup.ksh \
	functional/cli_root/zpool_reopen/zpool_reopen_001_pos.ksh \
	functional/cli_root/zpool_reopen/zpool_reopen_002_pos.ksh \
	functional/cli_root/zpool_reopen/zpool_reopen_003_pos.ksh \
	functional/cli_root/zpool_reopen/zpool_reopen_004_pos.ksh \
	functional/cli_root/zpool_reopen/zpool_reopen_005_pos.ksh \
	functional/cli_root/zpool_reopen/zpool_reopen_006_neg.ksh \
	functional/cli_root/zpool_reopen/zpool_reopen_007_pos.ksh \
	functional/cli_root/zpool_replace/cleanup.ksh \
	functional/cli_root/zpool_replace/replace-o_ashift.ksh \
	functional/cli_root/zpool_replace/replace_prop_ashift.ksh \
	functional/cli_root/zpool_replace/setup.ksh \
	functional/cli_root/zpool_replace/zpool_replace_001_neg.ksh \
	functional/cli_root/zpool_resilver/cleanup.ksh \
	functional/cli_root/zpool_resilver/setup.ksh \
	functional/cli_root/zpool_resilver/zpool_resilver_bad_args.ksh \
	functional/cli_root/zpool_resilver/zpool_resilver_restart.ksh \
	functional/cli_root/zpool_resilver/zpool_resilver_concurrent.ksh \
	functional/cli_root/zpool_scrub/cleanup.ksh \
	functional/cli_root/zpool_scrub/setup.ksh \
	functional/cli_root/zpool_scrub/zpool_scrub_001_neg.ksh \
	functional/cli_root/zpool_scrub/zpool_scrub_002_pos.ksh \
	functional/cli_root/zpool_scrub/zpool_scrub_003_pos.ksh \
	functional/cli_root/zpool_scrub/zpool_scrub_004_pos.ksh \
	functional/cli_root/zpool_scrub/zpool_scrub_005_pos.ksh \
	functional/cli_root/zpool_scrub/zpool_scrub_encrypted_unloaded.ksh \
	functional/cli_root/zpool_scrub/zpool_scrub_multiple_copies.ksh \
	functional/cli_root/zpool_scrub/zpool_scrub_offline_device.ksh \
	functional/cli_root/zpool_scrub/zpool_scrub_print_repairing.ksh \
	functional/cli_root/zpool_scrub/zpool_error_scrub_001_pos.ksh \
	functional/cli_root/zpool_scrub/zpool_error_scrub_002_pos.ksh \
	functional/cli_root/zpool_scrub/zpool_error_scrub_003_pos.ksh \
	functional/cli_root/zpool_scrub/zpool_error_scrub_004_pos.ksh \
	functional/cli_root/zpool_set/cleanup.ksh \
	functional/cli_root/zpool_set/setup.ksh \
	functional/cli_root/zpool/setup.ksh \
	functional/cli_root/zpool_set/vdev_set_001_pos.ksh \
	functional/cli_root/zpool_set/zpool_set_common.kshlib \
	functional/cli_root/zpool_set/zpool_set_001_pos.ksh \
	functional/cli_root/zpool_set/zpool_set_002_neg.ksh \
	functional/cli_root/zpool_set/zpool_set_003_neg.ksh \
	functional/cli_root/zpool_set/zpool_set_ashift.ksh \
	functional/cli_root/zpool_set/user_property_001_pos.ksh \
	functional/cli_root/zpool_set/user_property_002_neg.ksh \
	functional/cli_root/zpool_set/zpool_set_features.ksh \
	functional/cli_root/zpool_split/cleanup.ksh \
	functional/cli_root/zpool_split/setup.ksh \
	functional/cli_root/zpool_split/zpool_split_cliargs.ksh \
	functional/cli_root/zpool_split/zpool_split_devices.ksh \
	functional/cli_root/zpool_split/zpool_split_dryrun_output.ksh \
	functional/cli_root/zpool_split/zpool_split_encryption.ksh \
	functional/cli_root/zpool_split/zpool_split_indirect.ksh \
	functional/cli_root/zpool_split/zpool_split_props.ksh \
	functional/cli_root/zpool_split/zpool_split_resilver.ksh \
	functional/cli_root/zpool_split/zpool_split_vdevs.ksh \
	functional/cli_root/zpool_split/zpool_split_wholedisk.ksh \
	functional/cli_root/zpool_status/cleanup.ksh \
	functional/cli_root/zpool_status/setup.ksh \
	functional/cli_root/zpool_status/zpool_status_001_pos.ksh \
	functional/cli_root/zpool_status/zpool_status_002_pos.ksh \
	functional/cli_root/zpool_status/zpool_status_003_pos.ksh \
	functional/cli_root/zpool_status/zpool_status_004_pos.ksh \
	functional/cli_root/zpool_status/zpool_status_005_pos.ksh \
	functional/cli_root/zpool_status/zpool_status_006_pos.ksh \
	functional/cli_root/zpool_status/zpool_status_007_pos.ksh \
	functional/cli_root/zpool_status/zpool_status_008_pos.ksh \
	functional/cli_root/zpool_status/zpool_status_features_001_pos.ksh \
	functional/cli_root/zpool_sync/cleanup.ksh \
	functional/cli_root/zpool_sync/setup.ksh \
	functional/cli_root/zpool_sync/zpool_sync_001_pos.ksh \
	functional/cli_root/zpool_sync/zpool_sync_002_neg.ksh \
	functional/cli_root/zpool_trim/cleanup.ksh \
	functional/cli_root/zpool_trim/setup.ksh \
	functional/cli_root/zpool_trim/zpool_trim_attach_detach_add_remove.ksh \
	functional/cli_root/zpool_trim/zpool_trim_fault_export_import_online.ksh \
	functional/cli_root/zpool_trim/zpool_trim_import_export.ksh \
	functional/cli_root/zpool_trim/zpool_trim_multiple.ksh \
	functional/cli_root/zpool_trim/zpool_trim_neg.ksh \
	functional/cli_root/zpool_trim/zpool_trim_offline_export_import_online.ksh \
	functional/cli_root/zpool_trim/zpool_trim_online_offline.ksh \
	functional/cli_root/zpool_trim/zpool_trim_partial.ksh \
	functional/cli_root/zpool_trim/zpool_trim_rate.ksh \
	functional/cli_root/zpool_trim/zpool_trim_rate_neg.ksh \
	functional/cli_root/zpool_trim/zpool_trim_secure.ksh \
	functional/cli_root/zpool_trim/zpool_trim_split.ksh \
	functional/cli_root/zpool_trim/zpool_trim_start_and_cancel_neg.ksh \
	functional/cli_root/zpool_trim/zpool_trim_start_and_cancel_pos.ksh \
	functional/cli_root/zpool_trim/zpool_trim_suspend_resume.ksh \
	functional/cli_root/zpool_trim/zpool_trim_unsupported_vdevs.ksh \
	functional/cli_root/zpool_trim/zpool_trim_verify_checksums.ksh \
	functional/cli_root/zpool_trim/zpool_trim_verify_trimmed.ksh \
	functional/cli_root/zpool_upgrade/cleanup.ksh \
	functional/cli_root/zpool_upgrade/setup.ksh \
	functional/cli_root/zpool_upgrade/zpool_upgrade_001_pos.ksh \
	functional/cli_root/zpool_upgrade/zpool_upgrade_002_pos.ksh \
	functional/cli_root/zpool_upgrade/zpool_upgrade_003_pos.ksh \
	functional/cli_root/zpool_upgrade/zpool_upgrade_004_pos.ksh \
	functional/cli_root/zpool_upgrade/zpool_upgrade_005_neg.ksh \
	functional/cli_root/zpool_upgrade/zpool_upgrade_006_neg.ksh \
	functional/cli_root/zpool_upgrade/zpool_upgrade_007_pos.ksh \
	functional/cli_root/zpool_upgrade/zpool_upgrade_008_pos.ksh \
	functional/cli_root/zpool_upgrade/zpool_upgrade_009_neg.ksh \
	functional/cli_root/zpool_upgrade/zpool_upgrade_features_001_pos.ksh \
	functional/cli_root/zpool_wait/cleanup.ksh \
	functional/cli_root/zpool_wait/scan/cleanup.ksh \
	functional/cli_root/zpool_wait/scan/setup.ksh \
	functional/cli_root/zpool_wait/scan/zpool_wait_rebuild.ksh \
	functional/cli_root/zpool_wait/scan/zpool_wait_replace_cancel.ksh \
	functional/cli_root/zpool_wait/scan/zpool_wait_replace.ksh \
	functional/cli_root/zpool_wait/scan/zpool_wait_resilver.ksh \
	functional/cli_root/zpool_wait/scan/zpool_wait_scrub_basic.ksh \
	functional/cli_root/zpool_wait/scan/zpool_wait_scrub_cancel.ksh \
	functional/cli_root/zpool_wait/scan/zpool_wait_scrub_flag.ksh \
	functional/cli_root/zpool_wait/setup.ksh \
	functional/cli_root/zpool_wait/zpool_wait_discard.ksh \
	functional/cli_root/zpool_wait/zpool_wait_freeing.ksh \
	functional/cli_root/zpool_wait/zpool_wait_initialize_basic.ksh \
	functional/cli_root/zpool_wait/zpool_wait_initialize_cancel.ksh \
	functional/cli_root/zpool_wait/zpool_wait_initialize_flag.ksh \
	functional/cli_root/zpool_wait/zpool_wait_multiple.ksh \
	functional/cli_root/zpool_wait/zpool_wait_no_activity.ksh \
	functional/cli_root/zpool_wait/zpool_wait_remove_cancel.ksh \
	functional/cli_root/zpool_wait/zpool_wait_remove.ksh \
	functional/cli_root/zpool_wait/zpool_wait_trim_basic.ksh \
	functional/cli_root/zpool_wait/zpool_wait_trim_cancel.ksh \
	functional/cli_root/zpool_wait/zpool_wait_trim_flag.ksh \
	functional/cli_root/zpool_wait/zpool_wait_usage.ksh \
	functional/cli_root/zpool/zpool_001_neg.ksh \
	functional/cli_root/zpool/zpool_002_pos.ksh \
	functional/cli_root/zpool/zpool_003_pos.ksh \
	functional/cli_root/zpool/zpool_colors.ksh \
	functional/cli_user/misc/arcstat_001_pos.ksh \
	functional/cli_user/misc/arc_summary_001_pos.ksh \
	functional/cli_user/misc/arc_summary_002_neg.ksh \
	functional/cli_user/misc/zilstat_001_pos.ksh \
	functional/cli_user/misc/cleanup.ksh \
	functional/cli_user/misc/setup.ksh \
	functional/cli_user/misc/zdb_001_neg.ksh \
	functional/cli_user/misc/zfs_001_neg.ksh \
	functional/cli_user/misc/zfs_allow_001_neg.ksh \
	functional/cli_user/misc/zfs_clone_001_neg.ksh \
	functional/cli_user/misc/zfs_create_001_neg.ksh \
	functional/cli_user/misc/zfs_destroy_001_neg.ksh \
	functional/cli_user/misc/zfs_get_001_neg.ksh \
	functional/cli_user/misc/zfs_inherit_001_neg.ksh \
	functional/cli_user/misc/zfs_mount_001_neg.ksh \
	functional/cli_user/misc/zfs_promote_001_neg.ksh \
	functional/cli_user/misc/zfs_receive_001_neg.ksh \
	functional/cli_user/misc/zfs_rename_001_neg.ksh \
	functional/cli_user/misc/zfs_rollback_001_neg.ksh \
	functional/cli_user/misc/zfs_send_001_neg.ksh \
	functional/cli_user/misc/zfs_set_001_neg.ksh \
	functional/cli_user/misc/zfs_share_001_neg.ksh \
	functional/cli_user/misc/zfs_snapshot_001_neg.ksh \
	functional/cli_user/misc/zfs_unallow_001_neg.ksh \
	functional/cli_user/misc/zfs_unmount_001_neg.ksh \
	functional/cli_user/misc/zfs_unshare_001_neg.ksh \
	functional/cli_user/misc/zfs_upgrade_001_neg.ksh \
	functional/cli_user/misc/zpool_001_neg.ksh \
	functional/cli_user/misc/zpool_add_001_neg.ksh \
	functional/cli_user/misc/zpool_attach_001_neg.ksh \
	functional/cli_user/misc/zpool_clear_001_neg.ksh \
	functional/cli_user/misc/zpool_create_001_neg.ksh \
	functional/cli_user/misc/zpool_destroy_001_neg.ksh \
	functional/cli_user/misc/zpool_detach_001_neg.ksh \
	functional/cli_user/misc/zpool_export_001_neg.ksh \
	functional/cli_user/misc/zpool_get_001_neg.ksh \
	functional/cli_user/misc/zpool_history_001_neg.ksh \
	functional/cli_user/misc/zpool_import_001_neg.ksh \
	functional/cli_user/misc/zpool_import_002_neg.ksh \
	functional/cli_user/misc/zpool_offline_001_neg.ksh \
	functional/cli_user/misc/zpool_online_001_neg.ksh \
	functional/cli_user/misc/zpool_remove_001_neg.ksh \
	functional/cli_user/misc/zpool_replace_001_neg.ksh \
	functional/cli_user/misc/zpool_scrub_001_neg.ksh \
	functional/cli_user/misc/zpool_set_001_neg.ksh \
	functional/cli_user/misc/zpool_status_001_neg.ksh \
	functional/cli_user/misc/zpool_upgrade_001_neg.ksh \
	functional/cli_user/misc/zpool_wait_privilege.ksh \
	functional/cli_user/zfs_list/cleanup.ksh \
	functional/cli_user/zfs_list/setup.ksh \
	functional/cli_user/zfs_list/zfs_list_001_pos.ksh \
	functional/cli_user/zfs_list/zfs_list_002_pos.ksh \
	functional/cli_user/zfs_list/zfs_list_003_pos.ksh \
	functional/cli_user/zfs_list/zfs_list_004_neg.ksh \
	functional/cli_user/zfs_list/zfs_list_005_neg.ksh \
	functional/cli_user/zfs_list/zfs_list_007_pos.ksh \
	functional/cli_user/zfs_list/zfs_list_008_neg.ksh \
	functional/cli_user/zpool_iostat/cleanup.ksh \
	functional/cli_user/zpool_iostat/setup.ksh \
	functional/cli_user/zpool_iostat/zpool_iostat_001_neg.ksh \
	functional/cli_user/zpool_iostat/zpool_iostat_002_pos.ksh \
	functional/cli_user/zpool_iostat/zpool_iostat_003_neg.ksh \
	functional/cli_user/zpool_iostat/zpool_iostat_004_pos.ksh \
	functional/cli_user/zpool_iostat/zpool_iostat_005_pos.ksh \
	functional/cli_user/zpool_iostat/zpool_iostat_-c_disable.ksh \
	functional/cli_user/zpool_iostat/zpool_iostat_-c_homedir.ksh \
	functional/cli_user/zpool_iostat/zpool_iostat_-c_searchpath.ksh \
	functional/cli_user/zpool_list/cleanup.ksh \
	functional/cli_user/zpool_list/setup.ksh \
	functional/cli_user/zpool_list/zpool_list_001_pos.ksh \
	functional/cli_user/zpool_list/zpool_list_002_neg.ksh \
	functional/cli_user/zpool_status/cleanup.ksh \
	functional/cli_user/zpool_status/setup.ksh \
	functional/cli_user/zpool_status/zpool_status_003_pos.ksh \
	functional/cli_user/zpool_status/zpool_status_-c_disable.ksh \
	functional/cli_user/zpool_status/zpool_status_-c_homedir.ksh \
	functional/cli_user/zpool_status/zpool_status_-c_searchpath.ksh \
	functional/compression/cleanup.ksh \
	functional/compression/compress_001_pos.ksh \
	functional/compression/compress_002_pos.ksh \
	functional/compression/compress_003_pos.ksh \
	functional/compression/compress_004_pos.ksh \
	functional/compression/compress_zstd_bswap.ksh \
	functional/compression/l2arc_compressed_arc_disabled.ksh \
	functional/compression/l2arc_compressed_arc.ksh \
	functional/compression/l2arc_encrypted.ksh \
	functional/compression/l2arc_encrypted_no_compressed_arc.ksh \
	functional/compression/setup.ksh \
	functional/cp_files/cleanup.ksh \
	functional/cp_files/cp_files_001_pos.ksh \
	functional/cp_files/cp_files_002_pos.ksh \
	functional/cp_files/cp_stress.ksh \
	functional/cp_files/setup.ksh \
	functional/crtime/cleanup.ksh \
	functional/crtime/crtime_001_pos.ksh \
	functional/crtime/setup.ksh \
	functional/ctime/cleanup.ksh \
	functional/ctime/ctime_001_pos.ksh \
	functional/ctime/setup.ksh \
	functional/deadman/deadman_ratelimit.ksh \
	functional/deadman/deadman_sync.ksh \
	functional/deadman/deadman_zio.ksh \
	functional/delegate/cleanup.ksh \
	functional/delegate/setup.ksh \
	functional/delegate/zfs_allow_001_pos.ksh \
	functional/delegate/zfs_allow_002_pos.ksh \
	functional/delegate/zfs_allow_003_pos.ksh \
	functional/delegate/zfs_allow_004_pos.ksh \
	functional/delegate/zfs_allow_005_pos.ksh \
	functional/delegate/zfs_allow_006_pos.ksh \
	functional/delegate/zfs_allow_007_pos.ksh \
	functional/delegate/zfs_allow_008_pos.ksh \
	functional/delegate/zfs_allow_009_neg.ksh \
	functional/delegate/zfs_allow_010_pos.ksh \
	functional/delegate/zfs_allow_011_neg.ksh \
	functional/delegate/zfs_allow_012_neg.ksh \
	functional/delegate/zfs_unallow_001_pos.ksh \
	functional/delegate/zfs_unallow_002_pos.ksh \
	functional/delegate/zfs_unallow_003_pos.ksh \
	functional/delegate/zfs_unallow_004_pos.ksh \
	functional/delegate/zfs_unallow_005_pos.ksh \
	functional/delegate/zfs_unallow_006_pos.ksh \
	functional/delegate/zfs_unallow_007_neg.ksh \
	functional/delegate/zfs_unallow_008_neg.ksh \
	functional/devices/cleanup.ksh \
	functional/devices/devices_001_pos.ksh \
	functional/devices/devices_002_neg.ksh \
	functional/devices/devices_003_pos.ksh \
	functional/devices/setup.ksh \
	functional/dos_attributes/cleanup.ksh \
	functional/dos_attributes/read_dos_attrs_001.ksh \
	functional/dos_attributes/setup.ksh \
	functional/dos_attributes/write_dos_attrs_001.ksh \
	functional/events/cleanup.ksh \
	functional/events/events_001_pos.ksh \
	functional/events/events_002_pos.ksh \
	functional/events/setup.ksh \
	functional/events/zed_cksum_config.ksh \
	functional/events/zed_cksum_reported.ksh \
	functional/events/zed_fd_spill.ksh \
	functional/events/zed_io_config.ksh \
	functional/events/zed_rc_filter.ksh \
	+ functional/events/zed_slow_io.ksh \
	+ functional/events/zed_slow_io_many_vdevs.ksh \
	functional/exec/cleanup.ksh \
	functional/exec/exec_001_pos.ksh \
	functional/exec/exec_002_neg.ksh \
	functional/exec/setup.ksh \
	functional/fadvise/cleanup.ksh \
	functional/fadvise/fadvise_sequential.ksh \
	functional/fadvise/setup.ksh \
	functional/fallocate/cleanup.ksh \
	functional/fallocate/fallocate_prealloc.ksh \
	functional/fallocate/fallocate_punch-hole.ksh \
	functional/fallocate/fallocate_zero-range.ksh \
	functional/fallocate/setup.ksh \
	functional/fault/auto_offline_001_pos.ksh \
	functional/fault/auto_online_001_pos.ksh \
	functional/fault/auto_online_002_pos.ksh \
	functional/fault/auto_replace_001_pos.ksh \
	functional/fault/auto_replace_002_pos.ksh \
	functional/fault/auto_spare_001_pos.ksh \
	functional/fault/auto_spare_002_pos.ksh \
	functional/fault/auto_spare_ashift.ksh \
	functional/fault/auto_spare_multiple.ksh \
	functional/fault/auto_spare_shared.ksh \
	functional/fault/cleanup.ksh \
	functional/fault/decompress_fault.ksh \
	functional/fault/decrypt_fault.ksh \
	functional/fault/scrub_after_resilver.ksh \
	functional/fault/setup.ksh \
	functional/fault/zpool_status_-s.ksh \
	functional/features/async_destroy/async_destroy_001_pos.ksh \
	functional/features/async_destroy/cleanup.ksh \
	functional/features/async_destroy/setup.ksh \
	functional/features/large_dnode/cleanup.ksh \
	functional/features/large_dnode/large_dnode_001_pos.ksh \
	functional/features/large_dnode/large_dnode_002_pos.ksh \
	functional/features/large_dnode/large_dnode_003_pos.ksh \
	functional/features/large_dnode/large_dnode_004_neg.ksh \
	functional/features/large_dnode/large_dnode_005_pos.ksh \
	functional/features/large_dnode/large_dnode_006_pos.ksh \
	functional/features/large_dnode/large_dnode_007_neg.ksh \
	functional/features/large_dnode/large_dnode_008_pos.ksh \
	functional/features/large_dnode/large_dnode_009_pos.ksh \
	functional/features/large_dnode/setup.ksh \
	functional/grow/grow_pool_001_pos.ksh \
	functional/grow/grow_replicas_001_pos.ksh \
	functional/history/cleanup.ksh \
	functional/history/history_001_pos.ksh \
	functional/history/history_002_pos.ksh \
	functional/history/history_003_pos.ksh \
	functional/history/history_004_pos.ksh \
	functional/history/history_005_neg.ksh \
	functional/history/history_006_neg.ksh \
	functional/history/history_007_pos.ksh \
	functional/history/history_008_pos.ksh \
	functional/history/history_009_pos.ksh \
	functional/history/history_010_pos.ksh \
	functional/history/setup.ksh \
	functional/inheritance/cleanup.ksh \
	functional/inheritance/inherit_001_pos.ksh \
	functional/inuse/inuse_001_pos.ksh \
	functional/inuse/inuse_003_pos.ksh \
	functional/inuse/inuse_004_pos.ksh \
	functional/inuse/inuse_005_pos.ksh \
	functional/inuse/inuse_006_pos.ksh \
	functional/inuse/inuse_007_pos.ksh \
	functional/inuse/inuse_008_pos.ksh \
	functional/inuse/inuse_009_pos.ksh \
	functional/inuse/setup.ksh \
	functional/io/cleanup.ksh \
	functional/io/io_uring.ksh \
	functional/io/libaio.ksh \
	functional/io/mmap.ksh \
	functional/io/posixaio.ksh \
	functional/io/psync.ksh \
	functional/io/setup.ksh \
	functional/io/sync.ksh \
	functional/l2arc/cleanup.ksh \
	functional/l2arc/l2arc_arcstats_pos.ksh \
	functional/l2arc/l2arc_l2miss_pos.ksh \
	functional/l2arc/l2arc_mfuonly_pos.ksh \
	functional/l2arc/persist_l2arc_001_pos.ksh \
	functional/l2arc/persist_l2arc_002_pos.ksh \
	functional/l2arc/persist_l2arc_003_neg.ksh \
	functional/l2arc/persist_l2arc_004_pos.ksh \
	functional/l2arc/persist_l2arc_005_pos.ksh \
	functional/l2arc/setup.ksh \
	functional/large_files/cleanup.ksh \
	functional/large_files/large_files_001_pos.ksh \
	functional/large_files/large_files_002_pos.ksh \
	functional/large_files/setup.ksh \
	functional/largest_pool/largest_pool_001_pos.ksh \
	functional/libzfs/cleanup.ksh \
	functional/libzfs/libzfs_input.ksh \
	functional/libzfs/setup.ksh \
	functional/limits/cleanup.ksh \
	functional/limits/filesystem_count.ksh \
	functional/limits/filesystem_limit.ksh \
	functional/limits/setup.ksh \
	functional/limits/snapshot_count.ksh \
	functional/limits/snapshot_limit.ksh \
	functional/link_count/cleanup.ksh \
	functional/link_count/link_count_001.ksh \
	functional/link_count/link_count_root_inode.ksh \
	functional/link_count/setup.ksh \
	functional/log_spacemap/log_spacemap_import_logs.ksh \
	functional/migration/cleanup.ksh \
	functional/migration/migration_001_pos.ksh \
	functional/migration/migration_002_pos.ksh \
	functional/migration/migration_003_pos.ksh \
	functional/migration/migration_004_pos.ksh \
	functional/migration/migration_005_pos.ksh \
	functional/migration/migration_006_pos.ksh \
	functional/migration/migration_007_pos.ksh \
	functional/migration/migration_008_pos.ksh \
	functional/migration/migration_009_pos.ksh \
	functional/migration/migration_010_pos.ksh \
	functional/migration/migration_011_pos.ksh \
	functional/migration/migration_012_pos.ksh \
	functional/migration/setup.ksh \
	functional/mmap/cleanup.ksh \
	functional/mmap/mmap_libaio_001_pos.ksh \
	functional/mmap/mmap_mixed.ksh \
	functional/mmap/mmap_read_001_pos.ksh \
	functional/mmap/mmap_seek_001_pos.ksh \
	functional/mmap/mmap_sync_001_pos.ksh \
	functional/mmap/mmap_write_001_pos.ksh \
	functional/mmap/setup.ksh \
	functional/mmp/cleanup.ksh \
	functional/mmp/mmp_active_import.ksh \
	functional/mmp/mmp_exported_import.ksh \
	functional/mmp/mmp_hostid.ksh \
	functional/mmp/mmp_inactive_import.ksh \
	functional/mmp/mmp_interval.ksh \
	functional/mmp/mmp_on_off.ksh \
	functional/mmp/mmp_on_thread.ksh \
	functional/mmp/mmp_on_uberblocks.ksh \
	functional/mmp/mmp_on_zdb.ksh \
	functional/mmp/mmp_reset_interval.ksh \
	functional/mmp/mmp_write_distribution.ksh \
	+ functional/mmp/mmp_write_slow_disk.ksh \
	functional/mmp/mmp_write_uberblocks.ksh \
	functional/mmp/multihost_history.ksh \
	functional/mmp/setup.ksh \
	functional/mount/cleanup.ksh \
	functional/mount/setup.ksh \
	functional/mount/umount_001.ksh \
	functional/mount/umountall_001.ksh \
	functional/mount/umount_unlinked_drain.ksh \
	functional/mv_files/cleanup.ksh \
	functional/mv_files/mv_files_001_pos.ksh \
	functional/mv_files/mv_files_002_pos.ksh \
	functional/mv_files/random_creation.ksh \
	functional/mv_files/setup.ksh \
	functional/nestedfs/cleanup.ksh \
	functional/nestedfs/nestedfs_001_pos.ksh \
	functional/nestedfs/setup.ksh \
	functional/nopwrite/cleanup.ksh \
	functional/nopwrite/nopwrite_copies.ksh \
	functional/nopwrite/nopwrite_mtime.ksh \
	functional/nopwrite/nopwrite_negative.ksh \
	functional/nopwrite/nopwrite_promoted_clone.ksh \
	functional/nopwrite/nopwrite_recsize.ksh \
	functional/nopwrite/nopwrite_sync.ksh \
	functional/nopwrite/nopwrite_varying_compression.ksh \
	functional/nopwrite/nopwrite_volume.ksh \
	functional/nopwrite/setup.ksh \
	functional/no_space/cleanup.ksh \
	functional/no_space/enospc_001_pos.ksh \
	functional/no_space/enospc_002_pos.ksh \
	functional/no_space/enospc_003_pos.ksh \
	functional/no_space/enospc_df.ksh \
	functional/no_space/enospc_ganging.ksh \
	functional/no_space/enospc_rm.ksh \
	functional/no_space/setup.ksh \
	functional/online_offline/cleanup.ksh \
	functional/online_offline/online_offline_001_pos.ksh \
	functional/online_offline/online_offline_002_neg.ksh \
	functional/online_offline/online_offline_003_neg.ksh \
	functional/online_offline/setup.ksh \
	functional/pam/cleanup.ksh \
	functional/pam/pam_basic.ksh \
	functional/pam/pam_change_unmounted.ksh \
	functional/pam/pam_nounmount.ksh \
	functional/pam/pam_recursive.ksh \
	functional/pam/pam_short_password.ksh \
	functional/pam/setup.ksh \
	functional/pool_checkpoint/checkpoint_after_rewind.ksh \
	functional/pool_checkpoint/checkpoint_big_rewind.ksh \
	functional/pool_checkpoint/checkpoint_capacity.ksh \
	functional/pool_checkpoint/checkpoint_conf_change.ksh \
	functional/pool_checkpoint/checkpoint_discard_busy.ksh \
	functional/pool_checkpoint/checkpoint_discard.ksh \
	functional/pool_checkpoint/checkpoint_discard_many.ksh \
	functional/pool_checkpoint/checkpoint_indirect.ksh \
	functional/pool_checkpoint/checkpoint_invalid.ksh \
	functional/pool_checkpoint/checkpoint_lun_expsz.ksh \
	functional/pool_checkpoint/checkpoint_open.ksh \
	functional/pool_checkpoint/checkpoint_removal.ksh \
	functional/pool_checkpoint/checkpoint_rewind.ksh \
	functional/pool_checkpoint/checkpoint_ro_rewind.ksh \
	functional/pool_checkpoint/checkpoint_sm_scale.ksh \
	functional/pool_checkpoint/checkpoint_twice.ksh \
	functional/pool_checkpoint/checkpoint_vdev_add.ksh \
	functional/pool_checkpoint/checkpoint_zdb.ksh \
	functional/pool_checkpoint/checkpoint_zhack_feat.ksh \
	functional/pool_checkpoint/cleanup.ksh \
	functional/pool_checkpoint/setup.ksh \
	functional/pool_names/pool_names_001_pos.ksh \
	functional/pool_names/pool_names_002_neg.ksh \
	functional/poolversion/cleanup.ksh \
	functional/poolversion/poolversion_001_pos.ksh \
	functional/poolversion/poolversion_002_pos.ksh \
	functional/poolversion/setup.ksh \
	functional/privilege/cleanup.ksh \
	functional/privilege/privilege_001_pos.ksh \
	functional/privilege/privilege_002_pos.ksh \
	functional/privilege/setup.ksh \
	functional/procfs/cleanup.ksh \
	functional/procfs/pool_state.ksh \
	functional/procfs/procfs_list_basic.ksh \
	functional/procfs/procfs_list_concurrent_readers.ksh \
	functional/procfs/procfs_list_stale_read.ksh \
	functional/procfs/setup.ksh \
	functional/projectquota/cleanup.ksh \
	functional/projectquota/projectid_001_pos.ksh \
	functional/projectquota/projectid_002_pos.ksh \
	functional/projectquota/projectid_003_pos.ksh \
	functional/projectquota/projectquota_001_pos.ksh \
	functional/projectquota/projectquota_002_pos.ksh \
	functional/projectquota/projectquota_003_pos.ksh \
	functional/projectquota/projectquota_004_neg.ksh \
	functional/projectquota/projectquota_005_pos.ksh \
	functional/projectquota/projectquota_006_pos.ksh \
	functional/projectquota/projectquota_007_pos.ksh \
	functional/projectquota/projectquota_008_pos.ksh \
	functional/projectquota/projectquota_009_pos.ksh \
	functional/projectquota/projectspace_001_pos.ksh \
	functional/projectquota/projectspace_002_pos.ksh \
	functional/projectquota/projectspace_003_pos.ksh \
	functional/projectquota/projectspace_004_pos.ksh \
	functional/projectquota/projecttree_001_pos.ksh \
	functional/projectquota/projecttree_002_pos.ksh \
	functional/projectquota/projecttree_003_neg.ksh \
	functional/projectquota/setup.ksh \
	functional/quota/cleanup.ksh \
	functional/quota/quota_001_pos.ksh \
	functional/quota/quota_002_pos.ksh \
	functional/quota/quota_003_pos.ksh \
	functional/quota/quota_004_pos.ksh \
	functional/quota/quota_005_pos.ksh \
	functional/quota/quota_006_neg.ksh \
	functional/quota/setup.ksh \
	functional/raidz/cleanup.ksh \
	functional/raidz/raidz_001_neg.ksh \
	functional/raidz/raidz_002_pos.ksh \
	functional/raidz/raidz_003_pos.ksh \
	functional/raidz/raidz_004_pos.ksh \
	functional/raidz/setup.ksh \
	functional/redacted_send/cleanup.ksh \
	functional/redacted_send/redacted_compressed.ksh \
	functional/redacted_send/redacted_contents.ksh \
	functional/redacted_send/redacted_deleted.ksh \
	functional/redacted_send/redacted_disabled_feature.ksh \
	functional/redacted_send/redacted_embedded.ksh \
	functional/redacted_send/redacted_holes.ksh \
	functional/redacted_send/redacted_incrementals.ksh \
	functional/redacted_send/redacted_largeblocks.ksh \
	functional/redacted_send/redacted_many_clones.ksh \
	functional/redacted_send/redacted_mixed_recsize.ksh \
	functional/redacted_send/redacted_mounts.ksh \
	functional/redacted_send/redacted_negative.ksh \
	functional/redacted_send/redacted_origin.ksh \
	functional/redacted_send/redacted_panic.ksh \
	functional/redacted_send/redacted_props.ksh \
	functional/redacted_send/redacted_resume.ksh \
	functional/redacted_send/redacted_size.ksh \
	functional/redacted_send/redacted_volume.ksh \
	functional/redacted_send/setup.ksh \
	functional/redundancy/cleanup.ksh \
	functional/redundancy/redundancy_draid1.ksh \
	functional/redundancy/redundancy_draid2.ksh \
	functional/redundancy/redundancy_draid3.ksh \
	functional/redundancy/redundancy_draid_damaged1.ksh \
	functional/redundancy/redundancy_draid_damaged2.ksh \
	functional/redundancy/redundancy_draid.ksh \
	functional/redundancy/redundancy_draid_spare1.ksh \
	functional/redundancy/redundancy_draid_spare2.ksh \
	functional/redundancy/redundancy_draid_spare3.ksh \
	functional/redundancy/redundancy_mirror.ksh \
	functional/redundancy/redundancy_raidz1.ksh \
	functional/redundancy/redundancy_raidz2.ksh \
	functional/redundancy/redundancy_raidz3.ksh \
	functional/redundancy/redundancy_raidz.ksh \
	functional/redundancy/redundancy_stripe.ksh \
	functional/redundancy/setup.ksh \
	functional/refquota/cleanup.ksh \
	functional/refquota/refquota_001_pos.ksh \
	functional/refquota/refquota_002_pos.ksh \
	functional/refquota/refquota_003_pos.ksh \
	functional/refquota/refquota_004_pos.ksh \
	functional/refquota/refquota_005_pos.ksh \
	functional/refquota/refquota_006_neg.ksh \
	functional/refquota/refquota_007_neg.ksh \
	functional/refquota/refquota_008_neg.ksh \
	functional/refquota/setup.ksh \
	functional/refreserv/cleanup.ksh \
	functional/refreserv/refreserv_001_pos.ksh \
	functional/refreserv/refreserv_002_pos.ksh \
	functional/refreserv/refreserv_003_pos.ksh \
	functional/refreserv/refreserv_004_pos.ksh \
	functional/refreserv/refreserv_005_pos.ksh \
	functional/refreserv/refreserv_multi_raidz.ksh \
	functional/refreserv/refreserv_raidz.ksh \
	functional/refreserv/setup.ksh \
	functional/removal/cleanup.ksh \
	functional/removal/removal_all_vdev.ksh \
	functional/removal/removal_cancel.ksh \
	functional/removal/removal_check_space.ksh \
	functional/removal/removal_condense_export.ksh \
	functional/removal/removal_multiple_indirection.ksh \
	functional/removal/removal_nopwrite.ksh \
	functional/removal/removal_remap_deadlists.ksh \
	functional/removal/removal_reservation.ksh \
	functional/removal/removal_resume_export.ksh \
	functional/removal/removal_sanity.ksh \
	functional/removal/removal_with_add.ksh \
	functional/removal/removal_with_create_fs.ksh \
	functional/removal/removal_with_dedup.ksh \
	functional/removal/removal_with_errors.ksh \
	functional/removal/removal_with_export.ksh \
	functional/removal/removal_with_faulted.ksh \
	functional/removal/removal_with_ganging.ksh \
	functional/removal/removal_with_indirect.ksh \
	functional/removal/removal_with_remove.ksh \
	functional/removal/removal_with_scrub.ksh \
	functional/removal/removal_with_send.ksh \
	functional/removal/removal_with_send_recv.ksh \
	functional/removal/removal_with_snapshot.ksh \
	functional/removal/removal_with_write.ksh \
	functional/removal/removal_with_zdb.ksh \
	functional/removal/remove_attach_mirror.ksh \
	functional/removal/remove_expanded.ksh \
	functional/removal/remove_indirect.ksh \
	functional/removal/remove_mirror.ksh \
	functional/removal/remove_mirror_sanity.ksh \
	functional/removal/remove_raidz.ksh \
	functional/rename_dirs/cleanup.ksh \
	functional/rename_dirs/rename_dirs_001_pos.ksh \
	functional/rename_dirs/setup.ksh \
	functional/renameat2/cleanup.ksh \
	functional/renameat2/setup.ksh \
	functional/renameat2/renameat2_exchange.ksh \
	functional/renameat2/renameat2_noreplace.ksh \
	functional/renameat2/renameat2_whiteout.ksh \
	functional/replacement/attach_import.ksh \
	functional/replacement/attach_multiple.ksh \
	functional/replacement/attach_rebuild.ksh \
	functional/replacement/attach_resilver.ksh \
	functional/replacement/cleanup.ksh \
	functional/replacement/detach.ksh \
	functional/replacement/rebuild_disabled_feature.ksh \
	functional/replacement/rebuild_multiple.ksh \
	functional/replacement/rebuild_raidz.ksh \
	functional/replacement/replace_import.ksh \
	functional/replacement/replace_rebuild.ksh \
	functional/replacement/replace_resilver.ksh \
	functional/replacement/resilver_restart_001.ksh \
	functional/replacement/resilver_restart_002.ksh \
	functional/replacement/scrub_cancel.ksh \
	functional/replacement/setup.ksh \
	functional/reservation/cleanup.ksh \
	functional/reservation/reservation_001_pos.ksh \
	functional/reservation/reservation_002_pos.ksh \
	functional/reservation/reservation_003_pos.ksh \
	functional/reservation/reservation_004_pos.ksh \
	functional/reservation/reservation_005_pos.ksh \
	functional/reservation/reservation_006_pos.ksh \
	functional/reservation/reservation_007_pos.ksh \
	functional/reservation/reservation_008_pos.ksh \
	functional/reservation/reservation_009_pos.ksh \
	functional/reservation/reservation_010_pos.ksh \
	functional/reservation/reservation_011_pos.ksh \
	functional/reservation/reservation_012_pos.ksh \
	functional/reservation/reservation_013_pos.ksh \
	functional/reservation/reservation_014_pos.ksh \
	functional/reservation/reservation_015_pos.ksh \
	functional/reservation/reservation_016_pos.ksh \
	functional/reservation/reservation_017_pos.ksh \
	functional/reservation/reservation_018_pos.ksh \
	functional/reservation/reservation_019_pos.ksh \
	functional/reservation/reservation_020_pos.ksh \
	functional/reservation/reservation_021_neg.ksh \
	functional/reservation/reservation_022_pos.ksh \
	functional/reservation/setup.ksh \
	functional/rootpool/cleanup.ksh \
	functional/rootpool/rootpool_002_neg.ksh \
	functional/rootpool/rootpool_003_neg.ksh \
	functional/rootpool/rootpool_007_pos.ksh \
	functional/rootpool/setup.ksh \
	functional/rsend/cleanup.ksh \
	functional/rsend/recv_dedup_encrypted_zvol.ksh \
	functional/rsend/recv_dedup.ksh \
	functional/rsend/rsend_001_pos.ksh \
	functional/rsend/rsend_002_pos.ksh \
	functional/rsend/rsend_003_pos.ksh \
	functional/rsend/rsend_004_pos.ksh \
	functional/rsend/rsend_005_pos.ksh \
	functional/rsend/rsend_006_pos.ksh \
	functional/rsend/rsend_007_pos.ksh \
	functional/rsend/rsend_008_pos.ksh \
	functional/rsend/rsend_009_pos.ksh \
	functional/rsend/rsend_010_pos.ksh \
	functional/rsend/rsend_011_pos.ksh \
	functional/rsend/rsend_012_pos.ksh \
	functional/rsend/rsend_013_pos.ksh \
	functional/rsend/rsend_014_pos.ksh \
	functional/rsend/rsend_016_neg.ksh \
	functional/rsend/rsend_019_pos.ksh \
	functional/rsend/rsend_020_pos.ksh \
	functional/rsend/rsend_021_pos.ksh \
	functional/rsend/rsend_022_pos.ksh \
	functional/rsend/rsend_024_pos.ksh \
	functional/rsend/rsend_025_pos.ksh \
	functional/rsend/rsend_026_neg.ksh \
	functional/rsend/rsend_027_pos.ksh \
	functional/rsend/rsend_028_neg.ksh \
	functional/rsend/rsend_029_neg.ksh \
	functional/rsend/rsend_030_pos.ksh \
	functional/rsend/rsend_031_pos.ksh \
	functional/rsend/send-c_embedded_blocks.ksh \
	functional/rsend/send-c_incremental.ksh \
	functional/rsend/send-c_lz4_disabled.ksh \
	functional/rsend/send-c_mixed_compression.ksh \
	functional/rsend/send-c_props.ksh \
	functional/rsend/send-c_recv_dedup.ksh \
	functional/rsend/send-c_recv_lz4_disabled.ksh \
	functional/rsend/send-c_resume.ksh \
	functional/rsend/send-c_stream_size_estimate.ksh \
	functional/rsend/send-c_verify_contents.ksh \
	functional/rsend/send-c_verify_ratio.ksh \
	functional/rsend/send-c_volume.ksh \
	functional/rsend/send-c_zstream_recompress.ksh \
	functional/rsend/send-c_zstreamdump.ksh \
	functional/rsend/send-cpL_varied_recsize.ksh \
	functional/rsend/send_doall.ksh \
	functional/rsend/send_encrypted_incremental.ksh \
	functional/rsend/send_encrypted_files.ksh \
	functional/rsend/send_encrypted_freeobjects.ksh \
	functional/rsend/send_encrypted_hierarchy.ksh \
	functional/rsend/send_encrypted_props.ksh \
	functional/rsend/send_encrypted_truncated_files.ksh \
	functional/rsend/send_freeobjects.ksh \
	functional/rsend/send_holds.ksh \
	functional/rsend/send_hole_birth.ksh \
	functional/rsend/send_invalid.ksh \
	functional/rsend/send-L_toggle.ksh \
	functional/rsend/send_mixed_raw.ksh \
	functional/rsend/send_partial_dataset.ksh \
	functional/rsend/send_raw_ashift.ksh \
	functional/rsend/send_raw_spill_block.ksh \
	functional/rsend/send_raw_large_blocks.ksh \
	functional/rsend/send_realloc_dnode_size.ksh \
	functional/rsend/send_realloc_encrypted_files.ksh \
	functional/rsend/send_realloc_files.ksh \
	functional/rsend/send_spill_block.ksh \
	functional/rsend/send-wR_encrypted_zvol.ksh \
	functional/rsend/setup.ksh \
	functional/scrub_mirror/cleanup.ksh \
	functional/scrub_mirror/scrub_mirror_001_pos.ksh \
	functional/scrub_mirror/scrub_mirror_002_pos.ksh \
	functional/scrub_mirror/scrub_mirror_003_pos.ksh \
	functional/scrub_mirror/scrub_mirror_004_pos.ksh \
	functional/scrub_mirror/setup.ksh \
	functional/slog/cleanup.ksh \
	functional/slog/setup.ksh \
	functional/slog/slog_001_pos.ksh \
	functional/slog/slog_002_pos.ksh \
	functional/slog/slog_003_pos.ksh \
	functional/slog/slog_004_pos.ksh \
	functional/slog/slog_005_pos.ksh \
	functional/slog/slog_006_pos.ksh \
	functional/slog/slog_007_pos.ksh \
	functional/slog/slog_008_neg.ksh \
	functional/slog/slog_009_neg.ksh \
	functional/slog/slog_010_neg.ksh \
	functional/slog/slog_011_neg.ksh \
	functional/slog/slog_012_neg.ksh \
	functional/slog/slog_013_pos.ksh \
	functional/slog/slog_014_pos.ksh \
	functional/slog/slog_015_neg.ksh \
	functional/slog/slog_016_pos.ksh \
	functional/slog/slog_replay_fs_001.ksh \
	functional/slog/slog_replay_fs_002.ksh \
	functional/slog/slog_replay_volume.ksh \
	functional/snapshot/cleanup.ksh \
	functional/snapshot/clone_001_pos.ksh \
	functional/snapshot/rollback_001_pos.ksh \
	functional/snapshot/rollback_002_pos.ksh \
	functional/snapshot/rollback_003_pos.ksh \
	functional/snapshot/setup.ksh \
	functional/snapshot/snapshot_001_pos.ksh \
	functional/snapshot/snapshot_002_pos.ksh \
	functional/snapshot/snapshot_003_pos.ksh \
	functional/snapshot/snapshot_004_pos.ksh \
	functional/snapshot/snapshot_005_pos.ksh \
	functional/snapshot/snapshot_006_pos.ksh \
	functional/snapshot/snapshot_007_pos.ksh \
	functional/snapshot/snapshot_008_pos.ksh \
	functional/snapshot/snapshot_009_pos.ksh \
	functional/snapshot/snapshot_010_pos.ksh \
	functional/snapshot/snapshot_011_pos.ksh \
	functional/snapshot/snapshot_012_pos.ksh \
	functional/snapshot/snapshot_013_pos.ksh \
	functional/snapshot/snapshot_014_pos.ksh \
	functional/snapshot/snapshot_015_pos.ksh \
	functional/snapshot/snapshot_016_pos.ksh \
	functional/snapshot/snapshot_017_pos.ksh \
	functional/snapshot/snapshot_018_pos.ksh \
	functional/snapused/cleanup.ksh \
	functional/snapused/setup.ksh \
	functional/snapused/snapused_001_pos.ksh \
	functional/snapused/snapused_002_pos.ksh \
	functional/snapused/snapused_003_pos.ksh \
	functional/snapused/snapused_004_pos.ksh \
	functional/snapused/snapused_005_pos.ksh \
	functional/sparse/cleanup.ksh \
	functional/sparse/setup.ksh \
	functional/sparse/sparse_001_pos.ksh \
	functional/stat/cleanup.ksh \
	functional/stat/setup.ksh \
	functional/stat/stat_001_pos.ksh \
	functional/suid/cleanup.ksh \
	functional/suid/setup.ksh \
	functional/suid/suid_write_to_none.ksh \
	functional/suid/suid_write_to_sgid.ksh \
	functional/suid/suid_write_to_suid.ksh \
	functional/suid/suid_write_to_suid_sgid.ksh \
	functional/suid/suid_write_zil_replay.ksh \
	functional/trim/autotrim_config.ksh \
	functional/trim/autotrim_integrity.ksh \
	functional/trim/autotrim_trim_integrity.ksh \
	functional/trim/cleanup.ksh \
	functional/trim/setup.ksh \
	functional/trim/trim_config.ksh \
	functional/trim/trim_integrity.ksh \
	functional/trim/trim_l2arc.ksh \
	functional/truncate/cleanup.ksh \
	functional/truncate/setup.ksh \
	functional/truncate/truncate_001_pos.ksh \
	functional/truncate/truncate_002_pos.ksh \
	functional/truncate/truncate_timestamps.ksh \
	functional/upgrade/cleanup.ksh \
	functional/upgrade/setup.ksh \
	functional/upgrade/upgrade_projectquota_001_pos.ksh \
	functional/upgrade/upgrade_readonly_pool.ksh \
	functional/upgrade/upgrade_userobj_001_pos.ksh \
	functional/user_namespace/cleanup.ksh \
	functional/user_namespace/setup.ksh \
	functional/user_namespace/user_namespace_001.ksh \
	functional/user_namespace/user_namespace_002.ksh \
	functional/user_namespace/user_namespace_003.ksh \
	functional/user_namespace/user_namespace_004.ksh \
	functional/userquota/cleanup.ksh \
	functional/userquota/groupspace_001_pos.ksh \
	functional/userquota/groupspace_002_pos.ksh \
	functional/userquota/groupspace_003_pos.ksh \
	functional/userquota/setup.ksh \
	functional/userquota/userquota_001_pos.ksh \
	functional/userquota/userquota_002_pos.ksh \
	functional/userquota/userquota_003_pos.ksh \
	functional/userquota/userquota_004_pos.ksh \
	functional/userquota/userquota_005_neg.ksh \
	functional/userquota/userquota_006_pos.ksh \
	functional/userquota/userquota_007_pos.ksh \
	functional/userquota/userquota_008_pos.ksh \
	functional/userquota/userquota_009_pos.ksh \
	functional/userquota/userquota_010_pos.ksh \
	functional/userquota/userquota_011_pos.ksh \
	functional/userquota/userquota_012_neg.ksh \
	functional/userquota/userquota_013_pos.ksh \
	functional/userquota/userspace_001_pos.ksh \
	functional/userquota/userspace_002_pos.ksh \
	functional/userquota/userspace_003_pos.ksh \
	functional/userquota/userspace_encrypted.ksh \
	functional/userquota/userspace_send_encrypted.ksh \
	functional/userquota/userspace_encrypted_13709.ksh \
	functional/vdev_zaps/cleanup.ksh \
	functional/vdev_zaps/setup.ksh \
	functional/vdev_zaps/vdev_zaps_001_pos.ksh \
	functional/vdev_zaps/vdev_zaps_002_pos.ksh \
	functional/vdev_zaps/vdev_zaps_003_pos.ksh \
	functional/vdev_zaps/vdev_zaps_004_pos.ksh \
	functional/vdev_zaps/vdev_zaps_005_pos.ksh \
	functional/vdev_zaps/vdev_zaps_006_pos.ksh \
	functional/vdev_zaps/vdev_zaps_007_pos.ksh \
	functional/write_dirs/cleanup.ksh \
	functional/write_dirs/setup.ksh \
	functional/write_dirs/write_dirs_001_pos.ksh \
	functional/write_dirs/write_dirs_002_pos.ksh \
	functional/xattr/cleanup.ksh \
	functional/xattr/setup.ksh \
	functional/xattr/xattr_001_pos.ksh \
	functional/xattr/xattr_002_neg.ksh \
	functional/xattr/xattr_003_neg.ksh \
	functional/xattr/xattr_004_pos.ksh \
	functional/xattr/xattr_005_pos.ksh \
	functional/xattr/xattr_006_pos.ksh \
	functional/xattr/xattr_007_neg.ksh \
	functional/xattr/xattr_008_pos.ksh \
	functional/xattr/xattr_009_neg.ksh \
	functional/xattr/xattr_010_neg.ksh \
	functional/xattr/xattr_011_pos.ksh \
	functional/xattr/xattr_012_pos.ksh \
	functional/xattr/xattr_013_pos.ksh \
	functional/xattr/xattr_compat.ksh \
	functional/zpool_influxdb/cleanup.ksh \
	functional/zpool_influxdb/setup.ksh \
	functional/zpool_influxdb/zpool_influxdb.ksh \
	functional/zvol/zvol_cli/cleanup.ksh \
	functional/zvol/zvol_cli/setup.ksh \
	functional/zvol/zvol_cli/zvol_cli_001_pos.ksh \
	functional/zvol/zvol_cli/zvol_cli_002_pos.ksh \
	functional/zvol/zvol_cli/zvol_cli_003_neg.ksh \
	functional/zvol/zvol_ENOSPC/cleanup.ksh \
	functional/zvol/zvol_ENOSPC/setup.ksh \
	functional/zvol/zvol_ENOSPC/zvol_ENOSPC_001_pos.ksh \
	functional/zvol/zvol_misc/cleanup.ksh \
	functional/zvol/zvol_misc/setup.ksh \
	functional/zvol/zvol_misc/zvol_misc_001_neg.ksh \
	functional/zvol/zvol_misc/zvol_misc_002_pos.ksh \
	functional/zvol/zvol_misc/zvol_misc_003_neg.ksh \
	functional/zvol/zvol_misc/zvol_misc_004_pos.ksh \
	functional/zvol/zvol_misc/zvol_misc_005_neg.ksh \
	functional/zvol/zvol_misc/zvol_misc_006_pos.ksh \
	functional/zvol/zvol_misc/zvol_misc_fua.ksh \
	functional/zvol/zvol_misc/zvol_misc_hierarchy.ksh \
	functional/zvol/zvol_misc/zvol_misc_rename_inuse.ksh \
	functional/zvol/zvol_misc/zvol_misc_snapdev.ksh \
	functional/zvol/zvol_misc/zvol_misc_trim.ksh \
	functional/zvol/zvol_misc/zvol_misc_volmode.ksh \
	functional/zvol/zvol_misc/zvol_misc_zil.ksh \
	functional/zvol/zvol_stress/cleanup.ksh \
	functional/zvol/zvol_stress/setup.ksh \
	functional/zvol/zvol_stress/zvol_stress.ksh \
	functional/zvol/zvol_swap/cleanup.ksh \
	functional/zvol/zvol_swap/setup.ksh \
	functional/zvol/zvol_swap/zvol_swap_001_pos.ksh \
	functional/zvol/zvol_swap/zvol_swap_002_pos.ksh \
	functional/zvol/zvol_swap/zvol_swap_003_pos.ksh \
	functional/zvol/zvol_swap/zvol_swap_004_pos.ksh \
	functional/zvol/zvol_swap/zvol_swap_005_pos.ksh \
	functional/zvol/zvol_swap/zvol_swap_006_pos.ksh \
	functional/idmap_mount/cleanup.ksh \
	functional/idmap_mount/setup.ksh \
	functional/idmap_mount/idmap_mount_001.ksh \
	functional/idmap_mount/idmap_mount_002.ksh \
	functional/idmap_mount/idmap_mount_003.ksh \
	functional/idmap_mount/idmap_mount_004.ksh \
	functional/idmap_mount/idmap_mount_005.ksh
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/bclone/bclone_common.kshlib b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/bclone/bclone_common.kshlib
	index 3b8eaea5bb54..84b92b4dcdc9 100644
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/bclone/bclone_common.kshlib
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/bclone/bclone_common.kshlib
	@@ -1,286 +1,291 @@
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License (the "License").
	# You may not use this file except in compliance with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END
	#

	#
	# Copyright (c) 2023 by Pawel Jakub Dawidek
	#

	. $STF_SUITE/tests/functional/bclone/bclone.cfg

	export RECORDSIZE=$(zfs get -Hp -o value recordsize $TESTPOOL/$TESTFS)

	MINBLKSIZE1=512
	MINBLKSIZE2=1024

	function verify_block_cloning
	{
	if is_linux && [[ $(linux_version) -lt $(linux_version "4.5") ]]; then
	log_unsupported "copy_file_range not available before Linux 4.5"
	fi
	}

	function verify_crossfs_block_cloning
	{
	if is_linux && [[ $(linux_version) -lt $(linux_version "5.3") ]]; then
	log_unsupported "copy_file_range can't copy cross-filesystem before Linux 5.3"
	fi

	# Cross dataset block cloning only supported on FreeBSD 14+
	# https://github.com/freebsd/freebsd-src/commit/969071be938c
	if is_freebsd && [ $(freebsd_version) -lt $(freebsd_version 14.0) ] ; then
	log_unsupported "Cloning across datasets not supported in $(uname -r)"
	fi
	}

	# Unused.
	function size_to_dsize
	{
	typeset -r size=$1
	typeset -r dir=$2

	typeset -r dataset=$(df $dir \| tail -1 \| awk '{print $1}')
	typeset -r recordsize=$(get_prop recordsize $dataset)
	typeset -r copies=$(get_prop copies $dataset)
	typeset dsize

	if [[ $size -le $recordsize ]]; then
	dsize=$(( ((size - 1) / MINBLOCKSIZE + 1) * MINBLOCKSIZE ))
	else
	dsize=$(( ((size - 1) / recordsize + 1) * recordsize ))
	fi
	dsize=$((dsize*copies))

	echo $dsize
	}

	function test_file_integrity
	{
	typeset -r original_checksum=$1
	typeset -r clone=$2
	typeset -r filesize=$3

	typeset -r clone_checksum=$(sha256digest $clone)

	if [[ $original_checksum != $clone_checksum ]]; then
	log_fail "Clone $clone is corrupted with file size $filesize"
	fi
	}

	function verify_pool_prop_eq
	{
	typeset -r prop=$1
	typeset -r expected=$2

	typeset -r value=$(get_pool_prop $prop $TESTPOOL)
	if [[ $value != $expected ]]; then
	log_fail "Pool property $prop is incorrect: expected $expected, got $value"
	fi
	}

	function verify_pool_props
	{
	- typeset -r dsize=$1
	- typeset -r ratio=$2
	+ typeset -r oused=$1
	+ typeset -r osaved=$2
	+ typeset dsize=$3
	+ typeset ratio=$4

	if [[ $dsize -eq 0 ]]; then
	- verify_pool_prop_eq bcloneused 0
	- verify_pool_prop_eq bclonesaved 0
	- verify_pool_prop_eq bcloneratio 1.00
	- else
	- if [[ $ratio -eq 1 ]]; then
	- verify_pool_prop_eq bcloneused 0
	- else
	- verify_pool_prop_eq bcloneused $dsize
	- fi
	- verify_pool_prop_eq bclonesaved $((dsize*(ratio-1)))
	+ ratio=1
	+ elif [[ $ratio -eq 1 ]]; then
	+ dsize=0
	+ fi
	+ verify_pool_prop_eq bcloneused $(($oused+$dsize))
	+ verify_pool_prop_eq bclonesaved $(($osaved+dsize*(ratio-1)))
	+ if [[ $oused -eq 0 ]]; then
	verify_pool_prop_eq bcloneratio "${ratio}.00"
	fi
	}

	# Function to test file copying and integrity check.
	function bclone_test
	{
	typeset -r datatype=$1
	typeset filesize=$2
	typeset -r embedded=$3
	typeset -r srcdir=$4
	typeset -r dstdir=$5
	typeset dsize
	+ typeset oused
	+ typeset osaved

	typeset -r original="${srcdir}/original"
	typeset -r clone="${dstdir}/clone"

	log_note "Testing file copy with datatype $datatype, file size $filesize, embedded $embedded"

	+ # Save current block cloning stats for later use.
	+ sync_pool $TESTPOOL
	+ oused=$(get_pool_prop bcloneused $TESTPOOL)
	+ osaved=$(get_pool_prop bclonesaved $TESTPOOL)
	+
	# Create a test file with known content.
	case $datatype in
	random\|text)
	- sync_pool $TESTPOOL
	if [[ $datatype = "random" ]]; then
	dd if=/dev/urandom of=$original bs=$filesize count=1 2>/dev/null
	else
	filesize=$(((filesize/4)*4))
	dd if=/dev/urandom bs=$(((filesize/4)*3)) count=1 \| \
	openssl base64 -A > $original
	fi
	sync_pool $TESTPOOL
	clonefile -f $original "${clone}-tmp"
	sync_pool $TESTPOOL
	# It is hard to predict block sizes that will be used,
	# so just do one clone and take it from bcloneused.
	- filesize=$(zpool get -Hp -o value bcloneused $TESTPOOL)
	+ dsize=$(get_pool_prop bcloneused $TESTPOOL)
	+ dsize=$(($dsize-$oused))
	if [[ $embedded = "false" ]]; then
	- log_must test $filesize -gt 0
	+ log_must test $dsize -gt 0
	fi
	rm -f "${clone}-tmp"
	sync_pool $TESTPOOL
	- dsize=$filesize
	;;
	hole)
	log_must truncate_test -s $filesize -f $original
	dsize=0
	;;
	*)
	log_fail "Unknown datatype $datatype"
	;;
	esac
	if [[ $embedded = "true" ]]; then
	dsize=0
	fi

	typeset -r original_checksum=$(sha256digest $original)

	sync_pool $TESTPOOL

	# Create a first clone of the entire file.
	clonefile -f $original "${clone}0"
	# Try to clone the clone in the same transaction group.
	clonefile -f "${clone}0" "${clone}2"

	# Clone the original again...
	clonefile -f $original "${clone}1"
	# ...and overwrite it in the same transaction group.
	clonefile -f $original "${clone}1"

	# Clone the clone...
	clonefile -f "${clone}1" "${clone}3"
	sync_pool $TESTPOOL
	# ...and overwrite in the new transaction group.
	clonefile -f "${clone}1" "${clone}3"

	sync_pool $TESTPOOL

	# Test removal of the pending clones (before they are committed to disk).
	clonefile -f $original "${clone}4"
	clonefile -f "${clone}4" "${clone}5"
	rm -f "${clone}4" "${clone}5"

	# Clone into one file, but remove another file, but with the same data in
	# the same transaction group.
	clonefile -f $original "${clone}5"
	sync_pool $TESTPOOL
	clonefile -f $original "${clone}4"
	rm -f "${clone}5"
	test_file_integrity $original_checksum "${clone}4" $filesize
	sync_pool $TESTPOOL
	test_file_integrity $original_checksum "${clone}4" $filesize

	clonefile -f "${clone}4" "${clone}5"
	# Verify integrity of the cloned file before it is committed to disk.
	test_file_integrity $original_checksum "${clone}5" $filesize

	sync_pool $TESTPOOL

	# Verify integrity in the new transaction group.
	test_file_integrity $original_checksum "${clone}0" $filesize
	test_file_integrity $original_checksum "${clone}1" $filesize
	test_file_integrity $original_checksum "${clone}2" $filesize
	test_file_integrity $original_checksum "${clone}3" $filesize
	test_file_integrity $original_checksum "${clone}4" $filesize
	test_file_integrity $original_checksum "${clone}5" $filesize

	- verify_pool_props $dsize 7
	+ verify_pool_props $oused $osaved $dsize 7

	# Clear cache and test after fresh import.
	log_must zpool export $TESTPOOL
	log_must zpool import $TESTPOOL

	# Cloned uncached file.
	clonefile -f $original "${clone}6"
	# Cloned uncached clone.
	clonefile -f "${clone}6" "${clone}7"

	# Cache the file.
	cat $original >/dev/null
	clonefile -f $original "${clone}8"
	clonefile -f "${clone}8" "${clone}9"

	test_file_integrity $original_checksum "${clone}6" $filesize
	test_file_integrity $original_checksum "${clone}7" $filesize
	test_file_integrity $original_checksum "${clone}8" $filesize
	test_file_integrity $original_checksum "${clone}9" $filesize

	sync_pool $TESTPOOL

	- verify_pool_props $dsize 11
	+ verify_pool_props $oused $osaved $dsize 11

	log_must zpool export $TESTPOOL
	log_must zpool import $TESTPOOL

	test_file_integrity $original_checksum "${clone}0" $filesize
	test_file_integrity $original_checksum "${clone}1" $filesize
	test_file_integrity $original_checksum "${clone}2" $filesize
	test_file_integrity $original_checksum "${clone}3" $filesize
	test_file_integrity $original_checksum "${clone}4" $filesize
	test_file_integrity $original_checksum "${clone}5" $filesize
	test_file_integrity $original_checksum "${clone}6" $filesize
	test_file_integrity $original_checksum "${clone}7" $filesize
	test_file_integrity $original_checksum "${clone}8" $filesize
	test_file_integrity $original_checksum "${clone}9" $filesize

	rm -f $original
	rm -f "${clone}1" "${clone}3" "${clone}5" "${clone}7"

	sync_pool $TESTPOOL

	test_file_integrity $original_checksum "${clone}0" $filesize
	test_file_integrity $original_checksum "${clone}2" $filesize
	test_file_integrity $original_checksum "${clone}4" $filesize
	test_file_integrity $original_checksum "${clone}6" $filesize
	test_file_integrity $original_checksum "${clone}8" $filesize
	test_file_integrity $original_checksum "${clone}9" $filesize

	- verify_pool_props $dsize 6
	+ verify_pool_props $oused $osaved $dsize 6

	rm -f "${clone}0" "${clone}2" "${clone}4" "${clone}8" "${clone}9"

	sync_pool $TESTPOOL

	test_file_integrity $original_checksum "${clone}6" $filesize

	- verify_pool_props $dsize 1
	+ verify_pool_props $oused $osaved $dsize 1

	rm -f "${clone}6"

	sync_pool $TESTPOOL

	- verify_pool_props $dsize 1
	+ verify_pool_props $oused $osaved $dsize 1
	}
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/bclone/bclone_corner_cases.kshlib b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/bclone/bclone_corner_cases.kshlib
	index ddfbfc999c4e..aeb8efe91715 100644
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/bclone/bclone_corner_cases.kshlib
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/bclone/bclone_corner_cases.kshlib
	@@ -1,315 +1,325 @@
	#! /bin/ksh -p
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License (the "License").
	# You may not use this file except in compliance with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END
	#

	#
	# Copyright (c) 2023 by Pawel Jakub Dawidek
	#

	. $STF_SUITE/include/libtest.shlib
	. $STF_SUITE/include/math.shlib
	. $STF_SUITE/tests/functional/bclone/bclone_common.kshlib

	function first_half_checksum
	{
	typeset -r file=$1

	dd if=$file bs=$HALFRECORDSIZE count=1 2>/dev/null \| sha256digest
	}

	function second_half_checksum
	{
	typeset -r file=$1

	dd if=$file bs=$HALFRECORDSIZE count=1 skip=1 2>/dev/null \| sha256digest
	}

	function bclone_corner_cases_init
	{
	typeset -r srcdir=$1
	typeset -r dstdir=$2

	export RECORDSIZE=4096
	export HALFRECORDSIZE=$((RECORDSIZE / 2))

	export CLONE="$dstdir/clone0"
	export ORIG0="$srcdir/orig0"
	export ORIG1="$srcdir/orig1"
	export ORIG2="$srcdir/orig2"

	# Create source files.
	log_must dd if=/dev/urandom of="$ORIG0" bs=$RECORDSIZE count=1
	log_must dd if=/dev/urandom of="$ORIG1" bs=$RECORDSIZE count=1
	log_must dd if=/dev/urandom of="$ORIG2" bs=$RECORDSIZE count=1

	export FIRST_HALF_ORIG0_CHECKSUM=$(first_half_checksum $ORIG0)
	export FIRST_HALF_ORIG1_CHECKSUM=$(first_half_checksum $ORIG1)
	export FIRST_HALF_ORIG2_CHECKSUM=$(first_half_checksum $ORIG2)
	export SECOND_HALF_ORIG0_CHECKSUM=$(second_half_checksum $ORIG0)
	export SECOND_HALF_ORIG1_CHECKSUM=$(second_half_checksum $ORIG1)
	export SECOND_HALF_ORIG2_CHECKSUM=$(second_half_checksum $ORIG2)
	- export ZEROS_CHECKSUM=$(dd if=/dev/zero bs=$HALFRECORDSIZE count=1 \| sha256digest)
	+ export ZEROS_CHECKSUM=$(dd if=/dev/zero bs=$HALFRECORDSIZE count=1 2>/dev/null \| sha256digest)
	export FIRST_HALF_CHECKSUM=""
	export SECOND_HALF_CHECKSUM=""
	}

	function cache_clone
	{
	typeset -r cached=$1

	case "$cached" in
	"cached")
	dd if=$CLONE of=/dev/null bs=$RECORDSIZE 2>/dev/null
	;;
	"uncached")
	;;
	*)
	log_fail "invalid cached: $cached"
	;;
	esac
	}

	function create_existing
	{
	typeset -r existing=$1

	case "$existing" in
	"no")
	;;
	"small empty")
	log_must truncate_test -s $HALFRECORDSIZE -f $CLONE
	;;
	"full empty")
	log_must truncate_test -s $RECORDSIZE -f $CLONE
	;;
	"small data")
	log_must dd if=/dev/urandom of=$CLONE bs=$HALFRECORDSIZE count=1 \
	2>/dev/null
	;;
	"full data")
	log_must dd if=/dev/urandom of=$CLONE bs=$RECORDSIZE count=1 2>/dev/null
	;;
	*)
	log_fail "invalid existing: $existing"
	;;
	esac
	}

	function create_clone
	{
	typeset -r clone=$1
	typeset -r file=$2

	case "$clone" in
	"no")
	;;
	"yes")
	clonefile -f $file $CLONE
	case "$file" in
	$ORIG0)
	FIRST_HALF_CHECKSUM=$FIRST_HALF_ORIG0_CHECKSUM
	SECOND_HALF_CHECKSUM=$SECOND_HALF_ORIG0_CHECKSUM
	;;
	$ORIG2)
	FIRST_HALF_CHECKSUM=$FIRST_HALF_ORIG2_CHECKSUM
	SECOND_HALF_CHECKSUM=$SECOND_HALF_ORIG2_CHECKSUM
	;;
	*)
	log_fail "invalid file: $file"
	;;
	esac
	;;
	*)
	log_fail "invalid clone: $clone"
	;;
	esac
	}

	function overwrite_clone
	{
	typeset -r overwrite=$1

	case "$overwrite" in
	"no")
	;;
	"free")
	log_must truncate_test -s 0 -f $CLONE
	log_must truncate_test -s $RECORDSIZE -f $CLONE
	FIRST_HALF_CHECKSUM=$ZEROS_CHECKSUM
	SECOND_HALF_CHECKSUM=$ZEROS_CHECKSUM
	;;
	"full")
	log_must dd if=$ORIG1 of=$CLONE bs=$RECORDSIZE count=1 2>/dev/null
	FIRST_HALF_CHECKSUM=$FIRST_HALF_ORIG1_CHECKSUM
	SECOND_HALF_CHECKSUM=$SECOND_HALF_ORIG1_CHECKSUM
	;;
	"first half")
	log_must dd if=$ORIG1 of=$CLONE bs=$HALFRECORDSIZE skip=0 seek=0 \
	count=1 conv=notrunc 2>/dev/null
	FIRST_HALF_CHECKSUM=$FIRST_HALF_ORIG1_CHECKSUM
	;;
	"second half")
	log_must dd if=$ORIG1 of=$CLONE bs=$HALFRECORDSIZE skip=1 seek=1 \
	count=1 conv=notrunc 2>/dev/null
	SECOND_HALF_CHECKSUM=$SECOND_HALF_ORIG1_CHECKSUM
	;;
	*)
	log_fail "invalid overwrite: $overwrite"
	;;
	esac
	}

	function checksum_compare
	{
	typeset -r compare=$1
	typeset first_half_calculated_checksum second_half_calculated_checksum

	case "$compare" in
	"no")
	;;
	"yes")
	first_half_calculated_checksum=$(first_half_checksum $CLONE)
	second_half_calculated_checksum=$(second_half_checksum $CLONE)

	if [[ $first_half_calculated_checksum != $FIRST_HALF_CHECKSUM ]] \|\| \
	[[ $second_half_calculated_checksum != $SECOND_HALF_CHECKSUM ]]; then
	return 1
	fi
	;;
	*)
	log_fail "invalid compare: $compare"
	;;
	esac
	}

	function bclone_corner_cases_test
	{
	typeset cached existing
	typeset first_clone first_overwrite
	typeset read_after read_before
	typeset second_clone second_overwrite
	typeset -r srcdir=$1
	typeset -r dstdir=$2
	typeset limit=$3
	typeset -i count=0
	+ typeset oused
	+ typeset osaved

	if [[ $srcdir != "count" ]]; then
	if [[ -n "$limit" ]]; then
	typeset -r total_count=$(bclone_corner_cases_test count)
	limit=$(random_int_between 1 $total_count $((limit*2)) \| sort -nu \| head -n $limit \| xargs)
	fi
	bclone_corner_cases_init $srcdir $dstdir
	+
	+ # Save current block cloning stats for later use.
	+ sync_pool $TESTPOOL
	+ oused=$(get_pool_prop bcloneused $TESTPOOL)
	+ osaved=$(get_pool_prop bclonesaved $TESTPOOL)
	fi

	#
	# (create) / (cache) / (clone) / (overwrite) / (read) / (clone) / (overwrite) / (read) / read next txg
	#
	for existing in "no" "small empty" "full empty" "small data" "full data"; do
	for cached in "uncached" "cached"; do
	for first_clone in "no" "yes"; do
	for first_overwrite in "no" "free" "full" "first half" "second half"; do
	for read_before in "no" "yes"; do
	for second_clone in "no" "yes"; do
	for second_overwrite in "no" "free" "full" "first half" "second half"; do
	for read_after in "no" "yes"; do
	if [[ $first_clone = "no" ]] && \
	[[ $second_clone = "no" ]]; then
	continue
	fi
	if [[ $first_clone = "no" ]] && \
	[[ $read_before = "yes" ]]; then
	continue
	fi
	if [[ $second_clone = "no" ]] && \
	[[ $read_before = "yes" ]] && \
	[[ $read_after = "yes" ]]; then
	continue
	fi

	count=$((count+1))

	if [[ $srcdir = "count" ]]; then
	# Just counting.
	continue
	fi

	if [[ -n "$limit" ]]; then
	if ! echo " $limit " \| grep -q " $count "; then
	continue
	fi
	fi

	FIRST_HALF_CHECKSUM=""
	SECOND_HALF_CHECKSUM=""

	log_must zpool export $TESTPOOL
	log_must zpool import $TESTPOOL

	create_existing "$existing"

	log_must zpool export $TESTPOOL
	log_must zpool import $TESTPOOL

	cache_clone "$cached"

	create_clone "$first_clone" "$ORIG0"

	overwrite_clone "$first_overwrite"

	if checksum_compare $read_before; then
	log_note "existing: $existing / cached: $cached / first_clone: $first_clone / first_overwrite: $first_overwrite / read_before: $read_before"
	else
	log_fail "FAIL: existing: $existing / cached: $cached / first_clone: $first_clone / first_overwrite: $first_overwrite / read_before: $read_before"
	fi

	create_clone "$second_clone" "$ORIG2"

	overwrite_clone "$second_overwrite"

	if checksum_compare $read_after; then
	- log_note "existing: $existing / cached: $cached / first_clone: $first_clone / first_overwrite: $first_overwrite / read_before: $read_before / second_clone: $second_clone / read_after: $read_after"
	+ log_note "existing: $existing / cached: $cached / first_clone: $first_clone / first_overwrite: $first_overwrite / read_before: $read_before / second_clone: $second_clone / second_overwrite: $second_overwrite / read_after: $read_after"
	else
	- log_fail "FAIL: existing: $existing / cached: $cached / first_clone: $first_clone / first_overwrite: $first_overwrite / read_before: $read_before / second_clone: $second_clone / read_after: $read_after"
	+ log_fail "FAIL: existing: $existing / cached: $cached / first_clone: $first_clone / first_overwrite: $first_overwrite / read_before: $read_before / second_clone: $second_clone / second_overwrite: $second_overwrite / read_after: $read_after"
	fi

	log_must zpool export $TESTPOOL
	log_must zpool import $TESTPOOL

	if checksum_compare "yes"; then
	- log_note "existing: $existing / cached: $cached / first_clone: $first_clone / first_overwrite: $first_overwrite / read_before: $read_before / second_clone: $second_clone / read_after: $read_after / read_next_txg"
	+ log_note "existing: $existing / cached: $cached / first_clone: $first_clone / first_overwrite: $first_overwrite / read_before: $read_before / second_clone: $second_clone / second_overwrite: $second_overwrite / read_after: $read_after / read_next_txg"
	else
	- log_fail "FAIL: existing: $existing / cached: $cached / first_clone: $first_clone / first_overwrite: $first_overwrite / read_before: $read_before / second_clone: $second_clone / read_after: $read_after / read_next_txg"
	+ log_fail "FAIL: existing: $existing / cached: $cached / first_clone: $first_clone / first_overwrite: $first_overwrite / read_before: $read_before / second_clone: $second_clone / second_overwrite: $second_overwrite / read_after: $read_after / read_next_txg"
	fi

	rm -f "$CLONE"
	+ sync_pool $TESTPOOL
	+ verify_pool_prop_eq bcloneused $oused
	+ verify_pool_prop_eq bclonesaved $osaved
	done
	done
	done
	done
	done
	done
	done
	done

	if [[ $srcdir = "count" ]]; then
	echo $count
	fi
	}
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_mount/zfs_mount.cfg b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_mount/zfs_mount.cfg
	index 06d25faf0356..739baf16086a 100644
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_mount/zfs_mount.cfg
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_mount/zfs_mount.cfg
	@@ -1,39 +1,40 @@
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License (the "License").
	# You may not use this file except in compliance with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END
	#

	#
	# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
	# Use is subject to license terms.
	#

	#
	# Copyright (c) 2012 by Delphix. All rights reserved.
	#

	export mountcmd=mount
	export mountforce="$mountcmd -f"
	export mountall="$mountcmd -a"
	+export mountrecursive="$mountcmd -R"

	export unmountcmd=unmount
	export unmountforce="$unmountcmd -f"
	export unmountall="$unmountcmd -a"

	export NONEXISTFSNAME="nonexistfs50charslong_0123456789012345678901234567"
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_mount/zfs_mount_recursive.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_mount/zfs_mount_recursive.ksh
	new file mode 100755
	index 000000000000..0e5cc5d6955e
	--- /dev/null
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_mount/zfs_mount_recursive.ksh
	@@ -0,0 +1,146 @@
	+#!/bin/ksh -p
	+#
	+# CDDL HEADER START
	+#
	+# The contents of this file are subject to the terms of the
	+# Common Development and Distribution License (the "License").
	+# You may not use this file except in compliance with the License.
	+#
	+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	+# or https://opensource.org/licenses/CDDL-1.0.
	+# See the License for the specific language governing permissions
	+# and limitations under the License.
	+#
	+# When distributing Covered Code, include this CDDL HEADER in each
	+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	+# If applicable, add the following below this CDDL HEADER, with the
	+# fields enclosed by brackets "[]" replaced with your own identifying
	+# information: Portions Copyright [yyyy] [name of copyright owner]
	+#
	+# CDDL HEADER END
	+#
	+
	+#
	+# Copyright 2024, iXsystems Inc. All rights reserved.
	+#
	+
	+. $STF_SUITE/include/libtest.shlib
	+. $STF_SUITE/tests/functional/cli_root/zfs_mount/zfs_mount.kshlib
	+
	+#
	+# DESCRIPTION:
	+# Verify zfs mount -R <filesystems/s> functionality.
	+#
	+# STRATEGY:
	+# 1. Create nested datasets
	+# 2. Unmount all datasets
	+# 3. Recusrively mount root datasets, this should mount all datasets
	+# present in a pool
	+# 4. Unmount all datasets
	+# 5. Recusrsively mount child datasets with children. This should mount
	+# child datasets, but not the root dataset or parent datasets
	+# 6. Unmount all datasets
	+# 7. Mount root dataset recursively again and confirm all child
	+# datasets are mounted.
	+#
	+
	+verify_runnable "both"
	+
	+function cleanup
	+{
	+ log_must datasetexists $TESTPOOL/$TESTFS1 && \
	+ destroy_dataset $TESTPOOL/$TESTFS1 -R
	+ log_must datasetexists $TESTPOOL/$TESTFS2 && \
	+ destroy_dataset $TESTPOOL/$TESTFS2 -R
	+ log_must datasetexists $TESTPOOL/$TESTFS3 && \
	+ destroy_dataset $TESTPOOL/$TESTFS3 -R
	+}
	+
	+function setup_all
	+{
	+ log_must datasetexists $TESTPOOL/$TESTFS \|\| zfs create $TESTPOOL/$TESTFS
	+ log_must zfs create $TESTPOOL/$TESTFS1
	+ log_must zfs create $TESTPOOL/$TESTFS2
	+ log_must zfs create $TESTPOOL/$TESTFS3
	+ log_must zfs create $TESTPOOL/$TESTFS2/child1
	+ log_must zfs create $TESTPOOL/$TESTFS2/child2
	+ log_must zfs create $TESTPOOL/$TESTFS2/child3
	+ log_must zfs create $TESTPOOL/$TESTFS2/child2/subchild
	+ log_must zfs create $TESTPOOL/$TESTFS3/child
	+}
	+
	+log_assert "Verify that 'zfs $mountrecursive' successfully, " \
	+ "mounts the dataset along with all its children."
	+
	+log_onexit cleanup
	+
	+log_must setup_all
	+
	+log_must zfs $unmountall
	+
	+log_must zfs $mountrecursive $TESTPOOL
	+
	+log_must mounted $TESTPOOL
	+log_must mounted $TESTPOOL/$TESTFS
	+log_must mounted $TESTPOOL/$TESTFS1
	+log_must mounted $TESTPOOL/$TESTFS2
	+log_must mounted $TESTPOOL/$TESTFS3
	+log_must mounted $TESTPOOL/$TESTFS2/child1
	+log_must mounted $TESTPOOL/$TESTFS2/child2
	+log_must mounted $TESTPOOL/$TESTFS2/child3
	+log_must mounted $TESTPOOL/$TESTFS2/child2/subchild
	+log_must mounted $TESTPOOL/$TESTFS3/child
	+
	+log_must zfs $unmountall
	+
	+log_mustnot mounted $TESTPOOL
	+log_mustnot mounted $TESTPOOL/$TESTFS
	+log_mustnot mounted $TESTPOOL/$TESTFS1
	+log_mustnot mounted $TESTPOOL/$TESTFS2
	+log_mustnot mounted $TESTPOOL/$TESTFS3
	+log_mustnot mounted $TESTPOOL/$TESTFS2/child1
	+log_mustnot mounted $TESTPOOL/$TESTFS2/child2
	+log_mustnot mounted $TESTPOOL/$TESTFS2/child3
	+log_mustnot mounted $TESTPOOL/$TESTFS2/child2/subchild
	+log_mustnot mounted $TESTPOOL/$TESTFS3/child
	+
	+log_must zfs $mountrecursive $TESTPOOL/$TESTFS2 $TESTPOOL/$TESTFS3
	+
	+log_mustnot mounted $TESTPOOL
	+log_mustnot mounted $TESTPOOL/$TESTFS
	+log_mustnot mounted $TESTPOOL/$TESTFS1
	+log_must mounted $TESTPOOL/$TESTFS2
	+log_must mounted $TESTPOOL/$TESTFS3
	+log_must mounted $TESTPOOL/$TESTFS2/child1
	+log_must mounted $TESTPOOL/$TESTFS2/child2
	+log_must mounted $TESTPOOL/$TESTFS2/child3
	+log_must mounted $TESTPOOL/$TESTFS2/child2/subchild
	+log_must mounted $TESTPOOL/$TESTFS3/child
	+
	+log_must zfs $unmountall
	+
	+log_mustnot mounted $TESTPOOL
	+log_mustnot mounted $TESTPOOL/$TESTFS
	+log_mustnot mounted $TESTPOOL/$TESTFS1
	+log_mustnot mounted $TESTPOOL/$TESTFS2
	+log_mustnot mounted $TESTPOOL/$TESTFS3
	+log_mustnot mounted $TESTPOOL/$TESTFS2/child1
	+log_mustnot mounted $TESTPOOL/$TESTFS2/child2
	+log_mustnot mounted $TESTPOOL/$TESTFS2/child3
	+log_mustnot mounted $TESTPOOL/$TESTFS2/child2/subchild
	+log_mustnot mounted $TESTPOOL/$TESTFS3/child
	+
	+log_must zfs $mountrecursive $TESTPOOL/$TESTFS2/child2
	+
	+log_must mounted $TESTPOOL/$TESTFS2/child2
	+log_must mounted $TESTPOOL/$TESTFS2/child2/subchild
	+log_mustnot mounted $TESTPOOL
	+log_mustnot mounted $TESTPOOL/$TESTFS
	+log_mustnot mounted $TESTPOOL/$TESTFS1
	+log_mustnot mounted $TESTPOOL/$TESTFS2
	+log_mustnot mounted $TESTPOOL/$TESTFS3
	+log_mustnot mounted $TESTPOOL/$TESTFS2/child1
	+log_mustnot mounted $TESTPOOL/$TESTFS2/child3
	+log_mustnot mounted $TESTPOOL/$TESTFS3/child
	+
	+log_pass "'zfs $mountrecursive' behaves as expected."
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/add-o_ashift.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/add-o_ashift.ksh
	index 7ecaf849e44b..51871934dd22 100755
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/add-o_ashift.ksh
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/add-o_ashift.ksh
	@@ -1,103 +1,114 @@
	#!/bin/ksh -p
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License (the "License").
	# You may not use this file except in compliance with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END
	#

	#
	# Copyright 2017, loli10K. All rights reserved.
	-# Copyright (c) 2020 by Delphix. All rights reserved.
	+# Copyright (c) 2020, 2024 by Delphix. All rights reserved.
	#

	. $STF_SUITE/include/libtest.shlib
	. $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib

	#
	# DESCRIPTION:
	# 'zpool add -o ashift=<n> ...' should work with different ashift
	# values.
	#
	# STRATEGY:
	# 1. Create a pool with default values.
	# 2. Verify 'zpool add -o ashift=<n>' works with allowed values (9-16).
	# 3. Verify setting kernel tunable for file vdevs works correctly.
	# 4. Verify 'zpool add -o ashift=<n>' doesn't accept other invalid values.
	#

	verify_runnable "global"

	function cleanup
	{
	log_must set_tunable32 VDEV_FILE_PHYSICAL_ASHIFT $orig_ashift
	poolexists $TESTPOOL && destroy_pool $TESTPOOL
	rm -f $disk1 $disk2
	}

	log_assert "zpool add -o ashift=<n>' works with different ashift values"
	log_onexit cleanup

	disk1=$TEST_BASE_DIR/disk1
	disk2=$TEST_BASE_DIR/disk2
	log_must mkfile $SIZE $disk1
	log_must mkfile $SIZE $disk2

	logical_ashift=$(get_tunable VDEV_FILE_LOGICAL_ASHIFT)
	orig_ashift=$(get_tunable VDEV_FILE_PHYSICAL_ASHIFT)
	max_auto_ashift=$(get_tunable VDEV_MAX_AUTO_ASHIFT)
	+opt=""

	typeset ashifts=("9" "10" "11" "12" "13" "14" "15" "16")
	for ashift in ${ashifts[@]}
	do
	+ #
	+ # Need to add the --allow-ashift-mismatch option to disable the
	+ # ashift mismatch checks in zpool add.
	+ #
	+ if [[ $ashift -eq $orig_ashift ]]; then
	+ opt=""
	+ else
	+ opt="--allow-ashift-mismatch"
	+ fi
	+
	log_must zpool create $TESTPOOL $disk1
	- log_must zpool add -o ashift=$ashift $TESTPOOL $disk2
	+ log_must zpool add $opt -o ashift=$ashift $TESTPOOL $disk2
	log_must verify_ashift $disk2 $ashift

	# clean things for the next run
	log_must zpool destroy $TESTPOOL
	log_must zpool labelclear $disk1
	log_must zpool labelclear $disk2

	#
	# Make sure we can also set the ashift using the tunable.
	#
	log_must zpool create $TESTPOOL $disk1
	log_must set_tunable32 VDEV_FILE_PHYSICAL_ASHIFT $ashift
	- log_must zpool add $TESTPOOL $disk2
	+ log_must zpool add $opt $TESTPOOL $disk2
	exp=$(( (ashift <= max_auto_ashift) ? ashift : logical_ashift ))
	log_must verify_ashift $disk2 $exp

	# clean things for the next run
	log_must set_tunable32 VDEV_FILE_PHYSICAL_ASHIFT $orig_ashift
	log_must zpool destroy $TESTPOOL
	log_must zpool labelclear $disk1
	log_must zpool labelclear $disk2
	done

	typeset badvals=("off" "on" "1" "8" "17" "1b" "ff" "-")
	for badval in ${badvals[@]}
	do
	log_must zpool create $TESTPOOL $disk1
	log_mustnot zpool add -o ashift="$badval" $TESTPOOL $disk2
	# clean things for the next run
	log_must zpool destroy $TESTPOOL
	log_must zpool labelclear $disk1
	log_mustnot zpool labelclear $disk2
	done

	log_pass "zpool add -o ashift=<n>' works with different ashift values"
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/add_prop_ashift.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/add_prop_ashift.ksh
	index 228f62232aae..6a3283d0618f 100755
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/add_prop_ashift.ksh
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/add_prop_ashift.ksh
	@@ -1,96 +1,106 @@
	#!/bin/ksh -p
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License (the "License").
	# You may not use this file except in compliance with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END
	#

	#
	# Copyright 2017, loli10K. All rights reserved.
	-# Copyright (c) 2020 by Delphix. All rights reserved.
	+# Copyright (c) 2020, 2024 by Delphix. All rights reserved.
	#

	. $STF_SUITE/include/libtest.shlib
	. $STF_SUITE/tests/functional/cli_root/zpool_create/zpool_create.shlib

	#
	# DESCRIPTION:
	# 'zpool add' should use the ashift pool property value as default.
	#
	# STRATEGY:
	# 1. Create a pool with default values.
	# 2. Verify 'zpool add' uses the ashift pool property value when adding
	# a new device.
	# 3. Verify the default ashift value can still be overridden by manually
	# specifying '-o ashift=<n>' from the command line.
	#

	verify_runnable "global"

	function cleanup
	{
	log_must set_tunable32 VDEV_FILE_PHYSICAL_ASHIFT $orig_ashift
	poolexists $TESTPOOL && destroy_pool $TESTPOOL
	log_must rm -f $disk1 $disk2
	}

	log_assert "'zpool add' uses the ashift pool property value as default."
	log_onexit cleanup

	disk1=$TEST_BASE_DIR/disk1
	disk2=$TEST_BASE_DIR/disk2
	log_must mkfile $SIZE $disk1
	log_must mkfile $SIZE $disk2

	orig_ashift=$(get_tunable VDEV_FILE_PHYSICAL_ASHIFT)
	#
	# Set the file vdev's ashift to the max. Overriding
	# the ashift using the -o ashift property should still
	# be honored.
	#
	log_must set_tunable32 VDEV_FILE_PHYSICAL_ASHIFT 16

	typeset ashifts=("9" "10" "11" "12" "13" "14" "15" "16")
	for ashift in ${ashifts[@]}
	do
	+ if [ $ashift -eq $orig_ashift ];then
	+ opt=""
	+ else
	+ opt="--allow-ashift-mismatch"
	+ fi
	log_must zpool create -o ashift=$ashift $TESTPOOL $disk1
	- log_must zpool add $TESTPOOL $disk2
	+ log_must zpool add $opt $TESTPOOL $disk2
	log_must verify_ashift $disk2 $ashift

	# clean things for the next run
	log_must zpool destroy $TESTPOOL
	log_must zpool labelclear $disk1
	log_must zpool labelclear $disk2
	done

	for ashift in ${ashifts[@]}
	do
	for cmdval in ${ashifts[@]}
	do
	+ if [ $ashift -eq $cmdval ];then
	+ opt=""
	+ else
	+ opt="--allow-ashift-mismatch"
	+ fi
	log_must zpool create -o ashift=$ashift $TESTPOOL $disk1
	- log_must zpool add -o ashift=$cmdval $TESTPOOL $disk2
	+ log_must zpool add $opt -o ashift=$cmdval $TESTPOOL $disk2
	log_must verify_ashift $disk2 $cmdval

	# clean things for the next run
	log_must zpool destroy $TESTPOOL
	log_must zpool labelclear $disk1
	log_must zpool labelclear $disk2
	done
	done

	log_pass "'zpool add' uses the ashift pool property value."
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add--allow-ashift-mismatch.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add--allow-ashift-mismatch.ksh
	new file mode 100755
	index 000000000000..e69de29bb2d1
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_002_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_002_pos.ksh
	index c5c06f76340b..afee34a33469 100755
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_002_pos.ksh
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_002_pos.ksh
	@@ -1,68 +1,79 @@
	#!/bin/ksh -p
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License (the "License").
	# You may not use this file except in compliance with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END
	#

	#
	# Copyright 2009 Sun Microsystems, Inc. All rights reserved.
	# Use is subject to license terms.
	#

	#
	# Copyright (c) 2014, 2016 by Delphix. All rights reserved.
	#
	. $STF_SUITE/include/libtest.shlib
	. $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib

	#
	# DESCRIPTION:
	# 'zpool add -f <pool> <vdev> ...' can successfully add the specified
	# devices to given pool in some cases.
	#
	# STRATEGY:
	# 1. Create a mirrored pool
	# 2. Without -f option to add 1-way device the mirrored pool will fail
	# 3. Use -f to override the errors to add 1-way device to the mirrored
	# pool
	# 4. Verify the device is added successfully
	#

	verify_runnable "global"

	function cleanup
	{
	poolexists $TESTPOOL && destroy_pool $TESTPOOL
	}

	log_assert "'zpool add -f <pool> <vdev> ...' can successfully add" \
	"devices to the pool in some cases."

	log_onexit cleanup

	create_pool $TESTPOOL mirror $DISK0 $DISK1
	log_must poolexists $TESTPOOL

	log_mustnot zpool add $TESTPOOL $DISK2
	log_mustnot vdevs_in_pool $TESTPOOL $DISK2

	log_must zpool add -f $TESTPOOL $DISK2
	log_must vdevs_in_pool $TESTPOOL $DISK2

	+log_must zpool destroy $TESTPOOL
	+
	+create_pool $TESTPOOL mirror $DISK0 $DISK1
	+log_must poolexists $TESTPOOL
	+
	+log_mustnot zpool add $TESTPOOL $DISK2
	+log_mustnot vdevs_in_pool $TESTPOOL $DISK2
	+
	+log_must zpool add --allow-replication-mismatch $TESTPOOL $DISK2
	+log_must vdevs_in_pool $TESTPOOL $DISK2
	+
	log_pass "'zpool add -f <pool> <vdev> ...' executes successfully."
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_004_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_004_pos.ksh
	index 646edc1a4557..cecda56ab125 100755
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_004_pos.ksh
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_004_pos.ksh
	@@ -1,81 +1,81 @@
	#!/bin/ksh -p
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License (the "License").
	# You may not use this file except in compliance with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END
	#

	#
	# Copyright 2009 Sun Microsystems, Inc. All rights reserved.
	# Use is subject to license terms.
	#

	#
	# Copyright (c) 2014, 2016 by Delphix. All rights reserved.
	#

	. $STF_SUITE/include/libtest.shlib
	. $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib

	#
	# DESCRIPTION:
	# 'zpool add <pool> <vdev> ...' can successfully add a zfs volume
	# to the given pool
	#
	# STRATEGY:
	# 1. Create a storage pool and a zfs volume
	# 2. Add the volume to the pool
	# 3. Verify the devices are added to the pool successfully
	#

	verify_runnable "global"

	function cleanup
	{
	poolexists $TESTPOOL && destroy_pool $TESTPOOL
	poolexists $TESTPOOL1 && destroy_pool $TESTPOOL1
	if [ -n "$recursive" ]; then
	set_tunable64 VOL_RECURSIVE $recursive
	fi
	}

	log_assert "'zpool add <pool> <vdev> ...' can add zfs volume to the pool."

	log_onexit cleanup

	create_pool $TESTPOOL $DISK0
	log_must poolexists $TESTPOOL

	create_pool $TESTPOOL1 $DISK1
	log_must poolexists $TESTPOOL1
	log_must zfs create -V $VOLSIZE $TESTPOOL1/$TESTVOL
	block_device_wait

	if is_freebsd; then
	recursive=$(get_tunable VOL_RECURSIVE)
	log_must set_tunable64 VOL_RECURSIVE 1
	fi
	-log_must zpool add $TESTPOOL $ZVOL_DEVDIR/$TESTPOOL1/$TESTVOL
	+log_must zpool add --allow-ashift-mismatch $TESTPOOL $ZVOL_DEVDIR/$TESTPOOL1/$TESTVOL

	log_must vdevs_in_pool "$TESTPOOL" "$ZVOL_DEVDIR/$TESTPOOL1/$TESTVOL"

	# Give zed a chance to finish processing the event, otherwise
	# a race condition can lead to stuck "zpool destroy $TESTPOOL"
	sleep 1

	log_pass "'zpool add <pool> <vdev> ...' adds zfs volume to the pool successfully"
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_005_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_005_pos.ksh
	index 4990ef9d29b0..0e9d9f5f030f 100755
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_005_pos.ksh
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_005_pos.ksh
	@@ -1,91 +1,93 @@
	#!/bin/ksh -p
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License (the "License").
	# You may not use this file except in compliance with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END
	#

	#
	# Copyright 2009 Sun Microsystems, Inc. All rights reserved.
	# Use is subject to license terms.
	#

	#
	# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
	#

	. $STF_SUITE/include/libtest.shlib
	. $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib

	#
	# DESCRIPTION:
	# 'zpool add' should return fail if
	# 1. vdev is part of an active pool
	# 2. vdev is currently mounted
	# 3. vdev is in /etc/vfstab
	# 3. vdev is specified as the dedicated dump device
	#
	# STRATEGY:
	# 1. Create case scenarios
	# 2. For each scenario, try to add the device to the pool
	# 3. Verify the add operation get failed
	#

	verify_runnable "global"

	function cleanup
	{
	poolexists $TESTPOOL && destroy_pool $TESTPOOL
	poolexists $TESTPOOL1 && destroy_pool $TESTPOOL1

	if [[ -n $saved_dump_dev ]]; then
	log_must eval "dumpadm -u -d $saved_dump_dev > /dev/null"
	fi
	}

	log_assert "'zpool add' should fail with inapplicable scenarios."

	log_onexit cleanup

	mnttab_dev=$(find_mnttab_dev)
	vfstab_dev=$(find_vfstab_dev)
	saved_dump_dev=$(save_dump_dev)
	dump_dev=$DISK2

	create_pool $TESTPOOL $DISK0
	log_must poolexists $TESTPOOL

	create_pool $TESTPOOL1 $DISK1
	log_must poolexists $TESTPOOL1

	unset NOINUSE_CHECK
	log_mustnot zpool add -f $TESTPOOL $DISK1
	+log_mustnot zpool add --allow-in-use $TESTPOOL $DISK1
	log_mustnot zpool add -f $TESTPOOL $mnttab_dev
	+log_mustnot zpool add --allow-in-use $TESTPOOL $mnttab_dev
	if is_linux; then
	log_mustnot zpool add $TESTPOOL $vfstab_dev
	else
	log_mustnot zpool add -f $TESTPOOL $vfstab_dev
	fi

	if is_illumos; then
	log_must eval "new_fs ${DEV_DSKDIR}/$dump_dev > /dev/null 2>&1"
	log_must eval "dumpadm -u -d ${DEV_DSKDIR}/$dump_dev > /dev/null"
	log_mustnot zpool add -f $TESTPOOL $dump_dev
	fi

	log_pass "'zpool add' should fail with inapplicable scenarios."
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_009_neg.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_009_neg.ksh
	index d7f3a900e8fd..a13a27160e76 100755
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_009_neg.ksh
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_009_neg.ksh
	@@ -1,71 +1,73 @@
	#!/bin/ksh -p
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License (the "License").
	# You may not use this file except in compliance with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END
	#

	#
	# Copyright 2009 Sun Microsystems, Inc. All rights reserved.
	# Use is subject to license terms.
	#

	#
	# Copyright (c) 2016 by Delphix. All rights reserved.
	#

	. $STF_SUITE/include/libtest.shlib
	. $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib

	#
	# DESCRIPTION:
	# 'zpool add' should return fail if vdevs are the same or vdev is
	# contained in the given pool
	#
	# STRATEGY:
	# 1. Create a storage pool
	# 2. Add the device in pool A to pool A again
	# 3. Add the two devices to pool A in the loop, one of them already
	# added or same device added multiple times
	#

	verify_runnable "global"

	function cleanup
	{
	poolexists $TESTPOOL && destroy_pool $TESTPOOL
	}

	log_assert "'zpool add' should fail if vdevs are the same or vdev is " \
	"contained in the given pool."

	log_onexit cleanup

	create_pool $TESTPOOL $DISK0
	log_must poolexists $TESTPOOL

	log_mustnot zpool add -f $TESTPOOL $DISK0

	for type in "" "mirror" "raidz" "draid" "spare" "log" "dedup" "special" "cache"
	do
	log_mustnot zpool add -f $TESTPOOL $type $DISK0 $DISK1
	+ log_mustnot zpool add --allow-in-use $TESTPOOL $type $DISK0 $DISK1
	log_mustnot zpool add -f $TESTPOOL $type $DISK1 $DISK1
	+ log_mustnot zpool add --allow-in-use $TESTPOOL $type $DISK1 $DISK1
	done

	log_pass "'zpool add' get fail as expected if vdevs are the same or vdev is " \
	"contained in the given pool."
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_010_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_010_pos.ksh
	index b8b25db1b9f9..22860e9caf1d 100755
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_010_pos.ksh
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_010_pos.ksh
	@@ -1,210 +1,210 @@
	#!/bin/ksh -p
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License (the "License").
	# You may not use this file except in compliance with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END
	#

	#
	# Copyright 2009 Sun Microsystems, Inc. All rights reserved.
	# Use is subject to license terms.
	#

	#
	# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
	#

	. $STF_SUITE/include/libtest.shlib
	. $STF_SUITE/tests/functional/cli_root/zpool_create/zpool_create.shlib

	#
	# DESCRIPTION:
	# Verify zpool add succeed when adding vdevs with matching redundancy.
	#
	# STRATEGY:
	# 1. Create several files == $MINVDEVSIZE.
	# 2. Verify 'zpool add' succeeds with matching redundancy.
	# 3. Verify 'zpool add' warns with differing redundancy.
	# 4. Verify 'zpool add' warns with differing redundancy after removal.
	#

	verify_runnable "global"

	function cleanup
	{
	datasetexists $TESTPOOL1 && destroy_pool $TESTPOOL1

	typeset -i i=0
	while ((i < 10)); do
	rm -f $TEST_BASE_DIR/vdev$i
	((i += 1))
	done
	}


	log_assert "Verify 'zpool add' succeed with keywords combination."
	log_onexit cleanup

	# 1. Create several files == $MINVDEVSIZE.
	typeset -i i=0
	while ((i < 10)); do
	log_must truncate -s $MINVDEVSIZE $TEST_BASE_DIR/vdev$i

	eval vdev$i=$TEST_BASE_DIR/vdev$i
	((i += 1))
	done

	set -A redundancy0_create_args \
	"$vdev0"

	set -A redundancy1_create_args \
	"mirror $vdev0 $vdev1" \
	"raidz1 $vdev0 $vdev1"

	set -A redundancy2_create_args \
	"mirror $vdev0 $vdev1 $vdev2" \
	"raidz2 $vdev0 $vdev1 $vdev2"

	set -A redundancy3_create_args \
	"mirror $vdev0 $vdev1 $vdev2 $vdev3" \
	"raidz3 $vdev0 $vdev1 $vdev2 $vdev3"

	set -A redundancy0_add_args \
	"$vdev5" \
	"$vdev5 $vdev6"

	set -A redundancy1_add_args \
	"mirror $vdev5 $vdev6" \
	"raidz1 $vdev5 $vdev6" \
	"raidz1 $vdev5 $vdev6 mirror $vdev7 $vdev8" \
	"mirror $vdev5 $vdev6 raidz1 $vdev7 $vdev8"

	set -A redundancy2_add_args \
	"mirror $vdev5 $vdev6 $vdev7" \
	"raidz2 $vdev5 $vdev6 $vdev7"

	set -A redundancy3_add_args \
	"mirror $vdev5 $vdev6 $vdev7 $vdev8" \
	"raidz3 $vdev5 $vdev6 $vdev7 $vdev8"

	set -A log_args "log" "$vdev4"
	set -A cache_args "cache" "$vdev4"
	set -A spare_args "spare" "$vdev4"

	typeset -i j=0

	function zpool_create_add
	{
	typeset -n create_args=$1
	typeset -n add_args=$2

	i=0
	while ((i < ${#create_args[@]})); do
	j=0
	while ((j < ${#add_args[@]})); do
	log_must zpool create $TESTPOOL1 ${create_args[$i]}
	log_must zpool add $TESTPOOL1 ${add_args[$j]}
	log_must zpool destroy -f $TESTPOOL1

	((j += 1))
	done
	((i += 1))
	done
	}

	function zpool_create_forced_add
	{
	typeset -n create_args=$1
	typeset -n add_args=$2

	i=0
	while ((i < ${#create_args[@]})); do
	j=0
	while ((j < ${#add_args[@]})); do
	log_must zpool create $TESTPOOL1 ${create_args[$i]}
	log_mustnot zpool add $TESTPOOL1 ${add_args[$j]}
	- log_must zpool add -f $TESTPOOL1 ${add_args[$j]}
	+ log_must zpool add --allow-replication-mismatch $TESTPOOL1 ${add_args[$j]}
	log_must zpool destroy -f $TESTPOOL1

	((j += 1))
	done
	((i += 1))
	done
	}

	function zpool_create_rm_add
	{
	typeset -n create_args=$1
	typeset -n add_args=$2
	typeset -n rm_args=$3

	i=0
	while ((i < ${#create_args[@]})); do
	j=0
	while ((j < ${#add_args[@]})); do
	log_must zpool create $TESTPOOL1 ${create_args[$i]}
	log_must zpool add $TESTPOOL1 ${rm_args[0]} ${rm_args[1]}
	log_must zpool add $TESTPOOL1 ${add_args[$j]}
	log_must zpool remove $TESTPOOL1 ${rm_args[1]}
	log_mustnot zpool add $TESTPOOL1 ${rm_args[1]}
	log_must zpool add $TESTPOOL1 ${rm_args[0]} ${rm_args[1]}
	log_must zpool destroy -f $TESTPOOL1

	((j += 1))
	done
	((i += 1))
	done
	}

	# 2. Verify 'zpool add' succeeds with matching redundancy.
	zpool_create_add redundancy0_create_args redundancy0_add_args
	zpool_create_add redundancy1_create_args redundancy1_add_args
	zpool_create_add redundancy2_create_args redundancy2_add_args
	zpool_create_add redundancy3_create_args redundancy3_add_args

	# 3. Verify 'zpool add' warns with differing redundancy.
	zpool_create_forced_add redundancy0_create_args redundancy1_add_args
	zpool_create_forced_add redundancy0_create_args redundancy2_add_args
	zpool_create_forced_add redundancy0_create_args redundancy3_add_args

	zpool_create_forced_add redundancy1_create_args redundancy0_add_args
	zpool_create_forced_add redundancy1_create_args redundancy2_add_args
	zpool_create_forced_add redundancy1_create_args redundancy3_add_args

	zpool_create_forced_add redundancy2_create_args redundancy0_add_args
	zpool_create_forced_add redundancy2_create_args redundancy1_add_args
	zpool_create_forced_add redundancy2_create_args redundancy3_add_args

	zpool_create_forced_add redundancy3_create_args redundancy0_add_args
	zpool_create_forced_add redundancy3_create_args redundancy1_add_args
	zpool_create_forced_add redundancy3_create_args redundancy2_add_args

	# 4. Verify 'zpool add' warns with differing redundancy after removal.
	zpool_create_rm_add redundancy1_create_args redundancy1_add_args log_args
	zpool_create_rm_add redundancy2_create_args redundancy2_add_args log_args
	zpool_create_rm_add redundancy3_create_args redundancy3_add_args log_args

	zpool_create_rm_add redundancy1_create_args redundancy1_add_args cache_args
	zpool_create_rm_add redundancy2_create_args redundancy2_add_args cache_args
	zpool_create_rm_add redundancy3_create_args redundancy3_add_args cache_args

	zpool_create_rm_add redundancy1_create_args redundancy1_add_args spare_args
	zpool_create_rm_add redundancy2_create_args redundancy2_add_args spare_args
	zpool_create_rm_add redundancy3_create_args redundancy3_add_args spare_args

	log_pass "'zpool add' succeed with keywords combination."
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_get/vdev_get.cfg b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_get/vdev_get.cfg
	index 71a64d4fae7a..c3b9efd6464a 100644
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_get/vdev_get.cfg
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_get/vdev_get.cfg
	@@ -1,73 +1,75 @@
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License (the "License").
	# You may not use this file except in compliance with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END
	#

	#
	# Copyright (c) 2022, Klara Inc.
	#

	# Set the expected properties of a vdev
	typeset -a properties=(
	capacity
	state
	guid
	asize
	psize
	ashift
	size
	free
	allocated
	comment
	expandsize
	fragmentation
	bootsize
	parity
	path
	devid
	physpath
	encpath
	fru
	parent
	children
	numchildren
	read_errors
	write_errors
	checksum_errors
	initialize_errors
	null_ops
	read_ops
	write_ops
	free_ops
	claim_ops
	trim_ops
	null_bytes
	read_bytes
	write_bytes
	free_bytes
	claim_bytes
	trim_bytes
	removing
	allocating
	failfast
	checksum_n
	checksum_t
	io_n
	io_t
	+ slow_io_n
	+ slow_io_t
	)
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_import/zpool_import_status.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_import/zpool_import_status.ksh
	new file mode 100755
	index 000000000000..c96961bf6419
	--- /dev/null
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_import/zpool_import_status.ksh
	@@ -0,0 +1,132 @@
	+#!/bin/ksh -p
	+#
	+# CDDL HEADER START
	+#
	+# The contents of this file are subject to the terms of the
	+# Common Development and Distribution License (the "License").
	+# You may not use this file except in compliance with the License.
	+#
	+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	+# or https://opensource.org/licenses/CDDL-1.0.
	+# See the License for the specific language governing permissions
	+# and limitations under the License.
	+#
	+# When distributing Covered Code, include this CDDL HEADER in each
	+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	+# If applicable, add the following below this CDDL HEADER, with the
	+# fields enclosed by brackets "[]" replaced with your own identifying
	+# information: Portions Copyright [yyyy] [name of copyright owner]
	+#
	+# CDDL HEADER END
	+#
	+
	+#
	+# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
	+# Use is subject to license terms.
	+#
	+
	+#
	+# Copyright (c) 2023 Klara, Inc.
	+#
	+
	+. $STF_SUITE/include/libtest.shlib
	+. $STF_SUITE/tests/functional/cli_root/zpool_import/zpool_import.cfg
	+
	+#
	+# DESCRIPTION:
	+# During a pool import, the 'import_progress' kstat contains details
	+# on the import progress.
	+#
	+# STRATEGY:
	+# 1. Create test pool with several devices
	+# 2. Generate some ZIL records and spacemap logs
	+# 3. Export the pool
	+# 4. Import the pool in the background and monitor the kstat content
	+# 5. Check the zfs debug messages for import progress
	+#
	+
	+verify_runnable "global"
	+
	+function cleanup
	+{
	+ log_must set_tunable64 KEEP_LOG_SPACEMAPS_AT_EXPORT 0
	+ log_must set_tunable64 METASLAB_DEBUG_LOAD 0
	+
	+ destroy_pool $TESTPOOL1
	+}
	+
	+log_assert "During a pool import, the 'import_progress' kstat contains " \
	+ "notes on the progress"
	+
	+log_onexit cleanup
	+
	+log_must zpool create $TESTPOOL1 $VDEV0 $VDEV1 $VDEV2
	+typeset guid=$(zpool get -H -o value guid $TESTPOOL1)
	+
	+log_must zfs create -o recordsize=8k $TESTPOOL1/fs
	+#
	+# This dd command works around an issue where ZIL records aren't created
	+# after freezing the pool unless a ZIL header already exists. Create a file
	+# synchronously to force ZFS to write one out.
	+#
	+log_must dd if=/dev/zero of=/$TESTPOOL1/fs/sync conv=fsync bs=1 count=1
	+
	+#
	+# Overwrite some blocks to populate spacemap logs
	+#
	+log_must dd if=/dev/urandom of=/$TESTPOOL1/fs/00 bs=1M count=200
	+sync_all_pools
	+log_must dd if=/dev/urandom of=/$TESTPOOL1/fs/00 bs=1M count=200
	+sync_all_pools
	+
	+#
	+# Freeze the pool to retain intent log records
	+#
	+log_must zpool freeze $TESTPOOL1
	+
	+# fill_fs [destdir] [dirnum] [filenum] [bytes] [num_writes] [data]
	+log_must fill_fs /$TESTPOOL1/fs 1 2000 100 1024 R
	+
	+log_must zpool list -v $TESTPOOL1
	+
	+#
	+# Unmount filesystem and export the pool
	+#
	+# At this stage the zfs intent log contains
	+# a set of records to replay.
	+#
	+log_must zfs unmount /$TESTPOOL1/fs
	+
	+log_must set_tunable64 KEEP_LOG_SPACEMAPS_AT_EXPORT 1
	+log_must zpool export $TESTPOOL1
	+
	+log_must set_tunable64 METASLAB_DEBUG_LOAD 1
	+log_note "Starting zpool import in background at" $(date +'%H:%M:%S')
	+zpool import -d $DEVICE_DIR -f $guid &
	+pid=$!
	+
	+#
	+# capture progress until import is finished
	+#
	+log_note waiting for pid $pid to exit
	+kstat import_progress
	+while [[ -d /proc/"$pid" ]]; do
	+ line=$(kstat import_progress \| grep -v pool_guid)
	+ if [[ -n $line ]]; then
	+ echo $line
	+ fi
	+ if [[ -f /$TESTPOOL1/fs/00 ]]; then
	+ break;
	+ fi
	+ sleep 0.0001
	+done
	+log_note "zpool import completed at" $(date +'%H:%M:%S')
	+
	+entries=$(kstat dbgmsg \| grep "spa_import_progress_set_notes_impl(): 'testpool1'" \| wc -l)
	+log_note "found $entries progress notes in dbgmsg"
	+log_must test $entries -gt 20
	+
	+log_must zpool status $TESTPOOL1
	+
	+log_pass "During a pool import, the 'import_progress' kstat contains " \
	+ "notes on the progress"
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cp_files/cp_files_002_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cp_files/cp_files_002_pos.ksh
	index 60817449ab03..4db968ffae05 100755
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cp_files/cp_files_002_pos.ksh
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cp_files/cp_files_002_pos.ksh
	@@ -1,161 +1,161 @@
	#! /bin/ksh -p
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License (the "License").
	# You may not use this file except in compliance with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END
	#

	#
	# Copyright (c) 2024 by Lawrence Livermore National Security, LLC.
	#

	. $STF_SUITE/include/libtest.shlib
	. $STF_SUITE/tests/functional/bclone/bclone_common.kshlib

	#
	# DESCRIPTION:
	# Verify all cp --reflink modes work with modified file.
	#
	# STRATEGY:
	# 1. Verify "cp --reflink=never\|auto\|always" behaves as expected.
	# Two different modes of operation are tested.
	#
	# a. zfs_bclone_wait_dirty=0: FICLONE and FICLONERANGE fail with EINVAL
	# when there are dirty blocks which cannot be immediately cloned.
	# This is the default behavior.
	#
	# b. zfs_bclone_wait_dirty=1: FICLONE and FICLONERANGE wait for
	# dirty blocks to be written to disk allowing the clone to succeed.
	# The downside to this is it may be slow which depending on the
	# situtation may defeat the point of making a clone.
	#

	verify_runnable "global"
	verify_block_cloning

	if ! is_linux; then
	log_unsupported "cp --reflink is a GNU coreutils option"
	fi

	function cleanup
	{
	datasetexists $TESTPOOL/cp-reflink && \
	destroy_dataset $$TESTPOOL/cp-reflink -f
	log_must set_tunable32 BCLONE_WAIT_DIRTY 0
	}

	function verify_copy
	{
	src_cksum=$(sha256digest $1)
	dst_cksum=$(sha256digest $2)

	if [[ "$src_cksum" != "$dst_cksum" ]]; then
	log_must ls -l $CP_TESTDIR
	log_fail "checksum mismatch ($src_cksum != $dst_cksum)"
	fi
	}

	log_assert "Verify all cp --reflink modes work with modified file"

	log_onexit cleanup

	SRC_FILE=src.data
	DST_FILE=dst.data
	-SRC_SIZE=$(($RANDOM % 2048))
	+SRC_SIZE=$((1024 + $RANDOM % 1024))

	# A smaller recordsize is used merely to speed up the test.
	RECORDSIZE=4096

	log_must zfs create -o recordsize=$RECORDSIZE $TESTPOOL/cp-reflink
	CP_TESTDIR=$(get_prop mountpoint $TESTPOOL/cp-reflink)

	log_must cd $CP_TESTDIR

	# Never wait on dirty blocks (zfs_bclone_wait_dirty=0)
	log_must set_tunable32 BCLONE_WAIT_DIRTY 0

	for mode in "never" "auto" "always"; do
	log_note "Checking 'cp --reflink=$mode'"

	# Create a new file and immediately copy it.
	log_must dd if=/dev/urandom of=$SRC_FILE bs=$RECORDSIZE count=$SRC_SIZE

	if [[ "$mode" == "always" ]]; then
	log_mustnot cp --reflink=$mode $SRC_FILE $DST_FILE
	log_must ls -l $CP_TESTDIR
	else
	log_must cp --reflink=$mode $SRC_FILE $DST_FILE
	verify_copy $SRC_FILE $DST_FILE
	fi
	log_must rm -f $DST_FILE

	# Append to an existing file and immediately copy it.
	sync_pool $TESTPOOL
	log_must dd if=/dev/urandom of=$SRC_FILE bs=$RECORDSIZE seek=$SRC_SIZE \
	count=1 conv=notrunc
	if [[ "$mode" == "always" ]]; then
	log_mustnot cp --reflink=$mode $SRC_FILE $DST_FILE
	log_must ls -l $CP_TESTDIR
	else
	log_must cp --reflink=$mode $SRC_FILE $DST_FILE
	verify_copy $SRC_FILE $DST_FILE
	fi
	log_must rm -f $DST_FILE

	# Overwrite a random range of an existing file and immediately copy it.
	sync_pool $TESTPOOL
	log_must dd if=/dev/urandom of=$SRC_FILE bs=$((RECORDSIZE / 2)) \
	- seek=$(($RANDOM % $SRC_SIZE)) count=$(($RANDOM % 16)) conv=notrunc
	+ seek=$(($RANDOM % $SRC_SIZE)) count=$((1 + $RANDOM % 16)) conv=notrunc
	if [[ "$mode" == "always" ]]; then
	log_mustnot cp --reflink=$mode $SRC_FILE $DST_FILE
	log_must ls -l $CP_TESTDIR
	else
	log_must cp --reflink=$mode $SRC_FILE $DST_FILE
	verify_copy $SRC_FILE $DST_FILE
	fi
	log_must rm -f $SRC_FILE $DST_FILE
	done

	# Wait on dirty blocks (zfs_bclone_wait_dirty=1)
	log_must set_tunable32 BCLONE_WAIT_DIRTY 1

	for mode in "never" "auto" "always"; do
	log_note "Checking 'cp --reflink=$mode'"

	# Create a new file and immediately copy it.
	log_must dd if=/dev/urandom of=$SRC_FILE bs=$RECORDSIZE count=$SRC_SIZE
	log_must cp --reflink=$mode $SRC_FILE $DST_FILE
	verify_copy $SRC_FILE $DST_FILE
	log_must rm -f $DST_FILE

	# Append to an existing file and immediately copy it.
	log_must dd if=/dev/urandom of=$SRC_FILE bs=$RECORDSIZE seek=$SRC_SIZE \
	count=1 conv=notrunc
	log_must cp --reflink=$mode $SRC_FILE $DST_FILE
	verify_copy $SRC_FILE $DST_FILE
	log_must rm -f $DST_FILE

	# Overwrite a random range of an existing file and immediately copy it.
	log_must dd if=/dev/urandom of=$SRC_FILE bs=$((RECORDSIZE / 2)) \
	- seek=$(($RANDOM % $SRC_SIZE)) count=$(($RANDOM % 16)) conv=notrunc
	+ seek=$(($RANDOM % $SRC_SIZE)) count=$((1 + $RANDOM % 16)) conv=notrunc
	log_must cp --reflink=$mode $SRC_FILE $DST_FILE
	verify_copy $SRC_FILE $DST_FILE
	log_must rm -f $SRC_FILE $DST_FILE
	done

	log_pass
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/cleanup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/cleanup.ksh
	index ef6e098cf42a..669b8ae99456 100755
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/cleanup.ksh
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/cleanup.ksh
	@@ -1,33 +1,35 @@
	#!/bin/ksh -p
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License (the "License").
	# You may not use this file except in compliance with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END
	#

	#
	# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
	#

	. $STF_SUITE/include/libtest.shlib

	+zed_stop
	+
	zed_cleanup all-debug.sh all-syslog.sh all-dumpfds

	-zed_stop
	+zed_events_drain

	default_cleanup
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/zed_slow_io.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/zed_slow_io.ksh
	new file mode 100755
	index 000000000000..d9fabb2c3bc9
	--- /dev/null
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/zed_slow_io.ksh
	@@ -0,0 +1,205 @@
	+#!/bin/ksh -p
	+#
	+# CDDL HEADER START
	+#
	+# The contents of this file are subject to the terms of the
	+# Common Development and Distribution License (the "License").
	+# You may not use this file except in compliance with the License.
	+#
	+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	+# or https://opensource.org/licenses/CDDL-1.0.
	+# See the License for the specific language governing permissions
	+# and limitations under the License.
	+#
	+# When distributing Covered Code, include this CDDL HEADER in each
	+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	+# If applicable, add the following below this CDDL HEADER, with the
	+# fields enclosed by brackets "[]" replaced with your own identifying
	+# information: Portions Copyright [yyyy] [name of copyright owner]
	+#
	+# CDDL HEADER END
	+#
	+
	+#
	+# Copyright (c) 2023, Klara Inc.
	+#
	+
	+# DESCRIPTION:
	+# Verify that vdev properties, slow_io_n and slow_io_t, work with ZED.
	+#
	+# STRATEGY:
	+# 1. Create a pool with single vdev
	+# 2. Set slow_io_n/slow_io_t to non-default values
	+# 3. Inject slow io errors
	+# 4. Verify that ZED degrades vdev
	+#
	+
	+. $STF_SUITE/include/libtest.shlib
	+
	+TESTDIR="$TEST_BASE_DIR/zed_slow_io"
	+VDEV="$TEST_BASE_DIR/vdevfile.$$"
	+TESTPOOL="slow_io_pool"
	+FILEPATH="$TESTDIR/slow_io.testfile"
	+
	+OLD_SLOW_IO=$(get_tunable ZIO_SLOW_IO_MS)
	+OLD_SLOW_IO_EVENTS=$(get_tunable SLOW_IO_EVENTS_PER_SECOND)
	+
	+verify_runnable "both"
	+
	+function do_setup
	+{
	+ log_must truncate -s 1G $VDEV
	+ default_setup_noexit $VDEV
	+ zed_events_drain
	+ log_must zfs set compression=off $TESTPOOL
	+ log_must zfs set primarycache=none $TESTPOOL
	+ log_must zfs set prefetch=none $TESTPOOL
	+ log_must zfs set recordsize=512 $TESTPOOL
	+ for i in {1..10}; do
	+ dd if=/dev/urandom of=${FILEPATH}$i bs=512 count=1 2>/dev/null
	+ done
	+ zpool sync
	+}
	+
	+# intermediate cleanup
	+function do_clean
	+{
	+ log_must zinject -c all
	+ log_must zpool destroy $TESTPOOL
	+ log_must rm -f $VDEV
	+}
	+
	+# final cleanup
	+function cleanup
	+{
	+ log_must zinject -c all
	+
	+ # if pool still exists then something failed so log additional info
	+ if poolexists $TESTPOOL ; then
	+ log_note "$(zpool status -s $TESTPOOL)"
	+ echo "=================== zed log search ==================="
	+ grep "Diagnosis Engine" $ZEDLET_DIR/zed.log
	+ destroy_pool $TESTPOOL
	+ fi
	+ log_must zed_stop
	+
	+ log_must rm -f $VDEV
	+
	+ log_must set_tunable64 ZIO_SLOW_IO_MS $OLD_SLOW_IO
	+ log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND $OLD_SLOW_IO_EVENTS
	+}
	+
	+function start_slow_io
	+{
	+ zpool sync
	+ log_must set_tunable64 ZIO_SLOW_IO_MS 10
	+ log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND 1000
	+
	+ log_must zinject -d $VDEV -D10:1 -T read $TESTPOOL
	+ zpool sync
	+}
	+
	+function stop_slow_io
	+{
	+ log_must set_tunable64 ZIO_SLOW_IO_MS $OLD_SLOW_IO
	+ log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND $OLD_SLOW_IO_EVENTS
	+
	+ log_must zinject -c all
	+}
	+
	+# Test default ZED settings:
	+# inject 10 events over 2.5 seconds, should not degrade.
	+function default_degrade
	+{
	+ do_setup
	+
	+ start_slow_io
	+ for i in {1..10}; do
	+ dd if=${FILEPATH}$i of=/dev/null count=1 bs=512 2>/dev/null
	+ sleep 0.25
	+ done
	+ stop_slow_io
	+ log_note "$(zpool status -s $TESTPOOL)"
	+
	+ # give slow ZED a chance to process the delay events
	+ sleep 18
	+ log_note "$(zpool status -s $TESTPOOL)"
	+
	+ degrades=$(grep "zpool_vdev_degrade" $ZEDLET_DIR/zed.log \| wc -l)
	+ log_note $degrades vdev degrades in ZED log
	+ [ $degrades -eq "0" ] \|\| \
	+ log_fail "expecting no degrade events, found $degrades"
	+
	+ do_clean
	+}
	+
	+# change slow_io_n, slow_io_t to 5 events in 60 seconds
	+# fire more than 5 events, should degrade
	+function slow_io_degrade
	+{
	+ do_setup
	+
	+ zpool set slow_io_n=5 $TESTPOOL $VDEV
	+ zpool set slow_io_t=60 $TESTPOOL $VDEV
	+
	+ start_slow_io
	+ for i in {1..16}; do
	+ dd if=${FILEPATH}$i of=/dev/null count=1 bs=512 2>/dev/null
	+ sleep 0.5
	+ done
	+ stop_slow_io
	+ zpool sync
	+
	+ #
	+ # wait up to 60 seconds for kernel to produce at least 5 delay events
	+ #
	+ typeset -i i=0
	+ typeset -i events=0
	+ while [[ $i -lt 60 ]]; do
	+ events=$(zpool events \| grep "ereport\.fs\.zfs.delay" \| wc -l)
	+ [[ $events -ge "5" ]] && break
	+ i=$((i+1))
	+ sleep 1
	+ done
	+ log_note "$events delay events found"
	+
	+ if [[ $events -ge "5" ]]; then
	+ log_must wait_vdev_state $TESTPOOL $VDEV "DEGRADED" 10
	+ fi
	+
	+ do_clean
	+}
	+
	+# change slow_io_n, slow_io_t to 10 events in 1 second
	+# inject events spaced 0.5 seconds apart, should not degrade
	+function slow_io_no_degrade
	+{
	+ do_setup
	+
	+ zpool set slow_io_n=10 $TESTPOOL $VDEV
	+ zpool set slow_io_t=1 $TESTPOOL $VDEV
	+
	+ start_slow_io
	+ for i in {1..16}; do
	+ dd if=${FILEPATH}$i of=/dev/null count=1 bs=512 2>/dev/null
	+ sleep 0.5
	+ done
	+ stop_slow_io
	+ zpool sync
	+
	+ log_mustnot wait_vdev_state $TESTPOOL $VDEV "DEGRADED" 45
	+
	+ do_clean
	+}
	+
	+log_assert "Test ZED slow io configurability"
	+log_onexit cleanup
	+
	+log_must zed_events_drain
	+log_must zed_start
	+
	+default_degrade
	+slow_io_degrade
	+slow_io_no_degrade
	+
	+log_pass "Test ZED slow io configurability"
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/zed_slow_io_many_vdevs.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/zed_slow_io_many_vdevs.ksh
	new file mode 100755
	index 000000000000..3357ae2e3510
	--- /dev/null
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/zed_slow_io_many_vdevs.ksh
	@@ -0,0 +1,177 @@
	+#!/bin/ksh -p
	+#
	+# CDDL HEADER START
	+#
	+# The contents of this file are subject to the terms of the
	+# Common Development and Distribution License (the "License").
	+# You may not use this file except in compliance with the License.
	+#
	+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	+# or https://opensource.org/licenses/CDDL-1.0.
	+# See the License for the specific language governing permissions
	+# and limitations under the License.
	+#
	+# When distributing Covered Code, include this CDDL HEADER in each
	+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	+# If applicable, add the following below this CDDL HEADER, with the
	+# fields enclosed by brackets "[]" replaced with your own identifying
	+# information: Portions Copyright [yyyy] [name of copyright owner]
	+#
	+# CDDL HEADER END
	+#
	+
	+#
	+# Copyright (c) 2023, Klara Inc.
	+#
	+
	+# DESCRIPTION:
	+# Verify that delay events from multiple vdevs doesnt degrade
	+#
	+# STRATEGY:
	+# 1. Create a pool with a 3 disk raidz vdev
	+# 2. Inject slow io errors
	+# 3. Verify that ZED detects slow I/Os but doesn't degrade any vdevs
	+#
	+
	+. $STF_SUITE/include/libtest.shlib
	+
	+TESTDIR="$TEST_BASE_DIR/zed_slow_io"
	+VDEV1="$TEST_BASE_DIR/vdevfile1.$$"
	+VDEV2="$TEST_BASE_DIR/vdevfile2.$$"
	+VDEV3="$TEST_BASE_DIR/vdevfile3.$$"
	+VDEV4="$TEST_BASE_DIR/vdevfile4.$$"
	+VDEVS="$VDEV1 $VDEV2 $VDEV3 $VDEV4"
	+TESTPOOL="slow_io_pool"
	+FILEPATH="$TESTDIR/slow_io.testfile"
	+
	+OLD_SLOW_IO=$(get_tunable ZIO_SLOW_IO_MS)
	+OLD_SLOW_IO_EVENTS=$(get_tunable SLOW_IO_EVENTS_PER_SECOND)
	+
	+verify_runnable "both"
	+
	+function cleanup
	+{
	+ log_must zinject -c all
	+
	+ # if pool still exists then something failed so log additional info
	+ if poolexists $TESTPOOL ; then
	+ log_note "$(zpool status -s $TESTPOOL)"
	+ echo "=================== zed log search ==================="
	+ grep "Diagnosis Engine" $ZEDLET_DIR/zed.log
	+ destroy_pool $TESTPOOL
	+ fi
	+ log_must zed_stop
	+
	+ log_must rm -f $VDEVS
	+ log_must set_tunable64 ZIO_SLOW_IO_MS $OLD_SLOW_IO
	+ log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND $OLD_SLOW_IO_EVENTS
	+}
	+
	+function start_slow_io
	+{
	+ for vdev in $VDEVS
	+ do
	+ log_must zpool set slow_io_n=4 $TESTPOOL $vdev
	+ log_must zpool set slow_io_t=60 $TESTPOOL $vdev
	+ done
	+ zpool sync
	+
	+ log_must set_tunable64 ZIO_SLOW_IO_MS 10
	+ log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND 1000
	+
	+ for vdev in $VDEVS
	+ do
	+ log_must zinject -d $vdev -D10:1 $TESTPOOL
	+ done
	+ zpool sync
	+}
	+
	+function stop_slow_io
	+{
	+ log_must set_tunable64 ZIO_SLOW_IO_MS $OLD_SLOW_IO
	+ log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND $OLD_SLOW_IO_EVENTS
	+
	+ log_must zinject -c all
	+}
	+
	+function multiple_slow_vdevs_test
	+{
	+ log_must truncate -s 1G $VDEVS
	+ default_raidz_setup_noexit $VDEVS
	+
	+ log_must zpool events -c
	+ log_must zfs set compression=off $TESTPOOL
	+ log_must zfs set primarycache=none $TESTPOOL
	+ log_must zfs set recordsize=4K $TESTPOOL
	+
	+ log_must dd if=/dev/urandom of=$FILEPATH bs=1M count=20
	+ zpool sync
	+
	+ #
	+ # Read the file with slow io injected on the disks
	+ # This will cause multiple errors on each disk to trip ZED SERD
	+ #
	+ # pool: slow_io_pool
	+ # state: ONLINE
	+ # config:
	+ #
	+ # NAME STATE READ WRITE CKSUM SLOW
	+ # slow_io_pool ONLINE 0 0 0 -
	+ # raidz1-0 ONLINE 0 0 0 -
	+ # /var/tmp/vdevfile1.499278 ONLINE 0 0 0 113
	+ # /var/tmp/vdevfile2.499278 ONLINE 0 0 0 109
	+ # /var/tmp/vdevfile3.499278 ONLINE 0 0 0 96
	+ # /var/tmp/vdevfile4.499278 ONLINE 0 0 0 109
	+ #
	+ start_slow_io
	+ dd if=$FILEPATH of=/dev/null bs=1M count=20 2>/dev/null
	+ stop_slow_io
	+
	+ # count events available for processing
	+ typeset -i i=0
	+ typeset -i events=0
	+ while [[ $i -lt 60 ]]; do
	+ events=$(zpool events \| grep "ereport\.fs\.zfs.delay" \| wc -l)
	+ [[ $events -ge "50" ]] && break
	+ i=$((i+1))
	+ sleep 1
	+ done
	+ log_note "$events delay events found"
	+ if [[ $events -lt "50" ]]; then
	+ log_note "bailing: not enough events to complete the test"
	+ destroy_pool $TESTPOOL
	+ return
	+ fi
	+
	+ #
	+ # give slow ZED a chance to process the delay events
	+ #
	+ typeset -i i=0
	+ typeset -i skips=0
	+ while [[ $i -lt 75 ]]; do
	+ skips=$(grep "retiring case" \
	+ $ZEDLET_DIR/zed.log \| wc -l)
	+ [[ $skips -gt "0" ]] && break
	+ i=$((i+1))
	+ sleep 1
	+ done
	+
	+ log_note $skips degrade skips in ZED log after $i seconds
	+ [ $skips -gt "0" ] \|\| log_fail "expecting to see skips"
	+
	+ degrades=$(grep "zpool_vdev_degrade" $ZEDLET_DIR/zed.log \| wc -l)
	+ log_note $degrades vdev degrades in ZED log
	+ [ $degrades -eq "0" ] \|\| \
	+ log_fail "expecting no degrade events, found $degrades"
	+
	+ destroy_pool $TESTPOOL
	+}
	+
	+log_assert "Test ZED slow io across multiple vdevs"
	+log_onexit cleanup
	+
	+log_must zed_events_drain
	+log_must zed_start
	+multiple_slow_vdevs_test
	+
	+log_pass "Test ZED slow io across multiple vdevs"
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/cleanup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/cleanup.ksh
	index 654343c0cf00..2959236b59a3 100755
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/cleanup.ksh
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/cleanup.ksh
	@@ -1,36 +1,37 @@
	#!/bin/ksh -p
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License (the "License").
	# You may not use this file except in compliance with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END

	#
	# Copyright (c) 2016, 2017 by Intel Corporation. All rights reserved.
	#

	. $STF_SUITE/include/libtest.shlib
	. $STF_SUITE/tests/functional/fault/fault.cfg

	verify_runnable "global"

	cleanup_devices $DISKS

	zed_stop
	zed_cleanup resilver_finish-start-scrub.sh
	+zed_events_drain

	log_pass
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/setup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/setup.ksh
	index 62f1c8ab56cb..61b9206ec1a6 100755
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/setup.ksh
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/setup.ksh
	@@ -1,34 +1,35 @@
	#!/bin/ksh -p
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License (the "License").
	# You may not use this file except in compliance with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END

	#
	# Copyright (c) 2016, 2017 by Intel Corporation. All rights reserved.
	#

	. $STF_SUITE/include/libtest.shlib
	. $STF_SUITE/tests/functional/fault/fault.cfg

	verify_runnable "global"

	+zed_events_drain
	zed_setup resilver_finish-start-scrub.sh
	zed_start

	log_pass
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/mmp/mmp_write_slow_disk.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/mmp/mmp_write_slow_disk.ksh
	new file mode 100755
	index 000000000000..8b118684aa7f
	--- /dev/null
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/mmp/mmp_write_slow_disk.ksh
	@@ -0,0 +1,97 @@
	+#!/bin/ksh -p
	+#
	+# CDDL HEADER START
	+#
	+# This file and its contents are supplied under the terms of the
	+# Common Development and Distribution License ("CDDL"), version 1.0.
	+# You may only use this file in accordance with the terms of version
	+# 1.0 of the CDDL.
	+#
	+# A full copy of the text of the CDDL should have accompanied this
	+# source. A copy of the CDDL is also available via the Internet at
	+# http://www.illumos.org/license/CDDL.
	+#
	+# CDDL HEADER END
	+#
	+
	+#
	+# Copyright (c) 2024, Klara Inc
	+#
	+
	+# DESCRIPTION:
	+# Verify that long VDEV probes do not cause MMP checks to suspend pool
	+# Note: without PR-15839 fix, this test will suspend the pool.
	+#
	+# A device that is returning unexpected errors will trigger a vdev_probe.
	+# When the device additionally has slow response times, the probe can hold
	+# the spa config lock as a writer for a long period of time such that the
	+# mmp uberblock updates stall when trying to acquire the spa config lock.
	+#
	+# STRATEGY:
	+# 1. Create a pool with multiple leaf vdevs
	+# 2. Enable multihost and multihost_history
	+# 3. Delay for MMP writes to occur
	+# 4. Verify that a long VDEV probe didn't cause MMP check to suspend pool
	+#
	+
	+. $STF_SUITE/include/libtest.shlib
	+. $STF_SUITE/tests/functional/mmp/mmp.cfg
	+. $STF_SUITE/tests/functional/mmp/mmp.kshlib
	+
	+verify_runnable "both"
	+
	+function cleanup
	+{
	+ log_must zinject -c all
	+
	+ if [[ $(zpool list -H -o health $MMP_POOL) == "SUSPENDED" ]]; then
	+ log_must zpool clear $MMP_POOL
	+ zpool get state $MMP_POOL $MMP_DIR/file.3
	+ zpool events \| grep ".fs.zfs." \| grep -v "history_event"
	+ fi
	+
	+ poolexists $MMP_POOL && destroy_pool $MMP_POOL
	+ log_must rm -r $MMP_DIR
	+ log_must mmp_clear_hostid
	+}
	+
	+log_assert "A long VDEV probe doesn't cause a MMP check suspend"
	+log_onexit cleanup
	+
	+MMP_HISTORY_URL=/proc/spl/kstat/zfs/$MMP_POOL/multihost
	+
	+# Create a multiple drive pool
	+log_must zpool events -c
	+log_must mkdir -p $MMP_DIR
	+log_must truncate -s 128M $MMP_DIR/file.{0,1,2,3,4,5}
	+log_must zpool create -f $MMP_POOL \
	+ mirror $MMP_DIR/file.{0,1,2} \
	+ mirror $MMP_DIR/file.{3,4,5}
	+
	+# Enable MMP
	+log_must mmp_set_hostid $HOSTID1
	+log_must zpool set multihost=on $MMP_POOL
	+clear_mmp_history
	+
	+# Inject vdev write error along with a delay
	+log_must zinject -f 33 -e io -L pad2 -T write -d $MMP_DIR/file.3 $MMP_POOL
	+log_must zinject -f 50 -e io -L uber -T write -d $MMP_DIR/file.3 $MMP_POOL
	+log_must zinject -D 2000:4 -T write -d $MMP_DIR/file.3 $MMP_POOL
	+
	+log_must dd if=/dev/urandom of=/$MMP_POOL/data bs=1M count=5
	+sleep 10
	+sync_pool $MMP_POOL
	+
	+# Confirm mmp writes to the non-slow disks have taken place
	+for x in {0,1,2,4}; do
	+ write_count=$(grep -c file.${x} $MMP_HISTORY_URL)
	+ [[ $write_count -gt 0 ]] \|\| log_fail "expecting mmp writes"
	+done
	+
	+# Expect that the pool was not suspended
	+log_must check_state $MMP_POOL "" "ONLINE"
	+health=$(zpool list -H -o health $MMP_POOL)
	+log_note "$MMP_POOL health is $health"
	+[[ "$health" == "SUSPENDED" ]] && log_fail "$MMP_POOL $health unexpected"
	+
	+log_pass "A long VDEV probe doesn't cause a MMP check suspend"
	diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/user_namespace/user_namespace_004.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/user_namespace/user_namespace_004.ksh
	index 37ef84b72377..e6ad25f23f93 100755
	--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/user_namespace/user_namespace_004.ksh
	+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/user_namespace/user_namespace_004.ksh
	@@ -1,67 +1,67 @@
	#!/bin/ksh -p
	#
	# CDDL HEADER START
	#
	# The contents of this file are subject to the terms of the
	# Common Development and Distribution License (the "License").
	# You may not use this file except in compliance with the License.
	#
	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	# or https://opensource.org/licenses/CDDL-1.0.
	# See the License for the specific language governing permissions
	# and limitations under the License.
	#
	# When distributing Covered Code, include this CDDL HEADER in each
	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	# If applicable, add the following below this CDDL HEADER, with the
	# fields enclosed by brackets "[]" replaced with your own identifying
	# information: Portions Copyright [yyyy] [name of copyright owner]
	#
	# CDDL HEADER END
	#

	. $STF_SUITE/tests/functional/user_namespace/user_namespace_common.kshlib

	#
	# DESCRIPTION:
	# Regression test for safeguards around the delegation of datasets to
	# user namespaces.
	#
	# STRATEGY:
	# 1. Check that 'zfs zone' correctly handles the case of the first
	# argument being a non-namespace file.
	# 2. Check that 'zfs zone' correctly handles the case of the first
	# argument being a non-namespace and non-existent file.
	#

	verify_runnable "both"

	user_ns_cleanup() {
	if [ -n "$temp_file" ]; then
	log_must rm -f "$temp_file"
	fi

	log_must zfs destroy -r "$TESTPOOL/userns"
	}

	-log_onexit user_ns_cleanup
	-
	log_assert "Check zfs zone command handling of non-namespace files"

	# Pass if user namespaces are not supported.
	unshare -Urm echo test
	if [ "$?" -ne "0" ]; then
	log_unsupported "Failed to create user namespace"
	fi

	+log_onexit user_ns_cleanup
	+
	# Create the baseline datasets.
	log_must zfs create -o zoned=on "$TESTPOOL/userns"

	# 1. Try to pass a non-namespace file to zfs zone.
	temp_file="$(TMPDIR=$TEST_BASE_DIR mktemp)"
	log_mustnot zfs zone "$temp_file" "$TESTPOOL/userns"

	# 2. Try to pass a non-namespace and non-existent file to zfs zone.
	log_mustnot zfs zone "$TEMP_BASE_DIR/nonexistent" "$TESTPOOL/userns"

	log_pass "Check zfs zone command handling of non-namespace files"
	diff --git a/sys/contrib/openzfs/udev/zvol_id.c b/sys/contrib/openzfs/udev/zvol_id.c
	index 5960b978787a..609349594767 100644
	--- a/sys/contrib/openzfs/udev/zvol_id.c
	+++ b/sys/contrib/openzfs/udev/zvol_id.c
	@@ -1,86 +1,87 @@
	/*
	* CDDL HEADER START
	*
	* The contents of this file are subject to the terms of the
	* Common Development and Distribution License (the "License").
	* You may not use this file except in compliance with the License.
	*
	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	* or https://opensource.org/licenses/CDDL-1.0.
	* See the License for the specific language governing permissions
	* and limitations under the License.
	*
	* When distributing Covered Code, include this CDDL HEADER in each
	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	* If applicable, add the following below this CDDL HEADER, with the
	* fields enclosed by brackets "[]" replaced with your own identifying
	* information: Portions Copyright [yyyy] [name of copyright owner]
	*
	* CDDL HEADER END
	*/
	/*
	* Copyright (c) 2011, Fajar A. Nugraha. All rights reserved.
	* Use is subject to license terms.
	*/

	#include <ctype.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <stdio.h>
	#include <string.h>
	#include <unistd.h>
	#include <sys/fs/zfs.h>
	#include <sys/ioctl.h>
	#include <sys/stat.h>

	#if defined(ZFS_ASAN_ENABLED)
	/*
	* zvol_id is invoked by udev with the help of ptrace()
	* making sanitized binary with leak detection croak
	* because of tracing mechanisms collision
	*/
	extern const char *__asan_default_options(void);

	const char *__asan_default_options(void) {
	return ("abort_on_error=true:halt_on_error=true:"
	"allocator_may_return_null=true:disable_coredump=false:"
	"detect_stack_use_after_return=true:detect_leaks=false");
	}
	#endif

	int
	main(int argc, const char const argv)
	{
	- if (argc != 2) {
	+ if (argc != 2 \|\| strncmp(argv[1], "/dev/zd", 7) != 0) {
	fprintf(stderr, "usage: %s /dev/zdX\n", argv[0]);
	return (1);
	}
	const char *dev_name = argv[1];

	int fd;
	struct stat sb;
	if ((fd = open(dev_name, O_RDONLY\|O_CLOEXEC)) == -1 \|\|
	fstat(fd, &sb) != 0) {
	fprintf(stderr, "%s: %s\n", dev_name, strerror(errno));
	return (1);
	}

	char zvol_name[MAXNAMELEN + strlen("-part") + 10];
	if (ioctl(fd, BLKZNAME, zvol_name) == -1) {
	fprintf(stderr, "%s: BLKZNAME: %s\n",
	dev_name, strerror(errno));
	return (1);
	}

	- unsigned int dev_part = minor(sb.st_rdev) % ZVOL_MINORS;
	- if (dev_part != 0)
	- sprintf(zvol_name + strlen(zvol_name), "-part%u", dev_part);
	+ const char *dev_part = strrchr(dev_name, 'p');
	+ if (dev_part != NULL) {
	+ sprintf(zvol_name + strlen(zvol_name), "-part%s", dev_part + 1);
	+ }

	for (size_t i = 0; i < strlen(zvol_name); ++i)
	if (isblank(zvol_name[i]))
	zvol_name[i] = '+';

	puts(zvol_name);

	return (0);
	}
	diff --git a/sys/modules/zfs/zfs_config.h b/sys/modules/zfs/zfs_config.h
	index fe8580263185..35e6b4655832 100644
	--- a/sys/modules/zfs/zfs_config.h
	+++ b/sys/modules/zfs/zfs_config.h
	@@ -1,1191 +1,1215 @@
	/*
	*/

	/* zfs_config.h. Generated from zfs_config.h.in by configure. */
	/* zfs_config.h.in. Generated from configure.ac by autoheader. */

	/* Define to 1 if translation of program messages to the user's native
	language is requested. */
	/* #undef ENABLE_NLS */

	/* bio_end_io_t wants 1 arg */
	/* #undef HAVE_1ARG_BIO_END_IO_T */

	/* lookup_bdev() wants 1 arg */
	/* #undef HAVE_1ARG_LOOKUP_BDEV */

	/* submit_bio() wants 1 arg */
	/* #undef HAVE_1ARG_SUBMIT_BIO */

	/* bdi_setup_and_register() wants 2 args */
	/* #undef HAVE_2ARGS_BDI_SETUP_AND_REGISTER */

	/* vfs_getattr wants 2 args */
	/* #undef HAVE_2ARGS_VFS_GETATTR */

	/* zlib_deflate_workspacesize() wants 2 args */
	/* #undef HAVE_2ARGS_ZLIB_DEFLATE_WORKSPACESIZE */

	/* bdi_setup_and_register() wants 3 args */
	/* #undef HAVE_3ARGS_BDI_SETUP_AND_REGISTER */

	/* vfs_getattr wants 3 args */
	/* #undef HAVE_3ARGS_VFS_GETATTR */

	/* vfs_getattr wants 4 args */
	/* #undef HAVE_4ARGS_VFS_GETATTR */

	/* kernel has access_ok with 'type' parameter */
	/* #undef HAVE_ACCESS_OK_TYPE */

	/* posix_acl has refcount_t */
	/* #undef HAVE_ACL_REFCOUNT */

	/* add_disk() returns int */
	/* #undef HAVE_ADD_DISK_RET */

	/* Define if host toolchain supports AES */
	#define HAVE_AES 1

	/* Define if you have [rt] */
	#define HAVE_AIO_H 1

	#ifdef __amd64__
	#ifndef RESCUE
	/* Define if host toolchain supports AVX */
	#define HAVE_AVX 1
	#endif

	/* Define if host toolchain supports AVX2 */
	#define HAVE_AVX2 1

	/* Define if host toolchain supports AVX512BW */
	#define HAVE_AVX512BW 1

	/* Define if host toolchain supports AVX512CD */
	#define HAVE_AVX512CD 1

	/* Define if host toolchain supports AVX512DQ */
	#define HAVE_AVX512DQ 1

	/* Define if host toolchain supports AVX512ER */
	#define HAVE_AVX512ER 1

	/* Define if host toolchain supports AVX512F */
	#define HAVE_AVX512F 1

	/* Define if host toolchain supports AVX512IFMA */
	#define HAVE_AVX512IFMA 1

	/* Define if host toolchain supports AVX512PF */
	#define HAVE_AVX512PF 1

	/* Define if host toolchain supports AVX512VBMI */
	#define HAVE_AVX512VBMI 1

	/* Define if host toolchain supports AVX512VL */
	#define HAVE_AVX512VL 1
	#endif

	/* bdevname() is available */
	/* #undef HAVE_BDEVNAME */

	/* bdev_check_media_change() exists */
	/* #undef HAVE_BDEV_CHECK_MEDIA_CHANGE */

	+/* bdev_file_open_by_path() exists */
	+/* #undef HAVE_BDEV_FILE_OPEN_BY_PATH */
	+
	/* bdev__io_acct() available /
	/* #undef HAVE_BDEV_IO_ACCT_63 */

	/* bdev__io_acct() available /
	/* #undef HAVE_BDEV_IO_ACCT_OLD */

	/* bdev_kobj() exists */
	/* #undef HAVE_BDEV_KOBJ */

	/* bdev_max_discard_sectors() is available */
	/* #undef HAVE_BDEV_MAX_DISCARD_SECTORS */

	/* bdev_max_secure_erase_sectors() is available */
	/* #undef HAVE_BDEV_MAX_SECURE_ERASE_SECTORS */

	/* bdev_open_by_path() exists */
	/* #undef HAVE_BDEV_OPEN_BY_PATH */

	/* bdev_release() exists */
	/* #undef HAVE_BDEV_RELEASE */

	/* block_device_operations->submit_bio() returns void */
	/* #undef HAVE_BDEV_SUBMIT_BIO_RETURNS_VOID */

	/* bdev_whole() is available */
	/* #undef HAVE_BDEV_WHOLE */

	/* bio_alloc() takes 4 arguments */
	/* #undef HAVE_BIO_ALLOC_4ARG */

	/* bio->bi_bdev->bd_disk exists */
	/* #undef HAVE_BIO_BDEV_DISK */

	/* bio->bi_opf is defined */
	/* #undef HAVE_BIO_BI_OPF */

	/* bio->bi_status exists */
	/* #undef HAVE_BIO_BI_STATUS */

	/* bio has bi_iter */
	/* #undef HAVE_BIO_BVEC_ITER */

	/* bio__io_acct() available /
	/* #undef HAVE_BIO_IO_ACCT */

	/* bio_max_segs() is implemented */
	/* #undef HAVE_BIO_MAX_SEGS */

	/* bio_set_dev() is available */
	/* #undef HAVE_BIO_SET_DEV */

	/* bio_set_dev() GPL-only */
	/* #undef HAVE_BIO_SET_DEV_GPL_ONLY */

	/* bio_set_dev() is a macro */
	/* #undef HAVE_BIO_SET_DEV_MACRO */

	/* bio_set_op_attrs is available */
	/* #undef HAVE_BIO_SET_OP_ATTRS */

	/* blkdev_get_by_path() exists and takes 4 args */
	/* #undef HAVE_BLKDEV_GET_BY_PATH_4ARG */

	/* blkdev_get_by_path() handles ERESTARTSYS */
	/* #undef HAVE_BLKDEV_GET_ERESTARTSYS */

	-/* blkdev_issue_discard() is available */
	-/* #undef HAVE_BLKDEV_ISSUE_DISCARD */
	+/* __blkdev_issue_discard(flags) is available */
	+/* #undef HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS */

	/* __blkdev_issue_discard() is available */
	-/* #undef HAVE_BLKDEV_ISSUE_DISCARD_ASYNC */
	+/* #undef HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_NOFLAGS */
	+
	+/* blkdev_issue_discard(flags) is available */
	+/* #undef HAVE_BLKDEV_ISSUE_DISCARD_FLAGS */
	+
	+/* blkdev_issue_discard() is available */
	+/* #undef HAVE_BLKDEV_ISSUE_DISCARD_NOFLAGS */

	/* blkdev_issue_secure_erase() is available */
	/* #undef HAVE_BLKDEV_ISSUE_SECURE_ERASE */

	+/* blkdev_put() exists */
	+/* #undef HAVE_BLKDEV_PUT */
	+
	/* blkdev_put() accepts void* as arg 2 */
	/* #undef HAVE_BLKDEV_PUT_HOLDER */

	/* blkdev_reread_part() exists */
	/* #undef HAVE_BLKDEV_REREAD_PART */

	/* blkg_tryget() is available */
	/* #undef HAVE_BLKG_TRYGET */

	/* blkg_tryget() GPL-only */
	/* #undef HAVE_BLKG_TRYGET_GPL_ONLY */

	/* blk_alloc_disk() exists */
	/* #undef HAVE_BLK_ALLOC_DISK */

	+/* blk_alloc_disk() exists and takes 2 args */
	+/* #undef HAVE_BLK_ALLOC_DISK_2ARG */
	+
	/* blk_alloc_queue() expects request function */
	/* #undef HAVE_BLK_ALLOC_QUEUE_REQUEST_FN */

	/* blk_alloc_queue_rh() expects request function */
	/* #undef HAVE_BLK_ALLOC_QUEUE_REQUEST_FN_RH */

	/* blk_cleanup_disk() exists */
	/* #undef HAVE_BLK_CLEANUP_DISK */

	/* blk_mode_t is defined */
	/* #undef HAVE_BLK_MODE_T */

	/* block multiqueue is available */
	/* #undef HAVE_BLK_MQ */

	+/* block multiqueue hardware context is cached in struct request */
	+/* #undef HAVE_BLK_MQ_RQ_HCTX */
	+
	/* blk queue backing_dev_info is dynamic */
	/* #undef HAVE_BLK_QUEUE_BDI_DYNAMIC */

	/* blk_queue_discard() is available */
	/* #undef HAVE_BLK_QUEUE_DISCARD */

	/* blk_queue_flag_clear() exists */
	/* #undef HAVE_BLK_QUEUE_FLAG_CLEAR */

	/* blk_queue_flag_set() exists */
	/* #undef HAVE_BLK_QUEUE_FLAG_SET */

	/* blk_queue_flush() is available */
	/* #undef HAVE_BLK_QUEUE_FLUSH */

	/* blk_queue_flush() is GPL-only */
	/* #undef HAVE_BLK_QUEUE_FLUSH_GPL_ONLY */

	/* blk_queue_secdiscard() is available */
	/* #undef HAVE_BLK_QUEUE_SECDISCARD */

	/* blk_queue_secure_erase() is available */
	/* #undef HAVE_BLK_QUEUE_SECURE_ERASE */

	/* blk_queue_update_readahead() exists */
	/* #undef HAVE_BLK_QUEUE_UPDATE_READAHEAD */

	/* blk_queue_write_cache() exists */
	/* #undef HAVE_BLK_QUEUE_WRITE_CACHE */

	/* blk_queue_write_cache() is GPL-only */
	/* #undef HAVE_BLK_QUEUE_WRITE_CACHE_GPL_ONLY */

	/* BLK_STS_RESV_CONFLICT is defined */
	/* #undef HAVE_BLK_STS_RESV_CONFLICT */

	/* Define if release() in block_device_operations takes 1 arg */
	/* #undef HAVE_BLOCK_DEVICE_OPERATIONS_RELEASE_1ARG */

	/* Define if revalidate_disk() in block_device_operations */
	/* #undef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK */

	/* Define to 1 if you have the Mac OS X function CFLocaleCopyCurrent in the
	CoreFoundation framework. */
	/* #undef HAVE_CFLOCALECOPYCURRENT */

	/* Define to 1 if you have the Mac OS X function
	CFLocaleCopyPreferredLanguages in the CoreFoundation framework. */
	/* #undef HAVE_CFLOCALECOPYPREFERREDLANGUAGES */

	/* Define to 1 if you have the Mac OS X function CFPreferencesCopyAppValue in
	the CoreFoundation framework. */
	/* #undef HAVE_CFPREFERENCESCOPYAPPVALUE */

	/* check_disk_change() exists */
	/* #undef HAVE_CHECK_DISK_CHANGE */

	/* clear_inode() is available */
	/* #undef HAVE_CLEAR_INODE */

	/* dentry uses const struct dentry_operations */
	/* #undef HAVE_CONST_DENTRY_OPERATIONS */

	/* copy_from_iter() is available */
	/* #undef HAVE_COPY_FROM_ITER */

	/* copy_splice_read exists */
	/* #undef HAVE_COPY_SPLICE_READ */

	/* copy_to_iter() is available */
	/* #undef HAVE_COPY_TO_ITER */

	/* cpu_has_feature() is GPL-only */
	/* #undef HAVE_CPU_HAS_FEATURE_GPL_ONLY */

	/* yes */
	/* #undef HAVE_CPU_HOTPLUG */

	/* current_time() exists */
	/* #undef HAVE_CURRENT_TIME */

	/* Define if the GNU dcgettext() function is already present or preinstalled.
	*/
	/* #undef HAVE_DCGETTEXT */

	/* DECLARE_EVENT_CLASS() is available */
	/* #undef HAVE_DECLARE_EVENT_CLASS */

	/* dentry aliases are in d_u member */
	/* #undef HAVE_DENTRY_D_U_ALIASES */

	/* dequeue_signal() takes 4 arguments */
	/* #undef HAVE_DEQUEUE_SIGNAL_4ARG */

	/* lookup_bdev() wants dev_t arg */
	/* #undef HAVE_DEVT_LOOKUP_BDEV */

	/* sops->dirty_inode() wants flags */
	/* #undef HAVE_DIRTY_INODE_WITH_FLAGS */

	/* disk_check_media_change() exists */
	/* #undef HAVE_DISK_CHECK_MEDIA_CHANGE */

	/* disk__io_acct() available /
	/* #undef HAVE_DISK_IO_ACCT */

	/* disk_update_readahead() exists */
	/* #undef HAVE_DISK_UPDATE_READAHEAD */

	/* Define to 1 if you have the <dlfcn.h> header file. */
	#define HAVE_DLFCN_H 1

	/* d_make_root() is available */
	/* #undef HAVE_D_MAKE_ROOT */

	/* d_prune_aliases() is available */
	/* #undef HAVE_D_PRUNE_ALIASES */

	/* dops->d_revalidate() operation takes nameidata */
	/* #undef HAVE_D_REVALIDATE_NAMEIDATA */

	/* eops->encode_fh() wants child and parent inodes */
	/* #undef HAVE_ENCODE_FH_WITH_INODE */

	/* sops->evict_inode() exists */
	/* #undef HAVE_EVICT_INODE */

	/* FALLOC_FL_ZERO_RANGE is defined */
	/* #undef HAVE_FALLOC_FL_ZERO_RANGE */

	/* fault_in_iov_iter_readable() is available */
	/* #undef HAVE_FAULT_IN_IOV_ITER_READABLE */

	/* filemap_range_has_page() is available */
	/* #undef HAVE_FILEMAP_RANGE_HAS_PAGE */

	/* fops->aio_fsync() exists */
	/* #undef HAVE_FILE_AIO_FSYNC */

	/* file_dentry() is available */
	/* #undef HAVE_FILE_DENTRY */

	/* fops->fadvise() exists */
	/* #undef HAVE_FILE_FADVISE */

	/* file_inode() is available */
	/* #undef HAVE_FILE_INODE */

	/* flush_dcache_page() is GPL-only */
	/* #undef HAVE_FLUSH_DCACHE_PAGE_GPL_ONLY */

	/* iops->follow_link() cookie */
	/* #undef HAVE_FOLLOW_LINK_COOKIE */

	/* iops->follow_link() nameidata */
	/* #undef HAVE_FOLLOW_LINK_NAMEIDATA */

	/* Define if compiler supports -Wformat-overflow */
	/* #undef HAVE_FORMAT_OVERFLOW */

	/* fsync_bdev() is declared in include/blkdev.h */
	/* #undef HAVE_FSYNC_BDEV */

	/* fops->fsync() with range */
	/* #undef HAVE_FSYNC_RANGE */

	/* fops->fsync() without dentry */
	/* #undef HAVE_FSYNC_WITHOUT_DENTRY */

	/* yes */
	/* #undef HAVE_GENERIC_FADVISE */

	/* generic_fillattr requires struct mnt_idmap* */
	/* #undef HAVE_GENERIC_FILLATTR_IDMAP */

	/* generic_fillattr requires struct mnt_idmap* and u32 request_mask */
	/* #undef HAVE_GENERIC_FILLATTR_IDMAP_REQMASK */

	/* generic_fillattr requires struct user_namespace* */
	/* #undef HAVE_GENERIC_FILLATTR_USERNS */

	/* generic__io_acct() 3 arg available /
	/* #undef HAVE_GENERIC_IO_ACCT_3ARG */

	/* generic__io_acct() 4 arg available /
	/* #undef HAVE_GENERIC_IO_ACCT_4ARG */

	/* generic_readlink is global */
	/* #undef HAVE_GENERIC_READLINK */

	/* generic_setxattr() exists */
	/* #undef HAVE_GENERIC_SETXATTR */

	/* generic_write_checks() takes kiocb */
	/* #undef HAVE_GENERIC_WRITE_CHECKS_KIOCB */

	/* Define if the GNU gettext() function is already present or preinstalled. */
	/* #undef HAVE_GETTEXT */

	/* iops->get_acl() exists */
	/* #undef HAVE_GET_ACL */

	/* iops->get_acl() takes rcu */
	/* #undef HAVE_GET_ACL_RCU */

	/* has iops->get_inode_acl() */
	/* #undef HAVE_GET_INODE_ACL */

	/* iops->get_link() cookie */
	/* #undef HAVE_GET_LINK_COOKIE */

	/* iops->get_link() delayed */
	/* #undef HAVE_GET_LINK_DELAYED */

	/* group_info->gid exists */
	/* #undef HAVE_GROUP_INFO_GID */

	/* has_capability() is available */
	/* #undef HAVE_HAS_CAPABILITY */

	/* iattr->ia_vfsuid and iattr->ia_vfsgid exist */
	/* #undef HAVE_IATTR_VFSID */

	/* Define if you have the iconv() function and it works. */
	#define HAVE_ICONV 1

	/* iops->getattr() takes struct mnt_idmap* */
	/* #undef HAVE_IDMAP_IOPS_GETATTR */

	/* iops->setattr() takes struct mnt_idmap* */
	/* #undef HAVE_IDMAP_IOPS_SETATTR */

	/* APIs for idmapped mount are present */
	/* #undef HAVE_IDMAP_MNT_API */

	/* mnt_idmap does not have user_namespace */
	/* #undef HAVE_IDMAP_NO_USERNS */

	/* Define if compiler supports -Wimplicit-fallthrough */
	/* #undef HAVE_IMPLICIT_FALLTHROUGH */

	/* Define if compiler supports -Winfinite-recursion */
	/* #undef HAVE_INFINITE_RECURSION */

	/* inode_get_atime() exists in linux/fs.h */
	/* #undef HAVE_INODE_GET_ATIME */

	/* inode_get_ctime() exists in linux/fs.h */
	/* #undef HAVE_INODE_GET_CTIME */

	/* inode_get_mtime() exists in linux/fs.h */
	/* #undef HAVE_INODE_GET_MTIME */

	/* yes */
	/* #undef HAVE_INODE_LOCK_SHARED */

	/* inode_owner_or_capable() exists */
	/* #undef HAVE_INODE_OWNER_OR_CAPABLE */

	/* inode_owner_or_capable() takes mnt_idmap */
	/* #undef HAVE_INODE_OWNER_OR_CAPABLE_IDMAP */

	/* inode_owner_or_capable() takes user_ns */
	/* #undef HAVE_INODE_OWNER_OR_CAPABLE_USERNS */

	/* inode_set_atime_to_ts() exists in linux/fs.h */
	/* #undef HAVE_INODE_SET_ATIME_TO_TS */

	/* inode_set_ctime_to_ts() exists in linux/fs.h */
	/* #undef HAVE_INODE_SET_CTIME_TO_TS */

	/* inode_set_flags() exists */
	/* #undef HAVE_INODE_SET_FLAGS */

	/* inode_set_iversion() exists */
	/* #undef HAVE_INODE_SET_IVERSION */

	/* inode_set_mtime_to_ts() exists in linux/fs.h */
	/* #undef HAVE_INODE_SET_MTIME_TO_TS */

	/* inode->i_time's are timespec64 /
	/* #undef HAVE_INODE_TIMESPEC64_TIMES */

	/* timestamp_truncate() exists */
	/* #undef HAVE_INODE_TIMESTAMP_TRUNCATE */

	/* Define to 1 if you have the <inttypes.h> header file. */
	#define HAVE_INTTYPES_H 1

	/* in_compat_syscall() is available */
	/* #undef HAVE_IN_COMPAT_SYSCALL */

	/* iops->create() takes struct mnt_idmap* */
	/* #undef HAVE_IOPS_CREATE_IDMAP */

	/* iops->create() takes struct user_namespace* */
	/* #undef HAVE_IOPS_CREATE_USERNS */

	/* iops->mkdir() takes struct mnt_idmap* */
	/* #undef HAVE_IOPS_MKDIR_IDMAP */

	/* iops->mkdir() takes struct user_namespace* */
	/* #undef HAVE_IOPS_MKDIR_USERNS */

	/* iops->mknod() takes struct mnt_idmap* */
	/* #undef HAVE_IOPS_MKNOD_IDMAP */

	/* iops->mknod() takes struct user_namespace* */
	/* #undef HAVE_IOPS_MKNOD_USERNS */

	/* iops->permission() takes struct mnt_idmap* */
	/* #undef HAVE_IOPS_PERMISSION_IDMAP */

	/* iops->permission() takes struct user_namespace* */
	/* #undef HAVE_IOPS_PERMISSION_USERNS */

	/* iops->rename() takes struct mnt_idmap* */
	/* #undef HAVE_IOPS_RENAME_IDMAP */

	/* iops->rename() takes struct user_namespace* */
	/* #undef HAVE_IOPS_RENAME_USERNS */

	/* iops->setattr() exists */
	/* #undef HAVE_IOPS_SETATTR */

	/* iops->symlink() takes struct mnt_idmap* */
	/* #undef HAVE_IOPS_SYMLINK_IDMAP */

	/* iops->symlink() takes struct user_namespace* */
	/* #undef HAVE_IOPS_SYMLINK_USERNS */

	/* iov_iter_advance() is available */
	/* #undef HAVE_IOV_ITER_ADVANCE */

	/* iov_iter_count() is available */
	/* #undef HAVE_IOV_ITER_COUNT */

	/* iov_iter_fault_in_readable() is available */
	/* #undef HAVE_IOV_ITER_FAULT_IN_READABLE */

	/* iov_iter_revert() is available */
	/* #undef HAVE_IOV_ITER_REVERT */

	/* iov_iter_type() is available */
	/* #undef HAVE_IOV_ITER_TYPE */

	/* iov_iter types are available */
	/* #undef HAVE_IOV_ITER_TYPES */

	/* yes */
	/* #undef HAVE_IO_SCHEDULE_TIMEOUT */

	-/* Define to 1 if you have the `issetugid' function. */
	+/* Define to 1 if you have the 'issetugid' function. */
	#define HAVE_ISSETUGID 1

	/* iter_iov() is available */
	/* #undef HAVE_ITER_IOV */

	/* kernel has kernel_fpu_* functions */
	/* #undef HAVE_KERNEL_FPU */

	/* kernel has asm/fpu/api.h */
	/* #undef HAVE_KERNEL_FPU_API_HEADER */

	/* kernel fpu internal */
	/* #undef HAVE_KERNEL_FPU_INTERNAL */

	/* kernel has asm/fpu/internal.h */
	/* #undef HAVE_KERNEL_FPU_INTERNAL_HEADER */

	/* uncached_acl_sentinel() exists */
	/* #undef HAVE_KERNEL_GET_ACL_HANDLE_CACHE */

	/* Define if compiler supports -Winfinite-recursion */
	/* #undef HAVE_KERNEL_INFINITE_RECURSION */

	/* kernel has kernel_neon_* functions */
	/* #undef HAVE_KERNEL_NEON */

	/* kernel does stack verification */
	/* #undef HAVE_KERNEL_OBJTOOL */

	/* kernel has linux/objtool.h */
	/* #undef HAVE_KERNEL_OBJTOOL_HEADER */

	/* kernel_read() take loff_t pointer */
	/* #undef HAVE_KERNEL_READ_PPOS */

	/* strlcpy() exists */
	/* #undef HAVE_KERNEL_STRLCPY */

	/* strscpy() exists */
	/* #undef HAVE_KERNEL_STRSCPY */

	/* timer_list.function gets a timer_list */
	/* #undef HAVE_KERNEL_TIMER_FUNCTION_TIMER_LIST */

	/* struct timer_list has a flags member */
	/* #undef HAVE_KERNEL_TIMER_LIST_FLAGS */

	/* timer_setup() is available */
	/* #undef HAVE_KERNEL_TIMER_SETUP */

	/* kernel_write() take loff_t pointer */
	/* #undef HAVE_KERNEL_WRITE_PPOS */

	/* kmem_cache_create_usercopy() exists */
	/* #undef HAVE_KMEM_CACHE_CREATE_USERCOPY */

	/* kstrtoul() exists */
	/* #undef HAVE_KSTRTOUL */

	/* ktime_get_coarse_real_ts64() exists */
	/* #undef HAVE_KTIME_GET_COARSE_REAL_TS64 */

	/* ktime_get_raw_ts64() exists */
	/* #undef HAVE_KTIME_GET_RAW_TS64 */

	/* kvmalloc exists */
	/* #undef HAVE_KVMALLOC */

	/* Define if you have [aio] */
	/* #undef HAVE_LIBAIO */

	/* Define if you have [blkid] */
	/* #undef HAVE_LIBBLKID */

	/* Define if you have [crypto] */
	#define HAVE_LIBCRYPTO 1

	/* Define if you have [tirpc] */
	/* #undef HAVE_LIBTIRPC */

	/* Define if you have [udev] */
	/* #undef HAVE_LIBUDEV */

	/* Define if you have [uuid] */
	/* #undef HAVE_LIBUUID */

	/* linux/blk-cgroup.h exists */
	/* #undef HAVE_LINUX_BLK_CGROUP_HEADER */

	/* lseek_execute() is available */
	/* #undef HAVE_LSEEK_EXECUTE */

	/* makedev() is declared in sys/mkdev.h */
	/* #undef HAVE_MAKEDEV_IN_MKDEV */

	/* makedev() is declared in sys/sysmacros.h */
	/* #undef HAVE_MAKEDEV_IN_SYSMACROS */

	/* Noting that make_request_fn() returns blk_qc_t */
	/* #undef HAVE_MAKE_REQUEST_FN_RET_QC */

	/* Noting that make_request_fn() returns void */
	/* #undef HAVE_MAKE_REQUEST_FN_RET_VOID */

	/* iops->mkdir() takes umode_t */
	/* #undef HAVE_MKDIR_UMODE_T */

	-/* Define to 1 if you have the `mlockall' function. */
	+/* Define to 1 if you have the 'mlockall' function. */
	#define HAVE_MLOCKALL 1

	+/* page_size() is available */
	+/* #undef HAVE_MM_PAGE_SIZE */
	+
	/* lookup_bdev() wants mode arg */
	/* #undef HAVE_MODE_LOOKUP_BDEV */

	/* Define if host toolchain supports MOVBE */
	#define HAVE_MOVBE 1

	/* new_sync_read()/new_sync_write() are available */
	/* #undef HAVE_NEW_SYNC_READ */

	/* folio_wait_bit() exists */
	/* #undef HAVE_PAGEMAP_FOLIO_WAIT_BIT */

	/* part_to_dev() exists */
	/* #undef HAVE_PART_TO_DEV */

	/* iops->getattr() takes a path */
	/* #undef HAVE_PATH_IOPS_GETATTR */

	/* Define if host toolchain supports PCLMULQDQ */
	#define HAVE_PCLMULQDQ 1

	/* percpu_counter_add_batch() is defined */
	/* #undef HAVE_PERCPU_COUNTER_ADD_BATCH */

	/* percpu_counter_init() wants gfp_t */
	/* #undef HAVE_PERCPU_COUNTER_INIT_WITH_GFP */

	/* posix_acl_chmod() exists */
	/* #undef HAVE_POSIX_ACL_CHMOD */

	/* posix_acl_from_xattr() needs user_ns */
	/* #undef HAVE_POSIX_ACL_FROM_XATTR_USERNS */

	/* posix_acl_release() is available */
	/* #undef HAVE_POSIX_ACL_RELEASE */

	/* posix_acl_release() is GPL-only */
	/* #undef HAVE_POSIX_ACL_RELEASE_GPL_ONLY */

	/* posix_acl_valid() wants user namespace */
	/* #undef HAVE_POSIX_ACL_VALID_WITH_NS */

	/* proc_ops structure exists */
	/* #undef HAVE_PROC_OPS_STRUCT */

	/* iops->put_link() cookie */
	/* #undef HAVE_PUT_LINK_COOKIE */

	/* iops->put_link() delayed */
	/* #undef HAVE_PUT_LINK_DELAYED */

	/* iops->put_link() nameidata */
	/* #undef HAVE_PUT_LINK_NAMEIDATA */

	/* If available, contains the Python version number currently in use. */
	#define HAVE_PYTHON "3.7"

	/* qat is enabled and existed */
	/* #undef HAVE_QAT */

	/* struct reclaim_state has reclaimed */
	/* #undef HAVE_RECLAIM_STATE_RECLAIMED */

	/* register_shrinker is vararg */
	/* #undef HAVE_REGISTER_SHRINKER_VARARG */

	/* register_sysctl_table exists */
	/* #undef HAVE_REGISTER_SYSCTL_TABLE */

	/* iops->rename2() exists */
	/* #undef HAVE_RENAME2 */

	/* struct inode_operations_wrapper takes .rename2() */
	/* #undef HAVE_RENAME2_OPERATIONS_WRAPPER */

	/* iops->rename() wants flags */
	/* #undef HAVE_RENAME_WANTS_FLAGS */

	/* REQ_DISCARD is defined */
	/* #undef HAVE_REQ_DISCARD */

	/* REQ_FLUSH is defined */
	/* #undef HAVE_REQ_FLUSH */

	/* REQ_OP_DISCARD is defined */
	/* #undef HAVE_REQ_OP_DISCARD */

	/* REQ_OP_FLUSH is defined */
	/* #undef HAVE_REQ_OP_FLUSH */

	/* REQ_OP_SECURE_ERASE is defined */
	/* #undef HAVE_REQ_OP_SECURE_ERASE */

	/* REQ_PREFLUSH is defined */
	/* #undef HAVE_REQ_PREFLUSH */

	/* revalidate_disk() is available */
	/* #undef HAVE_REVALIDATE_DISK */

	/* revalidate_disk_size() is available */
	/* #undef HAVE_REVALIDATE_DISK_SIZE */

	/* struct rw_semaphore has member activity */
	/* #undef HAVE_RWSEM_ACTIVITY */

	/* struct rw_semaphore has atomic_long_t member count */
	/* #undef HAVE_RWSEM_ATOMIC_LONG_COUNT */

	/* linux/sched/signal.h exists */
	/* #undef HAVE_SCHED_SIGNAL_HEADER */

	/* Define to 1 if you have the <security/pam_modules.h> header file. */
	#define HAVE_SECURITY_PAM_MODULES_H 1

	/* setattr_prepare() accepts mnt_idmap */
	/* #undef HAVE_SETATTR_PREPARE_IDMAP */

	/* setattr_prepare() is available, doesn't accept user_namespace */
	/* #undef HAVE_SETATTR_PREPARE_NO_USERNS */

	/* setattr_prepare() accepts user_namespace */
	/* #undef HAVE_SETATTR_PREPARE_USERNS */

	/* iops->set_acl() exists, takes 3 args */
	/* #undef HAVE_SET_ACL */

	/* iops->set_acl() takes 4 args, arg1 is struct mnt_idmap * */
	/* #undef HAVE_SET_ACL_IDMAP_DENTRY */

	/* iops->set_acl() takes 4 args */
	/* #undef HAVE_SET_ACL_USERNS */

	/* iops->set_acl() takes 4 args, arg2 is struct dentry * */
	/* #undef HAVE_SET_ACL_USERNS_DENTRY_ARG2 */

	/* set_cached_acl() is usable */
	/* #undef HAVE_SET_CACHED_ACL_USABLE */

	/* set_special_state() exists */
	/* #undef HAVE_SET_SPECIAL_STATE */

	/* shrinker_register exists */
	/* #undef HAVE_SHRINKER_REGISTER */

	/* struct shrink_control exists */
	/* #undef HAVE_SHRINK_CONTROL_STRUCT */

	/* kernel_siginfo_t exists */
	/* #undef HAVE_SIGINFO */

	/* signal_stop() exists */
	/* #undef HAVE_SIGNAL_STOP */

	/* new shrinker callback wants 2 args */
	/* #undef HAVE_SINGLE_SHRINKER_CALLBACK */

	/* cs->count_objects exists */
	/* #undef HAVE_SPLIT_SHRINKER_CALLBACK */

	#if defined(__amd64__) \|\| defined(__i386__)
	/* Define if host toolchain supports SSE */
	#define HAVE_SSE 1

	/* Define if host toolchain supports SSE2 */
	#define HAVE_SSE2 1

	/* Define if host toolchain supports SSE3 */
	#define HAVE_SSE3 1

	/* Define if host toolchain supports SSE4.1 */
	#define HAVE_SSE4_1 1

	/* Define if host toolchain supports SSE4.2 */
	#define HAVE_SSE4_2 1

	/* Define if host toolchain supports SSSE3 */
	#define HAVE_SSSE3 1
	#endif

	/* STACK_FRAME_NON_STANDARD is defined */
	/* #undef HAVE_STACK_FRAME_NON_STANDARD */

	/* standalone <linux/stdarg.h> exists */
	/* #undef HAVE_STANDALONE_LINUX_STDARG */

	/* Define to 1 if you have the <stdint.h> header file. */
	#define HAVE_STDINT_H 1

	/* Define to 1 if you have the <stdio.h> header file. */
	#define HAVE_STDIO_H 1

	/* Define to 1 if you have the <stdlib.h> header file. */
	#define HAVE_STDLIB_H 1

	/* Define to 1 if you have the <strings.h> header file. */
	#define HAVE_STRINGS_H 1

	/* Define to 1 if you have the <string.h> header file. */
	#define HAVE_STRING_H 1

	-/* Define to 1 if you have the `strlcat' function. */
	+/* Define to 1 if you have the 'strlcat' function. */
	#define HAVE_STRLCAT 1

	-/* Define to 1 if you have the `strlcpy' function. */
	+/* Define to 1 if you have the 'strlcpy' function. */
	#define HAVE_STRLCPY 1

	/* submit_bio is member of struct block_device_operations */
	/* #undef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS */

	/* have super_block s_shrink */
	/* #undef HAVE_SUPER_BLOCK_S_SHRINK */

	/* have super_block s_shrink pointer */
	/* #undef HAVE_SUPER_BLOCK_S_SHRINK_PTR */

	/* super_setup_bdi_name() exits */
	/* #undef HAVE_SUPER_SETUP_BDI_NAME */

	/* super_block->s_user_ns exists */
	/* #undef HAVE_SUPER_USER_NS */

	/* sync_blockdev() is declared in include/blkdev.h */
	/* #undef HAVE_SYNC_BLOCKDEV */

	/* struct kobj_type has default_groups */
	/* #undef HAVE_SYSFS_DEFAULT_GROUPS */

	/* Define to 1 if you have the <sys/stat.h> header file. */
	#define HAVE_SYS_STAT_H 1

	/* Define to 1 if you have the <sys/types.h> header file. */
	#define HAVE_SYS_TYPES_H 1

	/* i_op->tmpfile() exists */
	/* #undef HAVE_TMPFILE */

	/* i_op->tmpfile() uses old dentry signature */
	/* #undef HAVE_TMPFILE_DENTRY */

	/* i_op->tmpfile() has mnt_idmap */
	/* #undef HAVE_TMPFILE_IDMAP */

	/* i_op->tmpfile() has userns */
	/* #undef HAVE_TMPFILE_USERNS */

	/* totalhigh_pages() exists */
	/* #undef HAVE_TOTALHIGH_PAGES */

	/* kernel has totalram_pages() */
	/* #undef HAVE_TOTALRAM_PAGES_FUNC */

	-/* Define to 1 if you have the `udev_device_get_is_initialized' function. */
	+/* Define to 1 if you have the 'udev_device_get_is_initialized' function. */
	/* #undef HAVE_UDEV_DEVICE_GET_IS_INITIALIZED */

	/* kernel has __kernel_fpu_* functions */
	/* #undef HAVE_UNDERSCORE_KERNEL_FPU */

	/* Define to 1 if you have the <unistd.h> header file. */
	#define HAVE_UNISTD_H 1

	/* iops->getattr() takes struct user_namespace* */
	/* #undef HAVE_USERNS_IOPS_GETATTR */

	/* iops->setattr() takes struct user_namespace* */
	/* #undef HAVE_USERNS_IOPS_SETATTR */

	/* user_namespace->ns.inum exists */
	/* #undef HAVE_USER_NS_COMMON_INUM */

	/* iops->getattr() takes a vfsmount */
	/* #undef HAVE_VFSMOUNT_IOPS_GETATTR */

	/* fops->clone_file_range() is available */
	/* #undef HAVE_VFS_CLONE_FILE_RANGE */

	/* fops->copy_file_range() is available */
	/* #undef HAVE_VFS_COPY_FILE_RANGE */

	/* fops->dedupe_file_range() is available */
	/* #undef HAVE_VFS_DEDUPE_FILE_RANGE */

	/* aops->direct_IO() uses iovec */
	/* #undef HAVE_VFS_DIRECT_IO_IOVEC */

	/* aops->direct_IO() uses iov_iter without rw */
	/* #undef HAVE_VFS_DIRECT_IO_ITER */

	/* aops->direct_IO() uses iov_iter with offset */
	/* #undef HAVE_VFS_DIRECT_IO_ITER_OFFSET */

	/* aops->direct_IO() uses iov_iter with rw and offset */
	/* #undef HAVE_VFS_DIRECT_IO_ITER_RW_OFFSET */

	/* filemap_dirty_folio exists */
	/* #undef HAVE_VFS_FILEMAP_DIRTY_FOLIO */

	/* file_operations_extend takes .copy_file_range() and .clone_file_range() */
	/* #undef HAVE_VFS_FILE_OPERATIONS_EXTEND */

	/* generic_copy_file_range() is available */
	/* #undef HAVE_VFS_GENERIC_COPY_FILE_RANGE */

	/* All required iov_iter interfaces are available */
	/* #undef HAVE_VFS_IOV_ITER */

	/* fops->iterate() is available */
	/* #undef HAVE_VFS_ITERATE */

	/* fops->iterate_shared() is available */
	/* #undef HAVE_VFS_ITERATE_SHARED */

	/* fops->readdir() is available */
	/* #undef HAVE_VFS_READDIR */

	/* address_space_operations->readpages exists */
	/* #undef HAVE_VFS_READPAGES */

	/* read_folio exists */
	/* #undef HAVE_VFS_READ_FOLIO */

	/* fops->remap_file_range() is available */
	/* #undef HAVE_VFS_REMAP_FILE_RANGE */

	/* fops->read/write_iter() are available */
	/* #undef HAVE_VFS_RW_ITERATE */

	/* __set_page_dirty_nobuffers exists */
	/* #undef HAVE_VFS_SET_PAGE_DIRTY_NOBUFFERS */

	+/* splice_copy_file_range() is available */
	+/* #undef HAVE_VFS_SPLICE_COPY_FILE_RANGE */
	+
	/* __vmalloc page flags exists */
	/* #undef HAVE_VMALLOC_PAGE_KERNEL */

	/* yes */
	/* #undef HAVE_WAIT_ON_BIT_ACTION */

	/* wait_queue_entry_t exists */
	/* #undef HAVE_WAIT_QUEUE_ENTRY_T */

	/* wq_head->head and wq_entry->entry exist */
	/* #undef HAVE_WAIT_QUEUE_HEAD_ENTRY */

	/* int (writepage_t)() takes struct folio */
	/* #undef HAVE_WRITEPAGE_T_FOLIO */

	/* xattr_handler->get() wants dentry */
	/* #undef HAVE_XATTR_GET_DENTRY */

	/* xattr_handler->get() wants both dentry and inode */
	/* #undef HAVE_XATTR_GET_DENTRY_INODE */

	/* xattr_handler->get() wants dentry and inode and flags */
	/* #undef HAVE_XATTR_GET_DENTRY_INODE_FLAGS */

	/* xattr_handler->get() wants xattr_handler */
	/* #undef HAVE_XATTR_GET_HANDLER */

	/* xattr_handler has name */
	/* #undef HAVE_XATTR_HANDLER_NAME */

	/* xattr_handler->list() wants dentry */
	/* #undef HAVE_XATTR_LIST_DENTRY */

	/* xattr_handler->list() wants xattr_handler */
	/* #undef HAVE_XATTR_LIST_HANDLER */

	/* xattr_handler->list() wants simple */
	/* #undef HAVE_XATTR_LIST_SIMPLE */

	/* xattr_handler->set() wants dentry */
	/* #undef HAVE_XATTR_SET_DENTRY */

	/* xattr_handler->set() wants both dentry and inode */
	/* #undef HAVE_XATTR_SET_DENTRY_INODE */

	/* xattr_handler->set() wants xattr_handler */
	/* #undef HAVE_XATTR_SET_HANDLER */

	/* xattr_handler->set() takes mnt_idmap */
	/* #undef HAVE_XATTR_SET_IDMAP */

	/* xattr_handler->set() takes user_namespace */
	/* #undef HAVE_XATTR_SET_USERNS */

	/* Define if host toolchain supports XSAVE */
	#define HAVE_XSAVE 1

	/* Define if host toolchain supports XSAVEOPT */
	#define HAVE_XSAVEOPT 1

	/* Define if host toolchain supports XSAVES */
	#define HAVE_XSAVES 1

	/* ZERO_PAGE() is GPL-only */
	/* #undef HAVE_ZERO_PAGE_GPL_ONLY */

	/* Define if you have [z] */
	#define HAVE_ZLIB 1

	/* __posix_acl_chmod() exists */
	/* #undef HAVE___POSIX_ACL_CHMOD */

	/* kernel exports FPU functions */
	/* #undef KERNEL_EXPORTS_X86_FPU */

	/* TBD: fetch(3) support */
	#if 0
	/* whether the chosen libfetch is to be loaded at run-time */
	#define LIBFETCH_DYNAMIC 1

	/* libfetch is fetch(3) */
	#define LIBFETCH_IS_FETCH 1

	/* libfetch is libcurl */
	#define LIBFETCH_IS_LIBCURL 0

	/* soname of chosen libfetch */
	#define LIBFETCH_SONAME "libfetch.so.6"
	#endif

	/* Define to the sub-directory where libtool stores uninstalled libraries. */
	#define LT_OBJDIR ".libs/"

	/* make_request_fn() return type */
	/* #undef MAKE_REQUEST_FN_RET */

	/* struct shrink_control has nid */
	/* #undef SHRINK_CONTROL_HAS_NID */

	/* using complete_and_exit() instead */
	/* #undef SPL_KTHREAD_COMPLETE_AND_EXIT */

	/* Defined for legacy compatibility. */
	#define SPL_META_ALIAS ZFS_META_ALIAS

	/* Defined for legacy compatibility. */
	#define SPL_META_RELEASE ZFS_META_RELEASE

	/* Defined for legacy compatibility. */
	#define SPL_META_VERSION ZFS_META_VERSION

	/* pde_data() is PDE_DATA() */
	/* #undef SPL_PDE_DATA */

	-/* Define to 1 if all of the C90 standard headers exist (not just the ones
	+/* Define to 1 if all of the C89 standard headers exist (not just the ones
	required in a freestanding environment). This macro is provided for
	backward compatibility; new code need not use it. */
	#define SYSTEM_FREEBSD 1

	/* True if ZFS is to be compiled for a Linux system */
	/* #undef SYSTEM_LINUX */

	/* Version number of package */
	/* #undef ZFS_DEBUG */

	/* /dev/zfs minor */
	/* #undef ZFS_DEVICE_MINOR */

	/* enum node_stat_item contains NR_FILE_PAGES */
	/* #undef ZFS_ENUM_NODE_STAT_ITEM_NR_FILE_PAGES */

	/* enum node_stat_item contains NR_INACTIVE_ANON */
	/* #undef ZFS_ENUM_NODE_STAT_ITEM_NR_INACTIVE_ANON */

	/* enum node_stat_item contains NR_INACTIVE_FILE */
	/* #undef ZFS_ENUM_NODE_STAT_ITEM_NR_INACTIVE_FILE */

	/* enum zone_stat_item contains NR_FILE_PAGES */
	/* #undef ZFS_ENUM_ZONE_STAT_ITEM_NR_FILE_PAGES */

	/* enum zone_stat_item contains NR_INACTIVE_ANON */
	/* #undef ZFS_ENUM_ZONE_STAT_ITEM_NR_INACTIVE_ANON */

	/* enum zone_stat_item contains NR_INACTIVE_FILE */
	/* #undef ZFS_ENUM_ZONE_STAT_ITEM_NR_INACTIVE_FILE */

	/* GENHD_FL_EXT_DEVT flag is not available */
	/* #undef ZFS_GENHD_FL_EXT_DEVT */

	/* GENHD_FL_NO_PART_SCAN flag is available */
	/* #undef ZFS_GENHD_FL_NO_PART */

	/* global_node_page_state() exists */
	/* #undef ZFS_GLOBAL_NODE_PAGE_STATE */

	/* global_zone_page_state() exists */
	/* #undef ZFS_GLOBAL_ZONE_PAGE_STATE */

	/* Define to 1 if GPL-only symbols can be used */
	/* #undef ZFS_IS_GPL_COMPATIBLE */

	/* Define the project alias string. */
	-#define ZFS_META_ALIAS "zfs-2.2.3-FreeBSD_gc883088df"
	+#define ZFS_META_ALIAS "zfs-2.2.4-FreeBSD_g256659204"

	/* Define the project author. */
	#define ZFS_META_AUTHOR "OpenZFS"

	/* Define the project release date. */
	/* #undef ZFS_META_DATA */

	/* Define the maximum compatible kernel version. */
	-#define ZFS_META_KVER_MAX "6.7"
	+#define ZFS_META_KVER_MAX "6.8"

	/* Define the minimum compatible kernel version. */
	#define ZFS_META_KVER_MIN "3.10"

	/* Define the project license. */
	#define ZFS_META_LICENSE "CDDL"

	/* Define the libtool library 'age' version information. */
	/* #undef ZFS_META_LT_AGE */

	/* Define the libtool library 'current' version information. */
	/* #undef ZFS_META_LT_CURRENT */

	/* Define the libtool library 'revision' version information. */
	/* #undef ZFS_META_LT_REVISION */

	/* Define the project name. */
	#define ZFS_META_NAME "zfs"

	/* Define the project release. */
	-#define ZFS_META_RELEASE "FreeBSD_gc883088df"
	+#define ZFS_META_RELEASE "FreeBSD_g256659204"

	/* Define the project version. */
	-#define ZFS_META_VERSION "2.2.3"
	+#define ZFS_META_VERSION "2.2.4"

	/* count is located in percpu_ref.data */
	/* #undef ZFS_PERCPU_REF_COUNT_IN_DATA */
	diff --git a/sys/modules/zfs/zfs_gitrev.h b/sys/modules/zfs/zfs_gitrev.h
	index 70b3c3310d8f..663c8a847fd9 100644
	--- a/sys/modules/zfs/zfs_gitrev.h
	+++ b/sys/modules/zfs/zfs_gitrev.h
	@@ -1 +1 @@
	-#define ZFS_META_GITREV "zfs-2.2.3-0-gc883088df"
	+#define ZFS_META_GITREV "zfs-2.2.4-0-g256659204"

File Metadata

Mime Type: application/octet-stream
Expires: Sun, Jun 30, 9:57 AM (2 d)
Storage Engine: chunks
Storage Format: Chunks
Storage Handle: WxJbnWIpenzR
Default Alt Text: (6 MB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions